dennybaa
9/8/2016 - 9:30 AM
foo.json
{
"Event": "SparkListenerTaskEnd",
"Stage ID": 40,
"Stage Attempt ID": 0,
"Task Type": "ShuffleMapTask",
"Task End Reason": {
"Reason": "ExceptionFailure",
"Class Name": "java.lang.OutOfMemoryError",
"Description": "Java heap space",
"Stack Trace": [{
"Declaring Class": "java.util.IdentityHashMap",
"Method Name": "resize",
"File Name": "IdentityHashMap.java",
"Line Number": 471
}, {
"Declaring Class": "java.util.IdentityHashMap",
"Method Name": "put",
"File Name": "IdentityHashMap.java",
"Line Number": 440
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$SearchState",
"Method Name": "enqueue",
"File Name": "SizeEstimator.scala",
"Line Number": 176
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$$anonfun$visitSingleObject$1",
"Method Name": "apply",
"File Name": "SizeEstimator.scala",
"Line Number": 224
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$$anonfun$visitSingleObject$1",
"Method Name": "apply",
"File Name": "SizeEstimator.scala",
"Line Number": 223
}, {
"Declaring Class": "scala.collection.immutable.List",
"Method Name": "foreach",
"File Name": "List.scala",
"Line Number": 318
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$",
"Method Name": "visitSingleObject",
"File Name": "SizeEstimator.scala",
"Line Number": 223
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$",
"Method Name": "org$apache$spark$util$SizeEstimator$$estimate",
"File Name": "SizeEstimator.scala",
"Line Number": 203
}, {
"Declaring Class": "org.apache.spark.util.SizeEstimator$",
"Method Name": "estimate",
"File Name": "SizeEstimator.scala",
"Line Number": 70
}, {
"Declaring Class": "org.apache.spark.util.collection.SizeTracker$class",
"Method Name": "takeSample",
"File Name": "SizeTracker.scala",
"Line Number": 78
}, {
"Declaring Class": "org.apache.spark.util.collection.SizeTracker$class",
"Method Name": "afterUpdate",
"File Name": "SizeTracker.scala",
"Line Number": 70
}, {
"Declaring Class": "org.apache.spark.util.collection.SizeTrackingVector",
"Method Name": "$plus$eq",
"File Name": "SizeTrackingVector.scala",
"Line Number": 31
}, {
"Declaring Class": "org.apache.spark.storage.MemoryStore",
"Method Name": "unrollSafely",
"File Name": "MemoryStore.scala",
"Line Number": 285
}, {
"Declaring Class": "org.apache.spark.storage.MemoryStore",
"Method Name": "putIterator",
"File Name": "MemoryStore.scala",
"Line Number": 169
}, {
"Declaring Class": "org.apache.spark.storage.MemoryStore",
"Method Name": "putIterator",
"File Name": "MemoryStore.scala",
"Line Number": 147
}, {
"Declaring Class": "org.apache.spark.storage.BlockManager",
"Method Name": "doPut",
"File Name": "BlockManager.scala",
"Line Number": 808
}, {
"Declaring Class": "org.apache.spark.storage.BlockManager",
"Method Name": "putIterator",
"File Name": "BlockManager.scala",
"Line Number": 655
}, {
"Declaring Class": "org.apache.spark.storage.BlockManager",
"Method Name": "putSingle",
"File Name": "BlockManager.scala",
"Line Number": 1013
}, {
"Declaring Class": "org.apache.spark.broadcast.TorrentBroadcast$$anonfun$readBroadcastBlock$1",
"Method Name": "apply",
"File Name": "TorrentBroadcast.scala",
"Line Number": 182
}, {
"Declaring Class": "org.apache.spark.util.Utils$",
"Method Name": "tryOrIOException",
"File Name": "Utils.scala",
"Line Number": 1205
}, {
"Declaring Class": "org.apache.spark.broadcast.TorrentBroadcast",
"Method Name": "readBroadcastBlock",
"File Name": "TorrentBroadcast.scala",
"Line Number": 165
}, {
"Declaring Class": "org.apache.spark.broadcast.TorrentBroadcast",
"Method Name": "_value$lzycompute",
"File Name": "TorrentBroadcast.scala",
"Line Number": 64
}, {
"Declaring Class": "org.apache.spark.broadcast.TorrentBroadcast",
"Method Name": "_value",
"File Name": "TorrentBroadcast.scala",
"Line Number": 64
}, {
"Declaring Class": "org.apache.spark.broadcast.TorrentBroadcast",
"Method Name": "getValue",
"File Name": "TorrentBroadcast.scala",
"Line Number": 88
}, {
"Declaring Class": "org.apache.spark.broadcast.Broadcast",
"Method Name": "value",
"File Name": "Broadcast.scala",
"Line Number": 70
}, {
"Declaring Class": "org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark$$anonfun$5",
"Method Name": "apply",
"File Name": "IndexedDatasetSpark.scala",
"Line Number": 93
}, {
"Declaring Class": "org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark$$anonfun$5",
"Method Name": "apply",
"File Name": "IndexedDatasetSpark.scala",
"Line Number": 92
}, {
"Declaring Class": "scala.collection.Iterator$$anon$11",
"Method Name": "next",
"File Name": "Iterator.scala",
"Line Number": 328
}, {
"Declaring Class": "org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter",
"Method Name": "write",
"File Name": "BypassMergeSortShuffleWriter.java",
"Line Number": 149
}, {
"Declaring Class": "org.apache.spark.scheduler.ShuffleMapTask",
"Method Name": "runTask",
"File Name": "ShuffleMapTask.scala",
"Line Number": 73
}, {
"Declaring Class": "org.apache.spark.scheduler.ShuffleMapTask",
"Method Name": "runTask",
"File Name": "ShuffleMapTask.scala",
"Line Number": 41
}, {
"Declaring Class": "org.apache.spark.scheduler.Task",
"Method Name": "run",
"File Name": "Task.scala",
"Line Number": 89
}],
"Full Stack Trace": "java.lang.OutOfMemoryError: Java heap space\n\tat java.util.IdentityHashMap.resize(IdentityHashMap.java:471)\n\tat java.util.IdentityHashMap.put(IdentityHashMap.java:440)\n\tat org.apache.spark.util.SizeEstimator$SearchState.enqueue(SizeEstimator.scala:176)\n\tat org.apache.spark.util.SizeEstimator$$anonfun$visitSingleObject$1.apply(SizeEstimator.scala:224)\n\tat org.apache.spark.util.SizeEstimator$$anonfun$visitSingleObject$1.apply(SizeEstimator.scala:223)\n\tat scala.collection.immutable.List.foreach(List.scala:318)\n\tat org.apache.spark.util.SizeEstimator$.visitSingleObject(SizeEstimator.scala:223)\n\tat org.apache.spark.util.SizeEstimator$.org$apache$spark$util$SizeEstimator$$estimate(SizeEstimator.scala:203)\n\tat org.apache.spark.util.SizeEstimator$.estimate(SizeEstimator.scala:70)\n\tat org.apache.spark.util.collection.SizeTracker$class.takeSample(SizeTracker.scala:78)\n\tat org.apache.spark.util.collection.SizeTracker$class.afterUpdate(SizeTracker.scala:70)\n\tat org.apache.spark.util.collection.SizeTrackingVector.$plus$eq(SizeTrackingVector.scala:31)\n\tat org.apache.spark.storage.MemoryStore.unrollSafely(MemoryStore.scala:285)\n\tat org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:169)\n\tat org.apache.spark.storage.MemoryStore.putIterator(MemoryStore.scala:147)\n\tat org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:808)\n\tat org.apache.spark.storage.BlockManager.putIterator(BlockManager.scala:655)\n\tat org.apache.spark.storage.BlockManager.putSingle(BlockManager.scala:1013)\n\tat org.apache.spark.broadcast.TorrentBroadcast$$anonfun$readBroadcastBlock$1.apply(TorrentBroadcast.scala:182)\n\tat org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1205)\n\tat org.apache.spark.broadcast.TorrentBroadcast.readBroadcastBlock(TorrentBroadcast.scala:165)\n\tat org.apache.spark.broadcast.TorrentBroadcast._value$lzycompute(TorrentBroadcast.scala:64)\n\tat org.apache.spark.broadcast.TorrentBroadcast._value(TorrentBroadcast.scala:64)\n\tat org.apache.spark.broadcast.TorrentBroadcast.getValue(TorrentBroadcast.scala:88)\n\tat org.apache.spark.broadcast.Broadcast.value(Broadcast.scala:70)\n\tat org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark$$anonfun$5.apply(IndexedDatasetSpark.scala:93)\n\tat org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark$$anonfun$5.apply(IndexedDatasetSpark.scala:92)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:149)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)\n\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:89)\n",
"Metrics": {
"Host Name": "ip-172-16-2-22.eu-central-1.compute.internal",
"Executor Deserialize Time": 0,
"Executor Run Time": 75266,
"Result Size": 0,
"JVM GC Time": 62931,
"Result Serialization Time": 0,
"Memory Bytes Spilled": 0,
"Disk Bytes Spilled": 0,
"Shuffle Write Metrics": {
"Shuffle Bytes Written": 0,
"Shuffle Write Time": 1512258,
"Shuffle Records Written": 0
},
"Input Metrics": {
"Data Read Method": "Memory",
"Bytes Read": 1121470368,
"Records Read": 1
}
}
},
"Task Info": {
"Task ID": 64,
"Index": 1,
"Attempt": 0,
"Launch Time": 1473267164711,
"Executor ID": "2",
"Host": "ip-172-16-2-22.eu-central-1.compute.internal",
"Locality": "PROCESS_LOCAL",
"Speculative": false,
"Getting Result Time": 0,
"Finish Time": 1473267240007,
"Failed": true,
"Accumulables": []
},
"Task Metrics": {
"Host Name": "ip-172-16-2-22.eu-central-1.compute.internal",
"Executor Deserialize Time": 0,
"Executor Run Time": 75266,
"Result Size": 0,
"JVM GC Time": 62931,
"Result Serialization Time": 0,
"Memory Bytes Spilled": 0,
"Disk Bytes Spilled": 0,
"Shuffle Write Metrics": {
"Shuffle Bytes Written": 0,
"Shuffle Write Time": 1512258,
"Shuffle Records Written": 0
},
"Input Metrics": {
"Data Read Method": "Memory",
"Bytes Read": 1121470368,
"Records Read": 1
}
}
}