Loading...

Type: Bug
Resolution: Cannot Reproduce
Priority: Critical - P2
Fix Version/s: None
Affects Version/s: 2.4.0
Component/s: API
Labels:
None

I met the same issue that were resolved in https://jira.mongodb.org/browse/SPARK-160

My versions are:

Spark: 2.4.1

Mongo: 4.0

Mongo Spark connector: org.mongodb.spark:mongo-spark-connector_2.11:2.4.0

>>> pyspark --version

Welcome to

____ __

/ _/ ___ _____/ /_

\ \/ _ \/ _ `/ __/ '/

/__/ ./_,// //_\ version 2.4.1

/_/

Using Scala version 2.11.12, OpenJDK 64-Bit Server VM, 1.8.0_151

Branch branch-2.4.1

Here is the code snippet:

df = spark.read.format("com.mongodb.spark.sql").options(uri="mongouri", database="db", collection="col").load()
df.printSchema()  # this works and can print intended results
df.show(1) # when doing show, it prints out the following error

Traceback (most recent call last):

File "<stdin>", line 1, in <module>

File "/usr/lib/spark-current/python/pyspark/sql/dataframe.py", line 378, in show

print(self._jdf.showString(n, 20, vertical))

File "/usr/lib/spark-current/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in _call_

File "/usr/lib/spark-current/python/pyspark/sql/utils.py", line 63, in deco

return f(*a, **kw)

File "/usr/lib/spark-current/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 328, in get_return_value

py4j.protocol.Py4JJavaError: An error occurred while calling o184.showString.

: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 2.0 failed 1 times, most recent failure: Lost task 0.0 in stage 2.0 (TID 2, localhost, executor driver): java.lang.NoClassDefFoundError: Could not initialize class com.mongodb.spark.sql.MapFunctions$isBsonNumber$

at com.mongodb.spark.sql.MapFunctions$.com$mongodb$spark$sql$MapFunctions$$convertToDataType(MapFunctions.scala:190)

at com.mongodb.spark.sql.MapFunctions$$anonfun$3.apply(MapFunctions.scala:39)

at com.mongodb.spark.sql.MapFunctions$$anonfun$3.apply(MapFunctions.scala:37)

at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)

at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)

at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)

at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)

at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:186)

at com.mongodb.spark.sql.MapFunctions$.documentToRow(MapFunctions.scala:37)

at com.mongodb.spark.sql.MongoRelation$$anonfun$buildScan$1.apply(MongoRelation.scala:58)

at scala.collection.Iterator$$anon$11.next(Iterator.scala:410)

at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)

at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)

at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$13$$anon$1.hasNext(WholeStageCodegenExec.scala:636)

at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:409)

at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:125)

at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)

at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:55)

at org.apache.spark.scheduler.Task.run(Task.scala:121)

at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403)

at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)

at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:

at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1893)

at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1881)

at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1880)

at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)

at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)

at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1880)

at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)

at scala.Option.foreach(Option.scala:257)

at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:926)

at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2114)

at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2063)

at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2052)

at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)

at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:737)

at org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)

at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)

at org.apache.spark.SparkContext.runJob(SparkContext.scala:2101)

at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:407)

at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)

at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3393)

at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2546)

at org.apache.spark.sql.Dataset$$anonfun$53.apply(Dataset.scala:3373)

at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:79)

at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:144)

at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:74)

at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3367)

at org.apache.spark.sql.Dataset.head(Dataset.scala:2546)

at org.apache.spark.sql.Dataset.take(Dataset.scala:2760)

at org.apache.spark.sql.Dataset.getRows(Dataset.scala:256)

at org.apache.spark.sql.Dataset.showString(Dataset.scala:293)

at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:498)

at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)

at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)

at py4j.Gateway.invoke(Gateway.java:282)

at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)

at py4j.commands.CallCommand.execute(CallCommand.java:79)

at py4j.GatewayConnection.run(GatewayConnection.java:238)

at java.lang.Thread.run(Thread.java:748)

Caused by: java.lang.NoClassDefFoundError: Could not initialize class com.mongodb.spark.sql.MapFunctions$isBsonNumber$