diff --git a/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/blaze/BlazeQuerySuite.scala b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/blaze/BlazeQuerySuite.scala new file mode 100644 index 000000000..64d9aa098 --- /dev/null +++ b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/blaze/BlazeQuerySuite.scala @@ -0,0 +1,15 @@ +package org.apache.spark.sql.blaze + +import org.apache.spark.sql.Row + +class BlazeQuerySuite extends org.apache.spark.sql.QueryTest with BaseBlazeSQLSuite { + + test("test partition path has url encoded character") { + withTable("t1") { + sql("create table t1 using parquet PARTITIONED BY (part) as select 1 as c1, 2 as c2, 'test test' as part") + val df = sql("select * from t1") + checkAnswer(df, Seq(Row(1, 2, "test test"))) + } + } + +} diff --git a/spark-extension/src/main/java/org/apache/spark/sql/blaze/JniBridge.java b/spark-extension/src/main/java/org/apache/spark/sql/blaze/JniBridge.java index 881850b55..f705ad4f8 100644 --- a/spark-extension/src/main/java/org/apache/spark/sql/blaze/JniBridge.java +++ b/spark-extension/src/main/java/org/apache/spark/sql/blaze/JniBridge.java @@ -17,6 +17,7 @@ import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; +import java.net.URI; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.fs.FileSystem; @@ -85,11 +86,12 @@ public static boolean isDriverSide() { } public static FSDataInputWrapper openFileAsDataInputWrapper(FileSystem fs, String path) throws Exception { - return FSDataInputWrapper$.MODULE$.wrap(fs.open(new Path(path))); + // the path is a URI string, so we need to convert it to a URI object, ref: org.apache.spark.paths.SparkPath.toPath + return FSDataInputWrapper$.MODULE$.wrap(fs.open(new Path(new URI(path)))); } public static FSDataOutputWrapper createFileAsDataOutputWrapper(FileSystem fs, String path) throws Exception { - return FSDataOutputWrapper$.MODULE$.wrap(fs.create(new Path(path))); + return FSDataOutputWrapper$.MODULE$.wrap(fs.create(new Path(new URI(path)))); } private static final List directMXBeans =