Skip to content

Commit ebdd43d

Browse files
yew1ebyew1eb
committed
[AURON #1956] Add initial compatibility support for Spark 4.1
1 parent 6a833df commit ebdd43d

File tree

48 files changed

+441
-109
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+441
-109
lines changed

‎.github/workflows/tpcds-reusable.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ jobs:
235235
if: steps.cache-spark-bin.outputs.cache-hit != 'true'
236236
run: |
237237
SPARK_PATH="spark/spark-${{ steps.get-dependency-version.outputs.sparkversion }}"
238-
if [ ${{ inputs.scalaver }} = "2.13" ]; then
238+
if [ ${{ inputs.scalaver }} = "2.13" && "${{ inputs.sparkver }}" != "spark-4.1" ]; then
239239
SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}-scala${{ inputs.scalaver }}.tgz"
240240
else
241241
SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}.tgz"

‎.github/workflows/tpcds.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,12 @@ jobs:
8787
javaver: '21'
8888
scalaver: '2.13'
8989
hadoop-profile: 'hadoop3'
90+
91+
test-spark-41-jdk21-scala-2-13:
92+
name: Test spark-4.1 JDK21 Scala-2.13
93+
uses: ./.github/workflows/tpcds-reusable.yml
94+
with:
95+
sparkver: spark-4.1
96+
javaver: '21'
97+
scalaver: '2.13'
98+
hadoop-profile: 'hadoop3'

‎auron-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
# Define constants for supported component versions
3131
# -----------------------------------------------------------------------------
3232
SUPPORTED_OS_IMAGES=("centos7" "ubuntu24" "rockylinux8" "debian11" "azurelinux3")
33-
SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5")
33+
SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "4.1")
3434
SUPPORTED_SCALA_VERSIONS=("2.12" "2.13")
3535
SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6")
3636
# Currently only one supported version, but kept plural for consistency

‎auron-spark-ui/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,19 @@
3636
<artifactId>spark-sql_${scalaVersion}</artifactId>
3737
<scope>provided</scope>
3838
</dependency>
39+
<!-- Required for XML processing (Scala 2.12+ split module, compatible with Spark 4.x) -->
40+
<dependency>
41+
<groupId>org.scala-lang.modules</groupId>
42+
<artifactId>scala-xml_${scalaVersion}</artifactId>
43+
<version>${scala-xml.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.apache.auron</groupId>
48+
<artifactId>spark-version-annotation-macros_${scalaVersion}</artifactId>
49+
<version>${project.version}</version>
50+
<scope>compile</scope>
51+
</dependency>
3952
</dependencies>
4053
<build>
4154
<plugins>

‎auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,44 @@
1616
*/
1717
package org.apache.spark.sql.execution.ui
1818

19-
import javax.servlet.http.HttpServletRequest
20-
2119
import scala.xml.{Node, NodeSeq}
2220

2321
import org.apache.spark.internal.Logging
2422
import org.apache.spark.ui.{UIUtils, WebUIPage}
2523

24+
import org.apache.auron.sparkver
25+
2626
private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage("") with Logging {
2727

2828
private val sqlStore = parent.sqlStore
2929

30-
override def render(request: HttpServletRequest): Seq[Node] = {
30+
@sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
31+
override def render(request: javax.servlet.http.HttpServletRequest): Seq[Node] = {
32+
val buildInfo = sqlStore.buildInfo()
33+
val infos =
34+
UIUtils.listingTable(propertyHeader, propertyRow, buildInfo.info, fixedWidth = true)
35+
val summary: NodeSeq =
36+
<div>
37+
<div>
38+
<span class="collapse-sql-properties collapse-table"
39+
onClick="collapseTable('collapse-sql-properties', 'sql-properties')">
40+
<h4>
41+
<span class="collapse-table-arrow arrow-open"></span>
42+
<a>Auron Build Information</a>
43+
</h4>
44+
</span>
45+
<div class="sql-properties collapsible-table">
46+
{infos}
47+
</div>
48+
</div>
49+
<br/>
50+
</div>
51+
52+
UIUtils.headerSparkPage(request, "Auron", summary, parent)
53+
}
54+
55+
@sparkver("4.1")
56+
override def render(request: jakarta.servlet.http.HttpServletRequest): Seq[Node] = {
3157
val buildInfo = sqlStore.buildInfo()
3258
val infos =
3359
UIUtils.listingTable(propertyHeader, propertyRow, buildInfo.info, fixedWidth = true)

‎dev/auron-it/pom.xml‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,45 @@
331331
</properties>
332332
</profile>
333333

334+
<profile>
335+
<id>spark-4.1</id>
336+
<properties>
337+
<shimName>spark-4.1</shimName>
338+
<sparkVersion>4.1.1</sparkVersion>
339+
</properties>
340+
<build>
341+
<plugins>
342+
<plugin>
343+
<groupId>org.apache.maven.plugins</groupId>
344+
<artifactId>maven-enforcer-plugin</artifactId>
345+
<version>${maven-enforcer-plugin.version}</version>
346+
<executions>
347+
<execution>
348+
<id>spark41-enforce-java-scala-version</id>
349+
<goals>
350+
<goal>enforce</goal>
351+
</goals>
352+
<configuration>
353+
<rules>
354+
<!-- Spark 4.1 requires JDK 17+ and Scala 2.13.x -->
355+
<requireJavaVersion>
356+
<version>[17,)</version>
357+
<message>Spark 4.1 requires JDK 17 or higher. Current: ${java.version}</message>
358+
</requireJavaVersion>
359+
<requireProperty>
360+
<property>scalaLongVersion</property>
361+
<regex>2\.13\.\d+</regex>
362+
<regexMessage>Spark 4.1 requires Scala 2.13.x. Current: ${scalaLongVersion}</regexMessage>
363+
</requireProperty>
364+
</rules>
365+
</configuration>
366+
</execution>
367+
</executions>
368+
</plugin>
369+
</plugins>
370+
</build>
371+
</profile>
372+
334373
<profile>
335374
<id>scala-2.12</id>
336375
<properties>
@@ -343,7 +382,7 @@
343382
<id>scala-2.13</id>
344383
<properties>
345384
<scalaVersion>2.13</scalaVersion>
346-
<scalaLongVersion>2.13.13</scalaLongVersion>
385+
<scalaLongVersion>2.13.17</scalaLongVersion>
347386
</properties>
348387
</profile>
349388
</profiles>

‎dev/auron-it/src/main/scala/org/apache/auron/integration/SessionManager.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ class SessionManager(val extraSparkConf: Map[String, String]) {
3535
private lazy val commonConf: Map[String, String] = Map(
3636
"spark.master" -> resolveMaster(),
3737
"spark.sql.shuffle.partitions" -> "100",
38+
"spark.sql.unionOutputPartitioning" -> "false",
39+
// Avoid the code size overflow error in Spark code generation.
40+
"spark.sql.codegen.wholeStage" -> "false",
3841
"spark.ui.enabled" -> "false",
3942
"spark.sql.sources.useV1SourceList" -> "parquet",
4043
"spark.sql.autoBroadcastJoinThreshold" -> "-1")

‎pom.xml‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@
5454
<protobufVersion>3.25.5</protobufVersion>
5555
<nettyVersion>4.2.7.Final</nettyVersion>
5656
<javaVersion>8</javaVersion>
57+
<maven.compiler.source>${javaVersion}</maven.compiler.source>
58+
<maven.compiler.target>${javaVersion}</maven.compiler.target>
5759
<scalaVersion>2.12</scalaVersion>
5860
<scalaLongVersion>2.12.18</scalaLongVersion>
61+
<scala-xml.version>2.1.0</scala-xml.version>
5962
<scalaJava8CompatVersion>1.0.2</scalaJava8CompatVersion>
6063
<maven.version>3.9.12</maven.version>
6164
<maven.plugin.scala.version>4.9.2</maven.plugin.scala.version>
@@ -462,6 +465,8 @@
462465
<version>${maven.plugin.surefire.version}</version>
463466
<!-- Note config is repeated in scalatest config -->
464467
<configuration>
468+
<forkCount>1</forkCount>
469+
<reuseForks>false</reuseForks>
465470
<skipTests>false</skipTests>
466471
<failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
467472
<argLine>${extraJavaTestArgs}</argLine>
@@ -483,6 +488,7 @@
483488
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
484489
<junitxml>.</junitxml>
485490
<filereports>TestSuite.txt</filereports>
491+
<forkMode>once</forkMode>
486492
<argLine>${extraJavaTestArgs}</argLine>
487493
<environmentVariables />
488494
<systemProperties>
@@ -824,6 +830,48 @@
824830
</properties>
825831
</profile>
826832

833+
<profile>
834+
<id>spark-4.1</id>
835+
<properties>
836+
<shimName>spark-4.1</shimName>
837+
<scalaTestVersion>3.2.9</scalaTestVersion>
838+
<sparkVersion>4.1.1</sparkVersion>
839+
<shortSparkVersion>4.1</shortSparkVersion>
840+
<nettyVersion>4.1.118.Final</nettyVersion>
841+
</properties>
842+
<build>
843+
<plugins>
844+
<plugin>
845+
<groupId>org.apache.maven.plugins</groupId>
846+
<artifactId>maven-enforcer-plugin</artifactId>
847+
<version>${maven-enforcer-plugin.version}</version>
848+
<executions>
849+
<execution>
850+
<id>spark41-enforce-java-scala-version</id>
851+
<goals>
852+
<goal>enforce</goal>
853+
</goals>
854+
<configuration>
855+
<rules>
856+
<!-- Spark 4.1 requires JDK 17+ and Scala 2.13.x -->
857+
<requireJavaVersion>
858+
<version>[17,)</version>
859+
<message>Spark 4.1 requires JDK 17 or higher. Current: ${java.version}</message>
860+
</requireJavaVersion>
861+
<requireProperty>
862+
<property>scalaLongVersion</property>
863+
<regex>2\.13\.\d+</regex>
864+
<regexMessage>Spark 4.1 requires Scala 2.13.x. Current: ${scalaLongVersion}</regexMessage>
865+
</requireProperty>
866+
</rules>
867+
</configuration>
868+
</execution>
869+
</executions>
870+
</plugin>
871+
</plugins>
872+
</build>
873+
</profile>
874+
827875
<profile>
828876
<id>jdk-8</id>
829877
<activation>
@@ -832,7 +880,7 @@
832880
<properties>
833881
<javaVersion>8</javaVersion>
834882
<spotless.plugin.version>2.30.0</spotless.plugin.version>
835-
<semanticdb.version>4.8.8</semanticdb.version>
883+
<semanticdb.version>4.14.5</semanticdb.version>
836884
<scalafmtVersion>3.0.0</scalafmtVersion>
837885
</properties>
838886
</profile>
@@ -845,7 +893,7 @@
845893
<properties>
846894
<javaVersion>11</javaVersion>
847895
<spotless.plugin.version>2.30.0</spotless.plugin.version>
848-
<semanticdb.version>4.8.8</semanticdb.version>
896+
<semanticdb.version>4.14.5</semanticdb.version>
849897
<scalafmtVersion>3.0.0</scalafmtVersion>
850898
</properties>
851899
</profile>
@@ -858,7 +906,7 @@
858906
<properties>
859907
<javaVersion>17</javaVersion>
860908
<spotless.plugin.version>2.45.0</spotless.plugin.version>
861-
<semanticdb.version>4.9.9</semanticdb.version>
909+
<semanticdb.version>4.14.5</semanticdb.version>
862910
<scalafmtVersion>3.9.9</scalafmtVersion>
863911
</properties>
864912
</profile>
@@ -871,7 +919,7 @@
871919
<properties>
872920
<javaVersion>21</javaVersion>
873921
<spotless.plugin.version>2.45.0</spotless.plugin.version>
874-
<semanticdb.version>4.9.9</semanticdb.version>
922+
<semanticdb.version>4.14.5</semanticdb.version>
875923
<scalafmtVersion>3.9.9</scalafmtVersion>
876924
</properties>
877925
</profile>
@@ -918,7 +966,7 @@
918966
</activation>
919967
<properties>
920968
<scalaVersion>2.13</scalaVersion>
921-
<scalaLongVersion>2.13.13</scalaLongVersion>
969+
<scalaLongVersion>2.13.17</scalaLongVersion>
922970
</properties>
923971
<build>
924972
<plugins>

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.auron.sparkver
2323

2424
object InterceptedValidateSparkPlan extends Logging {
2525

26-
@sparkver("3.2 / 3.3 / 3.4 / 3.5")
26+
@sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.1")
2727
def validate(plan: SparkPlan): Unit = {
2828
import org.apache.spark.sql.execution.adaptive.BroadcastQueryStageExec
2929
import org.apache.spark.sql.execution.auron.plan.NativeRenameColumnsBase
@@ -76,7 +76,7 @@ object InterceptedValidateSparkPlan extends Logging {
7676
throw new UnsupportedOperationException("validate is not supported in spark 3.0.3 or 3.1.3")
7777
}
7878

79-
@sparkver("3.2 / 3.3 / 3.4 / 3.5")
79+
@sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.1")
8080
private def errorOnInvalidBroadcastQueryStage(plan: SparkPlan): Unit = {
8181
import org.apache.spark.sql.execution.adaptive.InvalidAQEPlanException
8282
throw InvalidAQEPlanException("Invalid broadcast query stage", plan)

0 commit comments

Comments
 (0)