Skip to content

Commit 7e9247a

Browse files
yew1ebyew1eb
committed
[AURON #1956] Add initial compatibility support for Spark 4.1
1 parent 6a833df commit 7e9247a

51 files changed

Lines changed: 454 additions & 156 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/tpcds-reusable.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ jobs:
235235
if: steps.cache-spark-bin.outputs.cache-hit != 'true'
236236
run: |
237237
SPARK_PATH="spark/spark-${{ steps.get-dependency-version.outputs.sparkversion }}"
238-
if [ ${{ inputs.scalaver }} = "2.13" ]; then
238+
if [ ${{ inputs.scalaver }} = "2.13" && "${{ inputs.sparkver }}" != "spark-4.1" ]; then
239239
SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}-scala${{ inputs.scalaver }}.tgz"
240240
else
241241
SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}.tgz"

‎.github/workflows/tpcds.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,12 @@ jobs:
8787
javaver: '21'
8888
scalaver: '2.13'
8989
hadoop-profile: 'hadoop3'
90+
91+
test-spark-41-jdk21-scala-2-13:
92+
name: Test spark-4.1 JDK21 Scala-2.13
93+
uses: ./.github/workflows/tpcds-reusable.yml
94+
with:
95+
sparkver: spark-4.1
96+
javaver: '21'
97+
scalaver: '2.13'
98+
hadoop-profile: 'hadoop3'

‎auron-build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
# Define constants for supported component versions
3131
# -----------------------------------------------------------------------------
3232
SUPPORTED_OS_IMAGES=("centos7" "ubuntu24" "rockylinux8" "debian11" "azurelinux3")
33-
SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5")
33+
SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "4.1")
3434
SUPPORTED_SCALA_VERSIONS=("2.12" "2.13")
3535
SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6")
3636
# Currently only one supported version, but kept plural for consistency

‎auron-spark-ui/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,19 @@
3636
<artifactId>spark-sql_${scalaVersion}</artifactId>
3737
<scope>provided</scope>
3838
</dependency>
39+
<!-- Required for XML processing (Scala 2.12+ split module, compatible with Spark 4.x) -->
40+
<dependency>
41+
<groupId>org.scala-lang.modules</groupId>
42+
<artifactId>scala-xml_${scalaVersion}</artifactId>
43+
<version>${scala-xml.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.apache.auron</groupId>
48+
<artifactId>spark-version-annotation-macros_${scalaVersion}</artifactId>
49+
<version>${project.version}</version>
50+
<scope>compile</scope>
51+
</dependency>
3952
</dependencies>
4053
<build>
4154
<plugins>

‎auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,44 @@
1616
*/
1717
package org.apache.spark.sql.execution.ui
1818

19-
import javax.servlet.http.HttpServletRequest
20-
2119
import scala.xml.{Node, NodeSeq}
2220

2321
import org.apache.spark.internal.Logging
2422
import org.apache.spark.ui.{UIUtils, WebUIPage}
2523

24+
import org.apache.auron.sparkver
25+
2626
private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage("") with Logging {
2727

2828
private val sqlStore = parent.sqlStore
2929

30-
override def render(request: HttpServletRequest): Seq[Node] = {
30+
@sparkver("3.0 / 3.1 / 3.2 / 3.3 / 3.4 / 3.5")
31+
override def render(request: javax.servlet.http.HttpServletRequest): Seq[Node] = {
32+
val buildInfo = sqlStore.buildInfo()
33+
val infos =
34+
UIUtils.listingTable(propertyHeader, propertyRow, buildInfo.info, fixedWidth = true)
35+
val summary: NodeSeq =
36+
<div>
37+
<div>
38+
<span class="collapse-sql-properties collapse-table"
39+
onClick="collapseTable('collapse-sql-properties', 'sql-properties')">
40+
<h4>
41+
<span class="collapse-table-arrow arrow-open"></span>
42+
<a>Auron Build Information</a>
43+
</h4>
44+
</span>
45+
<div class="sql-properties collapsible-table">
46+
{infos}
47+
</div>
48+
</div>
49+
<br/>
50+
</div>
51+
52+
UIUtils.headerSparkPage(request, "Auron", summary, parent)
53+
}
54+
55+
@sparkver("4.1")
56+
override def render(request: jakarta.servlet.http.HttpServletRequest): Seq[Node] = {
3157
val buildInfo = sqlStore.buildInfo()
3258
val infos =
3359
UIUtils.listingTable(propertyHeader, propertyRow, buildInfo.info, fixedWidth = true)

‎common/src/test/scala/org/apache/auron/util/AuronTestUtils.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,5 @@ object AuronTestUtils {
2727
lazy val isSparkV33OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.3"
2828
lazy val isSparkV34OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.4"
2929
lazy val isSparkV35OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.5"
30+
lazy val isSparkV40OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "4.0"
3031
}

‎dev/auron-it/pom.xml‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,45 @@
331331
</properties>
332332
</profile>
333333

334+
<profile>
335+
<id>spark-4.1</id>
336+
<properties>
337+
<shimName>spark-4.1</shimName>
338+
<sparkVersion>4.1.1</sparkVersion>
339+
</properties>
340+
<build>
341+
<plugins>
342+
<plugin>
343+
<groupId>org.apache.maven.plugins</groupId>
344+
<artifactId>maven-enforcer-plugin</artifactId>
345+
<version>${maven-enforcer-plugin.version}</version>
346+
<executions>
347+
<execution>
348+
<id>spark41-enforce-java-scala-version</id>
349+
<goals>
350+
<goal>enforce</goal>
351+
</goals>
352+
<configuration>
353+
<rules>
354+
<!-- Spark 4.1 requires JDK 17+ and Scala 2.13.x -->
355+
<requireJavaVersion>
356+
<version>[17,)</version>
357+
<message>Spark 4.1 requires JDK 17 or higher. Current: ${java.version}</message>
358+
</requireJavaVersion>
359+
<requireProperty>
360+
<property>scalaLongVersion</property>
361+
<regex>2\.13\.\d+</regex>
362+
<regexMessage>Spark 4.1 requires Scala 2.13.x. Current: ${scalaLongVersion}</regexMessage>
363+
</requireProperty>
364+
</rules>
365+
</configuration>
366+
</execution>
367+
</executions>
368+
</plugin>
369+
</plugins>
370+
</build>
371+
</profile>
372+
334373
<profile>
335374
<id>scala-2.12</id>
336375
<properties>
@@ -343,7 +382,7 @@
343382
<id>scala-2.13</id>
344383
<properties>
345384
<scalaVersion>2.13</scalaVersion>
346-
<scalaLongVersion>2.13.13</scalaLongVersion>
385+
<scalaLongVersion>2.13.17</scalaLongVersion>
347386
</properties>
348387
</profile>
349388
</profiles>

‎dev/auron-it/src/main/resources/log4j2.properties‎

Lines changed: 11 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -15,54 +15,22 @@
1515
# limitations under the License.
1616
#
1717

18-
# Set everything to be logged to the file core/target/unit-tests.log
19-
rootLogger.level = warn
20-
rootLogger.appenderRef.stdout.ref = STDOUT
18+
rootLogger.level = info
19+
rootLogger.appenderRef.file.ref = File
20+
21+
#File Appender
22+
appender.file.type = File
23+
appender.file.name = File
24+
appender.file.fileName = target/unit-tests.log
25+
appender.file.layout.type = PatternLayout
26+
appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
2127

2228
#Console Appender
2329
appender.console.type = Console
2430
appender.console.name = STDOUT
2531
appender.console.target = SYSTEM_OUT
2632
appender.console.layout.type = PatternLayout
27-
appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n%ex
33+
appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %maxLen{%m}{512}%n%ex{8}%n
2834
appender.console.filter.threshold.type = ThresholdFilter
29-
appender.console.filter.threshold.level = debug
30-
31-
#File Appender
32-
#appender.file.type = File
33-
#appender.file.name = File
34-
#appender.file.fileName = target/unit-tests.log
35-
#appender.file.layout.type = PatternLayout
36-
#appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
37-
38-
# Set the logger level of File Appender to WARN
39-
# appender.file.filter.threshold.type = ThresholdFilter
40-
# appender.file.filter.threshold.level = info
41-
42-
# Some packages are noisy for no good reason.
43-
logger.parquet_recordreader.name = org.apache.parquet.hadoop.ParquetRecordReader
44-
logger.parquet_recordreader.additivity = false
45-
logger.parquet_recordreader.level = off
46-
47-
logger.parquet_outputcommitter.name = org.apache.parquet.hadoop.ParquetOutputCommitter
48-
logger.parquet_outputcommitter.additivity = false
49-
logger.parquet_outputcommitter.level = off
50-
51-
logger.hadoop_lazystruct.name = org.apache.hadoop.hive.serde2.lazy.LazyStruct
52-
logger.hadoop_lazystruct.additivity = false
53-
logger.hadoop_lazystruct.level = off
54-
55-
logger.hadoop_retryinghmshandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
56-
logger.hadoop_retryinghmshandler.additivity = false
57-
logger.hadoop_retryinghmshandler.level = off
58-
59-
logger.hive_metadata.name = hive.ql.metadata.Hive
60-
logger.hive_metadata.additivity = false
61-
logger.hive_metadata.level = off
62-
63-
# Parquet related logging
64-
logger.parquet1.name = org.apache.parquet.CorruptStatistics
65-
logger.parquet1.level = error
35+
appender.console.filter.threshold.level = warn
6636

67-
logger.parquet2.name = parquet.CorruptStatistics
68-
logger.parquet2.level = error

‎dev/auron-it/src/main/scala/org/apache/auron/integration/Main.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ object Main {
114114
|Spark Version: ${Shims.get.shimVersion}
115115
|Data: ${args.dataLocation}
116116
|Queries: [${args.queryFilter.mkString(", ")}] (${if (args.queryFilter.isEmpty)
117-
"all"
118-
else args.queryFilter.length} queries)
117+
"all"
118+
else args.queryFilter.length} queries)
119119
|Extra Spark Conf: ${args.extraSparkConf}""".stripMargin)
120120

121121
if (args.auronOnly) println("Mode: Auron-only (skip baseline)")

‎dev/auron-it/src/main/scala/org/apache/auron/integration/SessionManager.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class SessionManager(val extraSparkConf: Map[String, String]) {
3535
private lazy val commonConf: Map[String, String] = Map(
3636
"spark.master" -> resolveMaster(),
3737
"spark.sql.shuffle.partitions" -> "100",
38+
"spark.sql.unionOutputPartitioning" -> "false",
3839
"spark.ui.enabled" -> "false",
3940
"spark.sql.sources.useV1SourceList" -> "parquet",
4041
"spark.sql.autoBroadcastJoinThreshold" -> "-1")
@@ -77,7 +78,7 @@ class SessionManager(val extraSparkConf: Map[String, String]) {
7778
.appName(appName)
7879
.config(conf)
7980
.getOrCreate()
80-
session.sparkContext.setLogLevel("WARN")
81+
// session.sparkContext.setLogLevel("WARN")
8182
session
8283
}
8384

0 commit comments

Comments
 (0)