Skip to content

Commit 5241dec

Browse files
yew1ebyew1eb
committed
up
1 parent 9938629 commit 5241dec

18 files changed

Lines changed: 59 additions & 37 deletions

File tree

‎pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
<maven.plugin.templating.version>3.0.0</maven.plugin.templating.version>
7070
<maven-enforcer-plugin.version>3.5.0</maven-enforcer-plugin.version>
7171

72+
<silencer-plugin.version>1.7.19</silencer-plugin.version>
7273
<scalamacros.paradise.version>2.1.1</scalamacros.paradise.version>
7374
<junit.jupiter.version>5.13.4</junit.jupiter.version>
7475
<restrict-imports.enforcer.version>2.6.0</restrict-imports.enforcer.version>
@@ -374,6 +375,8 @@
374375
<target>${javaVersion}</target>
375376
<scalaVersion>${scalaLongVersion}</scalaVersion>
376377
<args>
378+
<arg>-deprecation</arg>
379+
<arg>-feature</arg>
377380
<arg>-Ywarn-unused</arg>
378381
<arg>-Xfatal-warnings</arg>
379382
</args>
@@ -835,12 +838,15 @@
835838
<artifactId>scala-maven-plugin</artifactId>
836839
<configuration>
837840
<args>
841+
<arg>-deprecation</arg>
842+
<arg>-feature</arg>
838843
<arg>-Ywarn-unused</arg>
839844
<!-- https://github.com/scalamacros/paradise is no longer actively developed. -->
840845
<!-- In Scala 2.13, the plugin's functionality has been included in the compiler -->
841846
<!-- directly under the -Ymacro-annotations flag. -->
842847
<arg>-Ymacro-annotations</arg>
843848
<arg>-Xfatal-warnings</arg>
849+
<arg>-Wconf:cat=deprecation:w</arg>
844850
</args>
845851
<compilerPlugins>
846852
<compilerPlugin>

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.auron
1919
import java.io.File
2020
import java.util.UUID
2121

22+
import scala.annotation.nowarn
2223
import scala.collection.mutable
2324

2425
import org.apache.commons.lang3.reflect.FieldUtils
@@ -962,6 +963,7 @@ class ShimsImpl extends Shims with Logging {
962963
}
963964
}
964965

966+
@nowarn("cat=unused") // Params temporarily unused
965967
@sparkver("3.4 / 3.5")
966968
private def convertPromotePrecision(
967969
e: Expression,

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
*/
1717
package org.apache.spark.sql.execution.auron.shuffle
1818

19+
import scala.annotation.nowarn
20+
1921
import org.apache.spark.{ShuffleDependency, SparkConf, TaskContext}
2022
import org.apache.spark.internal.Logging
2123
import org.apache.spark.shuffle._
2224
import org.apache.spark.sql.execution.auron.shuffle.AuronShuffleDependency.isArrowShuffle
2325

2426
import org.apache.auron.sparkver
2527

28+
@nowarn("cat=unused") // _conf temporarily unused
2629
abstract class AuronRssShuffleManagerBase(_conf: SparkConf) extends ShuffleManager with Logging {
2730
override def registerShuffle[K, V, C](
2831
shuffleId: Int,

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
278278
val dateTimeStampMin = format.parse(dateStringMin).getTime
279279
val dateTimeStampMax = format.parse(dateStringMax).getTime
280280
format = new SimpleDateFormat("yyyy-MM-dd")
281-
val dateString = "2015-01-01"
282-
val date = format.parse(dateString)
283281

284282
val functions =
285283
s"""
@@ -320,8 +318,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
320318
val dateTimeStampMin = format.parse(dateStringMin).getTime
321319
val dateTimeStampMax = format.parse(dateStringMax).getTime
322320
format = new SimpleDateFormat("yyyy-MM-dd")
323-
val dateString = "2015-07-01"
324-
val date = format.parse(dateString)
325321

326322
val functions =
327323
s"""

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
209209
withTable("t") {
210210
sql(s"CREATE EXTERNAL TABLE t(c3 INT, c2 INT) USING ORC LOCATION '$path'")
211211

212-
val expected = if (forcePositionalEvolution) {
212+
val _ = if (forcePositionalEvolution) {
213213
correctAnswer
214214
} else {
215215
Seq(Row(null, 2), Row(null, 4), Row(null, 6), Row(null, null))
@@ -246,7 +246,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
246246
|LOCATION '$path'
247247
|""".stripMargin)
248248
sql("MSCK REPAIR TABLE t")
249-
val expected = if (forcePositionalEvolution) {
249+
val _ = if (forcePositionalEvolution) {
250250
correctAnswer
251251
} else {
252252
Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3), Row(null, null, 4))

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.nio.file.Files
2222
import java.nio.file.StandardCopyOption
2323
import java.util.concurrent.atomic.AtomicReference
2424

25+
import scala.annotation.nowarn
2526
import scala.collection.mutable.ArrayBuffer
2627

2728
import org.apache.arrow.c.ArrowArray
@@ -53,6 +54,7 @@ import org.apache.auron.protobuf.TaskDefinition
5354
* This class has been deprecated and migrated to {@link
5455
* org.apache.auron.jni.AuronCallNativeWrapper}. Will be removed in the future.
5556
*/
57+
@nowarn("cat=deprecation")
5658
@Deprecated
5759
case class AuronCallNativeWrapper(
5860
nativePlan: PhysicalPlanNode,
@@ -193,6 +195,7 @@ case class AuronCallNativeWrapper(
193195
}
194196
}
195197

198+
@nowarn("cat=deprecation")
196199
object AuronCallNativeWrapper extends Logging {
197200
def initNative(): Unit = {
198201
lazyInitNative

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,18 @@
1717
package org.apache.spark.sql.auron
1818

1919
import java.util.ServiceLoader
20-
import scala.annotation.{nowarn, tailrec}
20+
21+
import scala.annotation.tailrec
2122
import scala.collection.JavaConverters._
2223
import scala.collection.mutable
24+
2325
import org.apache.commons.lang3.reflect.MethodUtils
2426
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
2527
import org.apache.spark.Partition
2628
import org.apache.spark.broadcast.Broadcast
27-
import org.apache.spark.internal.{Logging, config}
28-
import org.apache.spark.sql.auron.AuronConvertStrategy.{childOrderingRequiredTag, convertStrategyTag, convertToNonNativeTag, convertibleTag, isNeverConvert, joinSmallerSideTag, neverConvertReasonTag}
29-
import org.apache.spark.sql.auron.NativeConverters.{StubExpr, existTimestampType, isTypeSupported, roundRobinTypeSupported}
29+
import org.apache.spark.internal.{config, Logging}
30+
import org.apache.spark.sql.auron.AuronConvertStrategy.{childOrderingRequiredTag, convertibleTag, convertStrategyTag, convertToNonNativeTag, isNeverConvert, joinSmallerSideTag, neverConvertReasonTag}
31+
import org.apache.spark.sql.auron.NativeConverters.{existTimestampType, isTypeSupported, roundRobinTypeSupported, StubExpr}
3032
import org.apache.spark.sql.auron.util.AuronLogUtils.logDebugPlanConversion
3133
import org.apache.spark.sql.catalyst.expressions.AggregateWindowFunction
3234
import org.apache.spark.sql.catalyst.expressions.Alias
@@ -43,6 +45,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
4345
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
4446
import org.apache.spark.sql.catalyst.expressions.aggregate.Final
4547
import org.apache.spark.sql.catalyst.expressions.aggregate.Partial
48+
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
4649
import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
4750
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
4851
import org.apache.spark.sql.catalyst.plans.physical.RangePartitioning
@@ -70,6 +73,7 @@ import org.apache.spark.sql.hive.execution.InsertIntoHiveTable
7073
import org.apache.spark.sql.hive.execution.auron.plan.NativeHiveTableScanBase
7174
import org.apache.spark.sql.internal.SQLConf
7275
import org.apache.spark.sql.types.LongType
76+
7377
import org.apache.auron.configuration.AuronConfiguration
7478
import org.apache.auron.jni.AuronAdaptor
7579
import org.apache.auron.metric.SparkMetricNode
@@ -153,15 +157,6 @@ object AuronConverters extends Logging {
153157
supportedShuffleManagers.exists(name.contains)
154158
}
155159

156-
// format: off
157-
// scalafix:off
158-
// necessary imports for cross spark versions build
159-
@nowarn("cat=unused-imports")
160-
import org.apache.spark.sql.catalyst.plans._
161-
import org.apache.spark.sql.catalyst.optimizer._
162-
// scalafix:on
163-
// format: on
164-
165160
def convertSparkPlanRecursively(exec: SparkPlan): SparkPlan = {
166161
// convert
167162
var danglingConverted: Seq[SparkPlan] = Nil
@@ -552,7 +547,7 @@ object AuronConverters extends Logging {
552547
org.apache.spark.sql.execution.auron.plan.BuildLeft
553548
case Some(org.apache.spark.sql.execution.auron.plan.BuildRight) =>
554549
org.apache.spark.sql.execution.auron.plan.BuildRight
555-
case None =>
550+
case _ =>
556551
logWarning("JoinSmallerSideTag is missing, defaults to BuildRight")
557552
org.apache.spark.sql.execution.auron.plan.BuildRight
558553
}

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -993,12 +993,12 @@ object NativeConverters extends Logging {
993993
val children = e.children.map(Cast(_, e.dataType))
994994
buildScalarFunction(pb.ScalarFunction.Coalesce, children, e.dataType)
995995

996-
case e @ StringLPad(str, len, pad) =>
996+
case _ @StringLPad(str, len, pad) =>
997997
buildScalarFunction(
998998
pb.ScalarFunction.Lpad,
999999
Seq(str, castIfNecessary(len, LongType), pad),
10001000
StringType)
1001-
case e @ StringRPad(str, len, pad) =>
1001+
case _ @StringRPad(str, len, pad) =>
10021002
buildScalarFunction(
10031003
pb.ScalarFunction.Rpad,
10041004
Seq(str, castIfNecessary(len, LongType), pad),

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ object NativeHelper extends Logging {
100100
if (nativePlan == null) {
101101
return Iterator.empty
102102
}
103-
var auronCallNativeWrapper = new org.apache.auron.jni.AuronCallNativeWrapper(
103+
val auronCallNativeWrapper = new org.apache.auron.jni.AuronCallNativeWrapper(
104104
ROOT_ALLOCATOR,
105105
nativePlan,
106106
metrics,

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/Shims.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ package org.apache.spark.sql.auron
1818

1919
import java.io.File
2020

21+
import scala.annotation.nowarn
22+
2123
import org.apache.spark.ShuffleDependency
2224
import org.apache.spark.SparkContext
2325
import org.apache.spark.TaskContext
@@ -257,6 +259,7 @@ abstract class Shims {
257259

258260
def getMinPartitionNum(sparkSession: SparkSession): Int
259261

262+
@nowarn("cat=unused")
260263
def postTransform(plan: SparkPlan, sc: SparkContext): Unit = {}
261264

262265
def getAdaptiveInputPlan(exec: AdaptiveSparkPlanExec): SparkPlan

0 commit comments

Comments
 (0)