Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,16 @@ repos:
^Dockerfile\.ci$
pass_filenames: false
require_serial: true
- id: sync-java-sdk-supervisor-schema
name: Sync Java SDK supervisor schema with configured version
description: "Download the Supervisor Schema when Java SDK's bundled schema.json drifts"
entry: ./java-sdk/gradlew -p ./java-sdk :sdk:syncSupervisorSchema
language: system
pass_filenames: false
files: >
(?x)
^java-sdk/gradle\.properties$|
^java-sdk/sdk/schema/schema\.json$
- id: check-go-version-in-sync
name: Check Go toolchain version is consistent across build files
entry: ./scripts/ci/prek/check_go_version_in_sync.py
Expand Down
76 changes: 72 additions & 4 deletions java-sdk/sdk/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
* under the License.
*/

import java.io.File
import java.net.URI
import java.nio.file.Files
import java.nio.file.StandardCopyOption

val airflowSupervisorSchemaVersion: String by project

plugins {
Expand All @@ -30,8 +35,8 @@ plugins {
kotlin("plugin.serialization") version "2.3.0"
}

// TODO: Use a hosted file instead.
val schemaInput = rootProject.file("../task-sdk/src/airflow/sdk/execution_time/schema/schema.json")
val schemaBaseUrl = "https://airflow.staged.apache.org/schemas/supervisor-schema"
val schemaInput = layout.projectDirectory.file("schema/schema.json")
val pointersDir = layout.buildDirectory.dir("schema-pointers/main")
val jsonSchemaPackage = "org.apache.airflow.sdk.execution.comm"
val discriminatorDir = layout.buildDirectory.dir("generated-resources/main/src/main/kotlin")
Expand Down Expand Up @@ -150,16 +155,79 @@ abstract class GenerateDiscriminatorTask : DefaultTask() {
}
}

abstract class SyncSupervisorSchemaTask : DefaultTask() {
@get:Input
abstract val schemaVersion: Property<String>

@get:Input
abstract val baseUrl: Property<String>

@get:Internal
abstract val schemaFile: RegularFileProperty

private fun apiVersionOf(file: File): String =
if (file.exists()) {
com.fasterxml.jackson.databind
.ObjectMapper()
.readTree(file)
.path("api_version")
.asText()
} else {
""
}

@TaskAction
fun sync() {
val file = schemaFile.get().asFile
val version = schemaVersion.get()
if (apiVersionOf(file) == version) {
logger.lifecycle("Supervisor Schema is up-to-date (api_version=$version).")
return
}
Comment thread
uranusjr marked this conversation as resolved.
Resolved
val url = "${baseUrl.get()}/$version.json"
logger.lifecycle("Refreshing Supervisor Schema with $url")
file.parentFile.mkdirs()
val tempTarget = Files.createTempFile(file.parentFile.toPath(), "schema", ".json")
try {
val connection =
URI(url).toURL().openConnection().apply {
// Timeout values are arbitrary.
connectTimeout = 30_000
readTimeout = 30_000
}
connection.getInputStream().use { input ->
Files.copy(input, tempTarget, StandardCopyOption.REPLACE_EXISTING)
}
val downloaded = apiVersionOf(tempTarget.toFile())
if (downloaded != version) {
throw GradleException("Schema declares api_version='$downloaded' but expected '$version' ($url)")
}
Files.move(tempTarget, file.toPath(), StandardCopyOption.REPLACE_EXISTING)
} finally {
Files.deleteIfExists(tempTarget)
}
Comment thread
uranusjr marked this conversation as resolved.
Resolved
}
}

val syncSupervisorSchema by tasks.registering(SyncSupervisorSchemaTask::class) {
description = "Ensure the bundled Supervisor Schema is up-to-date with the Gradle property."
schemaVersion = airflowSupervisorSchemaVersion
baseUrl = schemaBaseUrl
schemaFile = schemaInput
}

tasks.register<GenerateDiscriminatorTask>("generateDiscriminator") {
dependsOn(syncSupervisorSchema)
description = "Generate Discriminator to wire type strings to model classes"
schemaFile = layout.file(provider { schemaInput })
schemaFile = schemaInput
modelPackage = jsonSchemaPackage
targetDirectory = discriminatorDir
}

tasks.register<GeneratePointersTask>("generatePointers") {
dependsOn(syncSupervisorSchema)
description = "Generate pointer files for jsonSchema2Pojo"
schemaFile = layout.file(provider { schemaInput })
schemaFile = schemaInput
targetDirectory = pointersDir
}

Expand Down
Loading
Loading