Skip to content

Commit 5766331

Browse files
author
Spenser Gilliland
committed
Merged in SIM-4289 (pull request apache#24)
SIM-4289: fix: build of downstream jobs Approved-by: Lam Nguyen Approved-by: Jeffrey Uong
2 parents 8087a7c + 1181027 commit 5766331

2 files changed

Lines changed: 147 additions & 46 deletions

File tree

‎ci_jobs/Jenkinsfile‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,45 @@
22
library('sima-jenkins-lib')
33

44
DOCKER_OPTS = '-m 32g --cpus 8 -v /jenkins/workspace/ref_repos:/jenkins/workspace/ref_repos:rw,z'
5+
NEXUS_DOCKER_DEV = "https://${env.NEXUS_URL}:5000"
6+
NEXUS_DOCKER_STAGING = "https://${env.NEXUS_URL}:5300"
7+
8+
/* promote - promote artifacts to staging repo on tagged builds
9+
*
10+
*/
11+
def promote(dev_image) {
12+
if (utils.isTagBuild()) {
13+
node('docker') {
14+
docker.withRegistry(NEXUS_DOCKER_DEV, "jenkins_user") {
15+
dev_image["image"].pull()
16+
dev_image["image"].inside(DOCKER_OPTS) {
17+
utils.installNexusCreds('jenkins_user')
18+
19+
stage("Promote") {
20+
unstash('dist')
21+
utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/dist/*.whl')
22+
utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/topi/dist/*.whl')
23+
}
24+
}
25+
}
26+
}
27+
}
28+
}
529

630
def main() {
731
def job_name = env.JOB_NAME.split('/')[1]
32+
def image
833

934
properties([
1035
parameters([
1136
booleanParam(
1237
name: 'SKIP_DOWNSTREAM_BUILDS',
1338
description: 'Skips building upstream jobs',
14-
defaultValue: false
39+
defaultValue: utils.isTagBuild()
1540
),
1641
booleanParam(
1742
name: "PACKAGE_ONLY",
18-
defaultValue: false,
43+
defaultValue: utils.isTagBuild(),
1944
description: 'Only package don\'t run tests'
2045
)
2146
]),
@@ -27,7 +52,6 @@ def main() {
2752
utils.setBuildMetadataFromVersionIn("python/VERSION.in")
2853
}
2954

30-
def image
3155
stage("DockerBuild") {
3256
image = utils.dockerBuild(
3357
"docker/Dockerfile",
@@ -63,6 +87,9 @@ python3 setup.py bdist_wheel
6387
"""
6488
}, 'sima')
6589
}
90+
dir("${env.WORKSPACE}") {
91+
stash(name: 'dist', includes: 'python/dist/*.whl, python/topi/dist/*.whl')
92+
}
6693
}
6794
}, "../sima-regres.cmake", "clean all")
6895
stage("Package") {
@@ -90,9 +117,13 @@ python3 setup.py bdist_wheel
90117
booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY)
91118
])
92119
}, psim_mla_flow: {
93-
utils.buildUpstream("psim_mla_flow", params.SKIP_UPSTREAM_BUILDS, [])
120+
utils.buildUpstream("psim_mla_flow", params.SKIP_DOWNSTREAM_BUILDS, [
121+
booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY)
122+
])
94123
}
95124
}
125+
126+
promote(image)
96127
}
97128

98129
utils.job_wrapper( {

‎ci_jobs/Jenkinsfile.release‎

Lines changed: 112 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,133 @@
1-
#!/usr/bin/env groovy
1+
#!groovy
22
library('sima-jenkins-lib')
33

44
DOCKER_OPTS = '-m 32g --cpus 8 -v /jenkins/workspace/ref_repos:/jenkins/workspace/ref_repos:rw,z'
5+
NEXUS_DOCKER_DEV = "https://${env.NEXUS_URL}:5000"
6+
NEXUS_DOCKER_STAGING = "https://${env.NEXUS_URL}:5300"
7+
8+
/* promote - promote artifacts to staging repo on tagged builds
9+
*
10+
*/
11+
def promote(dev_image) {
12+
if (utils.isTagBuild()) {
13+
node('docker') {
14+
docker.withRegistry(NEXUS_DOCKER_DEV, "jenkins_user") {
15+
dev_image["image"].pull()
16+
dev_image["image"].inside(DOCKER_OPTS) {
17+
utils.installNexusCreds('jenkins_user')
18+
19+
stage("Promote") {
20+
unstash('dist')
21+
utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/dist/*.whl')
22+
utils.uploadPythonPackages('jenkins_user', 'sima-staging', 'python/topi/dist/*.whl')
23+
}
24+
}
25+
}
26+
}
27+
}
28+
}
529

630
def main() {
731
def job_name = env.JOB_NAME.split('/')[1]
832
def image
933

10-
currentBuild.result = "SUCCESS"
34+
properties([
35+
parameters([
36+
booleanParam(
37+
name: 'SKIP_DOWNSTREAM_BUILDS',
38+
description: 'Skips building upstream jobs',
39+
defaultValue: utils.isTagBuild()
40+
),
41+
booleanParam(
42+
name: "PACKAGE_ONLY",
43+
defaultValue: utils.isTagBuild(),
44+
description: 'Only package don\'t run tests'
45+
)
46+
]),
47+
])
1148

12-
node('docker') {
13-
stage("Checkout"){
49+
node("docker") {
50+
stage("Checkout") {
1451
utils.checkoutBitbucket()
52+
utils.setBuildMetadataFromVersionIn("python/VERSION.in")
1553
}
1654

17-
timeout(120) {
18-
stage("Docker Pull") {
19-
docker.withRegistry('', 'docker_creds') {
20-
image = docker.image("simaai/n2a_compiler:latest")
21-
image.pull()
55+
stage("DockerBuild") {
56+
image = utils.dockerBuild(
57+
"docker/Dockerfile",
58+
"${env.NEXUS_URL}:5000/" + job_name,
59+
"jenkins_user",
60+
"docker_build.log",
61+
{},
62+
"https://${env.NEXUS_URL}:5000"
63+
)
64+
}
65+
66+
parallel push: {
67+
stage("DockerPush") {
68+
image['post']()
69+
}
70+
}, build: {
71+
image["image"].inside(DOCKER_OPTS) {
72+
utils.cmakeBuild("build", "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", {}, { src_dir ->
73+
stage("Python Bindings") {
74+
dir("${env.WORKSPACE}/python") {
75+
utils.setPythonBuildEnv([], {
76+
sh """#!/bin/bash -ex
77+
rm -rf dist build
78+
python3 setup.py bdist_wheel
79+
"""
80+
}, 'sima')
81+
}
82+
dir("${env.WORKSPACE}/topi/python") {
83+
utils.setPythonBuildEnv([], {
84+
sh """#!/bin/bash -ex
85+
rm -rf dist build
86+
python3 setup.py bdist_wheel
87+
"""
88+
}, 'sima')
89+
}
90+
dir("${env.WORKSPACE}") {
91+
stash(name: 'dist', includes: 'python/dist/*.whl, python/topi/dist/*.whl')
92+
}
93+
}
94+
}, "../sima-regres.cmake", "clean all")
95+
stage("Package") {
96+
tvm_pkg_dir = "python/dist/*.whl"
97+
archiveArtifacts(tvm_pkg_dir)
98+
utils.uploadPythonPackages('jenkins_user', 'sima-pypi', tvm_pkg_dir, 3)
99+
topi_pkg_dir = "topi/python/dist/*.whl"
100+
archiveArtifacts(topi_pkg_dir)
101+
utils.uploadPythonPackages('jenkins_user', 'sima-pypi', topi_pkg_dir, 3)
22102
}
23103
}
24104
}
25-
timeout(120) {
26-
image.inside(DOCKER_OPTS) {
27-
stage("Download Package") {
28-
utils.record('download.log', {
29-
sh "rm -rf *.whl"
30-
31-
tvm_pkg_name = "sima-tvm"
32-
vinfo = readYaml file: "${env.WORKSPACE}/python/VERSION.in"
33-
tvm_version = vinfo['major'] + "." + vinfo['minor'] + "." + vinfo['patch']
34105

35-
utils.downloadPythonPackage('jenkins_user', 'sima-pypi', tvm_pkg_name + "===" + tvm_version, 3)
36-
37-
topi_pkg_name = "sima-topi"
38-
vinfo = readYaml file: "${env.WORKSPACE}/topi/python/VERSION.in"
39-
topi_version = vinfo['major'] + "." + vinfo['minor'] + "." + vinfo['patch']
40-
41-
utils.downloadPythonPackage('jenkins_user', 'sima-pypi', topi_pkg_name + "===" + topi_version, 3)
106+
stage("Promotion") {
107+
if (env.BRANCH_NAME=="sima") {
108+
utils.docker_promote(image['image'], 'jenkins_user', "https://${env.NEXUS_URL}:5000")
109+
}
110+
}
42111

43-
sh "ls -alh"
44-
}, true)
45-
}
112+
}
46113

47-
stage("Validate Package") {
48-
//TODO: run tests on the package
49-
}
114+
stage("Upstream") {
115+
parallel n2a: {
116+
utils.buildUpstream("n2a_compiler", params.SKIP_DOWNSTREAM_BUILDS, [
117+
booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY)
118+
])
119+
}, psim_mla_flow: {
120+
utils.buildUpstream("psim_mla_flow", params.SKIP_DOWNSTREAM_BUILDS, [
121+
booleanParam(name: 'PACKAGE_ONLY', value: params.PACKAGE_ONLY)
122+
])
123+
}
124+
}
50125

51-
stage("Promote Package(Staging)") {
52-
utils.record('upload.log', {
53-
utils.uploadPythonPackages('jenkins_user', 'sima-staging', '*.whl', 3)
54-
}, true)
55-
}
56-
} // docker
57-
} // timeout
58-
} // node
59-
} // main
126+
promote(image)
127+
}
60128

61-
utils.job_wrapper({
129+
utils.job_wrapper( {
62130
main()
63-
}, 240)
131+
})
132+
133+
return this

0 commit comments

Comments
 (0)