Skip to content

Commit 6b72ca1

Browse files
author
Erlang/OTP
committed
Add OTP internal Jenkins configuration
1 parent 1797e50 commit 6b72ca1

37 files changed

+2758
-0
lines changed

‎Jenkinsfile‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
// -*- groovy -*-
2+
HTTP_PROXY = "http://www-proxy.otp.ericsson.se:8080"
3+
HTTPS_PROXY = "http://www-proxy.otp.ericsson.se:8080"
4+
NO_PROXY = "localhost,127.0.0.1,.otp.ericsson.se,otp.ericsson.se,otp,erlang,support,nessie,wiki,monitor,jenkins"
5+
6+
DOCKER_ARGS = "--ulimit core=-1 --ulimit nofile=1000000:1000000 --memory=4g --memory-swap=1g "
7+
DOCKER_RUN_ARGS = "--volume=\"/home/otptest/bin/:/home/otptest/bin/:rw\" " +
8+
"--volume=\"/home/otp/bin:/home/otp/bin:rw\" " +
9+
"--security-opt seccomp=unconfined --cap-add=SYS_PTRACE ${DOCKER_ARGS} "
10+
DOCKER_BUILD_ARGS = "${DOCKER_ARGS} " +
11+
"--build-arg MAKEFLAGS=-j16 " +
12+
"--build-arg https_proxy=${HTTP_PROXY} " +
13+
"--build-arg http_proxy=${HTTP_PROXY} " +
14+
"--build-arg HTTP_PROXY=${HTTP_PROXY} " +
15+
"--build-arg HTTPS_PROXY=${HTTP_PROXY} " +
16+
"--build-arg NO_PROXY=${NO_PROXY} " +
17+
"--build-arg no_proxy=${NO_PROXY} scripts"
18+
19+
DOCKER_INSIDE_ARGS = "${DOCKER_RUN_ARGS} --init --entrypoint=\"\" "
20+
21+
timestamp = ""
22+
sha = ""
23+
DOCKER_REGISTRY = "apollo.otp.ericsson.se:5000/"
24+
sendFailEmail64 = false
25+
sendFailEmail32 = false
26+
FAST_BUILD = false
27+
28+
otp = Collections.synchronizedMap([:]);
29+
30+
/*
31+
We use a scripted pipeline instead of a declarative pipeline here
32+
as we want to do parallel execution of dynamic stages (i386 vs i686)
33+
*/
34+
pipeline {
35+
36+
options {
37+
// Limit pipeline to only one build per branch at the same time
38+
disableConcurrentBuilds();
39+
timestamps();
40+
buildDiscarder(logRotator(numToKeepStr: '50'));
41+
}
42+
43+
agent { label 'docker' }
44+
45+
stages {
46+
stage('pack') {
47+
steps { script {
48+
timestamp = sh([script: 'date "+%Y-%m-%d_%H"', returnStdout: true]).trim()
49+
sh 'git archive --format=tar.gz --prefix=otp_src/ -o scripts/otp_src.tar.gz HEAD'
50+
sha = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
51+
stash name: "scripts", includes: "scripts/"
52+
} }
53+
}
54+
stage('build') {
55+
parallel {
56+
stage("64-bit") {
57+
options { timeout(time: 30, unit: 'MINUTES'); }
58+
steps { script {
59+
sendFailEmail64 = true;
60+
buildDockerImage(64)
61+
} }
62+
}
63+
stage("32-bit") {
64+
options { timeout(time: 30, unit: 'MINUTES'); }
65+
agent { label 'docker' }
66+
steps { script {
67+
sendFailEmail32 = true;
68+
buildDockerImage(32)
69+
} }
70+
}
71+
}
72+
}
73+
stage('check') {
74+
options { timeout(time: 60, unit: 'MINUTES'); }
75+
parallel {
76+
stage('cross') { steps { script { dockReg {
77+
stage("vxworks") {
78+
def targets = ["simlinux" : "i586-wrs-vxworks",
79+
"ppc32" : "powerpc-wrs-vxworks"];
80+
targets.each { target, triple ->
81+
otp[32].inside(inside(target) +
82+
" -v \"/net/duper/export/extra/vxworks/WindRiver:/WindRiver\"")
83+
{
84+
sh_log("cd \$ERL_TOP && \$ERL_TOP/scripts/build-otp-vxworks ${target} ${triple}", "Build VXWORKS");
85+
}
86+
}
87+
}
88+
stage("powerpc64el") {
89+
unstash "scripts";
90+
docker.build(
91+
"powerpc64el/otp-src-${BRANCH_NAME}",
92+
"-f scripts/Dockerfile.powerpc64el --pull " +
93+
DOCKER_BUILD_ARGS);
94+
}
95+
} } } }
96+
stage('build special types') { steps { script { dockReg {
97+
otp.each { arch, img ->
98+
stage("${arch}") {
99+
echo "${img.imageName()}";
100+
img.inside(inside("special-${arch}")) {
101+
sh_log('''
102+
if [ -f $ERL_TOP/erts/emulator/utils/beam_emu_vars ];
103+
then
104+
cd $ERL_TOP/erts/emulator
105+
make -f x86_64-unknown-linux-gnu/Makefile PROFILE=use check_emu_registers
106+
cd $ERL_TOP
107+
fi''',"check emu registers");
108+
}
109+
special = docker.build(
110+
"${getPrefix(arch)}otp-special-${BRANCH_NAME}:${BUILD_ID}",
111+
"-f scripts/Dockerfile.special " +
112+
"--build-arg \"BASE_IMAGE=${img.imageName()}\" " +
113+
"${DOCKER_BUILD_ARGS}"
114+
);
115+
special.push("latest");
116+
special.push(BUILD_ID);
117+
}
118+
}
119+
} } } }
120+
stage('check tests') { steps { script { dockReg {
121+
otp[64].inside(inside("check-tests")) {
122+
sh 'erl -s init stop'
123+
sh "mkdir /daily_build/test && \$ERL_TOP/otp_build tests /daily_build/test"
124+
def applications = sh([script: 'ls -d /daily_build/test/*_test/',
125+
returnStdout: true]).trim().split('[\\r\\n ]+')
126+
for (int i = 0; i < applications.size(); i++) {
127+
sh_log("\$ERL_TOP/scripts/build-otp-tests ${applications[i]}","Check ${applications[i]}s tests");
128+
}
129+
}
130+
} } } }
131+
stage('docs') { steps { script { dockReg {
132+
def install_img;
133+
stage("build") {
134+
install = docker.build(
135+
"otp-docs-${BRANCH_NAME}:${BUILD_ID}",
136+
"-f scripts/Dockerfile.docs " +
137+
"--build-arg \"FAST_BUILD=${FAST_BUILD}\" " +
138+
"--build-arg \"BASE_IMAGE=${otp[64].imageName()}\" " +
139+
"${DOCKER_BUILD_ARGS}");
140+
install.push("latest");
141+
install.push(BUILD_ID);
142+
}
143+
if (!FAST_BUILD) {
144+
stage("check") {
145+
install.inside(inside("check-docs")) {
146+
sh script: 'cd $ERL_TOP && make release_docs', label: "release_docs"
147+
sh label: "Render Chunks", script: '''#!/bin/bash
148+
cd $ERL_TOP
149+
if [ -f lib/stdlib/test/shell_docs_SUITE.erl ]; then
150+
erlc lib/stdlib/test/shell_docs_SUITE.erl
151+
ct_run -no_auto_compile -suite shell_docs_SUITE -case render
152+
fi
153+
'''
154+
sh '(cd $ERL_TOP/system/doc/ && make release_docs TESTROOT=$TARGET_DIR INSTALLROOT=$TARGET_DIR)'
155+
sh script: 'cd $ERL_TOP && $ERL_TOP/scripts/run-xml-lint', label: "xmllint"
156+
sh script: '''#!/bin/bash
157+
set -o pipefail
158+
cd $ERL_TOP/release/`erts/autoconf/config.guess`/
159+
if [ -f $ERL_TOP/lib/stdlib/src/shell_docs.erl ]; then
160+
$ERL_TOP/scripts/otp_html_check `pwd` doc/index.html | tee $WORKSPACE/otp_html_check.log
161+
else
162+
/home/otp/bin/otp_html_check `pwd` doc/index.html erts-*/doc/index.html lib/*/doc/index.html | tee $WORKSPACE/otp_html_check.log
163+
fi
164+
''', label: "html check"
165+
sh script: '''#!/bin/bash
166+
set -o pipefail
167+
cd $install_dir
168+
./Install -minimal `pwd` | tee $WORKSPACE/otp_man_check.log
169+
'''
170+
recordIssues sourceCodeEncoding: 'UTF-8',
171+
tool: groovyScript(parserId: 'otp_html_check_anchors',
172+
pattern:'**/*otp_html_check.log')
173+
recordIssues sourceCodeEncoding: 'UTF-8',
174+
tool: groovyScript(parserId: 'otp_html_check_links',
175+
pattern:'**/*otp_html_check.log')
176+
recordIssues sourceCodeEncoding: 'UTF-8',
177+
tool: groovyScript(parserId: 'otp_man_check',
178+
pattern:'**/*otp_man_check.log')
179+
180+
}
181+
}
182+
}
183+
} } } }
184+
stage('run dialyzer') { steps { script { dockReg {
185+
if (!FAST_BUILD) {
186+
otp[64].inside(inside("dialyzer")) {
187+
sh '''#!/bin/bash
188+
set -o pipefail
189+
cd $ERL_TOP
190+
if [ -f $ERL_TOP/scripts/run-dialyzer ]; then
191+
$ERL_TOP/scripts/run-dialyzer | tee $WORKSPACE/otp_dialyzer.log
192+
fi
193+
'''
194+
}
195+
}
196+
} } } }
197+
stage('benchmark') {
198+
steps {
199+
script {
200+
dockReg {
201+
otp[64].inside(DOCKER_INSIDE_ARGS) {
202+
OTP_VSN = sh([script: 'cat \$ERL_TOP/OTP_VERSION | sed "s|^\\([0-9]*\\)\\..*|\\1|"',
203+
returnStdout: true]).trim().toInteger();
204+
}
205+
}
206+
if (OTP_VSN > 20) {
207+
build job: "/benchmark",
208+
propagate: false, wait: false,
209+
parameters: [[$class: 'StringParameterValue', name: 'IMAGE_BRANCH', value: BRANCH_NAME]]
210+
}
211+
}
212+
}
213+
}
214+
stage('windows') { steps { script {
215+
if (fileExists("erts/etc/win32/wsl_tools")) {
216+
build job: "/WindowsWSL/${BRANCH_NAME}",
217+
propagate: true, wait: false,
218+
parameters: [[$class: 'BooleanParameterValue', name: 'QUICK', value: true]]
219+
} else {
220+
build job: "/windows/${BRANCH_NAME}", propagate: false,
221+
wait: false, parameters: []
222+
}
223+
} } }
224+
}
225+
}
226+
}
227+
post {
228+
success {
229+
recordIssues enabledForFailure: true, tools: [gcc4()],
230+
filters: [includeFile('.*\\.[cho]:?[0-9]*')]
231+
recordIssues enabledForFailure: true, tools: [erlc()],
232+
filters: [includeFile('.*\\.[he]rl:?[0-9]*')]
233+
}
234+
unsuccessful { script {
235+
if (sendFailEmail32 && sendFailEmail64) {
236+
/* See apollo:/ldisk/lukas/jenkins-data/email-templates/ for the template */
237+
emailext subject: '${DEFAULT_SUBJECT}',
238+
body: '${SCRIPT, template="pack-and-build.template"}',
239+
to: '[email protected]',
240+
recipientProviders: [[$class: 'CulpritsRecipientProvider']],
241+
presendScript: '${DEFAULT_PRESEND_SCRIPT}'
242+
}
243+
} }
244+
}
245+
}
246+
247+
def buildDockerImage(arch) {
248+
dockReg {
249+
250+
unstash "scripts";
251+
252+
def img = docker.build(
253+
"${getPrefix(arch)}otp-src-${BRANCH_NAME}",
254+
"-f scripts/Dockerfile.pack-otp --pull " +
255+
"--build-arg \"FAST_BUILD=${FAST_BUILD}\" " +
256+
"--build-arg \"BUILD_ID=${BUILD_ID}\" " +
257+
"--build-arg \"BRANCH=${BRANCH_NAME}\" " +
258+
"--build-arg \"TIMESTAMP=${timestamp}\" " +
259+
"--build-arg \"SHA=${sha}\" " +
260+
"--build-arg \"BASE=${getPrefix(arch)}ubuntu:16.04\" " +
261+
DOCKER_BUILD_ARGS);
262+
img.push("latest");
263+
img.push(BUILD_ID);
264+
265+
// Because of how img.inspect works the img needs to be the
266+
// build_id tagged image. See
267+
// https://github.com/jenkinsci/docker-workflow-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/docker/workflow/Docker.groovy#L124-L126
268+
img = docker.build(
269+
"${getPrefix(arch)}otp-${BRANCH_NAME}:${BUILD_ID}",
270+
"-f scripts/Dockerfile.run-tests " +
271+
"--build-arg \"FAST_BUILD=${arch == 32 ? true : FAST_BUILD}\" " +
272+
"--build-arg \"BASE_IMAGE=${img.imageName()}\" " +
273+
"${DOCKER_BUILD_ARGS}");
274+
275+
img.push("latest");
276+
img.push(BUILD_ID);
277+
otp[arch] = img;
278+
}
279+
}
280+
281+
def getPrefix(arch) {
282+
return arch == 64 ? "" : "i386/";
283+
}
284+
285+
def dockReg(body) {
286+
docker.withRegistry("http://" + DOCKER_REGISTRY) {
287+
body();
288+
}
289+
}
290+
291+
def sh_log(cmd, title = "") {
292+
if (title == "")
293+
return sh([script: cmd]);
294+
sh([script: cmd, label: title]);
295+
}
296+
297+
def inside(extra = "") {
298+
if (extra != "")
299+
extra += "-";
300+
return "${DOCKER_INSIDE_ARGS} --name=${extra}${BUILD_TAG}";
301+
}

‎Jenkinsfile.benchmark‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// -*- groovy -*-
2+
HTTP_PROXY = "http://www-proxy:8080"
3+
HTTPS_PROXY = "http://www-proxy:8080"
4+
NO_PROXY = "localhost,127.0.0.1,.otp.ericsson.se,otp.ericsson.se,otp,erlang,support,nessie,wiki,monitor,jenkins"
5+
6+
DOCKER_ARGS = "--ulimit core=-1 --ulimit nofile=1000000:1000000 "
7+
DOCKER_RUN_ARGS = "--security-opt seccomp=unconfined --cap-add=SYS_PTRACE ${DOCKER_ARGS} "
8+
DOCKER_BUILD_ARGS = "${DOCKER_ARGS} --build-arg MAKEFLAGS=-j16 --build-arg https_proxy=${HTTP_PROXY} --build-arg http_proxy=${HTTP_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --build-arg HTTPS_PROXY=${HTTP_PROXY} --build-arg NO_PROXY=${NO_PROXY} --build-arg no_proxy=${NO_PROXY} scripts"
9+
10+
DOCKER_INSIDE_ARGS = "${DOCKER_RUN_ARGS} --init --entrypoint=\"\" "
11+
12+
DOCKER_REGISTRY = "apollo.otp.ericsson.se:5000/"
13+
14+
slave = null
15+
16+
pipeline {
17+
parameters {
18+
string(name: 'IMAGE_BRANCH', defaultValue: 'master-opu', description: '')
19+
string(name: 'IMAGE_BUILD_ID', defaultValue: 'latest', description: '')
20+
string(name: 'APPS', defaultValue: "all",
21+
description: 'Which applications to test (comma separated),'+
22+
"defaults to ts:benchmarks() ++ edev.")
23+
}
24+
25+
options {
26+
// Limit pipeline to only one build per branch at the same time
27+
disableConcurrentBuilds();
28+
timestamps();
29+
buildDiscarder(logRotator(numToKeepStr: '50'));
30+
}
31+
32+
agent none
33+
34+
stages {
35+
stage('pack') {
36+
agent { label 'docker' }
37+
steps { script { dockReg {
38+
img = docker.image("${DOCKER_REGISTRY}otp-${params.IMAGE_BRANCH}:${params.IMAGE_BUILD_ID}");
39+
addBadge(icon: 'info.gif', text: params.IMAGE_BRANCH);
40+
img.pull();
41+
img.inside(DOCKER_INSIDE_ARGS) {
42+
sh "cd \$install_dir && cp /daily_build/build_info/sha . && " +
43+
"tar czf \$WORKSPACE/otp.tar.gz *"
44+
}
45+
stash name: "otp.tar.gz", includes: "otp.tar.gz"
46+
} } }
47+
}
48+
stage("build") {
49+
agent { label 'benchmark' }
50+
steps {
51+
unstash "otp.tar.gz";
52+
sh([script: "scripts/build-otp-benchmarks",
53+
label: "build benchmark framework"])
54+
script {
55+
def apps = params.APPS;
56+
if (apps == "all") {
57+
get_bench_apps = "find . -name '*_bench.spec' | sed -e 's:[\\./_]: :g' | awk '{ printf \$2 \" \" }'";
58+
apps = sh([script: get_bench_apps, returnStdout: true]).trim();
59+
apps += " edev";
60+
}
61+
apps.split("[, ]+").each({ app ->
62+
sh([script: "scripts/run-otp-benchmarks \"${app}\" \"${IMAGE_BRANCH}\" " +
63+
// We add 14000 to the build id in order to not clash with
64+
// the monitor bench db build ids for the previous jenkins
65+
"\"${BUILD_ID.toInteger() + 14000}\"",
66+
label: "benchmark ${app}"])
67+
})
68+
}
69+
}
70+
}
71+
}
72+
}
73+
74+
def dockReg(body) {
75+
docker.withRegistry("http://" + DOCKER_REGISTRY) {
76+
body();
77+
}
78+
}

0 commit comments

Comments
 (0)