From 880c7d9b2a7b4a1eaee3d7f2b613dbaa2842b91e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 2 Apr 2021 15:22:43 +0200
Subject: [PATCH] CI: add ability to generate Artifacts.toml for Julia
---
.gitlab-ci.yml | 100 +++++++++++-----------------
scripts/create-julia-artifacts-toml | 65 ++++++++++++++++++
2 files changed, 105 insertions(+), 60 deletions(-)
create mode 100755 scripts/create-julia-artifacts-toml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ebd9afef..f7b12abc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,13 +3,12 @@ variables:
MINGW32_BOOST_VERSION: 1.75.0-2
MINGW64_BOOST_VERSION: 1.75.0-2
WGET_OPTIONS: '--no-verbose --no-use-server-timestamps --retry-connrefused --retry-on-host-error'
-
-before_script:
- - 'export REMOTE_PATH=https://www.dynare.org/preprocessor/$CI_COMMIT_SHORT_SHA'
+ JULIA_VERSION: 1.6.0
stages:
- build
- - prepare
+ - aggregate
+ - deploy
build_linux_32:
stage: build
@@ -18,15 +17,11 @@ build_linux_32:
- './configure LDFLAGS="-m32 -static -static-libgcc -static-libstdc++" CXXFLAGS=-m32'
- make -j$(nproc)
- strip src/dynare-preprocessor
- - mkdir -p bin
- - mv src/dynare-preprocessor bin
- - mkdir -p $CI_COMMIT_SHORT_SHA/linux/32
- - tar cfz $CI_COMMIT_SHORT_SHA/linux/32/preprocessor.tar.gz bin
- - cd $CI_COMMIT_SHORT_SHA/linux/32 && sha256sum preprocessor.tar.gz > sha256sum
+ - mkdir -p $CI_COMMIT_SHA/linux-i686
+ - tar -caf $CI_COMMIT_SHA/linux-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/linux/32/*
- expire_in: 1 day
+ - $CI_COMMIT_SHA/linux-i686/*
build_linux_64:
stage: build
@@ -35,15 +30,11 @@ build_linux_64:
- './configure LDFLAGS="-static -static-libgcc -static-libstdc++"'
- make -j$(nproc)
- strip src/dynare-preprocessor
- - mkdir -p bin
- - mv src/dynare-preprocessor bin
- - mkdir -p $CI_COMMIT_SHORT_SHA/linux/64
- - tar cfz $CI_COMMIT_SHORT_SHA/linux/64/preprocessor.tar.gz bin
- - cd $CI_COMMIT_SHORT_SHA/linux/64 && sha256sum preprocessor.tar.gz > sha256sum
+ - mkdir -p $CI_COMMIT_SHA/linux-x86_64
+ - tar -caf $CI_COMMIT_SHA/linux-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/linux/64/*
- expire_in: 1 day
+ - $CI_COMMIT_SHA/linux-x86_64/*
build_windows_32:
stage: build
@@ -56,15 +47,11 @@ build_windows_32:
- './configure --host=i686-w64-mingw32 --with-boost=$(pwd)/deps/mingw32/include LDFLAGS="-static -static-libgcc -static-libstdc++"'
- make -j$(nproc)
- i686-w64-mingw32-strip src/dynare-preprocessor.exe
- - mkdir -p bin
- - mv src/dynare-preprocessor.exe bin
- - mkdir -p $CI_COMMIT_SHORT_SHA/windows/32
- - tar cfz $CI_COMMIT_SHORT_SHA/windows/32/preprocessor.tar.gz bin
- - cd $CI_COMMIT_SHORT_SHA/windows/32 && sha256sum preprocessor.tar.gz > sha256sum
+ - mkdir -p $CI_COMMIT_SHA/windows-i686
+ - tar -caf $CI_COMMIT_SHA/windows-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/windows/32/*
- expire_in: 1 day
+ - $CI_COMMIT_SHA/windows-i686/*
build_windows_64:
stage: build
@@ -77,15 +64,11 @@ build_windows_64:
- './configure --host=x86_64-w64-mingw32 --with-boost=$(pwd)/deps/mingw64/include LDFLAGS="-static -static-libgcc -static-libstdc++"'
- make -j$(nproc)
- x86_64-w64-mingw32-strip src/dynare-preprocessor.exe
- - mkdir -p bin
- - mv src/dynare-preprocessor.exe bin
- - mkdir -p $CI_COMMIT_SHORT_SHA/windows/64
- - tar cfz $CI_COMMIT_SHORT_SHA/windows/64/preprocessor.tar.gz bin
- - cd $CI_COMMIT_SHORT_SHA/windows/64 && sha256sum preprocessor.tar.gz > sha256sum
+ - mkdir -p $CI_COMMIT_SHA/windows-x86_64
+ - tar -caf $CI_COMMIT_SHA/windows-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/windows/64/*
- expire_in: 1 day
+ - $CI_COMMIT_SHA/windows-x86_64/*
build_macOS:
stage: build
@@ -96,18 +79,14 @@ build_macOS:
- './configure CXX=g++-10 CXXFLAGS=-static-libgcc LEX=/usr/local/opt/flex/bin/flex YACC=/usr/local/opt/bison/bin/bison'
- make -j$(nproc)
- strip src/dynare-preprocessor
- - mkdir -p bin
- - mv src/dynare-preprocessor bin
- - mkdir -p $CI_COMMIT_SHORT_SHA/macOS/64
- - tar cfz $CI_COMMIT_SHORT_SHA/macOS/64/preprocessor.tar.gz bin
- - cd $CI_COMMIT_SHORT_SHA/macOS/64 && sha256sum preprocessor.tar.gz > sha256sum
+ - mkdir -p $CI_COMMIT_SHA/macos-x86_64
+ - tar -caf $CI_COMMIT_SHA/macos-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/macOS/64/*
- expire_in: 1 day
+ - $CI_COMMIT_SHA/macos-x86_64/*
-aggregate_builds:
- stage: prepare
+aggregate_julia:
+ stage: aggregate
dependencies:
- build_linux_32
- build_linux_64
@@ -115,24 +94,25 @@ aggregate_builds:
- build_windows_64
- build_macOS
script:
- - linux32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/32/sha256sum)
- - linux64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/64/sha256sum)
- - windows32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/32/sha256sum)
- - windows64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/64/sha256sum)
- - macOS64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/macOS/64/sha256sum)
- - cd $CI_COMMIT_SHORT_SHA
- - echo 'PREPROCESSOR_VERSION = "'$CI_COMMIT_SHORT_SHA'"' > julia.conf
- - echo 'REMOTE_PATH = "https://www.dynare.org/preprocessor/'$CI_COMMIT_SHORT_SHA'"' >> julia.conf
- - echo -e "\n" >> julia.conf
- - echo "download_info = Dict(" >> julia.conf
- - echo " Linux(:i686, :glibc) => ("\"\$REMOTE_PATH/linux/32/preprocessor.tar.gz\"", "\"$linux32sha\"")," >> julia.conf
- - echo " Linux(:x86_64, :glibc) => ("\"\$REMOTE_PATH/linux/64/preprocessor.tar.gz\"", "\"$linux64sha\"")," >> julia.conf
- - echo " Windows(:i686) => ("\"\$REMOTE_PATH/windows/32/preprocessor.tar.gz\"", "\"$windows32sha\"")," >> julia.conf
- - echo " Windows(:x86_64) => ("\"\$REMOTE_PATH/windows/64/preprocessor.tar.gz\"", "\"$windows64sha\"")," >> julia.conf
- - echo " MacOS() => ("\"\$REMOTE_PATH/macOS/64/preprocessor.tar.gz\"", "\"$macOS64sha\"")," >> julia.conf
- - echo " )" >> julia.conf
+ - PATH=$PATH:/usr/local/julia/$JULIA_VERSION/bin scripts/create-julia-artifacts-toml $CI_COMMIT_SHA https://www.dynare.org/preprocessor
artifacts:
paths:
- - $CI_COMMIT_SHORT_SHA/*/*/*
- - $CI_COMMIT_SHORT_SHA/julia.conf
- expire_in: 1 week
+ - $CI_COMMIT_SHA/Artifacts.toml
+
+deploy_julia:
+ stage: deploy
+ dependencies:
+ - build_linux_32
+ - build_linux_64
+ - build_windows_32
+ - build_windows_64
+ - build_macOS
+ - aggregate_julia
+ tags:
+ - restricted
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "Dynare" && $CI_COMMIT_REF_NAME == "master"'
+ when: manual
+ - when: never
+ script:
+ - cp -R $CI_COMMIT_SHA /srv/www.dynare.org/preprocessor/
diff --git a/scripts/create-julia-artifacts-toml b/scripts/create-julia-artifacts-toml
new file mode 100755
index 00000000..c043a4c3
--- /dev/null
+++ b/scripts/create-julia-artifacts-toml
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Creates Julia’s Artifacts.toml for preprocessor binaries.
+# Used by the Gitlab CI file of the preprocessor.
+#
+# Must be given a git commit SHA as first argument, and a base download URL
+# as second argument.
+#
+# In the current directory, there must be a directory whose name is the SHA.
+# This directory must itself contain several directories corresponding to platforms
+# ("linux-i686", "windows-x86_64"…). Each of these directories must contain a
+# dynare-preprocessor.tar.gz file.
+#
+# The Julia executable must be in the path.
+#
+# The Artifacts.toml file is created in directory named after the git commit SHA.
+
+# Copyright © 2021 Dynare Team
+#
+# This file is part of Dynare.
+#
+# Dynare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dynare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dynare. If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+shopt -s extglob
+
+commit_sha=${1:?}
+base_url=${2:?}
+
+project_dir=$(mktemp --tmpdir -d)
+
+cd "$commit_sha"
+
+for platform in !(Artifacts.toml); do
+ os=${platform%-*}
+ arch=${platform#*-}
+ tarball=${platform}/dynare-preprocessor.tar.gz
+ sha256=$(sha256sum "$tarball")
+ sha256=${sha256%% *} # Strip filename from output
+ git_tree_sha1=$(julia --project="$project_dir" -q -e "import Pkg; Pkg.add([\"Tar\", \"Inflate\"]); using Tar, Inflate; println(Tar.tree_hash(IOBuffer(inflate_gzip(\"$tarball\"))))")
+ echo [dynare-preprocessor]
+ echo os = \"$os\"
+ echo arch = \"$arch\"
+ if [[ $os == linux ]]; then
+ echo libc = \"glibc\"
+ fi
+ echo git-tree-sha1 = \"$git_tree_sha1\"
+ echo
+ echo [[dynare-preprocessor.download]]
+ echo sha256 = \"$sha256\"
+ echo url = "$base_url/$commit_sha/$tarball"
+ echo
+ echo
+done > Artifacts.toml
--
GitLab