From 880c7d9b2a7b4a1eaee3d7f2b613dbaa2842b91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org> Date: Fri, 2 Apr 2021 15:22:43 +0200 Subject: [PATCH] CI: add ability to generate Artifacts.toml for Julia --- .gitlab-ci.yml | 100 +++++++++++----------------- scripts/create-julia-artifacts-toml | 65 ++++++++++++++++++ 2 files changed, 105 insertions(+), 60 deletions(-) create mode 100755 scripts/create-julia-artifacts-toml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebd9afef..f7b12abc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,13 +3,12 @@ variables: MINGW32_BOOST_VERSION: 1.75.0-2 MINGW64_BOOST_VERSION: 1.75.0-2 WGET_OPTIONS: '--no-verbose --no-use-server-timestamps --retry-connrefused --retry-on-host-error' - -before_script: - - 'export REMOTE_PATH=https://www.dynare.org/preprocessor/$CI_COMMIT_SHORT_SHA' + JULIA_VERSION: 1.6.0 stages: - build - - prepare + - aggregate + - deploy build_linux_32: stage: build @@ -18,15 +17,11 @@ build_linux_32: - './configure LDFLAGS="-m32 -static -static-libgcc -static-libstdc++" CXXFLAGS=-m32' - make -j$(nproc) - strip src/dynare-preprocessor - - mkdir -p bin - - mv src/dynare-preprocessor bin - - mkdir -p $CI_COMMIT_SHORT_SHA/linux/32 - - tar cfz $CI_COMMIT_SHORT_SHA/linux/32/preprocessor.tar.gz bin - - cd $CI_COMMIT_SHORT_SHA/linux/32 && sha256sum preprocessor.tar.gz > sha256sum + - mkdir -p $CI_COMMIT_SHA/linux-i686 + - tar -caf $CI_COMMIT_SHA/linux-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor artifacts: paths: - - $CI_COMMIT_SHORT_SHA/linux/32/* - expire_in: 1 day + - $CI_COMMIT_SHA/linux-i686/* build_linux_64: stage: build @@ -35,15 +30,11 @@ build_linux_64: - './configure LDFLAGS="-static -static-libgcc -static-libstdc++"' - make -j$(nproc) - strip src/dynare-preprocessor - - mkdir -p bin - - mv src/dynare-preprocessor bin - - mkdir -p $CI_COMMIT_SHORT_SHA/linux/64 - - tar cfz $CI_COMMIT_SHORT_SHA/linux/64/preprocessor.tar.gz bin - - cd $CI_COMMIT_SHORT_SHA/linux/64 && sha256sum preprocessor.tar.gz > sha256sum + - mkdir -p $CI_COMMIT_SHA/linux-x86_64 + - tar -caf $CI_COMMIT_SHA/linux-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor artifacts: paths: - - $CI_COMMIT_SHORT_SHA/linux/64/* - expire_in: 1 day + - $CI_COMMIT_SHA/linux-x86_64/* build_windows_32: stage: build @@ -56,15 +47,11 @@ build_windows_32: - './configure --host=i686-w64-mingw32 --with-boost=$(pwd)/deps/mingw32/include LDFLAGS="-static -static-libgcc -static-libstdc++"' - make -j$(nproc) - i686-w64-mingw32-strip src/dynare-preprocessor.exe - - mkdir -p bin - - mv src/dynare-preprocessor.exe bin - - mkdir -p $CI_COMMIT_SHORT_SHA/windows/32 - - tar cfz $CI_COMMIT_SHORT_SHA/windows/32/preprocessor.tar.gz bin - - cd $CI_COMMIT_SHORT_SHA/windows/32 && sha256sum preprocessor.tar.gz > sha256sum + - mkdir -p $CI_COMMIT_SHA/windows-i686 + - tar -caf $CI_COMMIT_SHA/windows-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe artifacts: paths: - - $CI_COMMIT_SHORT_SHA/windows/32/* - expire_in: 1 day + - $CI_COMMIT_SHA/windows-i686/* build_windows_64: stage: build @@ -77,15 +64,11 @@ build_windows_64: - './configure --host=x86_64-w64-mingw32 --with-boost=$(pwd)/deps/mingw64/include LDFLAGS="-static -static-libgcc -static-libstdc++"' - make -j$(nproc) - x86_64-w64-mingw32-strip src/dynare-preprocessor.exe - - mkdir -p bin - - mv src/dynare-preprocessor.exe bin - - mkdir -p $CI_COMMIT_SHORT_SHA/windows/64 - - tar cfz $CI_COMMIT_SHORT_SHA/windows/64/preprocessor.tar.gz bin - - cd $CI_COMMIT_SHORT_SHA/windows/64 && sha256sum preprocessor.tar.gz > sha256sum + - mkdir -p $CI_COMMIT_SHA/windows-x86_64 + - tar -caf $CI_COMMIT_SHA/windows-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe artifacts: paths: - - $CI_COMMIT_SHORT_SHA/windows/64/* - expire_in: 1 day + - $CI_COMMIT_SHA/windows-x86_64/* build_macOS: stage: build @@ -96,18 +79,14 @@ build_macOS: - './configure CXX=g++-10 CXXFLAGS=-static-libgcc LEX=/usr/local/opt/flex/bin/flex YACC=/usr/local/opt/bison/bin/bison' - make -j$(nproc) - strip src/dynare-preprocessor - - mkdir -p bin - - mv src/dynare-preprocessor bin - - mkdir -p $CI_COMMIT_SHORT_SHA/macOS/64 - - tar cfz $CI_COMMIT_SHORT_SHA/macOS/64/preprocessor.tar.gz bin - - cd $CI_COMMIT_SHORT_SHA/macOS/64 && sha256sum preprocessor.tar.gz > sha256sum + - mkdir -p $CI_COMMIT_SHA/macos-x86_64 + - tar -caf $CI_COMMIT_SHA/macos-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor artifacts: paths: - - $CI_COMMIT_SHORT_SHA/macOS/64/* - expire_in: 1 day + - $CI_COMMIT_SHA/macos-x86_64/* -aggregate_builds: - stage: prepare +aggregate_julia: + stage: aggregate dependencies: - build_linux_32 - build_linux_64 @@ -115,24 +94,25 @@ aggregate_builds: - build_windows_64 - build_macOS script: - - linux32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/32/sha256sum) - - linux64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/64/sha256sum) - - windows32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/32/sha256sum) - - windows64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/64/sha256sum) - - macOS64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/macOS/64/sha256sum) - - cd $CI_COMMIT_SHORT_SHA - - echo 'PREPROCESSOR_VERSION = "'$CI_COMMIT_SHORT_SHA'"' > julia.conf - - echo 'REMOTE_PATH = "https://www.dynare.org/preprocessor/'$CI_COMMIT_SHORT_SHA'"' >> julia.conf - - echo -e "\n" >> julia.conf - - echo "download_info = Dict(" >> julia.conf - - echo " Linux(:i686, :glibc) => ("\"\$REMOTE_PATH/linux/32/preprocessor.tar.gz\"", "\"$linux32sha\"")," >> julia.conf - - echo " Linux(:x86_64, :glibc) => ("\"\$REMOTE_PATH/linux/64/preprocessor.tar.gz\"", "\"$linux64sha\"")," >> julia.conf - - echo " Windows(:i686) => ("\"\$REMOTE_PATH/windows/32/preprocessor.tar.gz\"", "\"$windows32sha\"")," >> julia.conf - - echo " Windows(:x86_64) => ("\"\$REMOTE_PATH/windows/64/preprocessor.tar.gz\"", "\"$windows64sha\"")," >> julia.conf - - echo " MacOS() => ("\"\$REMOTE_PATH/macOS/64/preprocessor.tar.gz\"", "\"$macOS64sha\"")," >> julia.conf - - echo " )" >> julia.conf + - PATH=$PATH:/usr/local/julia/$JULIA_VERSION/bin scripts/create-julia-artifacts-toml $CI_COMMIT_SHA https://www.dynare.org/preprocessor artifacts: paths: - - $CI_COMMIT_SHORT_SHA/*/*/* - - $CI_COMMIT_SHORT_SHA/julia.conf - expire_in: 1 week + - $CI_COMMIT_SHA/Artifacts.toml + +deploy_julia: + stage: deploy + dependencies: + - build_linux_32 + - build_linux_64 + - build_windows_32 + - build_windows_64 + - build_macOS + - aggregate_julia + tags: + - restricted + rules: + - if: '$CI_PROJECT_NAMESPACE == "Dynare" && $CI_COMMIT_REF_NAME == "master"' + when: manual + - when: never + script: + - cp -R $CI_COMMIT_SHA /srv/www.dynare.org/preprocessor/ diff --git a/scripts/create-julia-artifacts-toml b/scripts/create-julia-artifacts-toml new file mode 100755 index 00000000..c043a4c3 --- /dev/null +++ b/scripts/create-julia-artifacts-toml @@ -0,0 +1,65 @@ +#!/bin/bash + +# Creates Julia’s Artifacts.toml for preprocessor binaries. +# Used by the Gitlab CI file of the preprocessor. +# +# Must be given a git commit SHA as first argument, and a base download URL +# as second argument. +# +# In the current directory, there must be a directory whose name is the SHA. +# This directory must itself contain several directories corresponding to platforms +# ("linux-i686", "windows-x86_64"…). Each of these directories must contain a +# dynare-preprocessor.tar.gz file. +# +# The Julia executable must be in the path. +# +# The Artifacts.toml file is created in directory named after the git commit SHA. + +# Copyright © 2021 Dynare Team +# +# This file is part of Dynare. +# +# Dynare is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Dynare is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Dynare. If not, see <http://www.gnu.org/licenses/>. + +set -e +shopt -s extglob + +commit_sha=${1:?} +base_url=${2:?} + +project_dir=$(mktemp --tmpdir -d) + +cd "$commit_sha" + +for platform in !(Artifacts.toml); do + os=${platform%-*} + arch=${platform#*-} + tarball=${platform}/dynare-preprocessor.tar.gz + sha256=$(sha256sum "$tarball") + sha256=${sha256%% *} # Strip filename from output + git_tree_sha1=$(julia --project="$project_dir" -q -e "import Pkg; Pkg.add([\"Tar\", \"Inflate\"]); using Tar, Inflate; println(Tar.tree_hash(IOBuffer(inflate_gzip(\"$tarball\"))))") + echo [dynare-preprocessor] + echo os = \"$os\" + echo arch = \"$arch\" + if [[ $os == linux ]]; then + echo libc = \"glibc\" + fi + echo git-tree-sha1 = \"$git_tree_sha1\" + echo + echo [[dynare-preprocessor.download]] + echo sha256 = \"$sha256\" + echo url = "$base_url/$commit_sha/$tarball" + echo + echo +done > Artifacts.toml -- GitLab