From 880c7d9b2a7b4a1eaee3d7f2b613dbaa2842b91e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Villemot?= <sebastien@dynare.org>
Date: Fri, 2 Apr 2021 15:22:43 +0200
Subject: [PATCH] CI: add ability to generate Artifacts.toml for Julia

---
 .gitlab-ci.yml                      | 100 +++++++++++-----------------
 scripts/create-julia-artifacts-toml |  65 ++++++++++++++++++
 2 files changed, 105 insertions(+), 60 deletions(-)
 create mode 100755 scripts/create-julia-artifacts-toml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ebd9afef..f7b12abc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,13 +3,12 @@ variables:
   MINGW32_BOOST_VERSION: 1.75.0-2
   MINGW64_BOOST_VERSION: 1.75.0-2
   WGET_OPTIONS: '--no-verbose --no-use-server-timestamps --retry-connrefused --retry-on-host-error'
-
-before_script:
-  - 'export REMOTE_PATH=https://www.dynare.org/preprocessor/$CI_COMMIT_SHORT_SHA'
+  JULIA_VERSION: 1.6.0
 
 stages:
   - build
-  - prepare
+  - aggregate
+  - deploy
 
 build_linux_32:
   stage: build
@@ -18,15 +17,11 @@ build_linux_32:
     - './configure LDFLAGS="-m32 -static -static-libgcc -static-libstdc++" CXXFLAGS=-m32'
     - make -j$(nproc)
     - strip src/dynare-preprocessor
-    - mkdir -p bin
-    - mv src/dynare-preprocessor bin
-    - mkdir -p $CI_COMMIT_SHORT_SHA/linux/32
-    - tar cfz $CI_COMMIT_SHORT_SHA/linux/32/preprocessor.tar.gz bin
-    - cd $CI_COMMIT_SHORT_SHA/linux/32 && sha256sum preprocessor.tar.gz > sha256sum
+    - mkdir -p $CI_COMMIT_SHA/linux-i686
+    - tar -caf $CI_COMMIT_SHA/linux-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/linux/32/*
-    expire_in: 1 day
+      - $CI_COMMIT_SHA/linux-i686/*
 
 build_linux_64:
   stage: build
@@ -35,15 +30,11 @@ build_linux_64:
     - './configure LDFLAGS="-static -static-libgcc -static-libstdc++"'
     - make -j$(nproc)
     - strip src/dynare-preprocessor
-    - mkdir -p bin
-    - mv src/dynare-preprocessor bin
-    - mkdir -p $CI_COMMIT_SHORT_SHA/linux/64
-    - tar cfz $CI_COMMIT_SHORT_SHA/linux/64/preprocessor.tar.gz bin
-    - cd $CI_COMMIT_SHORT_SHA/linux/64 && sha256sum preprocessor.tar.gz > sha256sum
+    - mkdir -p $CI_COMMIT_SHA/linux-x86_64
+    - tar -caf $CI_COMMIT_SHA/linux-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/linux/64/*
-    expire_in: 1 day
+      - $CI_COMMIT_SHA/linux-x86_64/*
 
 build_windows_32:
   stage: build
@@ -56,15 +47,11 @@ build_windows_32:
     - './configure --host=i686-w64-mingw32 --with-boost=$(pwd)/deps/mingw32/include LDFLAGS="-static -static-libgcc -static-libstdc++"'
     - make -j$(nproc)
     - i686-w64-mingw32-strip src/dynare-preprocessor.exe
-    - mkdir -p bin
-    - mv src/dynare-preprocessor.exe bin
-    - mkdir -p $CI_COMMIT_SHORT_SHA/windows/32
-    - tar cfz $CI_COMMIT_SHORT_SHA/windows/32/preprocessor.tar.gz bin
-    - cd $CI_COMMIT_SHORT_SHA/windows/32 && sha256sum preprocessor.tar.gz > sha256sum
+    - mkdir -p $CI_COMMIT_SHA/windows-i686
+    - tar -caf $CI_COMMIT_SHA/windows-i686/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/windows/32/*
-    expire_in: 1 day
+      - $CI_COMMIT_SHA/windows-i686/*
 
 build_windows_64:
   stage: build
@@ -77,15 +64,11 @@ build_windows_64:
     - './configure --host=x86_64-w64-mingw32 --with-boost=$(pwd)/deps/mingw64/include LDFLAGS="-static -static-libgcc -static-libstdc++"'
     - make -j$(nproc)
     - x86_64-w64-mingw32-strip src/dynare-preprocessor.exe
-    - mkdir -p bin
-    - mv src/dynare-preprocessor.exe bin
-    - mkdir -p $CI_COMMIT_SHORT_SHA/windows/64
-    - tar cfz $CI_COMMIT_SHORT_SHA/windows/64/preprocessor.tar.gz bin
-    - cd $CI_COMMIT_SHORT_SHA/windows/64 && sha256sum preprocessor.tar.gz > sha256sum
+    - mkdir -p $CI_COMMIT_SHA/windows-x86_64
+    - tar -caf $CI_COMMIT_SHA/windows-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor.exe
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/windows/64/*
-    expire_in: 1 day
+      - $CI_COMMIT_SHA/windows-x86_64/*
 
 build_macOS:
   stage: build
@@ -96,18 +79,14 @@ build_macOS:
     - './configure CXX=g++-10 CXXFLAGS=-static-libgcc LEX=/usr/local/opt/flex/bin/flex YACC=/usr/local/opt/bison/bin/bison'
     - make -j$(nproc)
     - strip src/dynare-preprocessor
-    - mkdir -p bin
-    - mv src/dynare-preprocessor bin
-    - mkdir -p $CI_COMMIT_SHORT_SHA/macOS/64
-    - tar cfz $CI_COMMIT_SHORT_SHA/macOS/64/preprocessor.tar.gz bin
-    - cd $CI_COMMIT_SHORT_SHA/macOS/64 && sha256sum preprocessor.tar.gz > sha256sum
+    - mkdir -p $CI_COMMIT_SHA/macos-x86_64
+    - tar -caf $CI_COMMIT_SHA/macos-x86_64/dynare-preprocessor.tar.gz -C src dynare-preprocessor
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/macOS/64/*
-    expire_in: 1 day
+      - $CI_COMMIT_SHA/macos-x86_64/*
 
-aggregate_builds:
-  stage: prepare
+aggregate_julia:
+  stage: aggregate
   dependencies:
     - build_linux_32
     - build_linux_64
@@ -115,24 +94,25 @@ aggregate_builds:
     - build_windows_64
     - build_macOS
   script:
-    - linux32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/32/sha256sum)
-    - linux64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/linux/64/sha256sum)
-    - windows32sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/32/sha256sum)
-    - windows64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/windows/64/sha256sum)
-    - macOS64sha=$(cut -f 1 -d ' ' < $CI_COMMIT_SHORT_SHA/macOS/64/sha256sum)
-    - cd $CI_COMMIT_SHORT_SHA
-    - echo 'PREPROCESSOR_VERSION = "'$CI_COMMIT_SHORT_SHA'"' > julia.conf
-    - echo 'REMOTE_PATH = "https://www.dynare.org/preprocessor/'$CI_COMMIT_SHORT_SHA'"' >> julia.conf
-    - echo -e "\n" >> julia.conf
-    - echo "download_info = Dict(" >> julia.conf
-    - echo "    Linux(:i686, :glibc) => ("\"\$REMOTE_PATH/linux/32/preprocessor.tar.gz\"", "\"$linux32sha\"")," >> julia.conf
-    - echo "    Linux(:x86_64, :glibc) => ("\"\$REMOTE_PATH/linux/64/preprocessor.tar.gz\"", "\"$linux64sha\"")," >> julia.conf
-    - echo "    Windows(:i686) => ("\"\$REMOTE_PATH/windows/32/preprocessor.tar.gz\"", "\"$windows32sha\"")," >> julia.conf
-    - echo "    Windows(:x86_64) => ("\"\$REMOTE_PATH/windows/64/preprocessor.tar.gz\"", "\"$windows64sha\"")," >> julia.conf
-    - echo "    MacOS() => ("\"\$REMOTE_PATH/macOS/64/preprocessor.tar.gz\"", "\"$macOS64sha\"")," >> julia.conf
-    - echo " )" >> julia.conf
+    - PATH=$PATH:/usr/local/julia/$JULIA_VERSION/bin scripts/create-julia-artifacts-toml $CI_COMMIT_SHA https://www.dynare.org/preprocessor
   artifacts:
     paths:
-      - $CI_COMMIT_SHORT_SHA/*/*/*
-      - $CI_COMMIT_SHORT_SHA/julia.conf
-    expire_in: 1 week
+      - $CI_COMMIT_SHA/Artifacts.toml
+
+deploy_julia:
+  stage: deploy
+  dependencies:
+    - build_linux_32
+    - build_linux_64
+    - build_windows_32
+    - build_windows_64
+    - build_macOS
+    - aggregate_julia
+  tags:
+    - restricted
+  rules:
+    - if: '$CI_PROJECT_NAMESPACE == "Dynare" && $CI_COMMIT_REF_NAME == "master"'
+      when: manual
+    - when: never
+  script:
+    - cp -R $CI_COMMIT_SHA /srv/www.dynare.org/preprocessor/
diff --git a/scripts/create-julia-artifacts-toml b/scripts/create-julia-artifacts-toml
new file mode 100755
index 00000000..c043a4c3
--- /dev/null
+++ b/scripts/create-julia-artifacts-toml
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Creates Julia’s Artifacts.toml for preprocessor binaries.
+# Used by the Gitlab CI file of the preprocessor.
+#
+# Must be given a git commit SHA as first argument, and a base download URL
+# as second argument.
+#
+# In the current directory, there must be a directory whose name is the SHA.
+# This directory must itself contain several directories corresponding to platforms
+# ("linux-i686", "windows-x86_64"…). Each of these directories must contain a
+# dynare-preprocessor.tar.gz file.
+#
+# The Julia executable must be in the path.
+#
+# The Artifacts.toml file is created in directory named after the git commit SHA.
+
+# Copyright © 2021 Dynare Team
+#
+# This file is part of Dynare.
+#
+# Dynare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Dynare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Dynare.  If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+shopt -s extglob
+
+commit_sha=${1:?}
+base_url=${2:?}
+
+project_dir=$(mktemp --tmpdir -d)
+
+cd "$commit_sha"
+
+for platform in !(Artifacts.toml); do
+    os=${platform%-*}
+    arch=${platform#*-}
+    tarball=${platform}/dynare-preprocessor.tar.gz
+    sha256=$(sha256sum "$tarball")
+    sha256=${sha256%% *} # Strip filename from output
+    git_tree_sha1=$(julia --project="$project_dir" -q -e "import Pkg; Pkg.add([\"Tar\", \"Inflate\"]); using Tar, Inflate; println(Tar.tree_hash(IOBuffer(inflate_gzip(\"$tarball\"))))")
+    echo [dynare-preprocessor]
+    echo os = \"$os\"
+    echo arch = \"$arch\"
+    if [[ $os == linux ]]; then
+	echo libc = \"glibc\"
+    fi
+    echo git-tree-sha1 = \"$git_tree_sha1\"
+    echo
+    echo [[dynare-preprocessor.download]]
+    echo sha256 = \"$sha256\"
+    echo url = "$base_url/$commit_sha/$tarball"
+    echo
+    echo
+done > Artifacts.toml
-- 
GitLab