Merge branch 'gitlabrunner' into 'devel'

Migration Jenkins -> Gitlabrunner

See merge request b2safe/B2SAFE-core!1

.gitlab-ci.yml

+---
+variables:
+  GIT_STRATEGY: clone
+  GIT_SUBMODULE_STRATEGY: recursive
+
+stages:
+  - build
+  - test
+  - deploy
+
+before_script:
+  - rm -f .variables
+  - touch .variables
+  - |
+      if [ "$CI_REGISTRY" = "git.ia.surfsara.nl:5050" ]
+      then
+        echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin  $CI_REGISTRY;
+        echo "export REGISTRY=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/irods-ci-server" >> .variables
+      else
+        echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USER --password-stdin  $REGISTRY;
+      fi
+  - source .variables
+  - docker-compose -f ci/docker-compose.yml down -v
+  - docker-compose -f ci/docker-compose.yml pull
+
+after_script:
+  - source .variables
+  - docker-compose -f ci/docker-compose.yml rm -f
+  - docker-compose -f ci/docker-compose.yml down -v
+
+# ###################################
+# build
+# ###################################
+build_4.2.6:
+  stage: build
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_2_6
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/build.sh ${VERSION}
+
+build_4.2.7:
+  stage: build
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_2_7
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/build.sh ${VERSION}
+
+# #####################################
+    
+build_4.1.11:
+  stage: build
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_1_11
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/build.sh ${VERSION}
+
+build_4.1.12:
+  stage: build
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_1_12
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/build.sh ${VERSION}
+
+# ###################################
+# test
+# ###################################
+test_4.2.6:
+  stage: test
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_2_6
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/test.sh ${VERSION}
+
+test_4.1.12:
+  stage: test
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_1_12
+  script:
+    - source .variables
+    - ./ci/shutdownall.sh
+    - ./ci/test.sh ${VERSION}
+
+# ###################################
+# build
+# ###################################
+deploy_4.2.6:
+  stage: deploy
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_2_6
+  script:
+    - source .variables
+    - ./ci/deploy.sh ${VERSION}
+
+deploy_4.2.7:
+  stage: deploy
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_2_7
+  script:
+    - source .variables
+    - ./ci/deploy.sh ${VERSION}
+
+# #####################################
+
+deploy_4.1.11:
+  stage: deploy
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_1_11
+  script:
+    - source .variables
+    - ./ci/deploy.sh ${VERSION}
+
+deploy_4.1.12:
+  stage: deploy
+  tags:
+    - irods
+  variables:
+    VERSION: centos7_4_1_12
+  script:
+    - source .variables
+    - ./ci/deploy.sh ${VERSION}

ci/app/4.1/setup_answers.txt

-tempZone
-1247
-20000
-20199
-/var/lib/irods/iRODS/Vault
-k
-12345678901234567890123456789012
-1248
-12345678901234567890123456789012
-
-rods
-test
-yes
-postgres
-5432
-ICAT
-irods
-test
-yes
-

ci/app/4.1/setup_irods.sh

-#!/bin/bash
-/app/wait_for_pg.sh
-set -x
-set -e
-
-if [ -f /etc/irods/service_account.config ]
-then
-    echo 'setup without service account'
-    cat /app/setup_answers.txt | /var/lib/irods/packaging/setup_irods.sh
-else
-    ( echo irods
-      echo irods
-      cat /app/setup_answers.txt
-    ) | /var/lib/irods/packaging/setup_irods.sh
-fi
-
-/app/sleep.sh

ci/app/4.1/surfsara-irods.repo

-[surfsara-irods]
-name=Local CentOS Repo
-baseurl=http://145.100.59.94/CentOS/7/irods-4.1.12/
-gpgcheck=0

ci/app/4.2/restart_irods.sh

-#!/bin/bash
-sudo -u irods python /var/lib/irods/scripts/irods_control.py restart

ci/app/4.2/setup_answers.txt

-irods
-irods
-1
-2
-postgres
-5432
-ICAT
-irods
-yes
-test
-
-tempZone
-1247
-20000
-20199
-1248
-
-rods
-yes
-k
-12345678901234567890123456789012
-12345678901234567890123456789012
-test
-/var/lib/irods/iRODS/Vault

ci/app/4.2/setup_irods.sh

-#!/bin/bash
-/app/wait_for_pg.sh
-
-set -x
-
-if [ -e  /var/lib/irods/iRODS/Vault/home ]
-then
-    echo "irods already installed"
-    /etc/init.d/irods start
-else
-    cat /app/setup_answers.txt | python /var/lib/irods/scripts/setup_irods.py
-fi
-
-/app/sleep.sh

ci/app/4.2/surfsara-irods.repo

-[surfsara-irods]
-name=Local CentOS Repo
-baseurl=http://145.100.59.94/CentOS/7/irods-4.2.6/
-gpgcheck=0

ci/app/create_rpm.sh

-#!/bin/bash
-set -x
-set -e
-BUILD=$1
-
-if [[ -z "$BUILD" ]]
-then
-    BUILD=0
-fi
-
-chown irods /var/lib/irods
-sudo -u irods cp -r /build /src/B2SAFE-core
-cd /src/B2SAFE-core/packaging
-sudo -u irods ./create_rpm_package.sh $BUILD
-chmod a+w /var/lib/irods/rpmbuild/RPMS/noarch/*.rpm
-
-
-

ci/app/irods_environment.json

-{
-    "irods_host": "localhost",
-    "irods_port": 1247,
-    "irods_user_name": "rods",
-    "irods_zone_name": "tempZone"
-}

ci/app/odbcinst.ini

-[PostgreSQL ANSI]
-Description=PostgreSQL ODBC driver (ANSI version)
-Driver=psqlodbca.so
-Setup=libodbcpsqlS.so
-Debug=0
-CommLog=1
-UsageCount=1
-
-[PostgreSQL Unicode]
-Description=PostgreSQL ODBC driver (Unicode version)
-Driver=psqlodbcw.so
-Setup=libodbcpsqlS.so
-Debug=0
-CommLog=1
-UsageCount=1

ci/app/sleep.sh

-#!/bin/bash
-
-while true; do sleep 10; done

ci/app/wait_for_pg.sh

-#!/bin/bash
-
-RETRIES=10
-PG_HOST=postgres
-PG_USER=irods
-PG_DATABASE=ICAT
-
-err=1
-while [ $err != 0 ]
-do
-    echo "attempt to connect to database"
-    PGPASSWORD=test psql --user $PG_USER --host=$PG_HOST --command='\q' $PG_DATABASE
-    err=$?
-    RETRIES=$(( RETRIES - 1 ))
-    if [ $err != 0 ];
-    then
-        if [ $RETRIES == 0 ]
-        then
-            echo "cannot connect to database"
-            exit 8
-        fi
-        echo "failed: try again ($attempts attempts)"
-        sleep 2;
-    fi
-done
-
-

ci/build.sh

 #!/bin/bash
 
-VERSION=centos7_4_2_6
+export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
+export VOLUME=$( dirname $( pwd ) )
+REGISTRY_USER=eudat-artie-readonly
+
+_VERSION=centos7_4_2_6
 CLEANUP="yes"
 BUILD=0
 GIT_BRANCH=""
-GIT_URL=""
 
 while [[ $# -gt 0 ]]
 do
@@ -14,11 +17,6 @@ do
             CLEANUP="no"
             shift
             ;;
-        "--url")
-            shift
-            GIT_URL=$1
-            shift
-            ;;
         "--branch")
             shift
             GIT_BRANCH=$1
@@ -30,34 +28,52 @@ do
             shift
             ;;
         *)
-            VERSION=$1
+            _VERSION=$1
             shift
             ;;
     esac
 done
 
+export VERSION=$_VERSION
+
 cleanup () {
     exit_code=$?
-    docker-compose -f ci/${VERSION}/docker-compose.yml down -v
+    echo "shuting down"
+    docker-compose -f ci/docker-compose.yml down -v
     exit $exit_code
 }
 
+is_logged_in() {
+  cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
+}
+
 set -x
 set -e
 
-trap cleanup EXIT ERR INT TERM
+if [[ "$(is_logged_in)" == "false" ]]; then
+    docker login -u $REGISTRY_USER $REGISTRY;
+fi
+
+if [ "$CLEANUP" = "yes" ]
+then
+    trap cleanup EXIT ERR INT TERM
+fi
 
 source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
 RPM_PACKAGE=`rpm_package $BUILD `
 IRODS_VERSION=`irods_version $VERSION`
-REPO_NAME=`repo_name $VERSION $GIT_URL $GIT_BRANCH `
-
+PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
 
-mkdir -p ci/RPMS/Centos/7/${REPO_NAME}
+mkdir -p $PACKAGE_DIR
 
-docker-compose -f ci/${VERSION}/docker-compose.yml build
-docker-compose -f ci/${VERSION}/docker-compose.yml up -d
+docker pull ${REGISTRY}/light_${VERSION}:latest
 
-docker exec ${VERSION}_icat_1 /app/create_rpm.sh $BUILD
+docker-compose -f ci/docker-compose.yml up -d
 
-docker exec ${VERSION}_icat_1 bash -c "set -x; set -e; cp /var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} /build/ci/RPMS/Centos/7/${REPO_NAME};" 
+docker exec ci_icat_1 mkdir -p /src/
+docker cp . ci_icat_1:/src/B2SAFE-core
+docker exec ci_icat_1 chown -R irods /src/B2SAFE-core
+docker exec ci_icat_1 sudo -u irods /src/B2SAFE-core/packaging/create_rpm_package.sh 0
+docker cp \
+       ci_icat_1:/var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} \
+       ${PACKAGE_DIR}/${RPM_PACKAGE}

ci/build_b2handle.sh

-#!/bin/bash
-
-workdir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
-YUM_SERVER='software@software.irodspoc-sara.surf-hosted.nl'
-SSH_OPTIONS='-oStrictHostKeyChecking=no'
-
-docker build $workdir/centos7_b2handle -t b2handle
-docker run -v $workdir/centos7_b2handle/dist:/opt/B2HANDLE/dist b2handle
-
-rpm=$workdir/centos7_b2handle/dist/b2handle-*.noarch.rpm
-rpm_file=$( basename $rpm )
-
-for REPO_NAME in irods-4.1.11  irods-4.1.12  irods-4.2.3  irods-4.2.4  irods-4.2.5  irods-4.2.6
-do
-    ssh $SSH_OPTIONS $YUM_SERVER "mkdir -p /repos/CentOS/7/${REPO_NAME}/Packages/"
-    scp $SSH_OPTIONS $rpm $YUM_SERVER:/repos/CentOS/7/${REPO_NAME}/Packages/$rpm_file
-    ssh $SSH_OPTIONS $YUM_SERVER createrepo --update /repos/CentOS/7/${REPO_NAME}
-done 
-

ci/centos7_4_1_12/Dockerfile

-FROM centos:7
-
-RUN yum update -y && \
-    yum install -y \
-    rsync wget less emacs make cmake gcc gcc-c++
-
-RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
-    rpm --import https://packages.irods.org/irods-signing-key.asc && \
-    wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
-    tee /etc/yum.repos.d/renci-irods.yum.repo
-
-RUN yum install -y \
-    sudo \
-    which \
-    python-jsonschema \
-    python-psutil \
-    python-pytest \
-    python-requests \
-    python-pip \
-    python-behave \
-    python-devel \
-    openssl-devel \
-    fuse-libs \
-    lsof \
-    openssl \
-    perl-JSON \
-    authd \
-    postgresql \
-    postgresql-odbc\
-    unixODBC \
-    boost-devel \
-    libcurl-devel \
-    supervisor \
-    rpm-build \
-    rpm-devel \
-    rpmlint \
-    rpmdevtools \
-    coreutils \
-    diffutils \
-    patch
-
-# Openstack
-RUN yum install -y centos-release-openstack-rocky
-
-RUN yum install -y \
-    python-swiftclient \
-    python2-keystoneclient \
-    python-openstackclient
-
-
-RUN pip install --upgrade pip && \
-    pip install flask \
-                flask_restful \
-                pytest \
-                s3cmd \
-                jsonschema \
-                python-irodsclient
-
-COPY ci/app/odbcinst.ini /etc/odbcinst.ini
-
-################################################################################
-#
-# irods
-#
-################################################################################
-RUN yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icat-4.1.12-centos7-x86_64.rpm &&\
-    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-database-plugin-postgres-1.12-centos7-x86_64.rpm && \
-    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-runtime-4.1.12-centos7-x86_64.rpm && \
-    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-dev-4.1.12-centos7-x86_64.rpm && \
-    rpm -i --replacefiles ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icommands-4.1.12-centos7-x86_64.rpm
-
-
-################################################################################
-#
-# irods setup preparation
-#
-################################################################################
-RUN useradd -rm -d /var/lib/irods irods
-RUN mkdir -m 777 /src
-ADD ci/app/4.1/setup_answers.txt /app/setup_answers.txt
-ADD ci/app/4.1/setup_irods.sh /app/setup_irods.sh
-ADD ci/app/4.1/surfsara-irods.repo /etc/yum.repos.d/surfsara-irods.repo
-ADD ci/app/irods_environment.json /root/.irods/irods_environment.json
-ADD ci/app/sleep.sh /app/sleep.sh
-ADD ci/app/wait_for_pg.sh /app/wait_for_pg.sh
-ADD ci/app/create_rpm.sh /app/create_rpm.sh
-ADD ci/app/update_install.py /app/update_install.py
-ADD scripts/tests/requirements.txt /app/test-requirements.txt
-
-RUN cd /app && \
-    wget https://github.com/irods/irods/archive/4.1.12.tar.gz && \
-    tar -xvf 4.1.12.tar.gz
-
-ADD scripts/tests/requirements.txt /app/test-requirements.txt
-
-RUN pip install -r /app/test-requirements.txt
-RUN yum install -y msi-persistent-id.x86_64
-
-# install B2HANDLE
-RUN pip install b2handle
-
-
-CMD ["/app/setup_irods.sh"]

ci/centos7_4_1_12/docker-compose.yml

-version: '2'
-
-volumes:
-  data-volume:
-
-  irods-vault:
-
-  irods-conf:
-
-
-
-services:
-  postgres:
-    build:
-      context: ../postgres
-      dockerfile: Dockerfile
-    ports:
-      - "5432"
-    hostname: postgresc7426.irods
-    volumes:
-      - data-volume:/var/lib/postgresql/9.5/main
-
-  icat:
-    build:
-      context: ../..
-      dockerfile: ci/centos7_4_1_12/Dockerfile
-    ports:
-      - "127.0.0.1:1247:1247"
-    depends_on:
-      - postgres
-    links:
-      - "postgres:postgres"
-    hostname: centos7_41_12.irods
-    volumes:
-      - "../../:/build"
-      - irods-vault:/var/lib/irods
-      - irods-conf:/etc/irods
-    command: /app/setup_irods.sh

ci/centos7_4_2_6/Dockerfile

-FROM centos:7
-
-RUN yum update -y && \
-    yum install -y \
-    rsync wget less emacs make cmake gcc gcc-c++
-
-RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
-    rpm --import https://packages.irods.org/irods-signing-key.asc && \
-    wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
-    tee /etc/yum.repos.d/renci-irods.yum.repo
-
-RUN yum install -y \
-    sudo \
-    which \
-    python-jsonschema \
-    python-psutil \
-    python-pytest \
-    python-requests \
-    python-pip \
-    python-behave \
-    python-devel  \
-    openssl-devel \
-    fuse-libs \
-    lsof \
-    openssl \
-    perl-JSON \
-    authd \
-    postgresql \
-    postgresql-odbc\
-    unixODBC \
-    boost-devel \
-    libcurl-devel \
-    supervisor \
-    rpm-build \
-    rpm-devel \
-    rpmlint \
-    rpmdevtools \
-    coreutils \
-    diffutils \
-    patch
-
-COPY ci/app/odbcinst.ini /etc/odbcinst.ini
-
-################################################################################
-#
-# irods
-#
-################################################################################
-RUN yum install -y irods-externals-boost1.60.0-0-1.0-1 \
-                   irods-externals-avro1.7.7-0 \
-                   irods-externals-clang-runtime3.8-0 \
-                   irods-externals-clang3.8-0 \
-                   irods-externals-cmake3.5.2-0 \
-                   irods-externals-cppzmq4.1-0 \
-                   irods-externals-epm4.2-0 irods-externals-imagemagick7.0.3-0 \
-                   irods-externals-jansson2.7-0 \
-                   irods-externals-json3.0.1-0 \
-                   irods-externals-libarchive3.1.2-0 \
-                   irods-externals-libarchive3.3.2-0 \
-                   irods-externals-qpid-with-proton0.34-0 \