Commit 38cf9fc3 authored by Stefan Wolfsheimer's avatar Stefan Wolfsheimer
Browse files

Merge branch 'gitlabrunner' into 'devel'

Migration Jenkins -> Gitlabrunner

See merge request b2safe/B2SAFE-core!1
parents 966e2f30 0acd2b28
---
variables:
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive
stages:
- build
- test
- deploy
before_script:
- rm -f .variables
- touch .variables
- |
if [ "$CI_REGISTRY" = "git.ia.surfsara.nl:5050" ]
then
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY;
echo "export REGISTRY=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/irods-ci-server" >> .variables
else
echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USER --password-stdin $REGISTRY;
fi
- source .variables
- docker-compose -f ci/docker-compose.yml down -v
- docker-compose -f ci/docker-compose.yml pull
after_script:
- source .variables
- docker-compose -f ci/docker-compose.yml rm -f
- docker-compose -f ci/docker-compose.yml down -v
# ###################################
# build
# ###################################
build_4.2.6:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
build_4.2.7:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
# #####################################
build_4.1.11:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
build_4.1.12:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
# ###################################
# test
# ###################################
test_4.2.6:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
test_4.1.12:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
# ###################################
# build
# ###################################
deploy_4.2.6:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.7:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
# #####################################
deploy_4.1.11:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.1.12:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
tempZone
1247
20000
20199
/var/lib/irods/iRODS/Vault
k
12345678901234567890123456789012
1248
12345678901234567890123456789012
rods
test
yes
postgres
5432
ICAT
irods
test
yes
#!/bin/bash
/app/wait_for_pg.sh
set -x
set -e
if [ -f /etc/irods/service_account.config ]
then
echo 'setup without service account'
cat /app/setup_answers.txt | /var/lib/irods/packaging/setup_irods.sh
else
( echo irods
echo irods
cat /app/setup_answers.txt
) | /var/lib/irods/packaging/setup_irods.sh
fi
/app/sleep.sh
[surfsara-irods]
name=Local CentOS Repo
baseurl=http://145.100.59.94/CentOS/7/irods-4.1.12/
gpgcheck=0
#!/bin/bash
sudo -u irods python /var/lib/irods/scripts/irods_control.py restart
irods
irods
1
2
postgres
5432
ICAT
irods
yes
test
tempZone
1247
20000
20199
1248
rods
yes
k
12345678901234567890123456789012
12345678901234567890123456789012
test
/var/lib/irods/iRODS/Vault
#!/bin/bash
/app/wait_for_pg.sh
set -x
if [ -e /var/lib/irods/iRODS/Vault/home ]
then
echo "irods already installed"
/etc/init.d/irods start
else
cat /app/setup_answers.txt | python /var/lib/irods/scripts/setup_irods.py
fi
/app/sleep.sh
[surfsara-irods]
name=Local CentOS Repo
baseurl=http://145.100.59.94/CentOS/7/irods-4.2.6/
gpgcheck=0
#!/bin/bash
set -x
set -e
BUILD=$1
if [[ -z "$BUILD" ]]
then
BUILD=0
fi
chown irods /var/lib/irods
sudo -u irods cp -r /build /src/B2SAFE-core
cd /src/B2SAFE-core/packaging
sudo -u irods ./create_rpm_package.sh $BUILD
chmod a+w /var/lib/irods/rpmbuild/RPMS/noarch/*.rpm
{
"irods_host": "localhost",
"irods_port": 1247,
"irods_user_name": "rods",
"irods_zone_name": "tempZone"
}
[PostgreSQL ANSI]
Description=PostgreSQL ODBC driver (ANSI version)
Driver=psqlodbca.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1
[PostgreSQL Unicode]
Description=PostgreSQL ODBC driver (Unicode version)
Driver=psqlodbcw.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1
#!/bin/bash
while true; do sleep 10; done
#!/bin/bash
RETRIES=10
PG_HOST=postgres
PG_USER=irods
PG_DATABASE=ICAT
err=1
while [ $err != 0 ]
do
echo "attempt to connect to database"
PGPASSWORD=test psql --user $PG_USER --host=$PG_HOST --command='\q' $PG_DATABASE
err=$?
RETRIES=$(( RETRIES - 1 ))
if [ $err != 0 ];
then
if [ $RETRIES == 0 ]
then
echo "cannot connect to database"
exit 8
fi
echo "failed: try again ($attempts attempts)"
sleep 2;
fi
done
#!/bin/bash
VERSION=centos7_4_2_6
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
export VOLUME=$( dirname $( pwd ) )
REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_6
CLEANUP="yes"
BUILD=0
GIT_BRANCH=""
GIT_URL=""
while [[ $# -gt 0 ]]
do
......@@ -14,11 +17,6 @@ do
CLEANUP="no"
shift
;;
"--url")
shift
GIT_URL=$1
shift
;;
"--branch")
shift
GIT_BRANCH=$1
......@@ -30,34 +28,52 @@ do
shift
;;
*)
VERSION=$1
_VERSION=$1
shift
;;
esac
done
export VERSION=$_VERSION
cleanup () {
exit_code=$?
docker-compose -f ci/${VERSION}/docker-compose.yml down -v
echo "shuting down"
docker-compose -f ci/docker-compose.yml down -v
exit $exit_code
}
is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
set -x
set -e
trap cleanup EXIT ERR INT TERM
if [[ "$(is_logged_in)" == "false" ]]; then
docker login -u $REGISTRY_USER $REGISTRY;
fi
if [ "$CLEANUP" = "yes" ]
then
trap cleanup EXIT ERR INT TERM
fi
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
REPO_NAME=`repo_name $VERSION $GIT_URL $GIT_BRANCH `
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
mkdir -p ci/RPMS/Centos/7/${REPO_NAME}
mkdir -p $PACKAGE_DIR
docker-compose -f ci/${VERSION}/docker-compose.yml build
docker-compose -f ci/${VERSION}/docker-compose.yml up -d
docker pull ${REGISTRY}/light_${VERSION}:latest
docker exec ${VERSION}_icat_1 /app/create_rpm.sh $BUILD
docker-compose -f ci/docker-compose.yml up -d
docker exec ${VERSION}_icat_1 bash -c "set -x; set -e; cp /var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} /build/ci/RPMS/Centos/7/${REPO_NAME};"
docker exec ci_icat_1 mkdir -p /src/
docker cp . ci_icat_1:/src/B2SAFE-core
docker exec ci_icat_1 chown -R irods /src/B2SAFE-core
docker exec ci_icat_1 sudo -u irods /src/B2SAFE-core/packaging/create_rpm_package.sh 0
docker cp \
ci_icat_1:/var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} \
${PACKAGE_DIR}/${RPM_PACKAGE}
#!/bin/bash
workdir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
YUM_SERVER='software@software.irodspoc-sara.surf-hosted.nl'
SSH_OPTIONS='-oStrictHostKeyChecking=no'
docker build $workdir/centos7_b2handle -t b2handle
docker run -v $workdir/centos7_b2handle/dist:/opt/B2HANDLE/dist b2handle
rpm=$workdir/centos7_b2handle/dist/b2handle-*.noarch.rpm
rpm_file=$( basename $rpm )
for REPO_NAME in irods-4.1.11 irods-4.1.12 irods-4.2.3 irods-4.2.4 irods-4.2.5 irods-4.2.6
do
ssh $SSH_OPTIONS $YUM_SERVER "mkdir -p /repos/CentOS/7/${REPO_NAME}/Packages/"
scp $SSH_OPTIONS $rpm $YUM_SERVER:/repos/CentOS/7/${REPO_NAME}/Packages/$rpm_file
ssh $SSH_OPTIONS $YUM_SERVER createrepo --update /repos/CentOS/7/${REPO_NAME}
done
FROM centos:7
RUN yum update -y && \
yum install -y \
rsync wget less emacs make cmake gcc gcc-c++
RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
rpm --import https://packages.irods.org/irods-signing-key.asc && \
wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
tee /etc/yum.repos.d/renci-irods.yum.repo
RUN yum install -y \
sudo \
which \
python-jsonschema \
python-psutil \
python-pytest \
python-requests \
python-pip \
python-behave \
python-devel \
openssl-devel \
fuse-libs \
lsof \
openssl \
perl-JSON \
authd \
postgresql \
postgresql-odbc\
unixODBC \
boost-devel \
libcurl-devel \
supervisor \
rpm-build \
rpm-devel \
rpmlint \
rpmdevtools \
coreutils \
diffutils \
patch
# Openstack
RUN yum install -y centos-release-openstack-rocky
RUN yum install -y \
python-swiftclient \
python2-keystoneclient \
python-openstackclient
RUN pip install --upgrade pip && \
pip install flask \
flask_restful \
pytest \
s3cmd \
jsonschema \
python-irodsclient
COPY ci/app/odbcinst.ini /etc/odbcinst.ini
################################################################################
#
# irods
#
################################################################################
RUN yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icat-4.1.12-centos7-x86_64.rpm &&\
yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-database-plugin-postgres-1.12-centos7-x86_64.rpm && \
yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-runtime-4.1.12-centos7-x86_64.rpm && \
yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-dev-4.1.12-centos7-x86_64.rpm && \
rpm -i --replacefiles ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icommands-4.1.12-centos7-x86_64.rpm
################################################################################
#
# irods setup preparation
#
################################################################################
RUN useradd -rm -d /var/lib/irods irods
RUN mkdir -m 777 /src
ADD ci/app/4.1/setup_answers.txt /app/setup_answers.txt
ADD ci/app/4.1/setup_irods.sh /app/setup_irods.sh
ADD ci/app/4.1/surfsara-irods.repo /etc/yum.repos.d/surfsara-irods.repo
ADD ci/app/irods_environment.json /root/.irods/irods_environment.json
ADD ci/app/sleep.sh /app/sleep.sh
ADD ci/app/wait_for_pg.sh /app/wait_for_pg.sh
ADD ci/app/create_rpm.sh /app/create_rpm.sh
ADD ci/app/update_install.py /app/update_install.py
ADD scripts/tests/requirements.txt /app/test-requirements.txt
RUN cd /app && \
wget https://github.com/irods/irods/archive/4.1.12.tar.gz && \
tar -xvf 4.1.12.tar.gz
ADD scripts/tests/requirements.txt /app/test-requirements.txt
RUN pip install -r /app/test-requirements.txt
RUN yum install -y msi-persistent-id.x86_64
# install B2HANDLE
RUN pip install b2handle
CMD ["/app/setup_irods.sh"]
version: '2'
volumes:
data-volume:
irods-vault:
irods-conf:
services:
postgres:
build:
context: ../postgres
dockerfile: Dockerfile
ports:
- "5432"
hostname: postgresc7426.irods
volumes:
- data-volume:/var/lib/postgresql/9.5/main
icat:
build:
context: ../..
dockerfile: ci/centos7_4_1_12/Dockerfile
ports:
- "127.0.0.1:1247:1247"
depends_on:
- postgres
links:
- "postgres:postgres"
hostname: centos7_41_12.irods
volumes:
- "../../:/build"
- irods-vault:/var/lib/irods
- irods-conf:/etc/irods
command: /app/setup_irods.sh
FROM centos:7
RUN yum update -y && \
yum install -y \
rsync wget less emacs make cmake gcc gcc-c++
RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
rpm --import https://packages.irods.org/irods-signing-key.asc && \
wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
tee /etc/yum.repos.d/renci-irods.yum.repo
RUN yum install -y \
sudo \
which \
python-jsonschema \
python-psutil \
python-pytest \
python-requests \
python-pip \
python-behave \
python-devel \
openssl-devel \
fuse-libs \
lsof \
openssl \
perl-JSON \
authd \
postgresql \
postgresql-odbc\
unixODBC \
boost-devel \
libcurl-devel \
supervisor \
rpm-build \
rpm-devel \
rpmlint \
rpmdevtools \
coreutils \
diffutils \
patch
COPY ci/app/odbcinst.ini /etc/odbcinst.ini
################################################################################
#
# irods
#
################################################################################
RUN yum install -y irods-externals-boost1.60.0-0-1.0-1 \
irods-externals-avro1.7.7-0 \
irods-externals-clang-runtime3.8-0 \
irods-externals-clang3.8-0 \
irods-externals-cmake3.5.2-0 \
irods-externals-cppzmq4.1-0 \
irods-externals-epm4.2-0 irods-externals-imagemagick7.0.3-0 \
irods-externals-jansson2.7-0 \
irods-externals-json3.0.1-0 \
irods-externals-libarchive3.1.2-0 \
irods-externals-libarchive3.3.2-0 \
irods-externals-qpid-with-proton0.34-0 \