Commit aff5d2aa authored by Stefan Wolfsheimer's avatar Stefan Wolfsheimer
Browse files

Merge branch 'devel' into 'v3.2-beta5'

# Conflicts:
#   ci/centos7_4_1_11/Dockerfile
parents 7d80b2dd 9f768067
---
variables:
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive
stages:
- build
- build_container
- test
- deploy
before_script:
- rm -f .variables
- touch .variables
- |
if [ "$CI_REGISTRY" = "git.ia.surfsara.nl:5050" ]
then
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY;
echo "export REGISTRY=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/irods-ci-server" >> .variables
else
echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USER --password-stdin $REGISTRY;
fi
- |
if [ -z "${SECRET_DIR}" ]
then
if [ -e secret ]
then
SECRET_DIR=$( pwd )/ci/secret
else
SECRET_DIR=$( realpath ~/secret )
fi
fi
echo "export SECRET_DIR=$SECRET_DIR" >> .variables
- source .variables
- docker-compose -f ci/docker-compose.yml down -v
after_script:
- source .variables
- docker-compose -f ci/docker-compose.yml rm -f
- docker-compose -f ci/docker-compose.yml down -v
# ###################################
# build
# ###################################
build_4.2.6:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.2.7:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
# #####################################
build_4.1.11:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.1.12:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
# #####################################
# Build container
# #####################################
container_4.1.11:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.1.12:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.6:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.7:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
# ###################################
# test
# ###################################
test_4.2.6:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
test_4.1.12:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
# ###################################
# deploy
# ###################################
deploy_4.1.11:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.1.12:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
# #####################################
deploy_4.2.6:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.7:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
tempZone
1247
20000
20199
/var/lib/irods/iRODS/Vault
k
12345678901234567890123456789012
1248
12345678901234567890123456789012
rods
test
yes
postgres
5432
ICAT
irods
test
yes
#!/bin/bash
/app/wait_for_pg.sh
set -x
set -e
if [ -f /etc/irods/service_account.config ]
then
echo 'setup without service account'
cat /app/setup_answers.txt | /var/lib/irods/packaging/setup_irods.sh
else
( echo irods
echo irods
cat /app/setup_answers.txt
) | /var/lib/irods/packaging/setup_irods.sh
fi
/app/sleep.sh
[surfsara-irods]
name=Local CentOS Repo
baseurl=http://145.100.59.94/CentOS/7/irods-4.1.12/
gpgcheck=0
#!/bin/bash
sudo -u irods python /var/lib/irods/scripts/irods_control.py restart
irods
irods
1
2
postgres
5432
ICAT
irods
yes
test
tempZone
1247
20000
20199
1248
rods
yes
k
12345678901234567890123456789012
12345678901234567890123456789012
test
/var/lib/irods/iRODS/Vault
#!/bin/bash
/app/wait_for_pg.sh
set -x
if [ -e /var/lib/irods/iRODS/Vault/home ]
then
echo "irods already installed"
/etc/init.d/irods start
else
cat /app/setup_answers.txt | python /var/lib/irods/scripts/setup_irods.py
fi
/app/sleep.sh
[surfsara-irods]
name=Local CentOS Repo
baseurl=http://145.100.59.94/CentOS/7/irods-4.2.6/
gpgcheck=0
#!/bin/bash
# setup irods
echo "---------------------------------------"
echo "-- setup irods --"
echo "---------------------------------------"
/app/setup_irods.sh /app/setup_b2safe.sh $@
#!/bin/bash
set -x
set -e
BUILD=$1
if [[ -z "$BUILD" ]]
then
BUILD=0
fi
chown irods /var/lib/irods
sudo -u irods cp -r /build /src/B2SAFE-core
cd /src/B2SAFE-core/packaging
sudo -u irods ./create_rpm_package.sh $BUILD
chmod a+w /var/lib/irods/rpmbuild/RPMS/noarch/*.rpm
{
"irods_host": "localhost",
"irods_port": 1247,
"irods_user_name": "rods",
"irods_zone_name": "tempZone"
}
[PostgreSQL ANSI]
Description=PostgreSQL ODBC driver (ANSI version)
Driver=psqlodbca.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1
[PostgreSQL Unicode]
Description=PostgreSQL ODBC driver (Unicode version)
Driver=psqlodbcw.so
Setup=libodbcpsqlS.so
Debug=0
CommLog=1
UsageCount=1
#!/bin/bash
for t in epic2 irods b2safe
do
cd /app/B2SAFE-core/scripts/tests/
export url_prefix_in_profile='true'
export prefix=21.T12996
export IRODS_ENV_PATH=/var/lib/irods/.irods/irods_environment.json
python3 testB2SafeCmd.py -test $t
done
#!/bin/bash
cmd=$1
shift
echo "---------------------------------------"
echo "-- setup b2safe --"
echo "---------------------------------------"
set -x
set -e
cp /secrets/308_21.T12995_TRAINING_certificate_only.pem /etc/irods/308_21.T12995_TRAINING_certificate_only.pem
cp /secrets/308_21.T12995_TRAINING_privkey.pem /etc/irods/308_21.T12995_TRAINING_privkey.pem
cp /secrets/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cp /secrets/epic2_credentials /app/B2SAFE-core/scripts/tests/resources/epic2_credentials
/app/update_install.py /opt/eudat/b2safe/packaging/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cd /opt/eudat/b2safe/packaging/
sudo -u irods python install.py
echo "---------------------------------------"
echo "-- b2safe set up --"
echo "---------------------------------------"
# execute command
if [ -z "$cmd" ]
then
/app/sleep.sh
else
$cmd $@
fi
#!/bin/bash
while true; do sleep 10; done
#!/bin/bash
RETRIES=10
PG_HOST=postgres
PG_USER=irods
PG_DATABASE=ICAT
err=1
while [ $err != 0 ]
do
echo "attempt to connect to database"
PGPASSWORD=test psql --user $PG_USER --host=$PG_HOST --command='\q' $PG_DATABASE
err=$?
RETRIES=$(( RETRIES - 1 ))
if [ $err != 0 ];
then
if [ $RETRIES == 0 ]
then
echo "cannot connect to database"
exit 8
fi
echo "failed: try again ($attempts attempts)"
sleep 2;
fi
done
#!/bin/bash
VERSION=centos7_4_2_6
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
export VOLUME=$( dirname $( pwd ) )
REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_6
CLEANUP="yes"
BUILD=0
GIT_BRANCH=""
GIT_URL=""
while [[ $# -gt 0 ]]
do
......@@ -14,11 +17,6 @@ do
CLEANUP="no"
shift
;;
"--url")
shift
GIT_URL=$1
shift
;;
"--branch")
shift
GIT_BRANCH=$1
......@@ -30,34 +28,56 @@ do
shift
;;
*)
VERSION=$1
_VERSION=$1
shift
;;
esac
done
export VERSION=$_VERSION
cleanup () {
exit_code=$?
docker-compose -f ci/${VERSION}/docker-compose.yml down -v
echo "shuting down"
docker-compose -f ci/docker-compose.yml down -v
exit $exit_code
}
is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
set -x
set -e
trap cleanup EXIT ERR INT TERM
if [[ "$(is_logged_in)" == "false" ]]; then
docker login -u $REGISTRY_USER $REGISTRY;
fi
if [ "$CLEANUP" = "yes" ]
then
trap cleanup EXIT ERR INT TERM
fi
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
REPO_NAME=`repo_name $VERSION $GIT_URL $GIT_BRANCH `
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
mkdir -p $PACKAGE_DIR
docker pull ${REGISTRY}/light_${VERSION}:latest
export DOCKER_IMAGE=light_${VERSION}:latest
mkdir -p ci/RPMS/Centos/7/${REPO_NAME}
docker-compose -f ci/${VERSION}/docker-compose.yml build
docker-compose -f ci/${VERSION}/docker-compose.yml up -d
docker exec ${VERSION}_icat_1 /app/create_rpm.sh $BUILD
docker-compose -f ci/docker-compose.yml up -d
docker exec ${VERSION}_icat_1 bash -c "set -x; set -e; cp /var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} /build/ci/RPMS/Centos/7/${REPO_NAME};"
docker exec ci_icat_1 mkdir -p /src/
docker cp . ci_icat_1:/src/B2SAFE-core
docker exec ci_icat_1 chown -R irods /src/B2SAFE-core
docker exec ci_icat_1 sudo -u irods /src/B2SAFE-core/packaging/create_rpm_package.sh 0
docker cp \
ci_icat_1:/var/lib/irods/rpmbuild/RPMS/noarch/${RPM_PACKAGE} \
${PACKAGE_DIR}/${RPM_PACKAGE}
#!/bin/bash
workdir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
YUM_SERVER='software@software.irodspoc-sara.surf-hosted.nl'
SSH_OPTIONS='-oStrictHostKeyChecking=no'
docker build $workdir/centos7_b2handle -t b2handle
docker run -v $workdir/centos7_b2handle/dist:/opt/B2HANDLE/dist b2handle
rpm=$workdir/centos7_b2handle/dist/b2handle-*.noarch.rpm
rpm_file=$( basename $rpm )
for REPO_NAME in irods-4.1.11 irods-4.1.12 irods-4.2.3 irods-4.2.4 irods-4.2.5 irods-4.2.6
do
ssh $SSH_OPTIONS $YUM_SERVER "mkdir -p /repos/CentOS/7/${REPO_NAME}/Packages/"
scp $SSH_OPTIONS $rpm $YUM_SERVER:/repos/CentOS/7/${REPO_NAME}/Packages/$rpm_file
ssh $SSH_OPTIONS $YUM_SERVER createrepo --update /repos/CentOS/7/${REPO_NAME}
done
#!/bin/bash
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
# export VOLUME=$( dirname $( pwd ) )
# REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_6
CLEANUP="yes"
BUILD=0
GIT_BRANCH=""
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
"--branch")
shift
GIT_BRANCH=$1
shift
;;
"--build")
shift
BUILD=$1
shift
;;
*)
_VERSION=$1
shift
;;
esac
done
export VERSION=$_VERSION
if [ -z "${GIT_BRANCH}" ]
then
GIT_BRANCH=${CI_COMMIT_REF_NAME}
fi
set -x
set -e
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
DOCKERFILE=ci/${VERSION}/Dockerfile
IMAGE_NAME=${REGISTRY}/b2safe_${GIT_BRANCH}_${VERSION}:latest
is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
if [[ "$(is_logged_in)" == "false" ]]
then
echo "docker login on the gitlab runner machine:"
echo "docker login eudat-docker-public.artie.ia.surfsara.nl"
fi
if [ -e ${DOCKERFILE} ]
then
rm -rf RPM
mkdir -p RPM
cp ${PACKAGE_DIR}/${RPM_PACKAGE} RPM/b2safe.rpm
sed "s/__IRODS_VERSION__/irods-$IRODS_VERSION/" ci/eudat-artifactory.repo | \
sed "s/__BRANCH__/$GIT_BRANCH/" > ci/${VERSION}/eudat-artifactory.repo
docker build -t ${IMAGE_NAME} -f ${DOCKERFILE} .
# [DEPRECATION NOTICE] registry v2 schema1 support will be removed in an upcoming release. Please contact admins of the eudat-docker-public.artie.ia.surfsara.nl registry NOW to avoid future disruption. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/
# unauthorized: Pushing Docker images with manifest v2 schema 1 to this repository is blocked. For more information visit https://www.jfrog.com/confluence/display/RTF/Advanced+Topics#AdvancedTopics-DockerManifestV2Schema1Deprecation
# ---> remove image manually
# curl -ueudat-artie:XXXXXXXXXX -X DELETE https://artie.ia.surfsara.nl:443/artifactory/eudat-docker-public/b2safe_${GIT_BRANCH}_${VERSION}
/home/gitlab-runner/rm_image.sh b2safe_${GIT_BRANCH}_${VERSION}
docker push ${IMAGE_NAME}
rm -rf RPM
else
echo "No Dockerfile ${DOCKERFILE}"
exit 8
fi
FROM centos:7
FROM eudat-docker-public.artie.ia.surfsara.nl/light_centos7_4_1_11:latest
RUN yum update -y && \
yum install -y \