Commit 2ee2444b authored by Stefan Wolfsheimer's avatar Stefan Wolfsheimer
Browse files

Merge branch 'b2container' into 'devel'

Generate b2safe container

See merge request b2safe/B2SAFE-core!2
parents 9cd8caf6 16d418a2
......@@ -5,9 +5,11 @@ variables:
stages:
- build
- build_container
- test
- deploy
before_script:
- rm -f .variables
- touch .variables
......@@ -19,6 +21,19 @@ before_script:
else
echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USER --password-stdin $REGISTRY;
fi
- |
if [ -z "${SECRET_DIR}" ]
then
if [ -e secret ]
then
SECRET_DIR=$( pwd )/ci/secret
else
SECRET_DIR=$( realpath ~/secret )
fi
fi
echo "export SECRET_DIR=$SECRET_DIR" >> .variables
- source .variables
- docker-compose -f ci/docker-compose.yml down -v
- docker-compose -f ci/docker-compose.yml pull
......@@ -39,8 +54,9 @@ build_4.2.6:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
- ./ci/build.sh --build 0 ${VERSION}
build_4.2.7:
stage: build
......@@ -51,10 +67,10 @@ build_4.2.7:
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
- ./ci/build.sh --build 0 ${VERSION}
# #####################################
build_4.1.11:
stage: build
tags:
......@@ -64,7 +80,7 @@ build_4.1.11:
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
- ./ci/build.sh --build 0 ${VERSION}
build_4.1.12:
stage: build
......@@ -75,7 +91,54 @@ build_4.1.12:
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh ${VERSION}
- ./ci/build.sh --build 0 ${VERSION}
# #####################################
# Build container
# #####################################
container_4.1.11:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.1.12:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.6:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.7:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
# ###################################
# test
......@@ -88,6 +151,7 @@ test_4.2.6:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
......@@ -103,46 +167,49 @@ test_4.1.12:
- ./ci/test.sh ${VERSION}
# ###################################
# build
# deploy
# ###################################
deploy_4.2.6:
deploy_4.1.11:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_6
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.7:
deploy_4.1.12:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_7
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
# #####################################
deploy_4.1.11:
deploy_4.2.6:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_11
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.1.12:
deploy_4.2.7:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_12
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
#!/bin/bash
# setup irods
echo "---------------------------------------"
echo "-- setup irods --"
echo "---------------------------------------"
/app/setup_irods.sh /app/setup_b2safe.sh $@
#!/bin/bash
for t in epic2 irods b2safe
do
cd /app/B2SAFE-core/scripts/tests/
export url_prefix_in_profile='true'
export prefix=21.T12996
export IRODS_ENV_PATH=/var/lib/irods/.irods/irods_environment.json
python3 testB2SafeCmd.py -test $t
done
#!/bin/bash
cmd=$1
shift
echo "---------------------------------------"
echo "-- setup b2safe --"
echo "---------------------------------------"
set -x
set -e
cp /secrets/308_21.T12995_TRAINING_certificate_only.pem /etc/irods/308_21.T12995_TRAINING_certificate_only.pem
cp /secrets/308_21.T12995_TRAINING_privkey.pem /etc/irods/308_21.T12995_TRAINING_privkey.pem
cp /secrets/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cp /secrets/epic2_credentials /app/B2SAFE-core/scripts/tests/resources/epic2_credentials
/app/update_install.py /opt/eudat/b2safe/packaging/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cd /opt/eudat/b2safe/packaging/
sudo -u irods python install.py
echo "---------------------------------------"
echo "-- b2safe set up --"
echo "---------------------------------------"
# execute command
if [ -z "$cmd" ]
then
/app/sleep.sh
else
$cmd $@
fi
#!/bin/bash
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
# export VOLUME=$( dirname $( pwd ) )
# REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_6
CLEANUP="yes"
BUILD=0
GIT_BRANCH=""
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
"--branch")
shift
GIT_BRANCH=$1
shift
;;
"--build")
shift
BUILD=$1
shift
;;
*)
_VERSION=$1
shift
;;
esac
done
export VERSION=$_VERSION
if [ -z "${GIT_BRANCH}" ]
then
GIT_BRANCH=${CI_COMMIT_REF_NAME}
fi
set -x
set -e
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
DOCKERFILE=ci/${VERSION}/Dockerfile
IMAGE_NAME=${REGISTRY}/b2safe_${GIT_BRANCH}_${VERSION}:latest
is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
if [[ "$(is_logged_in)" == "false" ]]
then
echo "docker login on the gitlab runner machine:"
echo "docker login eudat-docker-public.artie.ia.surfsara.nl"
fi
if [ -e ${DOCKERFILE} ]
then
rm -rf RPM
mkdir -p RPM
cp ${PACKAGE_DIR}/${RPM_PACKAGE} RPM/b2safe.rpm
sed "s/__IRODS_VERSION__/irods-$IRODS_VERSION/" ci/eudat-artifactory.repo | \
sed "s/__BRANCH__/$GIT_BRANCH/" > ci/${VERSION}/eudat-artifactory.repo
docker build -t ${IMAGE_NAME} -f ${DOCKERFILE} .
# [DEPRECATION NOTICE] registry v2 schema1 support will be removed in an upcoming release. Please contact admins of the eudat-docker-public.artie.ia.surfsara.nl registry NOW to avoid future disruption. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/
# unauthorized: Pushing Docker images with manifest v2 schema 1 to this repository is blocked. For more information visit https://www.jfrog.com/confluence/display/RTF/Advanced+Topics#AdvancedTopics-DockerManifestV2Schema1Deprecation
# ---> remove image manually
# curl -ueudat-artie:XXXXXXXXXX -X DELETE https://artie.ia.surfsara.nl:443/artifactory/eudat-docker-public/b2safe_${GIT_BRANCH}_${VERSION}
/home/gitlab-runner/rm_image.sh b2safe_${GIT_BRANCH}_${VERSION}
docker push ${IMAGE_NAME}
rm -rf RPM
else
echo "No Dockerfile ${DOCKERFILE}"
exit 8
fi
FROM eudat-docker-public.artie.ia.surfsara.nl/light_centos7_4_1_11:latest
ADD RPM/b2safe.rpm /tmp/b2safe.rpm
RUN rpm -i /tmp/b2safe.rpm
ADD ci/centos7_4_1_11/eudat-artifactory.repo /etc/yum.repos.d/eudat-artifactory.repo
ADD ci/app/update_install.py /app/update_install.py
ADD ci/app/setup_b2safe.sh /app/setup_b2safe.sh
ADD ci/app/run_b2safe_tests.sh /app/run_b2safe_tests.sh
ADD ci/app/b2safe_entry_point.sh /app/b2safe_entry_point.sh
ADD . /app/B2SAFE-core
RUN pip3 install -r /app/B2SAFE-core/scripts/tests/requirements.txt && \
pip3 install b2handle
RUN yum clean all && \
yum install -y msi-persistent-id
ENTRYPOINT [ "/app/b2safe_entry_point.sh" ]
CMD [ "/app/sleep.sh" ]
[EUDAT]
name=EUDAT
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/irods-4.1.11/devel
enabled=1
gpgcheck=0
FROM eudat-docker-public.artie.ia.surfsara.nl/light_centos7_4_1_12:latest
ADD RPM/b2safe.rpm /tmp/b2safe.rpm
RUN rpm -i /tmp/b2safe.rpm
ADD ci/centos7_4_1_12/eudat-artifactory.repo /etc/yum.repos.d/eudat-artifactory.repo
ADD ci/app/update_install.py /app/update_install.py
ADD ci/app/setup_b2safe.sh /app/setup_b2safe.sh
ADD ci/app/run_b2safe_tests.sh /app/run_b2safe_tests.sh
ADD ci/app/b2safe_entry_point.sh /app/b2safe_entry_point.sh
ADD . /app/B2SAFE-core
RUN pip3 install -r /app/B2SAFE-core/scripts/tests/requirements.txt && \
pip3 install b2handle
RUN yum clean all && \
yum install -y msi-persistent-id
ENTRYPOINT [ "/app/b2safe_entry_point.sh" ]
CMD [ "/app/sleep.sh" ]
[EUDAT]
name=EUDAT
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/irods-4.1.12/devel
enabled=1
gpgcheck=0
FROM eudat-docker-public.artie.ia.surfsara.nl/light_centos7_4_2_6:latest
ADD RPM/b2safe.rpm /tmp/b2safe.rpm
RUN rpm -i /tmp/b2safe.rpm
ADD ci/centos7_4_2_6/eudat-artifactory.repo /etc/yum.repos.d/eudat-artifactory.repo
ADD ci/app/update_install.py /app/update_install.py
ADD ci/app/setup_b2safe.sh /app/setup_b2safe.sh
ADD ci/app/run_b2safe_tests.sh /app/run_b2safe_tests.sh
ADD ci/app/b2safe_entry_point.sh /app/b2safe_entry_point.sh
ADD . /app/B2SAFE-core
RUN pip3 install -r /app/B2SAFE-core/scripts/tests/requirements.txt && \
pip3 install b2handle
RUN yum clean all && \
yum install -y msi-persistent-id
ENTRYPOINT [ "/app/b2safe_entry_point.sh" ]
CMD [ "/app/sleep.sh" ]
[EUDAT]
name=EUDAT
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/irods-4.2.6/devel
enabled=1
gpgcheck=0
FROM eudat-docker-public.artie.ia.surfsara.nl/light_centos7_4_2_7:latest
ADD RPM/b2safe.rpm /tmp/b2safe.rpm
RUN rpm -i /tmp/b2safe.rpm
ADD ci/centos7_4_2_7/eudat-artifactory.repo /etc/yum.repos.d/eudat-artifactory.repo
ADD ci/app/update_install.py /app/update_install.py
ADD ci/app/setup_b2safe.sh /app/setup_b2safe.sh
ADD ci/app/run_b2safe_tests.sh /app/run_b2safe_tests.sh
ADD ci/app/b2safe_entry_point.sh /app/b2safe_entry_point.sh
ADD . /app/B2SAFE-core
RUN pip3 install -r /app/B2SAFE-core/scripts/tests/requirements.txt && \
pip3 install b2handle
RUN yum clean all && \
yum install -y msi-persistent-id
ENTRYPOINT [ "/app/b2safe_entry_point.sh" ]
CMD [ "/app/sleep.sh" ]
[EUDAT]
name=EUDAT
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/irods-4.2.7/devel
enabled=1
gpgcheck=0
......@@ -2,7 +2,6 @@ version: '2'
services:
postgres:
image: ${REGISTRY}/postgres:latest
ports:
- "5432"
......@@ -15,7 +14,7 @@ services:
hostname: handle.irods
icat:
image: ${REGISTRY}/light_${VERSION}:latest
image: ${REGISTRY}/b2safe_${CI_COMMIT_REF_NAME}_${VERSION}:latest
ports:
- "127.0.0.1:1247:1247"
depends_on:
......@@ -27,6 +26,7 @@ services:
hostname: icat.irods
volumes:
- "${VOLUME}:/build"
- "${SECRET_DIR}:/secrets"
command: /app/sleep.sh
icommands:
......
[EUDAT]
name=EUDAT
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/__IRODS_VERSION__/devel
baseurl=https://artie.ia.surfsara.nl/artifactory/Eudat-RPM-Testing-Public/Centos/7/__IRODS_VERSION__/__BRANCH__
enabled=1
gpgcheck=0
......
#!/bin/bash
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
export VOLUME=$( dirname $( pwd ) )
REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_6
......@@ -9,6 +8,7 @@ CLEANUP="yes"
RUNTESTS="yes"
BUILD=0
GIT_BRANCH=""
SECRET_DIR=""
while [[ $# -gt 0 ]]
do
......@@ -28,9 +28,9 @@ do
GIT_BRANCH=$1
shift
;;
"--build")
"--secret")
shift
BUILD=$1
SECRET_DIR=$1
shift
;;
*)
......@@ -56,17 +56,6 @@ is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
copy_secret() {
SOURCE=$1
TARGET=$2
if [ -e ci/secret/$SOURCE ]
then
docker cp ci/secret/$SOURCE ci_icat_1:$TARGET
else
docker cp ~/secret/$SOURCE ci_icat_1:$TARGET
fi
}
if [[ "$(is_logged_in)" == "false" ]]; then
docker login -u $REGISTRY_USER $REGISTRY;
fi
......@@ -76,71 +65,28 @@ then
trap cleanup EXIT ERR INT TERM
fi
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
# start container
docker pull ${REGISTRY}/light_${VERSION}:latest
docker-compose -f ci/docker-compose.yml up -d
# install microservices
docker cp ci/eudat-artifactory.repo ci_icat_1:/etc/yum.repos.d/eudat-artifactory.repo
docker exec ci_icat_1 sed -i "s/__IRODS_VERSION__/irods-$IRODS_VERSION/" /etc/yum.repos.d/eudat-artifactory.repo
docker exec ci_icat_1 yum install -y msi-persistent-id
# install b2safe
docker cp ${PACKAGE_DIR}/${RPM_PACKAGE} ci_icat_1:/${RPM_PACKAGE}
docker exec ci_icat_1 rpm -i /${RPM_PACKAGE}
# configure the installation
docker cp ci/update_install.py ci_icat_1:/app/update_install.py
copy_secret 308_21.T12995_TRAINING_certificate_only.pem \
/etc/irods/308_21.T12995_TRAINING_certificate_only.pem
copy_secret 308_21.T12995_TRAINING_privkey.pem \
/etc/irods/308_21.T12995_TRAINING_privkey.pem
copy_secret install.json \
/opt/eudat/b2safe/packaging/install_overlay.json
docker exec ci_icat_1 /app/update_install.py \
/opt/eudat/b2safe/packaging/install.json \
/opt/eudat/b2safe/packaging/install_overlay.json
docker exec -u irods ci_icat_1 bash -c "cd /opt/eudat/b2safe/packaging/ ; python install.py;"
# install test scripts
docker exec ci_icat_1 mkdir -p /src/
docker cp . ci_icat_1:/src/B2SAFE-core
docker exec ci_icat_1 chown -R irods /src/B2SAFE-core
if [ -z "${GIT_BRANCH}" ]
then
GIT_BRANCH=${CI_COMMIT_REF_NAME}
fi
# epic2 credentials
copy_secret epic2_credentials /src/B2SAFE-core/scripts/tests/resources
IMAGE_NAME=${REGISTRY}/b2safe_${GIT_BRANCH}_${VERSION}:latest
# run tests
if [ "$RUNTESTS" = "yes" ]
if [ -z "${SECRET_DIR}" ]
then
# install test requirements
if [ -e secret ]
then
SECRET_DIR=$( pwd )/ci/secret
else
SECRET_DIR=$( realpath ~/secret )
fi
fi
echo "export SECRET_DIR=$SECRET_DIR" >> .variables
export VOLUME=$( dirname $( pwd ) )
export SECRET_DIR=${SECRET_DIR}
# todo remove pip
docker exec ci_icat_1 yum install -y python-pip
docker exec ci_icat_1 pip install -r /src/B2SAFE-core/scripts/tests/requirements.txt
docker exec ci_icat_1 pip install b2handle
#
# start container
docker pull ${IMAGE_NAME}
docker exec ci_icat_1 pip3 install -r /src/B2SAFE-core/scripts/tests/requirements.txt
docker exec ci_icat_1 pip3 install b2handle
for t in epic2 irods b2safe
do
echo "------ TEST -----"
echo " $t "
echo "-----------------"
TEST_DIR="/src/B2SAFE-core/scripts/tests/"
SET_ENV="set -x; set -e; export url_prefix_in_profile='true'; export prefix=21.T12996"
docker exec -u irods ci_icat_1 bash -c "$SET_ENV; cd $TEST_DIR; python testB2SafeCmd.py -test $t;"
done
fi
docker-compose -f ci/docker-compose.yml run --rm icat \
/app/b2safe_entry_point.sh sudo -u irods /app/run_b2safe_tests.sh
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment