Commit 8f1e8113 authored by Michele Carpene's avatar Michele Carpene
Browse files

Merge branch 'devel' into 'master'

Merge-Devel

See merge request b2safe/B2SAFE-core!6
parents bf29aa9e 9d27790e
# exclude python compiled files
*.pyc
*~
ci/RPMS
\ No newline at end of file
---
variables:
GIT_STRATEGY: clone
GIT_SUBMODULE_STRATEGY: recursive
stages:
- build
- build_container
- test
- deploy
before_script:
- rm -f .variables
- touch .variables
- |
if [ "$CI_REGISTRY" = "git.ia.surfsara.nl:5050" ]
then
echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY;
echo "export REGISTRY=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/irods-ci-server" >> .variables
else
echo $REGISTRY_PASSWORD | docker login -u $REGISTRY_USER --password-stdin $REGISTRY;
fi
- |
if [ -z "${SECRET_DIR}" ]
then
if [ -e secret ]
then
SECRET_DIR=$( pwd )/ci/secret
else
SECRET_DIR=$( realpath ~/secret )
fi
fi
echo "export SECRET_DIR=$SECRET_DIR" >> .variables
- source .variables
- docker-compose -f ci/docker-compose.yml down -v
after_script:
- source .variables
- docker-compose -f ci/docker-compose.yml rm -f
- docker-compose -f ci/docker-compose.yml down -v
# ###################################
# build
# ###################################
build_4.1.11:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.1.12:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.2.6:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.2.7:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
build_4.2.8:
stage: build
tags:
- irods
variables:
VERSION: centos7_4_2_8
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build.sh --build 0 ${VERSION}
# #####################################
# Build container
# #####################################
container_4.1.11:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.1.12:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.6:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.7:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
container_4.2.8:
stage: build_container
tags:
- irods
variables:
VERSION: centos7_4_2_8
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/build_container.sh --build 0 ${VERSION}
# ###################################
# test
# ###################################
test_4.1.12:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
test_4.2.6:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
test_4.2.7:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
test_4.2.8:
stage: test
tags:
- irods
variables:
VERSION: centos7_4_2_8
script:
- source .variables
- cat .variables
- ./ci/shutdownall.sh
- ./ci/test.sh ${VERSION}
# ###################################
# deploy
# ###################################
deploy_4.1.11:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_11
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.1.12:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_1_12
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.6:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_6
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.7:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_7
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
deploy_4.2.8:
stage: deploy
tags:
- irods
variables:
VERSION: centos7_4_2_8
script:
- source .variables
- ./ci/deploy.sh ${VERSION}
pipeline
{
agent any
environment {
BUILD_NUMBER = "${env.BUILD_NUMBER}"
GIT_URL = "${env.GIT_URL}"
GIT_BRANCH = "${env.GIT_BRANCH}"
YUM_SERVER = 'software@software.irodspoc-sara.surf-hosted.nl'
SSH_OPTIONS = '-oStrictHostKeyChecking=no'
}
stages
{
stage('Build_4.2.6')
{
steps
{
echo '-------------'
echo 'Shutting down'
echo '-------------'
sh './ci/shutdownall.sh'
echo '----------------------------'
echo 'Building against iRODS 4.2.6'
echo '----------------------------'
sh './ci/build.sh centos7_4_2_6 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Build_4.1.11')
{
steps
{
echo '-------------'
echo 'Shutting down'
echo '-------------'
sh './ci/shutdownall.sh'
echo '-----------------------------'
echo 'Building against iRODS 4.1.11'
echo '-----------------------------'
sh './ci/build.sh centos7_4_1_11 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Build_4.1.12')
{
steps
{
echo '-------------'
echo 'Shutting down'
echo '-------------'
sh './ci/shutdownall.sh'
echo '-----------------------------'
echo 'Building against iRODS 4.1.12'
echo '-----------------------------'
sh './ci/build.sh centos7_4_1_12 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Test_4.2.6')
{
steps
{
echo '----------------------------'
echo 'Testing against iRODS 4.2.6 '
echo '----------------------------'
sh './ci/test.sh centos7_4_2_6 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Test_4_1_12')
{
steps
{
echo '-----------------------------'
echo 'Testing against iRODS 4.1.12'
echo '-----------------------------'
sh './ci/test.sh centos7_4_1_12 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Deploy_4_1_11')
{
steps
{
echo '------------------------------'
echo 'Deploying.'
sh './ci/deploy.sh centos7_4_1_11 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Deploy_4_1_12')
{
steps
{
echo '------------------------------'
echo 'Deploying.'
sh './ci/deploy.sh centos7_4_1_12 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
stage('Deploy_4_2_6')
{
steps
{
echo '------------------------------'
echo 'Deploying.'
sh './ci/deploy.sh centos7_4_2_6 --build ${BUILD_NUMBER} --url ${GIT_URL} --branch ${GIT_BRANCH}'
}
}
}
}
\ No newline at end of file
......@@ -9,7 +9,9 @@ The EUDAT (http://www.eudat.eu) B2SAFE Service offers functionality to replicate
---------------
Deployment
---------------
See install.txt .
* [Deployment on Centos 7, see install_centos7.md](install_centos7.md)
* [Deployment on other systems, see install_other.txt](install_other.txt)
---------------
Documentation
......
# Building the package locally
![](dev-workflow.png)
## Requirements
* docker
* docker-compose
## Building and testing
The script can be used *./ci/build.sh* to build the RPM package.
### Build the RPM
Build the B2SAFE package for CentOS 7 and iRODS 4.2.8.
```
./ci/build.sh centos7_4_2_8
```
If the build process was successful, the RPM will be copied to the shared target folder:
```
ls ci/RPMS/Centos/7/irods-4.2.8/
irods-eudat-b2safe-4.3.0-0.noarch.rpm
```
### Configure the functional test
In order to run the tests make sure that you have access to a handle server with a dedicated test prefix.
You should have received a private key and a certificate PEM file
(in this case *308_21.T12995_TRAINING_privkey.pem* and *308_21.T12995_TRAINING_certificate_only.pem*)
Configure the following files:
*ci/secret/epic2_credentials*:
```
{
"handle_server_url": "https://epic5.storage.surfsara.nl:8003",
"private_key": "/etc/irods/308_21.T12995_TRAINING_privkey.pem",
"certificate_only": "/etc/irods/308_21.T12995_TRAINING_certificate_only.pem",
"prefix": "21.T12995",
"handleowner": "200:0.NA/21.T12995",
"reverselookup_username": "21.T12995",
"reverselookup_password": "<INSERT_PASSWORD_HERE>",
"HTTPS_verify": "False"
}
```
*ci/secret/install.json*
```
{
"server_id": "irods://{HOSTNAME}:1247",
"server_api_reg": "irods://{UUID}:1247",
"server_api_pub": "irods://{UUID}:1247",
"handle_server_url": "https://epic5.storage.surfsara.nl:8003",
"handle_private_key": "/etc/irods/308_21.T12995_TRAINING_privkey.pem",
"handle_certificate_only": "/etc/irods/308_21.T12995_TRAINING_certificate_only.pem",
"handle_prefix": "21.T12995",
"handle_owner": "200:0.NA/21.T12995",
"handle_reverse_lookup_name": "21.T12995",
"handle_reverse_lookup_password": "<INSERT_PASSWORD_HERE>",
"handle_https_verify": "False",
"handle_users": [ "*" ],
"handle_groups": []
}
```
### Run the functional test
```
./ci/test.sh centos7_4_2_8
```
### Automation
The details can be found in the Jenkinsfile in the root of the repository.
#!/bin/bash
# setup irods
echo "---------------------------------------"
echo "-- setup irods --"
echo "---------------------------------------"
/app/setup_irods.sh /app/setup_b2safe.sh $@
#!/bin/bash
for t in epic2 irods b2safe
do
cd /app/B2SAFE-core/scripts/tests/
export url_prefix_in_profile='true'
export prefix=21.T12996
export IRODS_ENV_PATH=/var/lib/irods/.irods/irods_environment.json
python3 testB2SafeCmd.py -test $t
done
#!/bin/bash
cmd=$1
shift
echo "---------------------------------------"
echo "-- setup b2safe --"
echo "---------------------------------------"
set -x
set -e
cp /secrets/308_21.T12995_TRAINING_certificate_only.pem /etc/irods/308_21.T12995_TRAINING_certificate_only.pem
cp /secrets/308_21.T12995_TRAINING_privkey.pem /etc/irods/308_21.T12995_TRAINING_privkey.pem
cp /secrets/TERENA-SSL-CA-3.pem /etc/pki/ca-trust/source/anchors/TERENA-SSL-CA-3.pem
/usr/bin/update-ca-trust
cp /secrets/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cp /secrets/epic2_credentials /app/B2SAFE-core/scripts/tests/resources/epic2_credentials
/app/update_install.py /opt/eudat/b2safe/packaging/install.json /opt/eudat/b2safe/packaging/install_overlay.json
cd /opt/eudat/b2safe/packaging/
sudo -u irods python install.py
# copy epicclient2.py credentials to make sure the correct epicclient2.py credentials are used
cd /opt/eudat/b2safe/conf
cp /secrets/epic2_credentials credentials
cd /opt/eudat/b2safe/packaging/
# end copy epicclient2.py credentials
echo "---------------------------------------"
echo "-- b2safe set up --"
echo "---------------------------------------"
# execute command
if [ -z "$cmd" ]
then
/app/sleep.sh
else
$cmd $@
fi
#!/usr/bin/env python
import sys
import json
import platform
import uuid
def update_hostname(value, id):
isstr = False
if sys.version_info >= (3, 0):
isstr = isinstance(value, str)
else:
isstr = isinstance(value, str) or isinstance(value, unicode)
if isstr:
return value.format(HOSTNAME=platform.node(),
UUID=id)
else:
return value
fname = sys.argv[1]
overlay_fname = sys.argv[2]
overlay_uuid = str(uuid.uuid4())
with open(overlay_fname, "r") as fp:
overlay = json.load(fp)
overlay = {k: update_hostname(v, overlay_uuid)
for k, v in overlay.items()}
with open(fname, "r") as fp:
data = json.load(fp)
data.update(overlay)
with open(fname, "w") as fp:
data = json.dump(data, fp)
#!/bin/bash
export REGISTRY=eudat-docker-public.artie.ia.surfsara.nl
export VOLUME=$( dirname $( pwd ) )
REGISTRY_USER=eudat-artie-readonly
_VERSION=centos7_4_2_8
CLEANUP="yes"
BUILD=0
GIT_BRANCH=""
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
"--nocleanup")
CLEANUP="no"
shift
;;
"--branch")
shift
GIT_BRANCH=$1
shift
;;
"--build")
shift
BUILD=$1
shift
;;
*)
_VERSION=$1
shift
;;
esac
done
export VERSION=$_VERSION
cleanup () {
exit_code=$?
echo "shuting down"
docker-compose -f ci/docker-compose.yml down -v
exit $exit_code
}
is_logged_in() {
cat ~/.docker/config.json | jq -r --arg url "${REGISTRY}" '.auths | has($url)'
}
set -x
set -e
if [[ "$(is_logged_in)" == "false" ]]; then
docker login -u $REGISTRY_USER $REGISTRY;
fi
if [ "$CLEANUP" = "yes" ]
then
trap cleanup EXIT ERR INT TERM
fi
source $(cd `dirname "${BASH_SOURCE[0]}"` && pwd)/version.sh
RPM_PACKAGE=`rpm_package $BUILD `
IRODS_VERSION=`irods_version $VERSION`
PACKAGE_DIR=`package_dir $VERSION $GIT_BRANCH`
mkdir -p $PACKAGE_DIR