Merge pull request #133 from stefan-wolfsheimer/devel

Bug Fixes and Jenkins Pipeline

.gitignore

 # exclude python compiled files
 *.pyc
+*~
+ci/RPMS
\ No newline at end of file

Jenkinsfile

+pipeline
+{
+    agent any
+
+    stages
+    {
+        stage('Build_4.2.6')
+        {
+            steps
+            {
+                echo '-------------'
+                echo 'Shutting down'
+                echo '-------------'
+                sh './ci/shutdownall.sh'
+
+                echo '----------------------------'
+                echo 'Building against iRODS 4.2.6'
+                echo '----------------------------'
+                sh './ci/build.sh centos7_4_2_6'
+
+            }
+        }
+        stage('Build_4.1.12')
+        {
+            steps
+            {
+                echo '-------------'
+                echo 'Shutting down'
+                echo '-------------'
+                sh './ci/shutdownall.sh'
+
+                echo '-----------------------------'
+                echo 'Building against iRODS 4.1.12'
+                echo '-----------------------------'
+                sh './ci/build.sh centos7_4_1_12'
+            }
+
+        }
+        stage('Test_4.2.6')
+        {
+            steps
+            {
+                echo '----------------------------'
+                echo 'Testing against iRODS 4.2.6 '
+                echo '----------------------------'
+                sh './ci/test.sh centos7_4_2_6'
+            }
+        }
+        stage('Test_4_1_12')
+        {
+            steps
+            {
+                echo '-----------------------------'
+                echo 'Testing against iRODS 4.1.12'
+                echo '-----------------------------'
+                sh './ci/test.sh centos7_4_1_12'
+            }
+        }
+
+        stage('Deploy')
+        {
+            steps
+            {
+                echo 'Deploying....'
+           }
+        }
+    }
+}
\ No newline at end of file

ci/app/4.1/setup_answers.txt

+tempZone
+1247
+20000
+20199
+/var/lib/irods/iRODS/Vault
+k
+12345678901234567890123456789012
+1248
+12345678901234567890123456789012
+
+rods
+test
+yes
+postgres
+5432
+ICAT
+irods
+test
+yes
+

ci/app/4.1/setup_irods.sh

+#!/bin/bash
+/app/wait_for_pg.sh
+set -x
+set -e
+
+if [ -f /etc/irods/service_account.config ]
+then
+    echo 'setup without service account'
+    cat /app/setup_answers.txt | /var/lib/irods/packaging/setup_irods.sh
+else
+    ( echo irods
+      echo irods
+      cat /app/setup_answers.txt
+    ) | /var/lib/irods/packaging/setup_irods.sh
+fi
+
+/app/sleep.sh

ci/app/4.1/surfsara-irods.repo

+[surfsara-irods]
+name=Local CentOS Repo
+baseurl=http://145.100.59.94/CentOS/7/irods-4.1.12/
+gpgcheck=0

ci/app/4.2/restart_irods.sh

+#!/bin/bash
+sudo -u irods python /var/lib/irods/scripts/irods_control.py restart

ci/app/4.2/setup_answers.txt

+irods
+irods
+1
+2
+postgres
+5432
+ICAT
+irods
+yes
+test
+
+tempZone
+1247
+20000
+20199
+1248
+
+rods
+yes
+k
+12345678901234567890123456789012
+12345678901234567890123456789012
+test
+/var/lib/irods/iRODS/Vault

ci/app/4.2/setup_irods.sh

+#!/bin/bash
+/app/wait_for_pg.sh
+
+set -x
+
+if [ -e  /var/lib/irods/iRODS/Vault/home ]
+then
+    echo "irods already installed"
+    /etc/init.d/irods start
+else
+    cat /app/setup_answers.txt | python /var/lib/irods/scripts/setup_irods.py
+fi
+
+/app/sleep.sh

ci/app/4.2/surfsara-irods.repo

+[surfsara-irods]
+name=Local CentOS Repo
+baseurl=http://145.100.59.94/CentOS/7/irods-4.2.6/
+gpgcheck=0

ci/app/create_rpm.sh

+#!/bin/bash
+set -x
+set -e
+
+chown irods /var/lib/irods
+sudo -u irods cp -r /build /src/B2SAFE-core
+cd /src/B2SAFE-core/packaging
+sudo -u irods ./create_rpm_package.sh
+chmod a+w /var/lib/irods/rpmbuild/RPMS/noarch/*.rpm
+
+
+

ci/app/irods_environment.json

+{
+    "irods_host": "localhost",
+    "irods_port": 1247,
+    "irods_user_name": "rods",
+    "irods_zone_name": "tempZone"
+}

ci/app/odbcinst.ini

+[PostgreSQL ANSI]
+Description=PostgreSQL ODBC driver (ANSI version)
+Driver=psqlodbca.so
+Setup=libodbcpsqlS.so
+Debug=0
+CommLog=1
+UsageCount=1
+
+[PostgreSQL Unicode]
+Description=PostgreSQL ODBC driver (Unicode version)
+Driver=psqlodbcw.so
+Setup=libodbcpsqlS.so
+Debug=0
+CommLog=1
+UsageCount=1

ci/app/sleep.sh

+#!/bin/bash
+
+while true; do sleep 10; done

ci/app/update_install.py

+#!/usr/bin/env python
+import sys
+import json
+import platform
+import uuid
+
+
+def update_hostname(value, id):
+    isstr = False
+    if sys.version_info >= (3, 0):
+        isstr = isinstance(value, str)
+    else:
+        isstr = isinstance(value, str) or isinstance(value, unicode)
+    if isstr:
+        return value.format(HOSTNAME=platform.node(),
+                            UUID=id)
+    else:
+        return value
+
+
+fname = sys.argv[1]
+overlay_fname = sys.argv[2]
+overlay_uuid = str(uuid.uuid4())
+
+with open(overlay_fname, "r") as fp:
+    overlay = json.load(fp)
+overlay = {k: update_hostname(v, overlay_uuid)
+           for k, v in overlay.items()}
+
+with open(fname, "r") as fp:
+    data = json.load(fp)
+
+data.update(overlay)
+
+with open(fname, "w") as fp:
+    data = json.dump(data, fp)

ci/app/wait_for_pg.sh

+#!/bin/bash
+
+RETRIES=10
+PG_HOST=postgres
+PG_USER=irods
+PG_DATABASE=ICAT
+
+err=1
+while [ $err != 0 ]
+do
+    echo "attempt to connect to database"
+    PGPASSWORD=test psql --user $PG_USER --host=$PG_HOST --command='\q' $PG_DATABASE
+    err=$?
+    RETRIES=$(( RETRIES - 1 ))
+    if [ $err != 0 ];
+    then
+        if [ $RETRIES == 0 ]
+        then
+            echo "cannot connect to database"
+            exit 8
+        fi
+        echo "failed: try again ($attempts attempts)"
+        sleep 2;
+    fi
+done
+
+

ci/build.sh

+#!/bin/bash
+set -x
+set -e
+
+VERSION=$1
+if [[ -z "$VERSION" ]]
+then
+    VERSION=centos7_4_2_6
+fi
+
+IRODS_VERSION=$( echo $VERSION | awk 'BEGIN{ FS="_"; }{ print $2"."$3"."$4; }' )
+
+cleanup () {
+    exit_code=$?
+    docker-compose -f ci/${VERSION}/docker-compose.yml down -v
+    exit $exit_code
+}
+
+trap cleanup EXIT ERR INT TERM
+
+mkdir -p ci/RPMS/Centos/7/irods-${IRODS_VERSION}
+
+docker-compose -f ci/${VERSION}/docker-compose.yml build
+docker-compose -f ci/${VERSION}/docker-compose.yml up -d
+
+docker exec ${VERSION}_icat_1 /app/create_rpm.sh
+docker exec ${VERSION}_icat_1 bash -c "set -x; set -e; cp /var/lib/irods/rpmbuild/RPMS/noarch/*.rpm /build/ci/RPMS/Centos/7/irods-${IRODS_VERSION};" 

ci/centos7_4_1_12/Dockerfile

+FROM centos:7
+
+RUN yum update -y && \
+    yum install -y \
+    rsync wget less emacs make cmake gcc gcc-c++
+
+RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
+    rpm --import https://packages.irods.org/irods-signing-key.asc && \
+    wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
+    tee /etc/yum.repos.d/renci-irods.yum.repo
+
+RUN yum install -y \
+    sudo \
+    which \
+    python-jsonschema \
+    python-psutil \
+    python-pytest \
+    python-requests \
+    python-pip \
+    python-behave \
+    python-devel \
+    openssl-devel \
+    fuse-libs \
+    lsof \
+    openssl \
+    perl-JSON \
+    authd \
+    postgresql \
+    postgresql-odbc\
+    unixODBC \
+    boost-devel \
+    libcurl-devel \
+    supervisor \
+    rpm-build \
+    rpm-devel \
+    rpmlint \
+    rpmdevtools \
+    coreutils \
+    diffutils \
+    patch
+
+# Openstack
+RUN yum install -y centos-release-openstack-rocky
+
+RUN yum install -y \
+    python-swiftclient \
+    python2-keystoneclient \
+    python-openstackclient
+
+
+RUN pip install --upgrade pip && \
+    pip install flask \
+                flask_restful \
+                pytest \
+                s3cmd \
+                jsonschema \
+                python-irodsclient
+
+COPY ci/app/odbcinst.ini /etc/odbcinst.ini
+
+################################################################################
+#
+# irods
+#
+################################################################################
+RUN yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icat-4.1.12-centos7-x86_64.rpm &&\
+    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-database-plugin-postgres-1.12-centos7-x86_64.rpm && \
+    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-runtime-4.1.12-centos7-x86_64.rpm && \
+    yum install -y ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-dev-4.1.12-centos7-x86_64.rpm && \
+    rpm -i --replacefiles ftp://ftp.renci.org/pub/irods/releases/4.1.12/centos7/irods-icommands-4.1.12-centos7-x86_64.rpm
+
+
+################################################################################
+#
+# irods setup preparation
+#
+################################################################################
+RUN useradd -rm -d /var/lib/irods irods
+RUN mkdir -m 777 /src
+ADD ci/app/4.1/setup_answers.txt /app/setup_answers.txt
+ADD ci/app/4.1/setup_irods.sh /app/setup_irods.sh
+ADD ci/app/4.1/surfsara-irods.repo /etc/yum.repos.d/surfsara-irods.repo
+ADD ci/app/irods_environment.json /root/.irods/irods_environment.json
+ADD ci/app/sleep.sh /app/sleep.sh
+ADD ci/app/wait_for_pg.sh /app/wait_for_pg.sh
+ADD ci/app/create_rpm.sh /app/create_rpm.sh
+ADD ci/app/update_install.py /app/update_install.py
+ADD scripts/tests/requirements.txt /app/test-requirements.txt
+
+RUN cd /app && \
+    wget https://github.com/irods/irods/archive/4.1.12.tar.gz && \
+    tar -xvf 4.1.12.tar.gz
+
+ADD scripts/tests/requirements.txt /app/test-requirements.txt
+
+RUN pip install -r /app/test-requirements.txt
+RUN yum install -y msi-persistent-id.x86_64
+
+# install B2HANDLE
+RUN pip install b2handle
+
+
+CMD ["/app/setup_irods.sh"]

ci/centos7_4_1_12/docker-compose.yml

+version: '2'
+
+volumes:
+  data-volume:
+
+  irods-vault:
+
+  irods-conf:
+
+
+
+services:
+  postgres:
+    build:
+      context: ../postgres
+      dockerfile: Dockerfile
+    ports:
+      - "5432"
+    hostname: postgresc7426.irods
+    volumes:
+      - data-volume:/var/lib/postgresql/9.5/main
+
+  icat:
+    build:
+      context: ../..
+      dockerfile: ci/centos7_4_1_12/Dockerfile
+    ports:
+      - "127.0.0.1:1247:1247"
+    depends_on:
+      - postgres
+    links:
+      - "postgres:postgres"
+    hostname: centos7_41_12.irods
+    volumes:
+      - "../../:/build"
+      - irods-vault:/var/lib/irods
+      - irods-conf:/etc/irods
+    command: /app/setup_irods.sh

ci/centos7_4_2_6/Dockerfile

+FROM centos:7
+
+RUN yum update -y && \
+    yum install -y \
+    rsync wget less emacs make cmake gcc gcc-c++
+
+RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
+    rpm --import https://packages.irods.org/irods-signing-key.asc && \
+    wget -qO - https://packages.irods.org/renci-irods.yum.repo | \
+    tee /etc/yum.repos.d/renci-irods.yum.repo
+
+RUN yum install -y \
+    sudo \
+    which \
+    python-jsonschema \
+    python-psutil \
+    python-pytest \
+    python-requests \
+    python-pip \
+    python-behave \
+    python-devel  \
+    openssl-devel \
+    fuse-libs \
+    lsof \
+    openssl \
+    perl-JSON \
+    authd \
+    postgresql \
+    postgresql-odbc\
+    unixODBC \
+    boost-devel \
+    libcurl-devel \
+    supervisor \
+    rpm-build \
+    rpm-devel \
+    rpmlint \
+    rpmdevtools \
+    coreutils \
+    diffutils \
+    patch
+
+COPY ci/app/odbcinst.ini /etc/odbcinst.ini
+
+################################################################################
+#
+# irods
+#
+################################################################################
+RUN yum install -y irods-externals-boost1.60.0-0-1.0-1 \
+                   irods-externals-avro1.7.7-0 \
+                   irods-externals-clang-runtime3.8-0 \
+                   irods-externals-clang3.8-0 \
+                   irods-externals-cmake3.5.2-0 \
+                   irods-externals-cppzmq4.1-0 \
+                   irods-externals-epm4.2-0 irods-externals-imagemagick7.0.3-0 \
+                   irods-externals-jansson2.7-0 \
+                   irods-externals-json3.0.1-0 \
+                   irods-externals-libarchive3.1.2-0 \
+                   irods-externals-libarchive3.3.2-0 \
+                   irods-externals-qpid-with-proton0.34-0 \
+                   irods-externals-redis4.0.8-0 \
+                   irods-externals-zeromq4-14.1.3-0 \
+                   irods-devel irods-externals-autoconf5ad3567c-0 \
+                   irods-externals-libs3a30e55e8-0-1.0-1 \
+                   irods-server-4.2.6 \
+                   irods-database-plugin-postgres-4.2.6 \
+                   irods-icommands%VERSION-4.2.6 \
+                   irods-rule-engine-plugin-audit-amqp-4.2.6 \
+                   irods-rule-engine-plugin-python-4.2.6 \
+                   irods-runtime-4.2.6 \
+                   irods-server-4.2.6 && \
+                   pip install --upgrade pip && \
+                   pip install python-irodsclient flask flask_restful pytest
+
+################################################################################
+#
+# irods setup preparation
+#
+################################################################################
+RUN useradd -rm -d /var/lib/irods irods
+RUN mkdir -m 777 /src
+# setup script from previous version
+ADD ci/app/4.2/setup_answers.txt /app/setup_answers.txt
+ADD ci/app/4.2/setup_irods.sh /app/setup_irods.sh
+ADD ci/app/4.2/restart_irods.sh /app/restart_irods.sh
+ADD ci/app/4.2/surfsara-irods.repo /etc/yum.repos.d/surfsara-irods.repo
+ADD ci/app/irods_environment.json /root/.irods/irods_environment.json
+ADD ci/app/sleep.sh /app/sleep.sh
+ADD ci/app/wait_for_pg.sh /app/wait_for_pg.sh
+ADD ci/app/create_rpm.sh /app/create_rpm.sh
+ADD ci/app/update_install.py /app/update_install.py
+ADD scripts/tests/requirements.txt /app/test-requirements.txt
+
+RUN pip install -r /app/test-requirements.txt
+RUN yum install -y msi-persistent-id.x86_64
+
+# install B2HANDLE
+RUN pip install b2handle
+
+CMD ["/app/setup_irods.sh"]

ci/centos7_4_2_6/docker-compose.yml

+version: '2'
+
+volumes:
+  data-volume:
+
+  irods-vault:
+
+  irods-conf:
+
+
+
+services:
+  postgres:
+    build:
+      context: ../postgres
+      dockerfile: Dockerfile
+    ports:
+      - "5432"
+    hostname: postgresc7426.irods
+    volumes:
+      - data-volume:/var/lib/postgresql/9.5/main
+
+  icat:
+    build:
+      context: ../..
+      dockerfile: ci/centos7_4_2_6/Dockerfile
+    ports:
+      - "127.0.0.1:1247:1247"
+    depends_on:
+      - postgres
+    links:
+      - "postgres:postgres"
+    hostname: centos7_42_6.irods
+    volumes:
+      - "../../:/build"
+      - irods-vault:/var/lib/irods
+      - irods-conf:/etc/irods
+    command: /app/setup_irods.sh
+
+    
+