Commit 1ecb9e32 authored by Claudio Cacciari's avatar Claudio Cacciari
Browse files
parents a6011fc4 1a254d23
#directory of the iRODS installation
#for irods3 it is, typically, /home/user/iRODS
#for irods4 it is, typically, /var/lib/irods/iRODS
IRODS_DIR: /var/lib/irods/iRODS
#directory of the iRODS configuration files
#for irods3 it is, typically, /home/user/iRODS/server/config/reConfigs
#for irods4 it is, typically, /etc/irods
IRODS_CONF_DIR: /etc/irods
#directory of the iRODS server configuration file (server.config)
#for irods3 it is, typically, /home/user/iRODS/server/config
#for irods4 it is, typically, /etc/irods
IRODS_SERVER_CONF_DIR: /etc/irods
#directory of the source package downloaded via git
SOURCE_DIR: /var/lib/irods/B2SAFE-core
#directory of iRODS module configuration files
#for irods3 it is, typically, /home/user/b2safe-core
#for irods4 it is, typically, /etc/irods/b2safe-core
CONF_DIR: /etc/irods/b2safe-core
#default iRODS resource
......@@ -20,6 +28,8 @@ DEFAULT_RESOURCE: demoResc
CRED_STORE_TYPE: os
#path to the credentials file for EPIC
#for irods3 it is, typically, /home/user/b2safe-core/credentials
#for irods4 it is, typically, /etc/irods/b2safe-core/credentials
CRED_FILE_PATH: /etc/irods/b2safe-core/credentials
#iRODS server id in the form <protocol>://<server_name>:<irods_port>
......@@ -42,6 +52,8 @@ USERS: *
LOG_LEVEL: DEBUG
#directory of the B2SAFE log files
#for irods3 it is, typically, /home/user/b2safe-core/log
#for irods4 it is, typically, /etc/irods/b2safe-core/log
LOG_DIR: /etc/irods/b2safe-core/log
#shared iRODS collection to store command files
......
......@@ -206,7 +206,14 @@ else:
+ BACK).lower()
chk = inpt(inp)
stat(SOURCE_DIR)
try:
subprocess.check_call(["stat", SOURCE_DIR])
except subprocess.CalledProcessError:
print RED + BOLD + 'Directory ' + SOURCE_DIR + ' does not exist.' \
'Please, check the SOURCE_DIR in 2safe.config. ' \
'It should point to the place where you have downloaded ' \
'the git repository' + BACK
exit()
try:
subprocess.call(["cp", "-r"] + glob.glob(os.path.join(SOURCE_DIR \
......@@ -238,7 +245,7 @@ symlink(SOURCE_DIR + "/rulebase/catchError.re", \
symlink(SOURCE_DIR + "/rulebase/local.re", \
IRODS_CONF_DIR + "/euloc.re", \
IRODS_CONF_DIR + "/euloc.re")
print '3. edit <irods>/server/config/server.config and append '\
',eudat,eurepl,eupids,eucerr,euloc'\
',to reRuleSet (make sure to include the comma and no spaces)'
......@@ -267,7 +274,20 @@ for line in source:
line1 = line.replace("replication","eurepl")
line1 = line1.replace("pid-service","eupids")
line1 = line1.replace("catchError","eucerr")
line1 = line1.replace(",eudat-authZ-filters","")
line1 = line1.replace("eudat-authZ-filters,","")
line1 = line1.replace(",authZ","")
line1 = line1.replace("authZ,","")
line1 = line1.replace("local","euloc")
line1 = line1.replace(",integritycheck","")
line1 = line1.replace("integritycheck,","")
destination.write(line1)
elif (line.find('euaf') > -1) or \
(line.find('euint') > -1):
line1 = line.replace(",euaf","")
line1 = line1.replace("euaf,","")
line1 = line1.replace(",euint","")
line1 = line1.replace("euint,","")
destination.write(line1)
else:
destination.write(line)
......@@ -521,7 +541,7 @@ except subprocess.CalledProcessError:
else:
subprocess.call(["mkdir", LOG_DIR])
else:
print RED + BOLD + 'Directory ' + LOG_DIR + ' exists. \n'
print RED + BOLD + 'Directory ' + LOG_DIR + ' exists. \n' + BACK
filename = CONF_DIR + "/log.manager.conf"
try:
......
......@@ -21,34 +21,27 @@ Enable the module
Automatically:
1.1. enable the module in "<irods>/modules/B2SAFE/info.txt"
If a previous version of the module is present, disable it before to install the new one, in "<irods>/
modules/OLD_MODULE/info.txt" (if the directory name is the same, change it)
1.2. rerun the irodssetup script
Manually:
1.1. <irods>/scripts/configure --enable-B2SAFE
If a previous version of the module is present, disable it before to install the new one (if the directory
name is the same, change it):
<irods>/scripts/configure --disable-OLD_MODULE --enable-B2SAFE
1.2. make clean
1.3. make
1.4 <irods>/irodsctl restart
0. download the git repository
from https://github.com/EUDAT-B2SAFE/B2SAFE-core/releases
1. copy source conf dir (/whre_you_have_downloaded/B2SAFE-core/conf)
to the future config directory of the b2safe module
(for irods4: /etc/irods/b2safe-core; for irods3: /home/user/b2safe-core)
2. create a symbolic link to the eudat rulebase
cd <irods>
ln -s <absolute-irods-path>/modules/B2SAFE/rulebase/*.re ./server/config/reConfigs/*.re
for irods3: ln -s /whre_you_have_downloaded/B2SAFE-core/rulebase/*.re <irods>/server/config/reConfigs/*.re
for irods4: ln -s /whre_you_have_downloaded/B2SAFE-core/rulebase/*.re /etc/irods/*.re
replacing the jolly character "*" with each file name available in the directory "rulebase".
3. edit <irods>/server/config/server.config and append to "reRuleSet" the list of ruleset file names written
3. edit <irods>/server/config/server.config (irods3) or /etc/irods/server.config (irods4)
and append to "reRuleSet" the list of ruleset file names written
in a single line and without the suffix".re", in this way ",filename1,filename2,filename3 ..."
(make sure to include the comma and no spaces)
4. configure iRODS hooks.
edit the <irods>/server/config/reConfigs/core.re file and add the following two acPostProcForPutHooks:
edit the <irods>/server/config/reConfigs/core.re (irods3) or /etc/irods/core.re (irods4) file
and add the following two acPostProcForPutHooks:
acPostProcForPut {
ON($objPath like "\*.replicate") {
......@@ -61,30 +54,40 @@ Manually:
}
}
5. properly configure the default resource in <irods>/server/config/reConfigs/core.re
5. properly configure the default resource in <irods>/server/config/reConfigs/core.re (irods3)
or /etc/irods/core.re (irods4)
6.1 install the python scripts
cd <irods>
ln -s <absolute-irods-path>/modules/B2SAFE/cmd/* ./server/bin/cmd/
for irods3: <irods_dir> is usually /home/user/iRODS
for irods4: <irods_dir> is usually /var/lib/irods/iRODS
ln -s /whre_you_have_downloaded/B2SAFE-core/cmd/* <irods_dir>/server/bin/cmd/
check permissions on the scripts and make sure they are executable by the irods user
e.g.chmod u+x cmd/*
6.2 update the "getEpicApiParameters" rule in "./server/config/reConfigs/local.re"
- Configure the credential storage type: "os": stored on the local filesystem or "irods": stored on de irods namespace.
6.2 update the "getEpicApiParameters" rule in "./server/config/reConfigs/euloc.re" (irods3)
or /etc/irods/euloc.re (irods4)
- Configure the credential storage type: "os": stored on the local filesystem
or "irods": stored on de irods namespace.
- Set the path to the credentials file
- set the correct serverID to include the fully qualified hostname. For instance: "irods://node.domain.com:1247"
- Set the proper values in the credentials file (see ./cmd/credentials_example for an example)
6.3 update the "getAuthZParameters" rule in "./server/config/reConfigs/local.re"
- Set the proper values in modules/B2SAFE/cmd/authz.map.json
6.4 update the "getLogParameters" rule in "./server/config/reConfigs/local.re"
- Set the proper values in modules/B2SAFE/cmd/log.manager.conf
- set the correct serverID to include the fully qualified hostname.
For instance: "irods://node.domain.com:1247"
- Set the proper values in the credentials file
(see /whre_you_have_downloaded/B2SAFE-core/cmd/credentials_example for an example)
6.3 update the "getAuthZParameters" rule in "./server/config/reConfigs/euloc.re" (irods3)
or /etc/irods/euloc.re (irods4)
- Set the proper values in <irods>/server/bin/cmd/authz.map.json
6.4 update the "getLogParameters" rule in "./server/config/reConfigs/euloc.re" (irods3)
or /etc/irods/euloc.re (irods4)
- Set the proper values in <irods>/server/bin/cmd/log.manager.conf
7. create a shared space in all zones as configured in the eudat.re rulebase getSharedCollection function.
- defaults to "<zone>/replicate"
- make sure all users involved in the replication can write in this collection.
7. create a shared space in all zones (local and remote), for example: "<zone>/replicate".
Grant all users involved in the replication write access to this collection.
8.0.1 change "#!/usr/bin/env python" in the python scripts in modules/B2SAFE/cmd/ to your python installation
8.0.2 install httplib2, simplejson, lxml, defusedxml, and pylint:
8.0.2 install httplib2, simplejson, lxml, defusedxml, queuelib, and pylint:
httplib2
download from http://code.google.com/p/httplib2
python setup.py install
......@@ -97,15 +100,25 @@ Manually:
pip install defusedxml
apt-get install pylint
yum install pylint
queuelib
download from https://pypi.python.org/pypi/queuelib
python setup.py install
8.1 test the epic api interaction by running the "./cmd/epicclient.py test" script manually and with "iexecmd epicclient.py"
8.2 test the replication by changing and triggering "replicate.r" rule in <irods>/modules/B2SAFE/rules
8.1 test the epic api interaction by running
"./cmd/epicclient.py <os_or_irods> <path_to_credentials> test" script manually
and with "iexecmd epicclient.py"
8.2 test the module by changing and triggering rules in <irods>/modules/B2SAFE/rules
For example, eudatRepl.r to replicate a data set. See the hints inside the rule.
DATACENTER only:
it is necessary to configure the authz.map.json file in order to allow user to execute rules, which calls external scripts (see documentation).
it is necessary to configure the <irods>/server/bin/cmd/authz.map.json file
in order to allow user to execute rules, which calls external scripts (see documentation).
it is necessary to configure the log.manager.conf file in order to enable the logging system (see documentation).
it is necessary to configure the <irods>/server/bin/cmd/log.manager.conf file
in order to enable the logging system (see documentation).
*** Command files ***
......
......@@ -11,9 +11,11 @@ RPM_BUILD_ROOT="${HOME}/debbuild/"
RPM_SOURCE_DIR=$B2SAFEHOME
PRODUCT="irods-eudat-b2safe"
IRODS_PACKAGE_DIR=`grep -i _irodsPackage ${PRODUCT}.spec | head -n 1 | awk '{print $3}'`
# retrieve parameters from spec file. So we only have to update the spec file.
VERSION=`grep -i "^Version:" ${PRODUCT}.spec | awk '{print $2}'`
RELEASE=`grep -i "^Release:" ${PRODUCT}.spec | awk '{print $2}'`
# retrieve parameters from local.re in tree
MAJOR_VERS=`grep "^\s*\*major_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'`
MINOR_VERS=`grep "^\s*\*minor_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'`
VERSION="${MAJOR_VERS}.${MINOR_VERS}"
RELEASE=`grep "^\s*\*sub_version" $B2SAFEHOME/rulebase/local.re | awk -F\" '{print $2}'`
PACKAGE="${PRODUCT}_${VERSION}-${RELEASE}"
if [ "$USERNAME" = "root" ]
......
......@@ -30,7 +30,16 @@ else
echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros
fi
# find directory where we are executing:
ABSOLUTE_PATH=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
#extract major_version, minor_version and subversion from local.re in tree
MAJOR_VERS=`grep "^\s*\*major_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'`
MINOR_VERS=`grep "^\s*\*minor_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'`
SUB_VERS=`grep "^\s*\*sub_version" $ABSOLUTE_PATH/../rulebase/local.re | awk -F\" '{print $2}'`
VERSION="${MAJOR_VERS}.${MINOR_VERS}"
# build rpm
rpmbuild -ba irods-eudat-b2safe.spec
rpmbuild -ba --define "_version $VERSION" --define "_release $SUB_VERS" irods-eudat-b2safe.spec
# done..
......@@ -42,6 +42,7 @@ SHARED_SPACE=
# end set default parameters for installation
#
DATE_TODAY=`date +%Y%m%d`
JSON_CONFIG="false"
# end default parameters for setup
......@@ -67,6 +68,11 @@ create_links() {
COUNT=0
EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server.config
if [ "$JSON_CONFIG" == "true" ]
then
EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server_config.json
EUDAT_SERVER_CONFIG_JSON_STRING=`awk 1 ORS=' ' ${EUDAT_SERVER_CONFIG} | sed 's/ //g'`
fi
#delete old symbolic links
for file in $IRODS_CONF_DIR/eudat*.re
......@@ -81,7 +87,12 @@ create_links() {
for file in $B2SAFE_PACKAGE_DIR/rulebase/*.re
do
LINK=eudat${COUNT}
grep "^reRuleSet.*$LINK.*" $EUDAT_SERVER_CONFIG > /dev/null
if [ "$JSON_CONFIG" == "true" ]
then
echo $EUDAT_SERVER_CONFIG_JSON_STRING | grep -e "re_rulebase_set.*$LINK.*" > /dev/null
else
cat $EUDAT_SERVER_CONFIG | grep -e "^reRuleSet.*$LINK.*" > /dev/null
fi
if [ $? -ne 0 ]
then
EUDAT_RULEFILES="$EUDAT_RULEFILES,$LINK"
......@@ -141,6 +152,52 @@ update_server_config() {
return $STATUS
}
update_server_config_json() {
EUDAT_SERVER_CONFIG=$IRODS_CONF_DIR/server_config.json
if [ -n "$EUDAT_RULEFILES" ]
then
if [ ! -e ${EUDAT_SERVER_CONFIG}.org.${DATE_TODAY} ]
then
cp ${EUDAT_SERVER_CONFIG} ${EUDAT_SERVER_CONFIG}.org.${DATE_TODAY}
fi
# put json file in a single string so it easier to process.
EUDAT_SERVER_CONFIG_JSON_STRING=`awk 1 ORS=' ' ${EUDAT_SERVER_CONFIG} | sed 's/ //g'`
JSON_RULESET=`echo $EUDAT_SERVER_CONFIG_JSON_STRING | sed 's/.*re_rulebase_set":/re_rulebase_set":/' | sed 's/}].*$/}]/'`
# append rules..
JSON_RULESET_WORK=`echo $JSON_RULESET | sed 's/]//'`
EUDAT_RULEFILES=`echo $EUDAT_RULEFILES | sed 's/,/ /g'`
for file in $EUDAT_RULEFILES
do
JSON_RULESET_WORK+=",{\"filename\":\"$file\"}"
done
JSON_RULESET_WORK+="]"
# substitute the new string for the old string
perl -pi -e "undef $/; s/re_rulebase_set[^\]]*\]/$JSON_RULESET_WORK/" $EUDAT_SERVER_CONFIG
if [ $? -eq 0 ]
then
echo "The file $EUDAT_SERVER_CONFIG has been updated!"
else
echo "ERROR: updating $EUDAT_SERVER_CONFIG failed!"
STATUS=1
fi
fi
echo "*********************************************************************"
echo ""
echo "Please check the file: ${EUDAT_SERVER_CONFIG} by hand "
echo " grep -A5 re_rulebase_set ${EUDAT_SERVER_CONFIG} "
echo ""
echo "It should only have the following eudat{n}.re files mentioned: "
echo " `cd $IRODS_CONF_DIR ; ls -C eudat*.re` "
echo ""
echo "*********************************************************************"
return $STATUS
}
configure_irods_hooks() {
IRODS_COREFILE=$IRODS_CONF_DIR/core.re
......@@ -344,6 +401,59 @@ update_get_auth_parameters() {
return $STATUS
}
update_authz_map_json() {
AUTH_MAP_PATH=$B2SAFE_PACKAGE_DIR/conf/authz.map.json
TMP_FILE=/tmp/authz.map.json
if [ ! -e ${AUTH_MAP_PATH}.org.${DATE_TODAY} ]
then
cp $AUTH_MAP_PATH ${AUTH_MAP_PATH}.org.${DATE_TODAY}
fi
# constrict a list of users to fill in the userlist
let count=0
for user in $USERS
do
if [ "$count" -lt "1" ]
then
userlist=$user
else
userlist+="\",\"$user"
fi
let count=$count+1
done
cat > $TMP_FILE << EOF
{
"assertion 1":
{ "subject":
[ "$userlist" ],
"action":
[ "read" ],
"target":
[ "${IRODS_DIR}/server/bin/cmd/*","${B2SAFE_PACKAGE_DIR}/conf/*" ]
}
}
EOF
if [ $? -eq 0 ]
then
mv $TMP_FILE $AUTH_MAP_PATH
if [ ! $? -eq 0 ]
then
echo "ERROR: moving $AUTH_MAP_PATH failed!"
STATUS=1
fi
else
echo "ERROR: creating $TMP_FILE failed!"
STATUS=1
fi
# set access mode to file
chmod 600 $AUTH_MAP_PATH
return $STATUS
}
update_get_log_parameters() {
LOG_MANAGER_CONF=$B2SAFE_PACKAGE_DIR/conf/log.manager.conf
......@@ -404,12 +514,24 @@ update_log_manager_conf() {
# main program
########################
#
# read parameter file
#
echo "read_parameters"
read_parameters
#
# check for json or normal config files
# irods 4.1 and higher use json config files
#
echo "check iRODS config files"
if [ -e "${IRODS_DIR}/../VERSION.json" ]
then
JSON_CONFIG="true"
echo "We have iRODS 4.1 or higher. The config files are in json format"
fi
#
# create symbolic links to the eudat rulebase
#
......@@ -420,7 +542,7 @@ then
fi
#
# edit /etc/irods/server.config
# edit /etc/irods/server.config or /etc/irods/server_config.json
# append eudat specific rules to to reRuleSet.
# we use links in the type of eudatxy.re.
# (make sure to include the comma and no spaces)
......@@ -428,7 +550,12 @@ fi
if [ $? -eq 0 ]
then
echo "update_server_config"
update_server_config
if [ "$JSON_CONFIG" == "false" ]
then
update_server_config
else
update_server_config_json
fi
fi
#
......
Name: irods-eudat-b2safe
Version: 3.0
Release: 0
Version: %{_version}
Release: %{_release}
#Release: 1%{?dist}
Summary: b2safe core application for iRODS v4
......@@ -147,6 +147,8 @@ cd %{_irodsPackage}/packaging
EOF
%changelog
* Mon Jul 07 2015 Robert Verkerk <robert.verkerk@surfsara.nl> 3.0
- assign version of package at build time with input parameters
* Fri Feb 13 2015 Robert Verkerk <robert.verkerk@surfsara.nl> 3.0
- Add files to b2safe package
* Wed Jan 28 2015 Robert Verkerk <robert.verkerk@surfsara.nl> 3.0
......
eudatCHMC{
# Walk through the collection. For each object in the collection
# create a PID and stores its value and the object checksum
# in the iCAT if it does not exist.
# IF the PID exists, modify its checksum value
EUDATeiPIDeiChecksumMgmtColl(*sourceColl);
}
INPUT *sourceColl='/vzRZGEUDAT/home/eudat/11'
OUTPUT ruleExecOut
eudatGetV{
# Provides version of the B2SAFE
getB2SAFEVersion(*version);
}
INPUT *source='1'
OUTPUT *version,ruleExecOut
eudatIC{
# Compare cheksums of data objects in the source and destination
# collection recursively
EUDATCheckIntegrityColl(*source, *destination, bool("true"), *response)
}
INPUT *source='/vzRZGEUDAT/home/eudat/11', *destination='/devRZG/home/eudat#vzRZGEUDAT/11-6'
OUTPUT ruleExecOut
eudatPidsColl{
# Create PIDs for all collections and objects in the collection recursively
# ROR is assumed to be "None"
EUDATPidsForColl(*coll_path);
}
INPUT *coll_path='/vzRZGEUDAT/home/eudat/11'
OUTPUT *newPID,ruleExecOut
eudatRepl{
# Data set replication
# registered data (with PID registration) (3rd argument - 1st bool("true"))
# recursive (4th argument 2nd bool("true"))
EUDATReplication(*source, *destination, bool("true"), bool("true"), *response)
}
INPUT *source='/vzRZGEUDAT/home/eudat/11', *destination='/devRZG/home/eudat#vzRZGEUDAT/11-6'
OUTPUT ruleExecOut
eudatTD{
# Calculate the difference between the creation time
# and the modification time of an object. In seconds.
EUDATgetObjectTimeDiff(*filePath, *mode, *age);
# Calculate the difference between the current time
# and the modification time of an object. In seconds.
EUDATgetObjectAge(*filePath, *age1);
}
INPUT *filePath='/vzRZGEUDAT/home/eudat/11/11.txt', *mode='1'
OUTPUT *age,*age1,ruleExecOut
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment