Merge pull request #130 from EUDAT-B2SAFE/devel

Devel

packaging/convert_b2safe_conf_to_json.sh

+#
+# procedure to convert the B2SAFE parameters to json using a bash script
+#
+# set tabstop=4
+# set expandtab
+#
+#set -x
+#
+#
+#
+# set default parameters for installation
+INSTALL_CONFIG=./install.conf
+
+# start default parameters for setup
+#===================================
+IRODS_CONF_DIR=/etc/irods
+#
+IRODS_DIR=/var/lib/irods
+#
+B2SAFE_PACKAGE_DIR=/opt/eudat/b2safe
+#
+# the default iRODS resource to use
+DEFAULT_RESOURCE=eudat
+#
+# credentials type and location
+CRED_STORE_TYPE=os
+CRED_FILE_PATH=$B2SAFE_PACKAGE_DIR/conf
+SERVER_ID=
+SERVERAPIREG=
+SERVERAPIPUB=
+
+#
+# credentials for epicclient2
+HANDLE_SERVER_URL=
+PRIVATE_KEY=
+CERTIFICATE_ONLY=
+PREFIX=
+HANDLEOWNER=
+REVERSELOOKUP_USERNAME=
+HTTPS_VERIFY=
+#
+USERS=
+LOG_LEVEL=
+LOG_DIR=
+SHARED_SPACE=
+#
+# EUDAT iRODS rules behavioral parameters. Because these are later added
+# sensible defaults are chosen. They can be overridden by adding them to
+# the configuration in install.conf
+AUTHZ_ENABLED=true
+MSG_QUEUE_ENABLED=false
+
+
+if [ -e $INSTALL_CONFIG ]
+then
+    source $INSTALL_CONFIG
+else
+    echo "ERROR: $INSTALL_CONFIG not present!"
+    STATUS=1
+fi
+
+string=$(echo "$HTTPS_VERIFY" | tr '[:upper:]' '[:lower:]')
+if [[ $string =~ .*true.* ]]
+then
+  HTTPS_VERIFY_STRING=true
+elif [[ $string =~ .*false.* ]]
+then
+  HTTPS_VERIFY_STRING=false
+else
+  HTTPS_VERIFY_STRING="\"${HTTPS_VERIFY}\""
+fi
+
+let count=0
+handle_users_array=(`echo ${USERS} | sed 's/[\t ]+/\n/g'`)
+handle_users_string=
+for each in ${handle_users_array[@]}
+do
+  if [ $count -eq 0 ]
+  then
+    handle_users_string=$(echo -n "\"$each\"")
+  else
+    handle_users_string=$handle_users_string$(echo -n ", \"$each\"")
+  fi
+  let count=$count+1
+done
+
+
+cat > install.json << EOT
+{
+  "b2safe_package_dir": "${B2SAFE_PACKAGE_DIR}",
+  "irods_conf_dir": "${IRODS_CONF_DIR}",
+  "irods_dir": "${IRODS_DIR}",
+  "irods_default_resource": "${DEFAULT_RESOURCE}",
+  "cred_store_type": "${CRED_STORE_TYPE}",
+  "cred_file_path": "${CRED_FILE_PATH}",
+  "server_id": "${SERVER_ID}",
+  "server_api_reg": "${SERVERAPIREG}",
+  "server_api_pub": "${SERVERAPIPUB}",
+  "handle_server_url": "${HANDLE_SERVER_URL}",
+  "handle_private_key": "${PRIVATE_KEY}",
+  "handle_certificate_only": "${CERTIFICATE_ONLY}",
+  "handle_prefix": "${PREFIX}",
+  "handle_owner": "${HANDLEOWNER}",
+  "handle_reverse_lookup_name": "${REVERSELOOKUP_USERNAME}",
+  "handle_https_verify": ${HTTPS_VERIFY_STRING},
+  "handle_users": [ ${handle_users_string} ],
+  "handle_groups": [  ],
+  "log_level": "${LOG_LEVEL}",
+  "log_directory": "${LOG_DIR}",
+  "shared_space": "${SHARED_SPACE}",
+  "authz_enabled": ${AUTHZ_ENABLED},
+  "msg_queue_enabled": ${MSG_QUEUE_ENABLED}
+}
+EOT
+
+chmod 600 install.json

packaging/create_deb_package.sh

@@ -50,7 +50,8 @@ cp $RPM_SOURCE_DIR/*.txt          $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}
 cp $RPM_SOURCE_DIR/LICENSE        $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}
 cp $RPM_SOURCE_DIR/cmd/*          $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/cmd
 cp $RPM_SOURCE_DIR/conf/*         $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf
-cp $RPM_SOURCE_DIR/packaging/install.sh $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
+cp $RPM_SOURCE_DIR/packaging/install.py $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
+cp $RPM_SOURCE_DIR/packaging/convert_b2safe_conf_to_json.sh $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
 cp $RPM_SOURCE_DIR/rulebase/*     $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/rulebase
 cp $RPM_SOURCE_DIR/rules/*        $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/testRules
 mkdir -p                          $RPM_BUILD_ROOT${PACKAGE}/var/log/irods
@@ -60,6 +61,7 @@ chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/cmd/*.py
 chmod 600 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf/*.json
 chmod 600 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf/*.conf
 chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging/*.sh
+chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging/*.py
 
 # create packaging directory
 mkdir -p  $RPM_BUILD_ROOT${PACKAGE}/DEBIAN
@@ -96,60 +98,37 @@ cat > $RPM_BUILD_ROOT${PACKAGE}/DEBIAN/postinst << EOF
 # script for postinstall actions
 
 # create configuration file if it does not exist yet
-INSTALL_CONF=${IRODS_PACKAGE_DIR}/packaging/install.conf
+INSTALL_CONF=${IRODS_PACKAGE_DIR}/packaging/install.json
 
 if [ ! -e \$INSTALL_CONF ]
 then
 
 cat > \$INSTALL_CONF << EOF2
-#
-# parameters for installation of irods module B2SAFE
-#
-# the absolute directory where the irods config is installed
-IRODS_CONF_DIR=/etc/irods
-#
-# the absolute directory where irods is installed
-IRODS_DIR=/var/lib/irods
-#
-# the directory where B2SAFE is installed as a package
-B2SAFE_PACKAGE_DIR=${IRODS_PACKAGE_DIR}
-#
-# the default iRODS resource to use. Will be set in core.re
-DEFAULT_RESOURCE=demoResc
-#
-# epic credentials type and location
-CRED_STORE_TYPE=os
-CRED_FILE_PATH=\\\$B2SAFE_PACKAGE_DIR/conf/credentials
-SERVER_ID="irods://<fully_qualified_hostname>:1247"
-#
-# epic credentials file usage parameters
-#
-# epicclient2 parameters
-HANDLE_SERVER_URL=<https://epic3.storage.surfsara.nl:8001>
-PRIVATE_KEY=</path/prefix_suffix_index_privkey.pem>
-CERTIFICATE_ONLY=</path/prefix_suffix_index_certificate_only.pem>
-PREFIX=<ZZZ>
-HANDLEOWNER="200:0.NA/\\\$PREFIX"
-REVERSELOOKUP_USERNAME=<ZZZ>
-HTTPS_VERIFY="True"
-#
-# users for msiexec command
-USERS="user0#Zone0 user1#Zone1"
-#
-# loglevel and log directory
-# possible log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
-LOG_LEVEL=DEBUG
-LOG_DIR=/var/log/irods
-#
-#
-# iRODS behavioral parameters
-#
-# check if user is authorized to perform several functions
-AUTHZ_ENABLED=true
-#
-# check if message queue is enabled
-MSG_QUEUE_ENABLED=false
-#
+{
+  "b2safe_package_dir": "${IRODS_PACKAGE_DIR}",
+  "irods_conf_dir": "/etc/irods",
+  "irods_dir": "/var/lib/irods/iRODS",
+  "irods_default_resource": "demoResc",
+  "cred_store_type": "os",
+  "cred_file_path": "${IRODS_PACKAGE_DIR}/conf/credentials",
+  "server_id": "irods://<fully_qualified_hostname>:1247",
+  "server_api_reg": "irods://<fully_qualified_hostname>:1247",
+  "server_api_pub": "irods://<fully_qualified_hostname>:1247",
+  "handle_server_url": "https://epic6.storage.surfsara.nl:8003",
+  "handle_private_key": "</path/prefix_suffix_index_privkey.pem>",
+  "handle_certificate_only": "</path/prefix_suffix_index_certificate_only.pem>",
+  "handle_prefix": "<ZZZ>",
+  "handle_owner": "200:0.NA/<ZZZ>",
+  "handle_reverse_lookup_name": "<ZZZ>",
+  "handle_reverse_lookup_password": "<reverse_lookup_password>",
+  "handle_https_verify": True,
+  "handle_users": [ "user0#Zone0", "user1#Zone1" ],
+  "log_level": "INFO",
+  "log_directory": "/var/log/irods",
+  "shared_space": "",
+  "authz_enabled": true,
+  "msg_queue_enabled": false
+}
 EOF2
 
 fi
@@ -162,11 +141,11 @@ The package b2safe has been installed in ${IRODS_PACKAGE_DIR}.
 To install/configure it in iRODS do following as the user who runs iRODS :
 
 # update install.conf with correct parameters with your favorite editor
-sudo vi ${IRODS_PACKAGE_DIR}/packaging/install.conf
+sudo vi ${IRODS_PACKAGE_DIR}/packaging/install.json
 
 # install/configure it as the user who runs iRODS
 source /etc/irods/service_account.config 
-sudo su - \\\$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd ${IRODS_PACKAGE_DIR}/packaging/ ; ./install.sh"
+sudo su - \\\$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd ${IRODS_PACKAGE_DIR}/packaging/ ; ./install.py"
 
 EOF1
 

packaging/irods-eudat-b2safe.spec

@@ -57,7 +57,8 @@ cp $RPM_SOURCE_DIR/*.txt      $RPM_BUILD_ROOT%{_irodsPackage}
 cp $RPM_SOURCE_DIR/LICENSE    $RPM_BUILD_ROOT%{_irodsPackage}
 cp $RPM_SOURCE_DIR/cmd/*      $RPM_BUILD_ROOT%{_irodsPackage}/cmd
 cp $RPM_SOURCE_DIR/conf/*     $RPM_BUILD_ROOT%{_irodsPackage}/conf
-cp $RPM_SOURCE_DIR/packaging/install.sh $RPM_BUILD_ROOT%{_irodsPackage}/packaging
+cp $RPM_SOURCE_DIR/packaging/install.py $RPM_BUILD_ROOT%{_irodsPackage}/packaging
+cp $RPM_SOURCE_DIR/packaging/convert_b2safe_conf_to_json.sh $RPM_BUILD_ROOT%{_irodsPackage}/packaging
 cp $RPM_SOURCE_DIR/rulebase/* $RPM_BUILD_ROOT%{_irodsPackage}/rulebase
 cp $RPM_SOURCE_DIR/rules/*    $RPM_BUILD_ROOT%{_irodsPackage}/testRules
 mkdir -p $RPM_BUILD_ROOT/var/log/irods
@@ -90,6 +91,7 @@ rm -rf %{buildroot}
 %attr(600,-,-) %{_irodsPackage}/conf/*.json
 %attr(600,-,-) %{_irodsPackage}/conf/*.conf
 %attr(700,-,-) %{_irodsPackage}/packaging/*.sh
+%attr(700,-,-) %{_irodsPackage}/packaging/*.py
 %attr(-,-,-)   /var/log/irods
 %doc
 # config files
@@ -99,59 +101,37 @@ rm -rf %{buildroot}
 
 %post
 # create configuration file if it does not exist yet
-INSTALL_CONF=%{_irodsPackage}/packaging/install.conf
+INSTALL_CONF=%{_irodsPackage}/packaging/install.json
 
 if [ ! -e $INSTALL_CONF ]
 then
 
 cat > $INSTALL_CONF << EOF
-#
-# parameters for installation of irods module B2SAFE
-#
-# the absolute directory where the irods config is installed
-IRODS_CONF_DIR=/etc/irods
-#
-# the absolute directory where irods is installed
-IRODS_DIR=/var/lib/irods/iRODS
-#
-# the directory where B2SAFE is installed as a package
-B2SAFE_PACKAGE_DIR=%{_irodsPackage}
-#
-# the default iRODS resource to use. Will be set in core.re
-DEFAULT_RESOURCE=demoResc
-#
-# epic credentials type and location
-CRED_STORE_TYPE=os
-CRED_FILE_PATH=\$B2SAFE_PACKAGE_DIR/conf/credentials
-SERVER_ID="irods://<fully_qualified_hostname>:1247"
-#
-# epicclient2 parameters
-HANDLE_SERVER_URL=<https://epic3.storage.surfsara.nl:8001>
-PRIVATE_KEY=</path/prefix_suffix_index_privkey.pem>
-CERTIFICATE_ONLY=</path/prefix_suffix_index_certificate_only.pem>
-PREFIX=<ZZZ>
-HANDLEOWNER="200:0.NA/\$PREFIX"
-REVERSELOOKUP_USERNAME=<ZZZ>
-HTTPS_VERIFY="True"
-#
-# users for msiexec command
-USERS="user0#Zone0 user1#Zone1"
-#
-# loglevel and log directory
-# possible log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
-LOG_LEVEL=DEBUG
-LOG_DIR=/var/log/irods
-#
-#
-# iRODS behavioral parameters
-#
-# check if user is authorized to perform several functions
-AUTHZ_ENABLED=true
-#
-# check if message queue is enabled
-MSG_QUEUE_ENABLED=false
-#
-
+{
+  "b2safe_package_dir": "%{_irodsPackage}",
+  "irods_conf_dir": "/etc/irods",
+  "irods_dir": "/var/lib/irods/iRODS",
+  "irods_default_resource": "demoResc",
+  "cred_store_type": "os",
+  "cred_file_path": "%{_irodsPackage}/conf/credentials",
+  "server_id": "irods://<fully_qualified_hostname>:1247",
+  "server_api_reg": "irods://<fully_qualified_hostname>:1247",
+  "server_api_pub": "irods://<fully_qualified_hostname>:1247",
+  "handle_server_url": "https://epic6.storage.surfsara.nl:8003",
+  "handle_private_key": "</path/prefix_suffix_index_privkey.pem>",
+  "handle_certificate_only": "</path/prefix_suffix_index_certificate_only.pem>",
+  "handle_prefix": "<ZZZ>",
+  "handle_owner": "200:0.NA/<ZZZ>",
+  "handle_reverse_lookup_name": "<ZZZ>",
+  "handle_reverse_lookup_password": "<reverse_lookup_password>",
+  "handle_https_verify": True,
+  "handle_users": [ "user0#Zone0", "user1#Zone1" ],
+  "log_level": "INFO",
+  "log_directory": "/var/log/irods",
+  "shared_space": "",
+  "authz_enabled": true,
+  "msg_queue_enabled": false
+}
 EOF
 
 fi
@@ -163,11 +143,11 @@ The package b2safe has been installed in %{_irodsPackage}.
 To install/configure it in iRODS do following as the user who runs iRODS :
 
 # update install.conf with correct parameters with your favorite editor
-sudo vi %{_irodsPackage}/packaging/install.conf
+sudo vi %{_irodsPackage}/packaging/install.json
 
 # install/configure it as the user who runs iRODS
 source /etc/irods/service_account.config 
-sudo su - \$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd %{_irodsPackage}/packaging/ ; ./install.sh"
+sudo su - \$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd %{_irodsPackage}/packaging/ ; ./install.py"
 
 EOF
 
@@ -182,6 +162,10 @@ fi
 
 
 %changelog
+* Tue Mar 26 2019  Robert Verkerk <robert.verkerk@surfsara.nl> 4.2.1
+- add python installation/setup script.
+* Fri Mar 8 2019  Robert Verkerk <robert.verkerk@surfsara.nl> 4.2.1
+- add new parameters. Convert to json input file for configuration.
 * Thu Nov 22 2018  Robert Verkerk <robert.verkerk@surfsara.nl> 4.2.0
 - add new parameters.
 * Wed Nov 22 2017  Robert Verkerk <robert.verkerk@surfsara.nl> 4.0.1

rulebase/eudat.re

@@ -479,7 +479,7 @@ EUDATiCHECKSUMretrieve(*path, *checksum, *modtime, *resource) {
             # other ones are "ichksum", "iput -k", "irepl" (on checksumed source) called by the user.
             EUDATiCHECKSUMdate(*coll, *name, *resc, *modtime);
             *status = bool("true");
-            if (*resource != "None" && *resource != "" && *resource == *resc) {
+            if (*resource == "null" || *resource == "" || *resource == "None" || *resource == *resc) {
                 break;
             }
         }

rulebase/local.re

@@ -138,7 +138,7 @@ getConfParameters(*authzEnabled) {
 #-------------------------------------------------------------------------------
 getB2SAFEVersion(*version) {
     *major_version = "4";
-    *minor_version = "2";
+    *minor_version = "3";
     *sub_version = "0";
     *version = *major_version ++ "." ++ *minor_version ++ "-" ++ *sub_version;
 }

scripts/tests/testB2SafeCmd.py

 #!/usr/bin/env python
 
-from testB2SafeCmd.epicapitest import EpicClientAPITestCase
-from testB2SafeCmd.epiccredtest import EpicClientCredentialsTestCase
-from testB2SafeCmd.epicclitest import EpicClientCLITestCase
-from testB2SafeCmd.epicintgtest import EpicClientIntegrationTests
 from testB2SafeCmd.testLogManager import LogManagerTestCase
 from testB2SafeCmd.testAuthzManager import AuthzManagerTestCase
 from testB2SafeCmd.irodsintgtest import IrodsIntegrationTests
 from testB2SafeCmd.irodsb2safetest import IrodsB2safeIntegrationTests
 from testB2SafeCmd.epic2intgtest import EpicClient2IntegrationTests 
+from testB2SafeCmd.msipidintgtest import MsiPidIntegrationTests
 
 import argparse
 import unittest
 
 __author__ = 'lphan'
 
-def create_epic_test_suite():
-    epic_api_suite = unittest.TestLoader().loadTestsFromTestCase(
-        EpicClientAPITestCase)
-    epic_cred_suite = unittest.TestLoader().loadTestsFromTestCase(
-        EpicClientCredentialsTestCase)
-    epic_cli_suite = unittest.TestLoader().loadTestsFromTestCase(
-        EpicClientCLITestCase)
-    epic_it_suite = unittest.TestLoader().loadTestsFromTestCase(
-        EpicClientIntegrationTests)
-    return unittest.TestSuite(
-        [epic_api_suite, epic_cred_suite, epic_cli_suite, epic_it_suite])
-
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Test script for epicclient '
-                                                 '(epic), '
-                                                 'epicclient2 (epic2), '
+    parser = argparse.ArgumentParser(description='epicclient2 (epic2), '
+                                                 'msiPid (msipid), '
                                                  'logging manager (log), '
                                                  'authorization manager (auth), '
                                                  'iRODS (irods), '
                                                  'B2SAFE (b2safe), '
                                                  'and all scripts (all)')
     parser.add_argument('-test', action='store', dest='script',
-                        help='[epic, epic2, log, auth, irods, b2safe, all]')
+                        help='[epic2, msipid, log, auth, irods, b2safe, all]')
 
     param = parser.parse_args()
 
-    if param.script == "epic":
-        # Test cases for B2Safe-Epicclient#
-        print "Test Epicclient Script"
-        epic_suite = create_epic_test_suite()
-        unittest.TextTestRunner(descriptions=2, verbosity=2).run(epic_suite)
-
-    elif param.script == "epic2":
+    if param.script == "epic2":
         # test cases for B2safe-epicclient2#
         print "Test Epicclient2 Script"
         epic2_suite = unittest.TestLoader().loadTestsFromTestCase(
         EpicClient2IntegrationTests)
         unittest.TextTestRunner(descriptions=2, verbosity=2).run(epic2_suite)
 
+    elif param.script == "msipid":
+        # test cases for B2safe-msipid#
+        print "Test msiPid Script"
+        msiPid_suite = unittest.TestLoader().loadTestsFromTestCase(
+        MsiPidIntegrationTests)
+        unittest.TextTestRunner(descriptions=2, verbosity=2).run(msiPid_suite)
+
     elif param.script == "log":
         # Test cases for B2Safe-LogManager#
         print "Test Logging Script"
@@ -88,7 +73,6 @@ if __name__ == '__main__':
         print "run authZmanager.py - if true then run epicclient2beta and log "\
               "with logmanager.py"
         all_suite = unittest.TestSuite()
-        all_suite.addTest(create_epic_test_suite())
         all_suite.addTest(LogManagerTestCase("test_case"))
         all_suite.addTest(AuthzManagerTestCase("test_case"))
         all_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
@@ -98,4 +82,4 @@ if __name__ == '__main__':
         unittest.TextTestRunner(descriptions=2, verbosity=2).run(all_suite)
 
     else:
-        print "Invalid Input; Valid example ./testB2SafeCmd -test epic"
+        print "Invalid Input; Valid example ./testB2SafeCmd -test epic2"

scripts/tests/testB2SafeCmd/irodsb2safetest.py

@@ -38,6 +38,8 @@ def subprocess_popen(cmd, input_string=None):
 
     arr = string.split(data_stdout, '\n')
     arr = map(string.strip, arr)
+    if arr and arr[-1] == '':
+        arr.pop()
     return arr
 
 def create_os_file(input_filename):