Merge pull request #125 from cookie33/handle_uService

Use pid uService

packaging/convert_b2safe_conf_to_json.sh

+#
+# procedure to convert the B2SAFE parameters to json using a bash script
+#
+# set tabstop=4
+# set expandtab
+#
+#set -x
+#
+#
+#
+# set default parameters for installation
+INSTALL_CONFIG=./install.conf
+
+# start default parameters for setup
+#===================================
+IRODS_CONF_DIR=/etc/irods
+#
+IRODS_DIR=/var/lib/irods
+#
+B2SAFE_PACKAGE_DIR=/opt/eudat/b2safe
+#
+# the default iRODS resource to use
+DEFAULT_RESOURCE=eudat
+#
+# credentials type and location
+CRED_STORE_TYPE=os
+CRED_FILE_PATH=$B2SAFE_PACKAGE_DIR/conf
+SERVER_ID=
+SERVERAPIREG=
+SERVERAPIPUB=
+
+#
+# credentials for epicclient2
+HANDLE_SERVER_URL=
+PRIVATE_KEY=
+CERTIFICATE_ONLY=
+PREFIX=
+HANDLEOWNER=
+REVERSELOOKUP_USERNAME=
+HTTPS_VERIFY=
+#
+USERS=
+LOG_LEVEL=
+LOG_DIR=
+SHARED_SPACE=
+#
+# EUDAT iRODS rules behavioral parameters. Because these are later added
+# sensible defaults are chosen. They can be overridden by adding them to
+# the configuration in install.conf
+AUTHZ_ENABLED=true
+MSG_QUEUE_ENABLED=false
+
+
+if [ -e $INSTALL_CONFIG ]
+then
+    source $INSTALL_CONFIG
+else
+    echo "ERROR: $INSTALL_CONFIG not present!"
+    STATUS=1
+fi
+
+string=$(echo "$HTTPS_VERIFY" | tr '[:upper:]' '[:lower:]')
+if [[ $string =~ .*true.* ]]
+then
+  HTTPS_VERIFY_STRING=true
+elif [[ $string =~ .*false.* ]]
+then
+  HTTPS_VERIFY_STRING=false
+else
+  HTTPS_VERIFY_STRING="\"${HTTPS_VERIFY}\""
+fi
+
+let count=0
+handle_users_array=(`echo ${USERS} | sed 's/[\t ]+/\n/g'`)
+handle_users_string=
+for each in ${handle_users_array[@]}
+do
+  if [ $count -eq 0 ]
+  then
+    handle_users_string=$(echo -n "\"$each\"")
+  else
+    handle_users_string=$handle_users_string$(echo -n ", \"$each\"")
+  fi
+  let count=$count+1
+done
+
+
+cat > install.json << EOT
+{
+  "b2safe_package_dir": "${B2SAFE_PACKAGE_DIR}",
+  "irods_conf_dir": "${IRODS_CONF_DIR}",
+  "irods_dir": "${IRODS_DIR}",
+  "irods_default_resource": "${DEFAULT_RESOURCE}",
+  "cred_store_type": "${CRED_STORE_TYPE}",
+  "cred_file_path": "${CRED_FILE_PATH}",
+  "server_id": "${SERVER_ID}",
+  "server_api_reg": "${SERVERAPIREG}",
+  "server_api_pub": "${SERVERAPIPUB}",
+  "handle_server_url": "${HANDLE_SERVER_URL}",
+  "handle_private_key": "${PRIVATE_KEY}",
+  "handle_certificate_only": "${CERTIFICATE_ONLY}",
+  "handle_prefix": "${PREFIX}",
+  "handle_owner": "${HANDLEOWNER}",
+  "handle_reverse_lookup_name": "${REVERSELOOKUP_USERNAME}",
+  "handle_https_verify": ${HTTPS_VERIFY_STRING},
+  "handle_users": [ ${handle_users_string} ],
+  "handle_groups": [  ],
+  "log_level": "${LOG_LEVEL}",
+  "log_directory": "${LOG_DIR}",
+  "shared_space": "${SHARED_SPACE}",
+  "authz_enabled": ${AUTHZ_ENABLED},
+  "msg_queue_enabled": ${MSG_QUEUE_ENABLED}
+}
+EOT
+
+chmod 600 install.json

packaging/create_deb_package.sh

@@ -50,7 +50,8 @@ cp $RPM_SOURCE_DIR/*.txt          $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}
 cp $RPM_SOURCE_DIR/LICENSE        $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}
 cp $RPM_SOURCE_DIR/cmd/*          $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/cmd
 cp $RPM_SOURCE_DIR/conf/*         $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf
-cp $RPM_SOURCE_DIR/packaging/install.sh $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
+cp $RPM_SOURCE_DIR/packaging/install.py $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
+cp $RPM_SOURCE_DIR/packaging/convert_b2safe_conf_to_json.sh $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging
 cp $RPM_SOURCE_DIR/rulebase/*     $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/rulebase
 cp $RPM_SOURCE_DIR/rules/*        $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/testRules
 mkdir -p                          $RPM_BUILD_ROOT${PACKAGE}/var/log/irods
@@ -60,6 +61,7 @@ chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/cmd/*.py
 chmod 600 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf/*.json
 chmod 600 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/conf/*.conf
 chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging/*.sh
+chmod 700 $RPM_BUILD_ROOT${PACKAGE}${IRODS_PACKAGE_DIR}/packaging/*.py
 
 # create packaging directory
 mkdir -p  $RPM_BUILD_ROOT${PACKAGE}/DEBIAN
@@ -96,60 +98,37 @@ cat > $RPM_BUILD_ROOT${PACKAGE}/DEBIAN/postinst << EOF
 # script for postinstall actions
 
 # create configuration file if it does not exist yet
-INSTALL_CONF=${IRODS_PACKAGE_DIR}/packaging/install.conf
+INSTALL_CONF=${IRODS_PACKAGE_DIR}/packaging/install.json
 
 if [ ! -e \$INSTALL_CONF ]
 then
 
 cat > \$INSTALL_CONF << EOF2
-#
-# parameters for installation of irods module B2SAFE
-#
-# the absolute directory where the irods config is installed
-IRODS_CONF_DIR=/etc/irods
-#
-# the absolute directory where irods is installed
-IRODS_DIR=/var/lib/irods
-#
-# the directory where B2SAFE is installed as a package
-B2SAFE_PACKAGE_DIR=${IRODS_PACKAGE_DIR}
-#
-# the default iRODS resource to use. Will be set in core.re
-DEFAULT_RESOURCE=demoResc
-#
-# epic credentials type and location
-CRED_STORE_TYPE=os
-CRED_FILE_PATH=\\\$B2SAFE_PACKAGE_DIR/conf/credentials
-SERVER_ID="irods://<fully_qualified_hostname>:1247"
-#
-# epic credentials file usage parameters
-#
-# epicclient2 parameters
-HANDLE_SERVER_URL=<https://epic3.storage.surfsara.nl:8001>
-PRIVATE_KEY=</path/prefix_suffix_index_privkey.pem>
-CERTIFICATE_ONLY=</path/prefix_suffix_index_certificate_only.pem>
-PREFIX=<ZZZ>
-HANDLEOWNER="200:0.NA/\\\$PREFIX"
-REVERSELOOKUP_USERNAME=<ZZZ>
-HTTPS_VERIFY="True"
-#
-# users for msiexec command
-USERS="user0#Zone0 user1#Zone1"
-#
-# loglevel and log directory
-# possible log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
-LOG_LEVEL=DEBUG
-LOG_DIR=/var/log/irods
-#
-#
-# iRODS behavioral parameters
-#
-# check if user is authorized to perform several functions
-AUTHZ_ENABLED=true
-#
-# check if message queue is enabled
-MSG_QUEUE_ENABLED=false
-#
+{
+  "b2safe_package_dir": "${IRODS_PACKAGE_DIR}",
+  "irods_conf_dir": "/etc/irods",
+  "irods_dir": "/var/lib/irods/iRODS",
+  "irods_default_resource": "demoResc",
+  "cred_store_type": "os",
+  "cred_file_path": "${IRODS_PACKAGE_DIR}/conf/credentials",
+  "server_id": "irods://<fully_qualified_hostname>:1247",
+  "server_api_reg": "irods://<fully_qualified_hostname>:1247",
+  "server_api_pub": "irods://<fully_qualified_hostname>:1247",
+  "handle_server_url": "https://epic6.storage.surfsara.nl:8003",
+  "handle_private_key": "</path/prefix_suffix_index_privkey.pem>",
+  "handle_certificate_only": "</path/prefix_suffix_index_certificate_only.pem>",
+  "handle_prefix": "<ZZZ>",
+  "handle_owner": "200:0.NA/<ZZZ>",
+  "handle_reverse_lookup_name": "<ZZZ>",
+  "handle_reverse_lookup_password": "<reverse_lookup_password>",
+  "handle_https_verify": True,
+  "handle_users": [ "user0#Zone0", "user1#Zone1" ],
+  "log_level": "INFO",
+  "log_directory": "/var/log/irods",
+  "shared_space": "",
+  "authz_enabled": true,
+  "msg_queue_enabled": false
+}
 EOF2
 
 fi
@@ -162,11 +141,11 @@ The package b2safe has been installed in ${IRODS_PACKAGE_DIR}.
 To install/configure it in iRODS do following as the user who runs iRODS :
 
 # update install.conf with correct parameters with your favorite editor
-sudo vi ${IRODS_PACKAGE_DIR}/packaging/install.conf
+sudo vi ${IRODS_PACKAGE_DIR}/packaging/install.json
 
 # install/configure it as the user who runs iRODS
 source /etc/irods/service_account.config 
-sudo su - \\\$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd ${IRODS_PACKAGE_DIR}/packaging/ ; ./install.sh"
+sudo su - \\\$IRODS_SERVICE_ACCOUNT_NAME -s "/bin/bash" -c "cd ${IRODS_PACKAGE_DIR}/packaging/ ; ./install.py"
 
 EOF1
 

packaging/install.py

+#!/usr/bin/env python
+#
+# epicclient.py
+#
+# * use 4 spaces!!! not tabs
+# * set tabstop=4
+# * set expandtab
+# * See PEP-8 Python style guide http://www.python.org/dev/peps/pep-0008/
+# * use pylint
+#
+
+"""EUDAT configuration of iRODS B2SAFE and msiPID """
+
+import getpass
+import glob
+import json
+import os
+import shutil
+from datetime import datetime
+
+CONFIG_FILE = 'install.json'
+CONFIG_PARAMETERS = ["b2safe_package_dir",
+                     "irods_conf_dir",
+                     "irods_dir",
+                     "irods_default_resource",
+                     "cred_store_type",
+                     "cred_file_path",
+                     "server_id",
+                     "server_api_reg",
+                     "server_api_pub",
+                     "handle_server_url",
+                     "handle_private_key",
+                     "handle_certificate_only",
+                     "handle_prefix",
+                     "handle_owner",
+                     "handle_reverse_lookup_name",
+                     "handle_reverse_lookup_password",
+                     "handle_https_verify",
+                     "handle_users",
+                     "handle_groups",
+                     "log_level",
+                     "log_directory",
+                     "shared_space",
+                     "authz_enabled",
+                     "msg_queue_enabled"]
+PID_DEFAULT_PROFILE = [
+    {
+        "entry": {
+            "index": 1,
+            "type": "URL",
+            "data": {
+                "format": "string",
+                "value": "{OBJECT}"
+            }
+        }
+    },
+    {
+        "entry": {
+            "index": 100,
+            "type": "HS_ADMIN",
+            "data": {
+                "format": "admin",
+                "value": {
+                    "handle": "0.NA/{HANDLE_PREFIX}",
+                    "index": 200,
+                    "permissions": "011111110011"
+                }
+            }
+        }
+    }
+]
+
+########################
+# Functions
+########################
+
+
+def add_irods_cmd_dir(json_config):
+    ''' add the iRODS path to the cmd directory. iRODS 4.1 and higher differs '''
+
+    base_irods_dir = os.path.dirname(json_config["irods_dir"])
+
+    if os.path.isdir(base_irods_dir+"/msiExecCmd_bin"):
+        irods_cmd_dir = '/msiExecCmd_bin'
+    else:
+        irods_cmd_dir = '/iRODS/server/bin/cmd'
+
+    json_config["irods_cmd_dir"] = base_irods_dir+irods_cmd_dir
+
+def check_user(json_config):
+    ''' check the user who is running this program. It has to be the iRODS user '''
+
+    username = getpass.getuser()
+    service_account_config_file = json_config["irods_conf_dir"]+"/service_account.config"
+    user_match = False
+
+    try:
+        with open(service_account_config_file) as service_account_config:
+            for line in service_account_config:
+                if line.rstrip() == 'IRODS_SERVICE_ACCOUNT_NAME='+username:
+                    user_match = True
+    except IOError:
+        print "Error, Unable to open file: %s" % service_account_config_file
+        exit(1)
+
+    if user_match:
+        print "The iRODS user matches the user who is running this script"
+    else:
+        print "The iRODS user  does not match the user who is running this script"
+        exit(1)
+
+def check_missing_parameters(json_config):
+    ''' check the json config for missing keys '''
+
+    parameters_match = True
+    missing_parameters = []
+
+    for parameter in CONFIG_PARAMETERS:
+        if parameter not in json_config:
+            parameters_match = False
+            missing_parameters.append(parameter)
+
+    if not parameters_match:
+        print "Not all parameters are present. Please add following parameter(s)"
+        for item in missing_parameters:
+            print "The missing parameter = %s" % item
+        exit(1)
+
+def create_b2safe_symbolic_links(json_config):
+    ''' create symbolic links for the b2safe rules, return files added to the config dict '''
+
+    count = 0
+    json_config["re_rulebase_set"] = []
+
+    # delete existing symlinks
+    for symlink_file in glob.glob(json_config["irods_conf_dir"]+'/eudat*.re'):
+        try:
+            os.unlink(symlink_file)
+        except IOError:
+            print "Error deleting symbolic link: %s" % symlink_file
+            exit(1)
+
+    # add new symlinks
+    for symlink_file in glob.glob(json_config["b2safe_package_dir"]+'/rulebase/*.re'):
+        try:
+            os.symlink(symlink_file, json_config["irods_conf_dir"]+'/eudat'+str(count)+'.re')
+            json_config["re_rulebase_set"].append('eudat'+str(count))
+        except IOError:
+            print "Error creating symbolic link: %s" % symlink_file
+            exit(1)
+
+        count = count + 1
+
+def install_python_b2safe_scripts(json_config):
+    ''' create symbolic links for python executables in b2safe package '''
+
+    # add new symlinks
+    for symlink_file in glob.glob(json_config["b2safe_package_dir"]+'/cmd/*.py'):
+        destination_path = json_config["irods_cmd_dir"]+'/'+os.path.basename(symlink_file)
+        try:
+            # remove existing links
+            if os.path.exists(destination_path):
+                os.unlink(destination_path)
+            # add new links
+            os.symlink(symlink_file, destination_path)
+        except IOError:
+            print "Error creating symbolic link: %s" % symlink_file
+            exit(1)
+
+def read_json_config(json_config_file):
+    ''' read the parameters from a json config file '''
+
+    try:
+        with open(json_config_file) as config_file:
+            json_config = json.load(config_file)
+    except IOError:
+        print "Error, Unable to open file: %s" % json_config_file
+        exit(1)
+
+    return dict(json_config)
+
+def secure_file(chmod_file):
+    ''' set the protection to rw for the owner only '''
+
+    if os.path.exists(chmod_file):
+        try:
+            os.chmod(chmod_file, 0o600)
+        except IOError:
+            print "Error, Unable to protect file: %s" % chmod_file
+            exit(1)
+    else:
+        print "Error, Unable to protect file: %s as it not present" % chmod_file
+        exit(1)
+
+def save_config_file(save_file):
+    ''' make first copy of the day of config file '''
+
+    save_file_copy = save_file+'.org.'+datetime.now().strftime('%Y%m%d')
+
+    # check if file exists and if not copy
+    if not os.path.exists(save_file_copy):
+        try:
+            shutil.copy2(save_file, save_file_copy)
+            secure_file(save_file_copy)
+        except IOError:
+            print "Error, Unable to copy file: %s" % save_file
+            exit(1)
+
+def update_authz_map(json_config):
+    ''' update the authz map json file '''
+
+    authz_map_file = json_config["b2safe_package_dir"]+'/conf/authz.map.json'
+
+    # save authz file
+    save_config_file(authz_map_file)
+
+    # read authz file
+    authz_map_config = read_json_config(authz_map_file)
+
+    print "TODO: Update %s not yet implemented !! " % authz_map_file
+
+    ### write authz config
+    ##write_json_config(authz_map_config, authz_map_file)
+
+def update_epicclient_credentials(json_config):
+    ''' create (if needed) and update epicclient2.py credentials file '''
+
+    credentials_file = json_config["cred_file_path"]
+
+    # create credentials file
+    if not os.path.exists(credentials_file):
+        shutil.copy2(json_config["b2safe_package_dir"]+'/conf/credentials_epicclient2_example',
+                     credentials_file)
+        secure_file(credentials_file)
+
+    # save credentails file
+    save_config_file(credentials_file)
+
+    # read credentials file
+    epicclient2_config = read_json_config(credentials_file)
+
+    # modify epicclient2 credentials
+    epicclient2_config["handle_server_url"] = json_config["handle_server_url"]
+    epicclient2_config["private_key"] = json_config["handle_private_key"]
+    epicclient2_config["certificate_only"] = json_config["handle_certificate_only"]
+    epicclient2_config["prefix"] = json_config["handle_prefix"]
+    epicclient2_config["handleowner"] = json_config["handle_owner"]
+    epicclient2_config["reverselookup_username"] = json_config["handle_reverse_lookup_name"]
+    epicclient2_config["reverselookup_password"] = json_config["handle_reverse_lookup_password"]
+    epicclient2_config["HTTPS_verify"] = json_config["handle_https_verify"]
+
+    # write credentials config
+    write_json_config(epicclient2_config, credentials_file)
+
+def update_flat_file_parameter(modify_file, mod_key, mod_value, irods_file=False):
+    ''' update a file and make it 'key = "value";' '''
+
+    lines = []
+    key_match = False
+
+    try:
+        # read file in list
+        with open(modify_file) as file_handle:
+            for line in file_handle:
+                line = line.rstrip()
+                lines.append(line)
+    except IOError:
+        print "Error, Unable to open file: %s" % modify_file
+        exit(1)
+
+    for idx, line in enumerate(lines):
+        stripped_line = line.lstrip()
+        if stripped_line.find(mod_key, 0, len(mod_key)) == 0:
+            key_match = True
+            if irods_file:
+                start_s = line.find('"')
+                end_s = line.find('"', start_s+1)
+                newline = line[0:start_s+1]+str(mod_value)+line[end_s:len(line)]
+                lines[idx] = newline
+
+    if key_match:
+        try:
+            with open(modify_file, "w+") as config_file:
+                config_file.write("\n".join(lines))
+        except IOError:
+            print "Error, Unable to open file: %s" % modify_file
+            exit(1)
+    else:
+        print "Parameter %s not found in file %s !!!" % (mod_key, modify_file)
+
+def update_irods_server_config(json_config):
+    ''' add the eudat rules to the iRODS server config '''
+
+    config_file = json_config["irods_conf_dir"]+'/server_config.json'
+
+    # save iRODS config file
+    save_config_file(config_file)
+
+    # read iRODS config
+    irods_config = read_json_config(config_file)
+
+    # modify iRODS config
+    for item in json_config["re_rulebase_set"]:
+        match_item = False
+        match_dict = False
+        for irods_config_item in irods_config["re_rulebase_set"]:
+            # in version 4.1.x it is a dict. In 4.2.1 and higher it is an single item.
+            if isinstance(irods_config_item, dict):
+                match_dict = True
+                if item == irods_config_item["filename"]:
+                    match_item = True
+            else:
+                if item == irods_config_item:
+                    match_item = True
+        # append to the list/array only if needed. Either dict or single item.
+        if not match_item and match_dict:
+            append_item_dict = {'filename': item}
+            irods_config["re_rulebase_set"].append(append_item_dict)
+        elif not match_item and not match_dict:
+            irods_config["re_rulebase_set"].append(item)
+
+    # write iRODS config
+    write_json_config(irods_config, config_file)
+
+def update_local_re_parameters(json_config):
+    ''' update parameters in local.re B2SAFE ruleset '''
+
+    config_file = json_config["b2safe_package_dir"]+'/rulebase/local.re'
+
+    # save B2SAFE rule file
+    save_config_file(config_file)
+
+    # update B2SAFE rule file
+
+    # getAuthZParameters
+    update_flat_file_parameter(config_file, '*authZMapPath',
+                               json_config["b2safe_package_dir"]+'/conf/authz.map.json', True)
+
+    # getConfParameters
+    update_flat_file_parameter(config_file, '*authzEnabled', str(json_config["authz_enabled"]).lower(), True)
+    update_flat_file_parameter(config_file, '*messageQueueEnabled',
+                               str(json_config["msg_queue_enabled"]).lower(), True)
+
+    # getEpicApiParameters
+    update_flat_file_parameter(config_file, '*credStoreType', json_config["cred_store_type"], True)
+    update_flat_file_parameter(config_file, '*credStorePath', json_config["cred_file_path"], True)
+    update_flat_file_parameter(config_file, '*serverID', json_config["server_id"], True)
+
+    # getHttpApiParameters
+    update_flat_file_parameter(config_file, '*serverApireg', json_config["server_api_reg"], True)
+    update_flat_file_parameter(config_file, '*serverApipub', json_config["server_api_pub"], True)
+
+    # getMetaParameters
+    update_flat_file_parameter(config_file, '*metaConfPath',
+                               json_config["b2safe_package_dir"]+'/conf/metadataManager.conf', True)
+
+    # getLogParameters
+    update_flat_file_parameter(config_file, '*logConfPath',
+                               json_config["b2safe_package_dir"]+'/conf/log.manager.json', True)
+
+def update_log_manager_conf(json_config):
+    ''' update log manager config file '''
+
+    log_manager_conf_file = json_config["b2safe_package_dir"]+'/conf/log.manager.json'
+    logmanager_config = {}
+
+    # save logmanager file
+    if os.path.exists(log_manager_conf_file):
+        save_config_file(log_manager_conf_file)
+
+    # read logmanager config
+    if os.path.exists(log_manager_conf_file):
+        logmanager_config = read_json_config(log_manager_conf_file)
+
+    # modify logmanager config
+    logmanager_config["log_level"] = json_config["log_level"]
+    logmanager_config["log_dir"] = json_config["log_directory"]
+
+    # write logmanager config
+    write_json_config(logmanager_config, log_manager_conf_file)
+
+def update_pid_uservice_config(json_config):
+    ''' update pid uService json file '''
+
+    pid_uservice_conf_file = json_config["irods_conf_dir"]+'/irods_pid.json'