merge: from master into b2share branch, so it is deployable again.

change: python3 migration for B2SHARE connector scripts + unit tests of it.

LICENSE

 Copyright (c) 2014, EUDAT project funded from the European Union under grant agreement n.283304.
 Copyright (c) 2015, EUDAT2020 project funded from the European Union under grant agreement n.654065.
-All rights reserved.
+Copyright (c) 2018, EUDAT CDI - www.eudat.eu.
 
 <Willem Elbers - Max Planck Institute for Psycholinguistics, contact willem.elbers@mpi.nl>
 <Claudio Cacciari - Cineca, contact c.cacciari@cineca.it>

changelog.txt

+B2SAFE package 4.2.0 [2019/01/07]
+- fixed bug related to redundant replica handles (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/120)
+- updated test rules
+- improved wiki docuentation
+- added support for oauth2 protocol via PAM module
+- added support to HTTP interfaces in the PID parameters
+
+B2SAFE package 4.1.1 [2018/02/22]
+- fixed bug related to EUDATTransferUsingFailLog rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/113)
+- updated test rules
+
+B2SAFE package 4.1.0 [2017/12/18]
+- tested sucessfully against iRODS v4.2.1. The test against iRODS 4.2.2 failed due to this bug: https://github.com/irods/irods/issues/3722, therefore the v4.2.2 is not supported yet.
+- fixed bug related to EUDATReplication rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/102)
+- improved install.txt documentation (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/103)
+- added EUDATeURLupdateColl(*PID, *newURL) to update the url in multiple PIDS associeted to a whole collection of objects.
+- many improvements in the logging
+- added the option to specify the destination iRODS resource in many rules, included the EUDATReplication.
+
 B2SAFE package 4.0.1 [2017/07/11]
 - added support for iRODS 4.2.1 (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/94)
 - fixed bug related to package deployment (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/85)

cmd/configuration.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-#############################################################################
-# Configuration Class                                                       #
-#############################################################################
-import configparser
+import json
 import logging.handlers
 import os
+from irodsUtility import IRODSUtils
 
 
 class Configuration():
-    """
-    Get properties from filesystem
-    """
-    defaultLogDir = os.getcwd() + os.sep + "logs"
-    defaultLogFileName = os.sep + "b2share_connection.log"
-    irods_home_dir = ""
-    irods_debug = ""
-
-    def __init__(self, conffile, debug, dryrun, logger,
-                 irodsenv=None):
-        self.conffile = conffile
-        self.debug = debug
-        self.dryrun = dryrun
+    def __init__(self, logger):
         self.logger = logger
-        self.access_token = ""
-        self.irodsenv = irodsenv
-        self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
-                          'ERROR': logging.ERROR, 'WARNING': logging.WARNING,
+        self.log_level = {'INFO': logging.INFO,
+                          'DEBUG': logging.DEBUG,
+                          'ERROR': logging.ERROR,
+                          'WARNING': logging.WARNING,
                           'CRITICAL': logging.CRITICAL}
-        self.config = configparser.RawConfigParser()
-
-    # return configuration
-    # missing exception
-    # if expected config is not in the config file
-    def parseConf(self):
-        """Parse the configuration file."""
-
-        # self.config = ConfigParser.RawConfigParser()
-        if not os.path.exists(self.conffile):
-            self.logger.error('missing configuration file %s:' %
-                              (self.conffile))
-            return  # error config file not found
-
-        with open(self.conffile, "r") as confFile:
-            try:
-                self.config.readfp(confFile)
-            except Exception as e:
-                self.logger.error(e)
-                return
-
-        logfilepath = self._getConfOption('Logging', 'log_file')
-        if not os.path.exists(logfilepath):
-            # if user did not specified a path for log file
-            # default to current dir + b2share_connection.log
-            # self.logger.error("no location for log file is specified,
-            # so try to default to
-            # <currend directory>/logs/b2share_connection.log")
-            try:
-                os.mkdir(self.defaultLogDir)
-                logfilepath = self.defaultLogDir + self.defaultLogFileName
-            except OSError:
-                # self.logger.error("no location for log file is specified,
-                # so try to default to
-                # <currend directory>/logs/b2share_connection.log")
-                logfilepath = os.getcwd() + self.defaultLogFileName
-                # print("Creation of the directory %s failed,
-                # so defaulting to <currend directory>/b2share_connection.log"
-                # % defaultLogDir)
-
-        loglevel = self._getConfOption('Logging', 'log_level')
-        if self.debug:
-            loglevel = 'DEBUG'
-        self.logger.setLevel(self.log_level[loglevel])
-
-        rfh = logging.handlers.RotatingFileHandler(logfilepath,
-                                                   maxBytes=8388608,
-                                                   backupCount=9)
-        formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
-                                      '[%(funcName)s] %(message)s')
-        rfh.setFormatter(formatter)
-        self.logger.addHandler(rfh)
-
-        self.b2share_scheme = self._getConfOption('B2SHARE', 'scheme')
-        self.b2share_addr = self._getConfOption('B2SHARE', 'address')
-        self.b2share_path = self._getConfOption('B2SHARE', 'path')
+        self.access_token = ""
+        self.config_path = ""
+        self.record_id = ""
+        self.dryrun = None
+        self.user = None
+        self.collection_path = None
+        self.title = None
+        self.community = None
 
-        self.b2share_host_name = self._getConfOption(
-            'B2SHARE_HTTP_API', 'host_name')
-        self.list_communities_endpoint = self._getConfOption(
-            'B2SHARE_HTTP_API', 'list_communities_endpoint')
-        self.records_endpoint = self._getConfOption(
-            'B2SHARE_HTTP_API', 'records_endpoint')
-        self.access_parameter = self._getConfOption(
-            'B2SHARE_HTTP_API', 'access_parameter')
-        self.get_community_schema_endpoint = self._getConfOption(
-            'B2SHARE_HTTP_API', 'get_community_schema_endpoint')
+    def loadConfigurarionsFrom(self, configuration_file_path):
+        with open(configuration_file_path, 'r') as json_file:
+            data = json.load(json_file)
+            loglevel = data["configurations"]["logging"]["loglevel"]
+            self.logger.setLevel(self.log_level[loglevel])
+            logfilepath = data["configurations"]["logging"]["logfile"]
+            # if logfile does not exist and not possible to create
+            # stop execution with ERROR
+            if not os.path.exists(logfilepath):
+                try:
+                    logDir = os.path.dirname(logfilepath)
+                    os.mkdir(logDir)
+                except OSError:
+                    return "Log file directory: " + logfilepath +\
+                           "specified in config file: " +\
+                           configuration_file_path +\
+                           " does NOT exist."
+            rfh = logging.handlers.RotatingFileHandler(logfilepath,
+                                                       maxBytes=8388608,
+                                                       backupCount=9)
+            formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
+                                          '[%(funcName)s] %(message)s')
+            rfh.setFormatter(formatter)
+            self.logger.addHandler(rfh)
 
-        self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir')
-        self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True)
+            b2share_http_api = data["configurations"]["b2share_http_api"]
+            self.b2share_host_name = b2share_http_api["host_name"]
+            self.list_communities_endpoint =\
+                b2share_http_api["list_communities_endpoint"]
+            self.records_endpoint = b2share_http_api["records_endpoint"]
+            self.access_parameter = b2share_http_api["access_parameter"]
+            self.get_community_schema_endpoint =\
+                b2share_http_api["get_community_schema_endpoint"]
 
-    def _getConfOption(self, section, option, boolean=False):
-        """
-        get the options from the configuration file
-        """
-        if self.config.has_section(section):
-            if (self.config.has_option(section, option)):
-                opt = self.config.get(section, option)
-                if boolean:
-                    if opt in ['True', 'true']:
+            irods = data["configurations"]["irods"]
+            self.irodsenv = irods["irods_env"]
+            if self.irodsenv:
+                self.irodsu = IRODSUtils(self.logger, irods_env=self.irodsenv)
+            self.irods_resources = irods["resources"]
+            self.irods_home_dir = irods["irods_home_dir"]
+            self.irods_debug = irods["irods_debug"]
+        return ""
\ No newline at end of file
-                        return True
-                    else:
-                        return False
-                return opt
-            else:
-                self.logger.error(
-                    'missing parameter %s: in section %s' % (section, option))
-        else:
-            self.logger.error(
-                'missing section %s: with parameter %s' % (section, option))
-        return None

cmd/create_md_file.py

@@ -5,79 +5,13 @@ import requests
 import os
 import logging.handlers
 import argparse
-import configparser
 import tempfile
+import json
 
-from manifest.irodsUtility import IRODSUtils
-logger = logging.getLogger('create_md_schema')
-
+from configuration import Configuration
+from irodsUtility import IRODSUtils
 
-class Configuration():
-    """
-    Get properties from filesystem
-    """
-
-    def __init__(self, conffile, debug, dryrun, logger):
-
-        self.conffile = conffile
-        self.debug = debug
-        self.dryrun = dryrun
-        self.logger = logger
-        self.access_token = ""
-        self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
-                          'ERROR': logging.ERROR, 'WARNING': logging.WARNING,
-                          'CRITICAL': logging.CRITICAL}
-
-    def parseConf(self):
-        """Parse the configuration file."""
-
-        self.config = configparser.RawConfigParser()
-        with open(self.conffile, "r") as confFile:
-            self.config.readfp(confFile)
-
-        logfilepath = self._getConfOption('Logging', 'log_file')
-        loglevel = self._getConfOption('Logging', 'log_level')
-        if self.debug:
-            loglevel = 'DEBUG'
-        logger.setLevel(self.log_level[loglevel])
-        rfh = logging.handlers.RotatingFileHandler(logfilepath,
-                                                   maxBytes=50000000,
-                                                   backupCount=10)
-        formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
-                                      '[%(funcName)s] %(message)s')
-        rfh.setFormatter(formatter)
-        logger.addHandler(rfh)
-
-        self.b2share_host_name = self._getConfOption(
-            'B2SHARE_HTTP_API', 'host_name')
-        self.list_communities_endpoint = self._getConfOption(
-            'B2SHARE_HTTP_API', 'list_communities_endpoint')
-        self.access_parameter = self._getConfOption(
-            'B2SHARE_HTTP_API', 'access_parameter')
-        self.get_community_schema_endpoint = self._getConfOption(
-            'B2SHARE_HTTP_API', 'get_community_schema_endpoint')
-
-        self.irods_zone_name = self._getConfOption('iRODS', 'zone_name')
-        self.irods_res = self._getConfOption('iRODS', 'resources')
-        self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir')
-        self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True)
-
-    def _getConfOption(self, section, option, boolean=False):
-        """
-        get the options from the configuration file
-        """
-
-        if (self.config.has_option(section, option)):
-            opt = self.config.get(section, option)
-            if boolean:
-                if opt in ['True', 'true']:
-                    return True
-                else:
-                    return False
-            return opt
-        else:
-            self.logger.warning('missing parameter %s:%s' % (section, option))
-            return None
+logger = logging.getLogger('create_md_schema')
 
 
 def getAllCommunities(configuration):
@@ -98,112 +32,215 @@ def getAllCommunities(configuration):
         name = community_object["name"]
         community_id = community_object["id"]
         communities[name] = community_id
-
     return communities
 
 
-def create_md_schema(args):
-    commName = args.communityName
+def getVerbosePrintMethod(args):
+    if args.verbouse:
+        def verboseprint(*args):
+            # Print each argument separately so caller doesn't need to
+            # stuff everything to be printed into a single string
+            for arg in args:
+                print(arg)
+            return
+    else:
+        def verboseprint(*args):
+            # do-nothing
+            return
+    return verboseprint
+
+
+def getAccessTokenWithConfigs(configuration):
+    # get access_token from users metadata in iRODS
+    if configuration.irodsu:
+        users_metadata = \
+            configuration.irodsu.getUserMetadata(
+                configuration.user, "access_token")
+        if users_metadata:
+            return users_metadata[0]
+        else:
+            return None
+    else:
+        return None
 
-    configuration = Configuration(
-        args.confpath, args.debug, args.dryrun, logger)
-    configuration.parseConf()
 
-    # get access_token from collection metadata
-    irodsu = IRODSUtils(configuration.irods_home_dir,
-                        logger, configuration.irods_debug)
-    access_token = irodsu.getMetadata(args.userName, "access_token", '-u')[0]
-    configuration.access_token = access_token
+DEFAULT_CONFIG_FILENAME = "b2share_client.json"
 
+
+def getConfigs(verboseprint):
+    config_path = None
+    if not args.confpath:
+        config_path = os.path.dirname(os.getcwd()) + os.sep +\
+            "conf" + os.sep + DEFAULT_CONFIG_FILENAME
+    else:
+        config_path = args.confpath
+    if not os.path.exists(config_path):
+        verboseprint('missing configuration file %s:' % (config_path))
+        logger.error('missing configuration file %s:' % (config_path))
+        return None
+    configuration = Configuration(logger)
+    configuration.config_path = config_path
+    configuration.loadConfigurarionsFrom(config_path)
+    configuration.user = args.user
+    configuration.community = args.community
+    configuration.collection_path = args.collection_path
+
+    logger.info("Start creating draft ...")
+    accessToken = getAccessTokenWithConfigs(configuration)
+    if accessToken is None:
+        verboseprint("Drafting FAILED. \
+            No B2SHARE access token found in users meta data.")
+        logger.error(
+            "Drafting FAILED. \
+            No B2SHARE access token found in users meta data.")
+        return None
+    configuration.access_token = accessToken
+    if 'metadata_file_name' in args:
+        configuration.metadata_file_name = args.metadata_file_name
+    return configuration
+
+
+def getCommunitySchema(configuration, verboseprint):
+    # get metadata schema
     communities_list = getAllCommunities(configuration)
+    if configuration.community not in communities_list.keys():
+        logger.error("No communityName specified. " +
+                     "Please select and specify " +
+                     "one of the communities names: " +
+                     str(communities_list.keys()))
+        verboseprint("No communityName specified. " +
+                     "Please select and specify " +
+                     "one of the communities names: " +
+                     str(communities_list.keys()))
+        return None
+    community_id = communities_list[configuration.community]
+    host = configuration.b2share_host_name
+    community_endpoint = configuration.list_communities_endpoint
+    get_schema_endpoint = configuration.get_community_schema_endpoint
+    acces_part = \
+        configuration.access_parameter + \
+        "=" + configuration.access_token
+    get_community_schema_url = host + community_endpoint + \
+        community_id + get_schema_endpoint + acces_part
+    community_schema = None
+    try:
+        response = requests.get(url=get_community_schema_url)
+        verboseprint("response status code: " + str(response.status_code))
+        logger.debug("response status code: " + str(response.status_code))
+        if ((str(response.status_code) == "200") |
+                (str(response.status_code) == "201")):
+            community_schema = response.json()["json_schema"]["allOf"][0]
+            logger.info("Get schema for community: " + configuration.community)
+        else:
+            verboseprint("NO schema for community: " +
+                         configuration.community +
+                         "Response: " + str(response.json()))
+            logger.error("NO schema for community: " +
+                         configuration.community +
+                         "Response: " + str(response.json()))
+    except requests.exceptions.RequestException as e:
+        logger.error(e)
+    return community_schema
 
-    if (commName is not None) and (commName != ''):
-        logger.info(
-            'Start creating metadata schema for the community'+str(commName))
-
-        # get metadata schema
-        # api/communities/
-        community_id = communities_list[commName]
-        host = configuration.b2share_host_name
-        community_endpoint = configuration.list_communities_endpoint
-        get_schema_endpoint = configuration.get_community_schema_endpoint
-        acces_part = \
-            configuration.access_parameter + \
-            "=" + configuration.access_token
-        get_community_schema_url = host + community_endpoint + \
-            community_id + get_schema_endpoint + acces_part
 
-        response = requests.get(url=get_community_schema_url)
-        # May be parsing for manifest extention
-        community_schema = response.json()["json_schema"]["allOf"][0]
-        requiredProperties = \
-            community_schema["b2share"]["presentation"]["major"]
-        optionalProperties = \
-            community_schema["b2share"]["presentation"]["minor"]
-
-        mdPatchSceleton = \
-            '#please fill out at least the required fields with' + \
-            'values as JSON strings in the line after the property name.' + \
-            '\n'
-        mdPatchSceleton = \
-            mdPatchSceleton + \
-            '#e.g. for community - EUDAT, ' + \
-            'for open_access - true, for contributors - ' + \
-            '[{"contributor_name":"Hulk",' + \
-            ' "contributor_type": "Editor"},' + \
-            ' {"contributor_name":"Banner",' + \
-            ' "contributor_type": "ContactPerson"}]' + \
-            "\n\n"
-        mdPatchSceleton = mdPatchSceleton + "[required]" + "\n"
-        for requiredProperty in requiredProperties:
-            mdPatchSceleton = mdPatchSceleton + requiredProperty + "\n\n"
-
-        mdPatchSceleton = mdPatchSceleton + "\n" + "[optional]" + "\n"
-        for optionalPropertiy in optionalProperties:
-            mdPatchSceleton = mdPatchSceleton + optionalPropertiy + "\n\n"
-
-        if args.dryrun:
-            print(mdPatchSceleton)
+def create_md_schema(args):
+    verboseprint = getVerbosePrintMethod(args)
+    configuration = getConfigs(verboseprint)
+    if not configuration:
+        return None
+
+    logger.info('Start creating metadata schema for the community' +
+                str(configuration.community))
+    community_schema = getCommunitySchema(configuration, verboseprint)
+    if not community_schema:
+        return
+    requiredProperties_list = \
+        community_schema["b2share"]["presentation"]["major"]
+    optionalProperties_list = \
+        community_schema["b2share"]["presentation"]["minor"]
+
+    requiredProperties = {}
+    for property_name in requiredProperties_list:
+        property_type = \
+            community_schema["properties"][property_name]["type"]
+        requiredProperties[property_name] = property_type
+    optionalProperties = {}
+    for property_name in optionalProperties_list:
+        property_type = \
+            community_schema["properties"][property_name]["type"]
+        optionalProperties[property_name] = property_type
+
+    mdPatchSceleton = ""
+    mdPatchSceleton = mdPatchSceleton + '{"metadata": {"required":['
+    for requiredProperty in requiredProperties:
+        mdPatchSceleton = mdPatchSceleton + \
+            '{"option_name": "' + requiredProperty + '",' \
+            '"value":"", ' + \
+            '"type":"' + requiredProperties[requiredProperty] +\
+            '"},'
+    mdPatchSceleton = mdPatchSceleton[:-1]
+    mdPatchSceleton = mdPatchSceleton + "],"
+    mdPatchSceleton = mdPatchSceleton + '"optional":['
+    for optionalPropertiy in optionalProperties.keys():
+        mdPatchSceleton = mdPatchSceleton + \
+            '{"option_name": "' + optionalPropertiy + '",' \
+            '"value":"", ' + \
+            '"type":"' + optionalProperties[optionalPropertiy] +\
+            '"},'
+    mdPatchSceleton = mdPatchSceleton[:-1]
+    mdPatchSceleton = mdPatchSceleton + "]"
+    mdPatchSceleton = mdPatchSceleton + '} }'
+
+    json_formated = json.dumps(json.loads(mdPatchSceleton), indent=4)
+    mdPatchSceleton = json_formated
+    if args.dryrun:
+        print(mdPatchSceleton)
+    else:
+        logger.info('Writing the metadata to a file')
+        # file_path = configuration.collection_path + os.sep + \
+        #     configuration.metadata_file_name
+        metadata_file_path = ""
+        if configuration.metadata_file_name:
+            metadata_file_path = configuration.collection_path + os.sep + \
+                                 configuration.metadata_file_name
         else:
-            logger.info('Writing the metadata to a file')
-            file_path = args.collectionName + os.sep + \
-                "b2share_metadata.json"  # TODO: config, argument?
-            temp = tempfile.NamedTemporaryFile()
+            metadata_file_path = configuration.collection_path + os.sep + \
+                              "b2share_metadata.json"
+        temp = tempfile.NamedTemporaryFile()
+        try:
+            temp.write(bytes(mdPatchSceleton, 'UTF-8'))
+            temp.flush()
             try:
-                temp.write(mdPatchSceleton)
-                temp.flush()
-                try:
-                    irodsu.putFile(temp.name, file_path,
-                                   configuration.irods_resource)
-                except:
-                    out = irodsu.putFile(temp.name, file_path)
-                    print(str(out))
-            finally:
-                temp.close()
-
-        logger.info(
-            'Finish creating metadata schema for the community'+commName)
-    else:
-        logger.error("No communityName specified. " +
-                     "Please select one of the communities names: " +
-                     str(communities_list.keys()))
-        print(str(communities_list.keys()))
+                configuration.irodsu.putFile(temp.name, metadata_file_path)
+            except Exception:
+                # try again
+                out = configuration.irodsu.putFile(temp.name,
+                                                   metadata_file_path)
+                print(str(out))
+        finally:
+            temp.close()
 
+    logger.info('Finish creating metadata schema for the community' +
+                configuration.community)
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='B2SAFE B2SHARE client')
-    parser.add_argument("--confpath", help="path to the configuration file")
-    parser.add_argument("-c", "--communityName", help="B2Share user name")
 
-    parser.add_argument("-dbg", "--debug",
-                        action="store_true", help="enable debug")
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Metadata creator')
+    parser.add_argument("-u", "--user", required=True,
+                        help="irods user to get B2SHARE access token")
+    parser.add_argument("-p", "--collection_path", required=True,
+                        help="irods path to the collection")
+    parser.add_argument('-comm', '--community', required=True,
+                        help='community name toget the schema of')
+    parser.add_argument("--confpath",
+                        help="path to the configuration file")
     parser.add_argument("-d", "--dryrun", action="store_true",
                         help="run without performing any real change")