Commit 6ef85871 authored by Julia Kaufhold's avatar Julia Kaufhold
Browse files

merge: from master into b2share branch, so it is deployable again.

change: python3 migration for B2SHARE connector scripts + unit tests of it.
parent 6d6eb156
Copyright (c) 2014, EUDAT project funded from the European Union under grant agreement n.283304. Copyright (c) 2014, EUDAT project funded from the European Union under grant agreement n.283304.
Copyright (c) 2015, EUDAT2020 project funded from the European Union under grant agreement n.654065. Copyright (c) 2015, EUDAT2020 project funded from the European Union under grant agreement n.654065.
All rights reserved. Copyright (c) 2018, EUDAT CDI - www.eudat.eu.
<Willem Elbers - Max Planck Institute for Psycholinguistics, contact willem.elbers@mpi.nl> <Willem Elbers - Max Planck Institute for Psycholinguistics, contact willem.elbers@mpi.nl>
<Claudio Cacciari - Cineca, contact c.cacciari@cineca.it> <Claudio Cacciari - Cineca, contact c.cacciari@cineca.it>
......
B2SAFE package 4.2.0 [2019/01/07]
- fixed bug related to redundant replica handles (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/120)
- updated test rules
- improved wiki docuentation
- added support for oauth2 protocol via PAM module
- added support to HTTP interfaces in the PID parameters
B2SAFE package 4.1.1 [2018/02/22]
- fixed bug related to EUDATTransferUsingFailLog rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/113)
- updated test rules
B2SAFE package 4.1.0 [2017/12/18]
- tested sucessfully against iRODS v4.2.1. The test against iRODS 4.2.2 failed due to this bug: https://github.com/irods/irods/issues/3722, therefore the v4.2.2 is not supported yet.
- fixed bug related to EUDATReplication rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/102)
- improved install.txt documentation (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/103)
- added EUDATeURLupdateColl(*PID, *newURL) to update the url in multiple PIDS associeted to a whole collection of objects.
- many improvements in the logging
- added the option to specify the destination iRODS resource in many rules, included the EUDATReplication.
B2SAFE package 4.0.1 [2017/07/11] B2SAFE package 4.0.1 [2017/07/11]
- added support for iRODS 4.2.1 (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/94) - added support for iRODS 4.2.1 (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/94)
- fixed bug related to package deployment (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/85) - fixed bug related to package deployment (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/85)
......
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
############################################################################# import json
# Configuration Class #
#############################################################################
import configparser
import logging.handlers import logging.handlers
import os import os
from irodsUtility import IRODSUtils
class Configuration(): class Configuration():
""" def __init__(self, logger):
Get properties from filesystem
"""
defaultLogDir = os.getcwd() + os.sep + "logs"
defaultLogFileName = os.sep + "b2share_connection.log"
irods_home_dir = ""
irods_debug = ""
def __init__(self, conffile, debug, dryrun, logger,
irodsenv=None):
self.conffile = conffile
self.debug = debug
self.dryrun = dryrun
self.logger = logger self.logger = logger
self.access_token = "" self.log_level = {'INFO': logging.INFO,
self.irodsenv = irodsenv 'DEBUG': logging.DEBUG,
self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG, 'ERROR': logging.ERROR,
'ERROR': logging.ERROR, 'WARNING': logging.WARNING, 'WARNING': logging.WARNING,
'CRITICAL': logging.CRITICAL} 'CRITICAL': logging.CRITICAL}
self.config = configparser.RawConfigParser() self.access_token = ""
self.config_path = ""
# return configuration self.record_id = ""
# missing exception self.dryrun = None
# if expected config is not in the config file self.user = None
def parseConf(self): self.collection_path = None
"""Parse the configuration file.""" self.title = None
self.community = None
# self.config = ConfigParser.RawConfigParser()
if not os.path.exists(self.conffile):
self.logger.error('missing configuration file %s:' %
(self.conffile))
return # error config file not found
with open(self.conffile, "r") as confFile:
try:
self.config.readfp(confFile)
except Exception as e:
self.logger.error(e)
return
logfilepath = self._getConfOption('Logging', 'log_file')
if not os.path.exists(logfilepath):
# if user did not specified a path for log file
# default to current dir + b2share_connection.log
# self.logger.error("no location for log file is specified,
# so try to default to
# <currend directory>/logs/b2share_connection.log")
try:
os.mkdir(self.defaultLogDir)
logfilepath = self.defaultLogDir + self.defaultLogFileName
except OSError:
# self.logger.error("no location for log file is specified,
# so try to default to
# <currend directory>/logs/b2share_connection.log")
logfilepath = os.getcwd() + self.defaultLogFileName
# print("Creation of the directory %s failed,
# so defaulting to <currend directory>/b2share_connection.log"
# % defaultLogDir)
loglevel = self._getConfOption('Logging', 'log_level')
if self.debug:
loglevel = 'DEBUG'
self.logger.setLevel(self.log_level[loglevel])
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=8388608,
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
self.logger.addHandler(rfh)
self.b2share_scheme = self._getConfOption('B2SHARE', 'scheme')
self.b2share_addr = self._getConfOption('B2SHARE', 'address')
self.b2share_path = self._getConfOption('B2SHARE', 'path')
self.b2share_host_name = self._getConfOption( def loadConfigurarionsFrom(self, configuration_file_path):
'B2SHARE_HTTP_API', 'host_name') with open(configuration_file_path, 'r') as json_file:
self.list_communities_endpoint = self._getConfOption( data = json.load(json_file)
'B2SHARE_HTTP_API', 'list_communities_endpoint') loglevel = data["configurations"]["logging"]["loglevel"]
self.records_endpoint = self._getConfOption( self.logger.setLevel(self.log_level[loglevel])
'B2SHARE_HTTP_API', 'records_endpoint') logfilepath = data["configurations"]["logging"]["logfile"]
self.access_parameter = self._getConfOption( # if logfile does not exist and not possible to create
'B2SHARE_HTTP_API', 'access_parameter') # stop execution with ERROR
self.get_community_schema_endpoint = self._getConfOption( if not os.path.exists(logfilepath):
'B2SHARE_HTTP_API', 'get_community_schema_endpoint') try:
logDir = os.path.dirname(logfilepath)
os.mkdir(logDir)
except OSError:
return "Log file directory: " + logfilepath +\
"specified in config file: " +\
configuration_file_path +\
" does NOT exist."
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=8388608,
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
self.logger.addHandler(rfh)
self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir') b2share_http_api = data["configurations"]["b2share_http_api"]
self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True) self.b2share_host_name = b2share_http_api["host_name"]
self.list_communities_endpoint =\
b2share_http_api["list_communities_endpoint"]
self.records_endpoint = b2share_http_api["records_endpoint"]
self.access_parameter = b2share_http_api["access_parameter"]
self.get_community_schema_endpoint =\
b2share_http_api["get_community_schema_endpoint"]
def _getConfOption(self, section, option, boolean=False): irods = data["configurations"]["irods"]
""" self.irodsenv = irods["irods_env"]
get the options from the configuration file if self.irodsenv:
""" self.irodsu = IRODSUtils(self.logger, irods_env=self.irodsenv)
if self.config.has_section(section): self.irods_resources = irods["resources"]
if (self.config.has_option(section, option)): self.irods_home_dir = irods["irods_home_dir"]
opt = self.config.get(section, option) self.irods_debug = irods["irods_debug"]
if boolean: return ""
if opt in ['True', 'true']: \ No newline at end of file
return True
else:
return False
return opt
else:
self.logger.error(
'missing parameter %s: in section %s' % (section, option))
else:
self.logger.error(
'missing section %s: with parameter %s' % (section, option))
return None
...@@ -5,79 +5,13 @@ import requests ...@@ -5,79 +5,13 @@ import requests
import os import os
import logging.handlers import logging.handlers
import argparse import argparse
import configparser
import tempfile import tempfile
import json
from manifest.irodsUtility import IRODSUtils from configuration import Configuration
logger = logging.getLogger('create_md_schema') from irodsUtility import IRODSUtils
class Configuration(): logger = logging.getLogger('create_md_schema')
"""
Get properties from filesystem
"""
def __init__(self, conffile, debug, dryrun, logger):
self.conffile = conffile
self.debug = debug
self.dryrun = dryrun
self.logger = logger
self.access_token = ""
self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR, 'WARNING': logging.WARNING,
'CRITICAL': logging.CRITICAL}
def parseConf(self):
"""Parse the configuration file."""
self.config = configparser.RawConfigParser()
with open(self.conffile, "r") as confFile:
self.config.readfp(confFile)
logfilepath = self._getConfOption('Logging', 'log_file')
loglevel = self._getConfOption('Logging', 'log_level')
if self.debug:
loglevel = 'DEBUG'
logger.setLevel(self.log_level[loglevel])
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=50000000,
backupCount=10)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
logger.addHandler(rfh)
self.b2share_host_name = self._getConfOption(
'B2SHARE_HTTP_API', 'host_name')
self.list_communities_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'list_communities_endpoint')
self.access_parameter = self._getConfOption(
'B2SHARE_HTTP_API', 'access_parameter')
self.get_community_schema_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'get_community_schema_endpoint')
self.irods_zone_name = self._getConfOption('iRODS', 'zone_name')
self.irods_res = self._getConfOption('iRODS', 'resources')
self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir')
self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True)
def _getConfOption(self, section, option, boolean=False):
"""
get the options from the configuration file
"""
if (self.config.has_option(section, option)):
opt = self.config.get(section, option)
if boolean:
if opt in ['True', 'true']:
return True
else:
return False
return opt
else:
self.logger.warning('missing parameter %s:%s' % (section, option))
return None
def getAllCommunities(configuration): def getAllCommunities(configuration):
...@@ -98,112 +32,215 @@ def getAllCommunities(configuration): ...@@ -98,112 +32,215 @@ def getAllCommunities(configuration):
name = community_object["name"] name = community_object["name"]
community_id = community_object["id"] community_id = community_object["id"]
communities[name] = community_id communities[name] = community_id
return communities return communities
def create_md_schema(args): def getVerbosePrintMethod(args):
commName = args.communityName if args.verbouse:
def verboseprint(*args):
# Print each argument separately so caller doesn't need to
# stuff everything to be printed into a single string
for arg in args:
print(arg)
return
else:
def verboseprint(*args):
# do-nothing
return
return verboseprint
def getAccessTokenWithConfigs(configuration):
# get access_token from users metadata in iRODS
if configuration.irodsu:
users_metadata = \
configuration.irodsu.getUserMetadata(
configuration.user, "access_token")
if users_metadata:
return users_metadata[0]
else:
return None
else:
return None
configuration = Configuration(
args.confpath, args.debug, args.dryrun, logger)
configuration.parseConf()
# get access_token from collection metadata DEFAULT_CONFIG_FILENAME = "b2share_client.json"
irodsu = IRODSUtils(configuration.irods_home_dir,
logger, configuration.irods_debug)
access_token = irodsu.getMetadata(args.userName, "access_token", '-u')[0]
configuration.access_token = access_token
def getConfigs(verboseprint):
config_path = None
if not args.confpath:
config_path = os.path.dirname(os.getcwd()) + os.sep +\
"conf" + os.sep + DEFAULT_CONFIG_FILENAME
else:
config_path = args.confpath
if not os.path.exists(config_path):
verboseprint('missing configuration file %s:' % (config_path))
logger.error('missing configuration file %s:' % (config_path))
return None
configuration = Configuration(logger)
configuration.config_path = config_path
configuration.loadConfigurarionsFrom(config_path)
configuration.user = args.user
configuration.community = args.community
configuration.collection_path = args.collection_path
logger.info("Start creating draft ...")
accessToken = getAccessTokenWithConfigs(configuration)
if accessToken is None:
verboseprint("Drafting FAILED. \
No B2SHARE access token found in users meta data.")
logger.error(
"Drafting FAILED. \
No B2SHARE access token found in users meta data.")
return None
configuration.access_token = accessToken
if 'metadata_file_name' in args:
configuration.metadata_file_name = args.metadata_file_name
return configuration
def getCommunitySchema(configuration, verboseprint):
# get metadata schema
communities_list = getAllCommunities(configuration) communities_list = getAllCommunities(configuration)
if configuration.community not in communities_list.keys():
logger.error("No communityName specified. " +
"Please select and specify " +
"one of the communities names: " +
str(communities_list.keys()))
verboseprint("No communityName specified. " +
"Please select and specify " +
"one of the communities names: " +
str(communities_list.keys()))
return None
community_id = communities_list[configuration.community]
host = configuration.b2share_host_name
community_endpoint = configuration.list_communities_endpoint
get_schema_endpoint = configuration.get_community_schema_endpoint
acces_part = \
configuration.access_parameter + \
"=" + configuration.access_token
get_community_schema_url = host + community_endpoint + \
community_id + get_schema_endpoint + acces_part
community_schema = None
try:
response = requests.get(url=get_community_schema_url)
verboseprint("response status code: " + str(response.status_code))
logger.debug("response status code: " + str(response.status_code))
if ((str(response.status_code) == "200") |
(str(response.status_code) == "201")):
community_schema = response.json()["json_schema"]["allOf"][0]
logger.info("Get schema for community: " + configuration.community)
else:
verboseprint("NO schema for community: " +
configuration.community +
"Response: " + str(response.json()))
logger.error("NO schema for community: " +
configuration.community +
"Response: " + str(response.json()))
except requests.exceptions.RequestException as e:
logger.error(e)
return community_schema
if (commName is not None) and (commName != ''):
logger.info(
'Start creating metadata schema for the community'+str(commName))
# get metadata schema
# api/communities/
community_id = communities_list[commName]
host = configuration.b2share_host_name
community_endpoint = configuration.list_communities_endpoint
get_schema_endpoint = configuration.get_community_schema_endpoint
acces_part = \
configuration.access_parameter + \
"=" + configuration.access_token
get_community_schema_url = host + community_endpoint + \
community_id + get_schema_endpoint + acces_part
response = requests.get(url=get_community_schema_url) def create_md_schema(args):
# May be parsing for manifest extention verboseprint = getVerbosePrintMethod(args)
community_schema = response.json()["json_schema"]["allOf"][0] configuration = getConfigs(verboseprint)
requiredProperties = \ if not configuration:
community_schema["b2share"]["presentation"]["major"] return None
optionalProperties = \
community_schema["b2share"]["presentation"]["minor"] logger.info('Start creating metadata schema for the community' +
str(configuration.community))
mdPatchSceleton = \ community_schema = getCommunitySchema(configuration, verboseprint)
'#please fill out at least the required fields with' + \ if not community_schema:
'values as JSON strings in the line after the property name.' + \ return
'\n' requiredProperties_list = \
mdPatchSceleton = \ community_schema["b2share"]["presentation"]["major"]
mdPatchSceleton + \ optionalProperties_list = \
'#e.g. for community - EUDAT, ' + \ community_schema["b2share"]["presentation"]["minor"]
'for open_access - true, for contributors - ' + \
'[{"contributor_name":"Hulk",' + \ requiredProperties = {}
' "contributor_type": "Editor"},' + \ for property_name in requiredProperties_list:
' {"contributor_name":"Banner",' + \ property_type = \
' "contributor_type": "ContactPerson"}]' + \ community_schema["properties"][property_name]["type"]
"\n\n" requiredProperties[property_name] = property_type
mdPatchSceleton = mdPatchSceleton + "[required]" + "\n" optionalProperties = {}
for requiredProperty in requiredProperties: for property_name in optionalProperties_list:
mdPatchSceleton = mdPatchSceleton + requiredProperty + "\n\n" property_type = \
community_schema["properties"][property_name]["type"]
mdPatchSceleton = mdPatchSceleton + "\n" + "[optional]" + "\n" optionalProperties[property_name] = property_type
for optionalPropertiy in optionalProperties:
mdPatchSceleton = mdPatchSceleton + optionalPropertiy + "\n\n" mdPatchSceleton = ""
mdPatchSceleton = mdPatchSceleton + '{"metadata": {"required":['
if args.dryrun: for requiredProperty in requiredProperties:
print(mdPatchSceleton) mdPatchSceleton = mdPatchSceleton + \
'{"option_name": "' + requiredProperty + '",' \
'"value":"", ' + \
'"type":"' + requiredProperties[requiredProperty] +\
'"},'
mdPatchSceleton = mdPatchSceleton[:-1]
mdPatchSceleton = mdPatchSceleton + "],"
mdPatchSceleton = mdPatchSceleton + '"optional":['
for optionalPropertiy in optionalProperties.keys():
mdPatchSceleton = mdPatchSceleton + \
'{"option_name": "' + optionalPropertiy + '",' \
'"value":"", ' + \
'"type":"' + optionalProperties[optionalPropertiy] +\
'"},'
mdPatchSceleton = mdPatchSceleton[:-1]
mdPatchSceleton = mdPatchSceleton + "]"
mdPatchSceleton = mdPatchSceleton + '} }'
json_formated = json.dumps(json.loads(mdPatchSceleton), indent=4)
mdPatchSceleton = json_formated
if args.dryrun:
print(mdPatchSceleton)
else:
logger.info('Writing the metadata to a file')
# file_path = configuration.collection_path + os.sep + \
# configuration.metadata_file_name
metadata_file_path = ""
if configuration.metadata_file_name:
metadata_file_path = configuration.collection_path + os.sep + \
configuration.metadata_file_name
else: else:
logger.info('Writing the metadata to a file') metadata_file_path = configuration.collection_path + os.sep + \
file_path = args.collectionName + os.sep + \ "b2share_metadata.json"
"b2share_metadata.json" # TODO: config, argument? temp = tempfile.NamedTemporaryFile()
temp = tempfile.NamedTemporaryFile() try:
temp.write(bytes(mdPatchSceleton, 'UTF-8'))
temp.flush()
try: try:
temp.write(mdPatchSceleton) configuration.irodsu.putFile(temp.name, metadata_file_path)
temp.flush() except Exception:
try: # try again
irodsu.putFile(temp.name, file_path, out = configuration.irodsu.putFile(temp.name,
configuration.irods_resource) metadata_file_path)
except: print(str(out))
out = irodsu.putFile(temp.name, file_path) finally:
print(str(out)) temp.close()
finally:
temp.close()
logger.info(
'Finish creating metadata schema for the community'+commName)
else:
logger.error("No communityName specified. " +
"Please select one of the communities names: " +
str(communities_list.keys()))
print(str(communities_list.keys()))
logger.info('Finish creating metadata schema for the community' +
configuration.community)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='B2SAFE B2SHARE client')
parser.add_argument("--confpath", help="path to the configuration file")
parser.add_argument("-c", "--communityName", help="B2Share user name")
parser.add_argument("-dbg", "--debug", if __name__ == "__main__":
action="store_true", help="enable debug") parser = argparse.ArgumentParser(description='Metadata creator')
parser.add_argument("-u", "--user", required=True,
help="irods user to get B2SHARE access token")
parser.add_argument("-p", "--collection_path", required=True,
help="irods path to the collection")
parser.add_argument('-comm', '--community', required=True,
help='community name toget the schema of')
parser.add_argument("--confpath",
help="path to the configuration file")
parser.add_argument("-d", "--dryrun", action="store_true", parser.add_argument("-d", "--dryrun", action="store_true",
help="run without performing any real change")