Commit 6ef85871 authored by Julia Kaufhold's avatar Julia Kaufhold
Browse files

merge: from master into b2share branch, so it is deployable again.

change: python3 migration for B2SHARE connector scripts + unit tests of it.
parent 6d6eb156
Copyright (c) 2014, EUDAT project funded from the European Union under grant agreement n.283304.
Copyright (c) 2015, EUDAT2020 project funded from the European Union under grant agreement n.654065.
All rights reserved.
Copyright (c) 2018, EUDAT CDI - www.eudat.eu.
<Willem Elbers - Max Planck Institute for Psycholinguistics, contact willem.elbers@mpi.nl>
<Claudio Cacciari - Cineca, contact c.cacciari@cineca.it>
......
B2SAFE package 4.2.0 [2019/01/07]
- fixed bug related to redundant replica handles (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/120)
- updated test rules
- improved wiki docuentation
- added support for oauth2 protocol via PAM module
- added support to HTTP interfaces in the PID parameters
B2SAFE package 4.1.1 [2018/02/22]
- fixed bug related to EUDATTransferUsingFailLog rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/113)
- updated test rules
B2SAFE package 4.1.0 [2017/12/18]
- tested sucessfully against iRODS v4.2.1. The test against iRODS 4.2.2 failed due to this bug: https://github.com/irods/irods/issues/3722, therefore the v4.2.2 is not supported yet.
- fixed bug related to EUDATReplication rule (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/102)
- improved install.txt documentation (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/103)
- added EUDATeURLupdateColl(*PID, *newURL) to update the url in multiple PIDS associeted to a whole collection of objects.
- many improvements in the logging
- added the option to specify the destination iRODS resource in many rules, included the EUDATReplication.
B2SAFE package 4.0.1 [2017/07/11]
- added support for iRODS 4.2.1 (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/94)
- fixed bug related to package deployment (https://github.com/EUDAT-B2SAFE/B2SAFE-core/issues/85)
......
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#############################################################################
# Configuration Class #
#############################################################################
import configparser
import json
import logging.handlers
import os
from irodsUtility import IRODSUtils
class Configuration():
"""
Get properties from filesystem
"""
defaultLogDir = os.getcwd() + os.sep + "logs"
defaultLogFileName = os.sep + "b2share_connection.log"
irods_home_dir = ""
irods_debug = ""
def __init__(self, conffile, debug, dryrun, logger,
irodsenv=None):
self.conffile = conffile
self.debug = debug
self.dryrun = dryrun
def __init__(self, logger):
self.logger = logger
self.access_token = ""
self.irodsenv = irodsenv
self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR, 'WARNING': logging.WARNING,
self.log_level = {'INFO': logging.INFO,
'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR,
'WARNING': logging.WARNING,
'CRITICAL': logging.CRITICAL}
self.config = configparser.RawConfigParser()
# return configuration
# missing exception
# if expected config is not in the config file
def parseConf(self):
"""Parse the configuration file."""
# self.config = ConfigParser.RawConfigParser()
if not os.path.exists(self.conffile):
self.logger.error('missing configuration file %s:' %
(self.conffile))
return # error config file not found
with open(self.conffile, "r") as confFile:
try:
self.config.readfp(confFile)
except Exception as e:
self.logger.error(e)
return
logfilepath = self._getConfOption('Logging', 'log_file')
if not os.path.exists(logfilepath):
# if user did not specified a path for log file
# default to current dir + b2share_connection.log
# self.logger.error("no location for log file is specified,
# so try to default to
# <currend directory>/logs/b2share_connection.log")
try:
os.mkdir(self.defaultLogDir)
logfilepath = self.defaultLogDir + self.defaultLogFileName
except OSError:
# self.logger.error("no location for log file is specified,
# so try to default to
# <currend directory>/logs/b2share_connection.log")
logfilepath = os.getcwd() + self.defaultLogFileName
# print("Creation of the directory %s failed,
# so defaulting to <currend directory>/b2share_connection.log"
# % defaultLogDir)
loglevel = self._getConfOption('Logging', 'log_level')
if self.debug:
loglevel = 'DEBUG'
self.logger.setLevel(self.log_level[loglevel])
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=8388608,
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
self.logger.addHandler(rfh)
self.b2share_scheme = self._getConfOption('B2SHARE', 'scheme')
self.b2share_addr = self._getConfOption('B2SHARE', 'address')
self.b2share_path = self._getConfOption('B2SHARE', 'path')
self.access_token = ""
self.config_path = ""
self.record_id = ""
self.dryrun = None
self.user = None
self.collection_path = None
self.title = None
self.community = None
self.b2share_host_name = self._getConfOption(
'B2SHARE_HTTP_API', 'host_name')
self.list_communities_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'list_communities_endpoint')
self.records_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'records_endpoint')
self.access_parameter = self._getConfOption(
'B2SHARE_HTTP_API', 'access_parameter')
self.get_community_schema_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'get_community_schema_endpoint')
def loadConfigurarionsFrom(self, configuration_file_path):
with open(configuration_file_path, 'r') as json_file:
data = json.load(json_file)
loglevel = data["configurations"]["logging"]["loglevel"]
self.logger.setLevel(self.log_level[loglevel])
logfilepath = data["configurations"]["logging"]["logfile"]
# if logfile does not exist and not possible to create
# stop execution with ERROR
if not os.path.exists(logfilepath):
try:
logDir = os.path.dirname(logfilepath)
os.mkdir(logDir)
except OSError:
return "Log file directory: " + logfilepath +\
"specified in config file: " +\
configuration_file_path +\
" does NOT exist."
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=8388608,
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
self.logger.addHandler(rfh)
self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir')
self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True)
b2share_http_api = data["configurations"]["b2share_http_api"]
self.b2share_host_name = b2share_http_api["host_name"]
self.list_communities_endpoint =\
b2share_http_api["list_communities_endpoint"]
self.records_endpoint = b2share_http_api["records_endpoint"]
self.access_parameter = b2share_http_api["access_parameter"]
self.get_community_schema_endpoint =\
b2share_http_api["get_community_schema_endpoint"]
def _getConfOption(self, section, option, boolean=False):
"""
get the options from the configuration file
"""
if self.config.has_section(section):
if (self.config.has_option(section, option)):
opt = self.config.get(section, option)
if boolean:
if opt in ['True', 'true']:
return True
else:
return False
return opt
else:
self.logger.error(
'missing parameter %s: in section %s' % (section, option))
else:
self.logger.error(
'missing section %s: with parameter %s' % (section, option))
return None
irods = data["configurations"]["irods"]
self.irodsenv = irods["irods_env"]
if self.irodsenv:
self.irodsu = IRODSUtils(self.logger, irods_env=self.irodsenv)
self.irods_resources = irods["resources"]
self.irods_home_dir = irods["irods_home_dir"]
self.irods_debug = irods["irods_debug"]
return ""
\ No newline at end of file
......@@ -5,79 +5,13 @@ import requests
import os
import logging.handlers
import argparse
import configparser
import tempfile
import json
from manifest.irodsUtility import IRODSUtils
logger = logging.getLogger('create_md_schema')
from configuration import Configuration
from irodsUtility import IRODSUtils
class Configuration():
"""
Get properties from filesystem
"""
def __init__(self, conffile, debug, dryrun, logger):
self.conffile = conffile
self.debug = debug
self.dryrun = dryrun
self.logger = logger
self.access_token = ""
self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR, 'WARNING': logging.WARNING,
'CRITICAL': logging.CRITICAL}
def parseConf(self):
"""Parse the configuration file."""
self.config = configparser.RawConfigParser()
with open(self.conffile, "r") as confFile:
self.config.readfp(confFile)
logfilepath = self._getConfOption('Logging', 'log_file')
loglevel = self._getConfOption('Logging', 'log_level')
if self.debug:
loglevel = 'DEBUG'
logger.setLevel(self.log_level[loglevel])
rfh = logging.handlers.RotatingFileHandler(logfilepath,
maxBytes=50000000,
backupCount=10)
formatter = logging.Formatter('%(asctime)s %(levelname)s: ' +
'[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
logger.addHandler(rfh)
self.b2share_host_name = self._getConfOption(
'B2SHARE_HTTP_API', 'host_name')
self.list_communities_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'list_communities_endpoint')
self.access_parameter = self._getConfOption(
'B2SHARE_HTTP_API', 'access_parameter')
self.get_community_schema_endpoint = self._getConfOption(
'B2SHARE_HTTP_API', 'get_community_schema_endpoint')
self.irods_zone_name = self._getConfOption('iRODS', 'zone_name')
self.irods_res = self._getConfOption('iRODS', 'resources')
self.irods_home_dir = self._getConfOption('iRODS', 'irods_home_dir')
self.irods_debug = self._getConfOption('iRODS', 'irods_debug', True)
def _getConfOption(self, section, option, boolean=False):
"""
get the options from the configuration file
"""
if (self.config.has_option(section, option)):
opt = self.config.get(section, option)
if boolean:
if opt in ['True', 'true']:
return True
else:
return False
return opt
else:
self.logger.warning('missing parameter %s:%s' % (section, option))
return None
logger = logging.getLogger('create_md_schema')
def getAllCommunities(configuration):
......@@ -98,112 +32,215 @@ def getAllCommunities(configuration):
name = community_object["name"]
community_id = community_object["id"]
communities[name] = community_id
return communities
def create_md_schema(args):
commName = args.communityName
def getVerbosePrintMethod(args):
if args.verbouse:
def verboseprint(*args):
# Print each argument separately so caller doesn't need to
# stuff everything to be printed into a single string
for arg in args:
print(arg)
return
else:
def verboseprint(*args):
# do-nothing
return
return verboseprint
def getAccessTokenWithConfigs(configuration):
# get access_token from users metadata in iRODS
if configuration.irodsu:
users_metadata = \
configuration.irodsu.getUserMetadata(
configuration.user, "access_token")
if users_metadata:
return users_metadata[0]
else:
return None
else:
return None
configuration = Configuration(
args.confpath, args.debug, args.dryrun, logger)
configuration.parseConf()
# get access_token from collection metadata
irodsu = IRODSUtils(configuration.irods_home_dir,
logger, configuration.irods_debug)
access_token = irodsu.getMetadata(args.userName, "access_token", '-u')[0]
configuration.access_token = access_token
DEFAULT_CONFIG_FILENAME = "b2share_client.json"
def getConfigs(verboseprint):
config_path = None
if not args.confpath:
config_path = os.path.dirname(os.getcwd()) + os.sep +\
"conf" + os.sep + DEFAULT_CONFIG_FILENAME
else:
config_path = args.confpath
if not os.path.exists(config_path):
verboseprint('missing configuration file %s:' % (config_path))
logger.error('missing configuration file %s:' % (config_path))
return None
configuration = Configuration(logger)
configuration.config_path = config_path
configuration.loadConfigurarionsFrom(config_path)
configuration.user = args.user
configuration.community = args.community
configuration.collection_path = args.collection_path
logger.info("Start creating draft ...")
accessToken = getAccessTokenWithConfigs(configuration)
if accessToken is None:
verboseprint("Drafting FAILED. \
No B2SHARE access token found in users meta data.")
logger.error(
"Drafting FAILED. \
No B2SHARE access token found in users meta data.")
return None
configuration.access_token = accessToken
if 'metadata_file_name' in args:
configuration.metadata_file_name = args.metadata_file_name
return configuration
def getCommunitySchema(configuration, verboseprint):
# get metadata schema
communities_list = getAllCommunities(configuration)
if configuration.community not in communities_list.keys():
logger.error("No communityName specified. " +
"Please select and specify " +
"one of the communities names: " +
str(communities_list.keys()))
verboseprint("No communityName specified. " +
"Please select and specify " +
"one of the communities names: " +
str(communities_list.keys()))
return None
community_id = communities_list[configuration.community]
host = configuration.b2share_host_name
community_endpoint = configuration.list_communities_endpoint
get_schema_endpoint = configuration.get_community_schema_endpoint
acces_part = \
configuration.access_parameter + \
"=" + configuration.access_token
get_community_schema_url = host + community_endpoint + \
community_id + get_schema_endpoint + acces_part
community_schema = None
try:
response = requests.get(url=get_community_schema_url)
verboseprint("response status code: " + str(response.status_code))
logger.debug("response status code: " + str(response.status_code))
if ((str(response.status_code) == "200") |
(str(response.status_code) == "201")):
community_schema = response.json()["json_schema"]["allOf"][0]
logger.info("Get schema for community: " + configuration.community)
else:
verboseprint("NO schema for community: " +
configuration.community +
"Response: " + str(response.json()))
logger.error("NO schema for community: " +
configuration.community +
"Response: " + str(response.json()))
except requests.exceptions.RequestException as e:
logger.error(e)
return community_schema
if (commName is not None) and (commName != ''):
logger.info(
'Start creating metadata schema for the community'+str(commName))
# get metadata schema
# api/communities/
community_id = communities_list[commName]
host = configuration.b2share_host_name
community_endpoint = configuration.list_communities_endpoint
get_schema_endpoint = configuration.get_community_schema_endpoint
acces_part = \
configuration.access_parameter + \
"=" + configuration.access_token
get_community_schema_url = host + community_endpoint + \
community_id + get_schema_endpoint + acces_part
response = requests.get(url=get_community_schema_url)
# May be parsing for manifest extention
community_schema = response.json()["json_schema"]["allOf"][0]
requiredProperties = \
community_schema["b2share"]["presentation"]["major"]
optionalProperties = \
community_schema["b2share"]["presentation"]["minor"]
mdPatchSceleton = \
'#please fill out at least the required fields with' + \
'values as JSON strings in the line after the property name.' + \
'\n'
mdPatchSceleton = \
mdPatchSceleton + \
'#e.g. for community - EUDAT, ' + \
'for open_access - true, for contributors - ' + \
'[{"contributor_name":"Hulk",' + \
' "contributor_type": "Editor"},' + \
' {"contributor_name":"Banner",' + \
' "contributor_type": "ContactPerson"}]' + \
"\n\n"
mdPatchSceleton = mdPatchSceleton + "[required]" + "\n"
for requiredProperty in requiredProperties:
mdPatchSceleton = mdPatchSceleton + requiredProperty + "\n\n"
mdPatchSceleton = mdPatchSceleton + "\n" + "[optional]" + "\n"
for optionalPropertiy in optionalProperties:
mdPatchSceleton = mdPatchSceleton + optionalPropertiy + "\n\n"
if args.dryrun:
print(mdPatchSceleton)
def create_md_schema(args):
verboseprint = getVerbosePrintMethod(args)
configuration = getConfigs(verboseprint)
if not configuration:
return None
logger.info('Start creating metadata schema for the community' +
str(configuration.community))
community_schema = getCommunitySchema(configuration, verboseprint)
if not community_schema:
return
requiredProperties_list = \
community_schema["b2share"]["presentation"]["major"]
optionalProperties_list = \
community_schema["b2share"]["presentation"]["minor"]
requiredProperties = {}
for property_name in requiredProperties_list:
property_type = \
community_schema["properties"][property_name]["type"]
requiredProperties[property_name] = property_type
optionalProperties = {}
for property_name in optionalProperties_list:
property_type = \
community_schema["properties"][property_name]["type"]
optionalProperties[property_name] = property_type
mdPatchSceleton = ""
mdPatchSceleton = mdPatchSceleton + '{"metadata": {"required":['
for requiredProperty in requiredProperties:
mdPatchSceleton = mdPatchSceleton + \
'{"option_name": "' + requiredProperty + '",' \
'"value":"", ' + \
'"type":"' + requiredProperties[requiredProperty] +\
'"},'
mdPatchSceleton = mdPatchSceleton[:-1]
mdPatchSceleton = mdPatchSceleton + "],"
mdPatchSceleton = mdPatchSceleton + '"optional":['
for optionalPropertiy in optionalProperties.keys():
mdPatchSceleton = mdPatchSceleton + \
'{"option_name": "' + optionalPropertiy + '",' \
'"value":"", ' + \
'"type":"' + optionalProperties[optionalPropertiy] +\
'"},'
mdPatchSceleton = mdPatchSceleton[:-1]
mdPatchSceleton = mdPatchSceleton + "]"
mdPatchSceleton = mdPatchSceleton + '} }'
json_formated = json.dumps(json.loads(mdPatchSceleton), indent=4)
mdPatchSceleton = json_formated
if args.dryrun:
print(mdPatchSceleton)
else:
logger.info('Writing the metadata to a file')
# file_path = configuration.collection_path + os.sep + \
# configuration.metadata_file_name
metadata_file_path = ""
if configuration.metadata_file_name:
metadata_file_path = configuration.collection_path + os.sep + \
configuration.metadata_file_name
else:
logger.info('Writing the metadata to a file')
file_path = args.collectionName + os.sep + \
"b2share_metadata.json" # TODO: config, argument?
temp = tempfile.NamedTemporaryFile()
metadata_file_path = configuration.collection_path + os.sep + \
"b2share_metadata.json"
temp = tempfile.NamedTemporaryFile()
try:
temp.write(bytes(mdPatchSceleton, 'UTF-8'))
temp.flush()
try:
temp.write(mdPatchSceleton)
temp.flush()
try:
irodsu.putFile(temp.name, file_path,
configuration.irods_resource)
except:
out = irodsu.putFile(temp.name, file_path)
print(str(out))
finally:
temp.close()
logger.info(
'Finish creating metadata schema for the community'+commName)
else:
logger.error("No communityName specified. " +
"Please select one of the communities names: " +
str(communities_list.keys()))
print(str(communities_list.keys()))
configuration.irodsu.putFile(temp.name, metadata_file_path)
except Exception:
# try again
out = configuration.irodsu.putFile(temp.name,
metadata_file_path)
print(str(out))
finally:
temp.close()
logger.info('Finish creating metadata schema for the community' +
configuration.community)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='B2SAFE B2SHARE client')
parser.add_argument("--confpath", help="path to the configuration file")
parser.add_argument("-c", "--communityName", help="B2Share user name")
parser.add_argument("-dbg", "--debug",
action="store_true", help="enable debug")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Metadata creator')
parser.add_argument("-u", "--user", required=True,
help="irods user to get B2SHARE access token")
parser.add_argument("-p", "--collection_path", required=True,
help="irods path to the collection")
parser.add_argument('-comm', '--community', required=True,
help='community name toget the schema of')
parser.add_argument("--confpath",
help="path to the configuration file")
parser.add_argument("-d", "--dryrun", action="store_true",
help="run without performing any real change")
parser.add_argument("-u", "--userName", help="B2Share user name")
parser.add_argument(
"--collectionName",
help="path to the collection where to create the metadata")
parser.add_argument("-v", "--verbouse", action="store_true",
help="enable printouts for debug")
parser.add_argument('-md', '--metadata_file_name',
help='file name of the collection describing \
metadata')
parser.set_defaults(func=create_md_schema)
args = parser.parse_args()
args.func(args)