Skip to content

GitLab

Commit 15fc075d authored by Claudio Cacciari's avatar Claudio Cacciari
Browse files

fixed conflict

parents 28783826 e0e10d7a
Expand all Hide whitespace changes
Inline Side-by-side
cmd/argo.py
View file @ 15fc075d
......@@ -8,11 +8,10 @@ import json
import base64
import argparse
import ConfigParser
import hashlib
logger = logging.getLogger('messageManager')
session = requests.Session()
#payload = {'key': '2b655947ec32f2352c8bcb4d4619bb77b3ad1cbc'}
#endpoint = 'https://messaging-devel.argo.grnet.gr/v1/projects/EUDAT2020'
class Configuration():
"""
......@@ -88,7 +87,10 @@ def createTopic(name):
data=data, headers=headers, params=payload)
logger.debug('Status code: {}'.format(str(res.status_code)))
logger.debug('Response: {}'.format(res.text))
return res.text
if res.status_code == 409:
return name.strip()
resJson = json.loads(res.text)
return resJson['name'].rsplit('/',1)[1]
def deleteTopic(name):
......@@ -192,10 +194,13 @@ def manageTopic(args):
if args.action == 'list':
print listTopics()
elif args.name is not None:
name = args.name
if args.md5hash:
name = hashlib.md5(args.name).hexdigest()
if args.action == 'create':
print createTopic(args.name)
print createTopic(name)
elif args.action == 'delete':
print deleteTopic(args.name)
print deleteTopic(name)
else:
print 'topic name is missing'
......@@ -215,8 +220,11 @@ def manageSubscription(args):
if not ('topic' in args_dict.keys()):
print 'topic name is missing'
sys.exit(1)
topicName = args.topic
if args.md5hash:
topicName = hashlib.md5(args.name).hexdigest()
if args.action == 'create':
print createSub(args.name, args.topic)
print createSub(args.name, topicName)
def _initializeLogger(args):
......@@ -233,6 +241,12 @@ def _initializeLogger(args):
logger.addHandler(han)
def _setToControlQueue(queue):
"""add the new queue to the control queue."""
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='EUDAT B2SAFE message manager.')
......@@ -261,6 +275,8 @@ if __name__ == "__main__":
help='topic action')
parser_topic.add_argument("name", nargs='?', default=argparse.SUPPRESS,
help='topic name for create and delete actions')
parser_topic.add_argument("-m", "--md5hash", help="create the topic using"
+ " the md5 hash of the name", action="store_true")
parser_topic.set_defaults(func=manageTopic)
parser_sub = subparsers.add_parser('sub', help='subscription management')
......@@ -269,7 +285,9 @@ if __name__ == "__main__":
parser_sub.add_argument("name", nargs='?', default=argparse.SUPPRESS,
help='subscription name for create and delete actions')
parser_sub.add_argument("topic", nargs='?', default=argparse.SUPPRESS,
help='topic name for create action')
help='topic name for create action')
parser_sub.add_argument("-m", "--md5hash", help="the topic real name is"
+ " the md5 hash of the name", action="store_true")
parser_sub.set_defaults(func=manageSubscription)
_args = parser.parse_args()
......
cmd/metadataManager.py deleted 100755 → 0
View file @ 28783826
#!/usr/bin/env python
# -*- python -*-
import logging
import logging.handlers
import json
import ConfigParser
import argparse
import subprocess
import os
import tempfile
import pprint
import sys
logger = logging.getLogger('Metadata')
##############################################################################
# Metadata management Class #
##############################################################################
class Metadata():
"""Class implementing the metadata management."""
def __init__( self, conf, user ):
"""Initialize object with configuration parameters."""
self.conf = conf
self.user = user
self.irodsu = IRODSUtils()
def store(self, path, keyValuePairs, dryrun):
if not path.endswith('.metadata'):
logger.info("Going to store metadata about " + path)
query = "SELECT COLL_NAME WHERE COLL_NAME = '" + path + "'"
(rc, out1) = self.irodsu.execute_icommand(["iquest", query],self.user)
if out1 is not None:
collPath = path
objPath = None
mcoll = MetadataItem(collPath, objPath, self.user, dryrun)
mcoll.store(keyValuePairs)
else:
(parent, child) = path.rsplit('/',1)
query = "SELECT DATA_NAME WHERE COLL_NAME = '" + parent + "' and "\
+ "DATA_NAME = '" + child + "'"
(rc, out2) = self.irodsu.execute_icommand(["iquest", query],self.user)
if out2 is not None:
mobj = MetadataItem(parent, child, self.user, dryrun)
mobj.store(keyValuePairs)
else:
logger.error('Wrong path: ' + path)
else:
logger.debug('skipping directory metadata: %s', path)
class MetadataItem():
"""Class implementing the single metadata record"""
def __init__( self, collPath, objPath, user, dryrun):
"""Initialize object with iRODS path parameters."""
self.dryrun = dryrun
self.irodsu = IRODSUtils()
self.objPath = objPath
self.user = user
# verify that .metadata exists, if not create it
self.metaPath = collPath + '/.metadata'
(rc, out1) = self.irodsu.execute_icommand(["ils", self.metaPath],
self.user)
if out1 is None:
(rc, out2) = self.irodsu.execute_icommand(["imkdir", self.metaPath],
self.user)
if out2 is None:
logger.error("impossible to create the path " + self.metaPath)
sys.exit()
def store(self, keyValuePairs):
"""Store the metadata key-value pairs."""
logger.info("Going to store the following metadata")
logger.info(pprint.pformat(keyValuePairs, indent=4))
if self.objPath is None:
# check if json of root_collection exists
logger.debug("check if json of root_collection exists")
metaObj = self.metaPath + '/root_collection.json'
else:
metaObj = self.metaPath + '/' + self.objPath + '_metadata.json'
(rc, out) = self.irodsu.execute_icommand(["ils", metaObj], self.user)
if out is not None:
# get the json of root_collection
logger.debug("get the json of root_collection")
(rc, out2) = self.irodsu.execute_icommand(["iget", metaObj, "-"],
self.user)
if out2:
rootCollectionMetaData = json.loads(out2)
else:
rootCollectionMetaData = {}
logger.debug("loaded the following metadata")
logger.debug(pprint.pformat(rootCollectionMetaData, indent=4))
for key, value in keyValuePairs.iteritems():
logger.debug("modifying the pair (%s,%s)" % (key,value))
rootCollectionMetaData[key] = value
if not self.dryrun:
self._writeMetadata(rootCollectionMetaData, metaObj)
else:
logger.info("writing the following metadata to " + metaObj)
logger.info(pprint.pformat(rootCollectionMetaData,indent=4))
else:
if not self.dryrun:
self._writeMetadata(keyValuePairs, metaObj)
else:
logger.info("writing the following metadata to " + metaObj)
logger.info(pprint.pformat(keyValuePairs,indent=4))
def _writeMetadata(self, keyValuePairs, irodsObj):
"""Write the metadata key-value pair to a file."""
with tempfile.NamedTemporaryFile() as tempJsonFile:
logger.debug("writing the metadata to file " + tempJsonFile.name)
tempJsonFile.write(json.dumps(keyValuePairs, indent=2))
tempJsonFile.seek(0)
logger.debug("Temp file content: " + tempJsonFile.read())
logger.debug("uploading the file %s to irods location %s"
% (tempJsonFile.name, irodsObj))
(rc, out) = self.irodsu.execute_icommand(["iput", "-f",
tempJsonFile.name,
irodsObj],self.user)
if out is None:
logger.error("impossible to upload the object: " + irodsObj)
##############################################################################
# iRODS Admin Utility Class #
##############################################################################
class IRODSUtils():
"""
utility for irods management
"""
def __init__(self):
"""initialize the object"""
logger.debug("iRODS utility class initialization")
def execute_icommand(self, command, user):
"""Execute a shell command and manage error conditions"""
envKey = 'clientUserName'
(rc, output) = self._shell_command(command, envKey, user)
if rc != 0:
logger.error('Error running %s, rc = %d' % (' '.join(command),
rc))
logger.error("output: %s", output[1])
if output[0] is not None and len(output[0].strip()) > 0:
logger.error("error message: %s", output[0])
return (rc, None)
logger.debug('executed %s, rc = %d' % (' '.join(command), rc))
logger.debug('output: %s', output[1])
if (output[1].startswith('CAT_NO_ROWS_FOUND')): return (rc, None)
return (rc, output[1])
def _shell_command(self, command_list, envKey, envValue):
"""
Performs a shell command using the subprocess object
input list of strings that represent the argv of the process to create
return tuple (return code, output object from subprocess.communicate)
"""
if not command_list:
return None
try:
d = dict(os.environ)
d[envKey] = envValue
process = subprocess.Popen(command_list, env=d,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = process.communicate()
return (process.returncode, [err, out])
except Exception, e:
logger.debug('Failure with error: ' + str(e))
return (-1, [None, None])
###############################################################################
# Configuration Class #
###############################################################################
class Configuration():
"""
Get properties from filesystem
"""
def __init__(self, file, logger):
self.file = file
self.log_level = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG, \
'ERROR': logging.ERROR, 'WARNING': logging.WARNING, \
'CRITICAL': logging.CRITICAL}
def parseConf(self):
"""Parse the configuration file."""
self.config = ConfigParser.RawConfigParser()
self.config.readfp(open(self.file))
logfilepath = self._getConfOption('Logging', 'log_file')
loglevel = self._getConfOption('Logging', 'log_level')
logger.setLevel(self.log_level[loglevel])
rfh = logging.handlers.RotatingFileHandler(logfilepath, \
maxBytes=8388608, \
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s: '
+ '[%(funcName)s] %(message)s')
rfh.setFormatter(formatter)
logger.addHandler(rfh)
def _getConfOption(self, section, option, boolean=False):
"""
get the options from the configuration file
"""
if (self.config.has_option(section, option)):
opt = self.config.get(section, option)
if boolean:
if opt in ['True', 'true']: return True
else: return False
return opt
else:
logger.warning('missing parameter %s:%s' % (section,option))
return None
###############################################################################
# Metadata script Command Line Interface #
###############################################################################
def store(args):
configuration = Configuration(args.confpath, logger)
configuration.parseConf();
meta = Metadata(configuration, args.user)
kv = {}
if args.pid: kv['pid'] = args.pid
if args.checksum: kv['checksum'] = args.checksum
if args.ror: kv['ror'] = args.ror
if args.checksum_timestamp:
kv['checksum_timestamp'] = args.checksum_timestamp
logger.info("Metadata storing starting ...")
meta.store(args.path, kv, args.dryrun)
logger.info("Metadata storing completed")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='EUDAT metadata management '
+ 'tool')
parser.add_argument("confpath",default="NULL",help='path to the '
+ 'configuration file')
parser.add_argument("-d", "--dryrun", action="store_true",\
help="execute a command without performing any real"
+ "change")
parser.add_argument("user", help="the owner of the metadata")
subparsers = parser.add_subparsers(title='Actions', \
description='metadata '
+ 'management operations', \
help='allowed operations')
parser_store = subparsers.add_parser('store', help='store metadata')
parser_store.add_argument('path', help='the iRODS object/collection path')
parser_store.add_argument('-i', '--pid', help='persistent identifier')
parser_store.add_argument('-c', '--checksum', help='checksum')
parser_store.add_argument('-r', '--ror', help='Repository of Records')
parser_store.add_argument('-t', '--checksum_timestamp', \
help='timestamp of the last checksum calculation')
parser_store.set_defaults(func=store)
args = parser.parse_args()
if not (args.pid or args.checksum or args.ror or args.checksum_timestamp):
parser.error('No key value pair in input, add at least one among '
+'pid, checksum, ror, checksum timestamp.')
args.func(args)
conf/metadataManager.conf deleted 100644 → 0
View file @ 28783826
# section containing the logging options
[Logging]
# possible values: INFO, DEBUG, ERROR, WARNING, CRITICAL
log_level=INFO
log_file=/var/log/irods/metadataManager.log
install.txt
View file @ 15fc075d
......@@ -2,6 +2,11 @@
The following describes the procedure to enable B2SAFE release-4.x.y
NOTE: This B2SAFE version makes use of the b2HANDLE library.
It can be found at: https://github.com/EUDAT-B2SAFE/B2HANDLE
Please use the included instructions in README.md to build and install B2HANDLE.
At the end of this installation the installation of B2HANDLE is described.
NOTE: iRODS is running as a normal user process. NOT as root. The package can
be build by any user. During installation of the package it will use:
"/etc/irods/service_account.config" to set the ownership of the files.
......@@ -88,6 +93,27 @@ NOTE: following needs to be updated/added in "/opt/eudat/b2safe/packaging/instal
Try to install missing packages with the standard package manager like apt, yum, zypper etc.
If packages are not within the standards use pip and install the missing packages with pip.
DONE
DONE
== installation of B2HANDLE ==
The customers need to install the b2handle library on the b2safe system and
create public/private keypairs and certificates and get the public key binary
uploaded before the upgrades.
This entails following:
- Download the b2handle code from: https://github.com/EUDAT-B2SAFE/B2HANDLE
- Create an rpm and install it on the b2safe system. With the necessary
dependencies. See the github page:
a> python setup.py bdist_rpm
b> yum install <created_rpm in dist directory>
- Ask the handle hosting service which user to use for a certificate.
- Create a private/public keypair and create a derived certificate as described
in http://eudat-b2safe.github.io/B2HANDLE/creatingclientcertificates.html.:
a> Generate public/private key pair
i> First download the software needed to create the private/public keypair.
It can be found on: http://www.handle.net/download_hnr.html.
ii> Execute ./hdl-keygen from hsj-8.1.1/bin directory
b> Send public key (.bin file) to your hosting service.
c> Step 2: Upload the user’s public key to the.... is executed by the hosting service
- Ask hosting service which username/password to use for reverselooup.
- Test using curl
rulebase/catchError.re
View file @ 15fc075d
......@@ -6,37 +6,48 @@
# List of the functions:
#
# EUDATCatchErrorChecksum(*source,*destination)
# EUDATCatchErrorSize(*source,*destination)
# EUDATCatchErrorChecksum(*source, *source_res, *destination, *dest_res, *response)
# EUDATCatchErrorSize(*source, *source_res, *destination, *dest_res, *response)
# EUDATCatchErrorDataOwner(*path,*msg)
# EUDATCheckOwnershipObj(*path)
# EUDATCheckOwnershipColl(*path)
# Check if 2 replicas have the same checksum
#
# Parameters:
# *source [IN] path source of data object
# *destination [IN] path destination of replicated data object
# *source [IN] path source of data object
# *source_res [IN] iRODS resource name of the source
# *destination [IN] path destination of replicated data object
# *dest_res [IN] iRODS resource name of the destination
# *response [OUT]
# return true/false
#
# Author: Long Phan, JSC
# Author: Long Phan (JSC), Claudio Cacciari (Cineca)
#-------------------------------------------------------------------------------
EUDATCatchErrorChecksum(*source,*destination) {
EUDATCatchErrorChecksum(*source, *source_res, *destination, *dest_res, *response) {
logInfo("[EUDATCatchErrorChecksum] Check if 2 replicas have the same checksum. " ++
"Source = *source, destination = *destination");
*b = bool("true");
if (!EUDATObjExist(*source, *response) || !EUDATObjExist(*destination, *response)) {
failmsg(-1, *response);
}
*b = bool("true");
*checksum0 = "";
EUDATiCHECKSUMget(*source, *checksum0, *modtime);
EUDATiCHECKSUMget(*source, *checksum0, *modtime0, *source_res);
*checksum1 = "";
EUDATiCHECKSUMget(*destination, *checksum1, *modtime);
EUDATiCHECKSUMget(*destination, *checksum1, *modtime1, *dest_res);
*response = "checksum(*source):*checksum0 = checksum(*destination):*checksum1"
if(*checksum0 != *checksum1) {
EUDATSearchPID(*source, *pid);
logDebug("*checksum0 != *checksum1, existing_pid = *pid");
*response = "checksum(*source):*checksum0 != checksum(*destination):*checksum1,"
++ " checksum(*source) modification time: *modtime0,"
++ " checksum(*destination) modification time: *modtime1"
logDebug("[EUDATCatchErrorChecksum] *response");
*b = bool("false");
}
logInfo("[EUDATCatchErrorChecksum] *response");
*b;
}
......@@ -45,36 +56,52 @@ EUDATCatchErrorChecksum(*source,*destination) {
#
# Parameters:
# *source [IN] path source of data object
# *source_res [IN] iRODS resource name of the source
# *destination [IN] path destination of replicated data object
# *dest_res [IN] iRODS resource name of the destination
#
# Author: Long Phan, JSC
#-------------------------------------------------------------------------------
EUDATCatchErrorSize(*source,*destination) {
EUDATCatchErrorSize(*source, *source_res, *destination, *dest_res, *response) {
logInfo("[EUDATCatchErrorSize] Check if 2 replicas have the same size." ++
"Source = *source, destination = *destination");
"Source = *source:*source_res:, destination = *destination:*dest_res:");
if (!EUDATObjExist(*source, *response)) {
failmsg(-1, *response);
}
if (!EUDATObjExist(*destination, *response)) {
failmsg(-1, *response);
}
*b = bool("true");
*size0 = "";
*s_res = *source_res;
msiSplitPath(*source,*parentS,*childS);
foreach ( *BS in SELECT DATA_SIZE WHERE COLL_NAME = '*parentS' AND DATA_NAME = '*childS' ) {
foreach ( *BS in SELECT DATA_SIZE,DATA_RESC_NAME WHERE COLL_NAME = '*parentS'
AND DATA_NAME = '*childS' ) {
*size0 = *BS.DATA_SIZE;
logDebug("Size *source = *size0");
logVerbose("[EUDATCatchErrorSize] Size *source = *size0");
*s_res = *BS.DATA_RESC_NAME;
if (*BS.DATA_RESC_NAME == '*source_res') { break; }
}
*size1 = "";
*d_res = *dest_res;
msiSplitPath(*destination,*parentD,*childD);
foreach ( *BD in SELECT DATA_SIZE WHERE COLL_NAME = '*parentD' AND DATA_NAME = '*childD') {
foreach ( *BD in SELECT DATA_SIZE,DATA_RESC_NAME WHERE COLL_NAME = '*parentD'
AND DATA_NAME = '*childD' ) {
*size1 = *BD.DATA_SIZE;
logDebug("Size *destination = *size1");
logVerbose("[EUDATCatchErrorSize] Size *destination = *size1");
*d_res = *BD.DATA_RESC_NAME;
if (*BD.DATA_RESC_NAME == '*dest_res') { break; }
}
*response = "size(*source:*s_res:):*size0 = size(*destination:*d_res:):*size1";
if(*size0 != *size1) {
EUDATSearchPID(*source, *pid);
logDebug("*size0 != *size1, existing_pid = *pid");
*response = "size(*source:*s_res:):*size0 != size(*destination:*d_res:):*size1";
logDebug("[EUDATCatchErrorSize] *response");
*b = bool("false");
}
logInfo("[EUDATCatchErrorSize] *response");
*b;
}
# Check if a user is or is not owner of the data object/collection
......@@ -87,14 +114,20 @@ EUDATCatchErrorSize(*source,*destination) {
# Author: Long Phan, JSC;
# Author: Claudio Cacciari, Cineca;
#-------------------------------------------------------------------------------
EUDATCatchErrorDataOwner(*path,*msg) {
*msg = "ownership check passed";
EUDATCatchErrorDataOwner(*path, *msg) {
if (!EUDATObjExist(*path, *response)) {
failmsg(-1, *response);
}
*msg = "ownership check passed";
msiGetObjType(*path ,*type);
if (*type == '-c') {
# check the ownership of the root collection
logDebug("checking the ownership of the collection *path for the user $userNameClient");
logDebug( "[EUDATCatchErrorDataOwner] checking the ownership of the collection"
++ " *path for the user $userNameClient");
msiCheckAccess(*path,"own",*result);
logVerbose("[EUDATCatchErrorDataOwner] msiCheckAccess(*path): *result");
if (*result != 1) {
if (EUDATCheckOwnershipColl(*path) != 1 ) {
*msg = "ownership check failed on path: *path";
......@@ -102,10 +135,11 @@ EUDATCatchErrorDataOwner(*path,*msg) {
}
}
# check the ownership of the sub-collections and their related data objects
logDebug("checking the ownership of the collections under the path *path");
logDebug("[EUDATCatchErrorDataOwner] checking the ownership of the collections under the path *path");
foreach(*Row in SELECT DATA_NAME,COLL_NAME WHERE COLL_NAME = '*path' || like '*path/%') {
if (*Row.COLL_NAME != *path) {
msiCheckAccess(*Row.COLL_NAME,"own",*result);
logVerbose("[EUDATCatchErrorDataOwner] msiCheckAccess(" ++ *Row.COLL_NAME ++ "): *result");
if (*result != 1) {
if (EUDATCheckOwnershipColl(*Row.COLL_NAME) != 1 ) {
*msg = "ownership check failed on path: *Row.COLL_NAME";
......@@ -115,7 +149,7 @@ EUDATCatchErrorDataOwner(*path,*msg) {
}
*objPath = *Row.COLL_NAME ++ '/' ++ *Row.DATA_NAME;
msiCheckAccess(*objPath,"own",*result);
logDebug("the object *objPath has result: *result");
logVerbose("[EUDATCatchErrorDataOwner] the object *objPath has result: *result");
if (*result != 1) {
if (EUDATCheckOwnershipObj(*objPath) != 1 ) {
*msg = "ownership check failed on path: *objPath";
......@@ -125,7 +159,8 @@ EUDATCatchErrorDataOwner(*path,*msg) {
}
}
else if (*type == '-d') {