Commit dd02356f authored by Claudio Cacciari's avatar Claudio Cacciari
Browse files

preparation for the new release

parent 005e170d
B2SAFE package 4.0.0 [2017/04/12]
Tested with iRODS 4.1.x and 4.2.0.
Compatible with EPIC API and HSv8 API.
- added support for iRODS v4.2.0
- added support for the EUDAT PID profile
- changed epic client to support new b2handle library
- python script code cleaning
- initial support of the metadata module (added a couple of rules)
- various bug fixing
- initial support for the ARGO messaging system
B2SAFE package 3.1.3 [2017/01/04]
bug fixing release:
- fixed many epicclents bugs
......
......@@ -12,7 +12,7 @@
# EUDATCheckOwnershipObj(*path)
# EUDATCheckOwnershipColl(*path)
#
# Check if 2 replicas have the same checksum
#
# Parameters:
......@@ -20,7 +20,7 @@
# *destination [IN] path destination of replicated data object
#
# Author: Long Phan, JSC
#
#-------------------------------------------------------------------------------
EUDATCatchErrorChecksum(*source,*destination) {
logInfo("[EUDATCatchErrorChecksum] Check if 2 replicas have the same checksum. " ++
"Source = *source, destination = *destination");
......@@ -41,7 +41,6 @@ EUDATCatchErrorChecksum(*source,*destination) {
}
#
# Check if 2 replicas have the same size.
#
# Parameters:
......@@ -49,7 +48,7 @@ EUDATCatchErrorChecksum(*source,*destination) {
# *destination [IN] path destination of replicated data object
#
# Author: Long Phan, JSC
#
#-------------------------------------------------------------------------------
EUDATCatchErrorSize(*source,*destination) {
logInfo("[EUDATCatchErrorSize] Check if 2 replicas have the same size." ++
"Source = *source, destination = *destination");
......@@ -78,7 +77,6 @@ EUDATCatchErrorSize(*source,*destination) {
}
#
# Check if a user is or is not owner of the data object/collection
# (Reference: https://www.irods.org/index.php/iRODS_Error_Codes or /iRODS/lib/core/include/rodsErrorTable.h)
#
......@@ -88,7 +86,7 @@ EUDATCatchErrorSize(*source,*destination) {
#
# Author: Long Phan, JSC;
# Author: Claudio Cacciari, Cineca;
#----------------------------------------------------
#-------------------------------------------------------------------------------
EUDATCatchErrorDataOwner(*path,*msg) {
*msg = "ownership check passed";
......@@ -138,7 +136,6 @@ EUDATCatchErrorDataOwner(*path,*msg) {
}
}
# Check if a user is or is not owner of the data object,
# but comparing the session var "userNameClient" with the owners of the object
#
......@@ -147,7 +144,7 @@ EUDATCatchErrorDataOwner(*path,*msg) {
# *message [OUT] response message
#
# Author: Claudio Cacciari, Cineca;
#----------------------------------------------------
#-------------------------------------------------------------------------------
EUDATCheckOwnershipObj(*path) {
msiSplitPath(*path, *collPath, *objPath);
......@@ -168,7 +165,7 @@ EUDATCheckOwnershipObj(*path) {
# *message [OUT] response message
#
# Author: Claudio Cacciari, Cineca;
# ----------------------------------------------------
#-------------------------------------------------------------------------------
EUDATCheckOwnershipColl(*path) {
*owner = 0;
......
......@@ -10,6 +10,7 @@
# EUDATAuthZ(*user, *action, *target, *response)
#---- utility ---
# EUDATisMetadata(*path)
# EUDATPushMetadata(*path, *queue)
# EUDATMessage(*queue, *message)
# EUDATLog(*message, *level)
# EUDATQueue(*action, *message, *number)
......@@ -31,6 +32,8 @@
# EUDATfileInPath(*path,*subColl)
# EUDATCreateAVU(*Key,*Value,*Path)
# EUDATgetLastAVU(*Path, *Key, *Value)
# EUDATgetCollAVU(*path, *res)
# EUDATgetBulkMetadata(*path, *res)
# EUDATcountMetaKeys( *Path, *Key, *Value )
# EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime)
#---- repository packages ---
......@@ -45,7 +48,6 @@
# #
################################################################################
#
# Return a boolean value:
# True, if the authorization request matches against, at least
# one assertion listed in the authz.map.json file
......@@ -58,7 +60,7 @@
# *response [OUT] True or False depending on authorization rights
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATAuthZ(*user, *action, *target, *response) {
getAuthZParameters(*authZMapPath);
logDebug("checking authorization for *user to perform: *action *target");
......@@ -99,7 +101,7 @@ EUDATAuthZ(*user, *action, *target, *response) {
# *path [IN] the path of the object/collection
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATisMetadata(*path) {
*isMeta = bool("false");
if (*path like regex ".*\\.metadata.*") {
......@@ -115,7 +117,7 @@ EUDATisMetadata(*path) {
# *path [IN] the path of the object/collection
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATPushMetadata(*path, *queue) {
logInfo("[EUDATPushMetadata] pushing metadata of object *path to topic *queue")
......@@ -141,7 +143,7 @@ EUDATPushMetadata(*path, *queue) {
# *message [IN] the message to be sent
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATMessage(*queue, *message) {
logInfo("[EUDATMessage] pushing the message to topic *queue");
......@@ -157,7 +159,6 @@ EUDATMessage(*queue, *message) {
}
}
# It manages the writing and reading of log messages to/from external log services.
# The current implementation writes the logs to specific log file.
#
......@@ -169,7 +170,7 @@ EUDATMessage(*queue, *message) {
# *level [IN] the logging level
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATLog(*message, *level) {
getLogParameters(*logConfPath);
logInfo("logging message '*message'");
......@@ -181,7 +182,6 @@ EUDATLog(*message, *level) {
}
}
# It implements a FIFO queue for messages to/from external log services.
#
# Return
......@@ -196,7 +196,7 @@ EUDATLog(*message, *level) {
# *number [IN] the number of elements to be extracted
#
# Author: Claudio Cacciari, Cineca
#
#-------------------------------------------------------------------------------
EUDATQueue(*action, *message, *number) {
getLogParameters(*logConfPath);
*options = "";
......@@ -222,9 +222,11 @@ EUDATQueue(*action, *message, *number) {
}
}
#
# Logging policies
#
################################################################################
# #
# Logging policies #
# #
################################################################################
logDebug(*msg) {
getEUDATLoggerLevel(*level);
......@@ -253,7 +255,6 @@ logWithLevel(*level, *msg) {
on (*level == "error") { writeLine("serverLog","ERROR: *msg");}
}
#-----------------------------------------------
# Function: trsnsfrom string to boolean value
#
# Author: Claudio Cacciari (Cineca)
......@@ -269,11 +270,10 @@ EUDATtoBoolean(*var) {
*status
}
#
# Function: replace epicclient function
#
# Author: Robert Verkerk SURFsara
#
#-------------------------------------------------------------------------------
EUDATReplaceHash(*path, *out) {
# replace #
......@@ -289,18 +289,16 @@ EUDATReplaceHash(*path, *out) {
}
}
#
# Function: replace microservice msiGetZoneNameFromPath (eudat.c)
#
# Author: Long Phan, JSC
#
#-------------------------------------------------------------------------------
EUDATGetZoneNameFromPath(*path, *out) {
*list = split("*path","/");
*out = elem(*list,0);
}
#
# Gets the connection details of an iRODS Zone.
#
# Arguments:
......@@ -308,7 +306,7 @@ EUDATGetZoneNameFromPath(*path, *out) {
# *conn [OUT] the connection details related to the input iRODS Zone (hostname:port)
#
# Author: Claudio Cacciari, Cineca
#----------------------------------------------------------
#-------------------------------------------------------------------------------
EUDATGetZoneHostFromZoneName(*zoneName, *conn) {
*conn = ""
......@@ -329,7 +327,6 @@ EUDATGetZoneHostFromZoneName(*zoneName, *conn) {
}
}
#
# Checks if date of the last computation of iCHECKSUM was set and set the date if not.
# The date is stored as metadata attribute of name 'eudat_dpm_checksum_date:<name of resc>'
#
......@@ -343,7 +340,7 @@ EUDATGetZoneHostFromZoneName(*zoneName, *conn) {
# - will be assumed as time of the first computation of the iCHECKSUM
#
# Author: Michal Jankowski, PSNC
#
#-------------------------------------------------------------------------------
EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) {
*metaName = 'eudat_dpm_checksum_date:*resc';
......@@ -361,8 +358,6 @@ EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) {
}
}
#
# The function retrieve iCHECKSUM for a given object.
#
# Environment variable used:
......@@ -374,7 +369,7 @@ EUDATiCHECKSUMdate(*coll, *name, *resc, *modTime) {
# *status [REI] false if no value is found, true elsewhere
#
# Author: Giacomo Mariani, CINECA, Michal Jankowski PSNC
#
#-------------------------------------------------------------------------------
EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) {
*status = bool("false");
*checksum = "";
......@@ -402,7 +397,6 @@ EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) {
*status;
}
# The function obtain iCHECKSUM for a given object creating it if necessary.
#
# Arguments:
......@@ -411,7 +405,7 @@ EUDATiCHECKSUMretrieve(*path, *checksum, *modtime) {
# *modtime [OUT] modification time of the checksum
#
# Author: Giacomo Mariani, CINECA, Michal Jankowski PSNC
#-------------------------------------------------------------------------------
EUDATiCHECKSUMget(*path, *checksum, *modtime) {
if (!EUDATiCHECKSUMretrieve(*path, *checksum, *modtime)) {
#If it is a collection, do not calculate the checksum
......@@ -424,7 +418,6 @@ EUDATiCHECKSUMget(*path, *checksum, *modtime) {
}
}
# Calculate the difference between the creation time or the current time
# and the modification time of an object. In seconds.
#
......@@ -437,7 +430,7 @@ EUDATiCHECKSUMget(*path, *checksum, *modtime) {
# *age = -1 if the object does not exist
#
# Author: Giacomo Mariani, CINECA
#
#-------------------------------------------------------------------------------
EUDATgetObjectTimeDiff(*filePath, *mode, *age) {
*age = -1;
# Check if the file exists
......@@ -464,7 +457,6 @@ EUDATgetObjectTimeDiff(*filePath, *mode, *age) {
}
}
#
# Calculate the difference between the current time and the modification time of an object.
# In seconds.
#
......@@ -474,7 +466,7 @@ EUDATgetObjectTimeDiff(*filePath, *mode, *age) {
# *age = -1 if the object does not exist
#
# Author: Claudio Cacciari, CINECA
#
#-------------------------------------------------------------------------------
EUDATgetObjectAge(*filePath, *age) {
*age = -1;
# Check if the file exists
......@@ -487,7 +479,6 @@ EUDATgetObjectAge(*filePath, *age) {
*age;
}
#
# Rules to check if a file is in a given path.
#
# Arguments:
......@@ -496,7 +487,7 @@ EUDATgetObjectAge(*filePath, *age) {
# *b [REI] False if no value is found, trou elsewhere
#
# Author: Hao Xu, DICE; Giacomo Mariani, CINECA
#
#-------------------------------------------------------------------------------
EUDATfileInPath(*path,*subColl) {
logInfo("conditional acPostProcForCopy -> EUDATfileInPath");
msiSplitPath(*path, *coll, *name);
......@@ -511,7 +502,6 @@ EUDATfileInPath(*path,*subColl) {
*b;
}
#
# Create AVU with INPUT *Key, *Value for DataObj *Path
#
# Parameters:
......@@ -521,7 +511,7 @@ EUDATfileInPath(*path,*subColl) {
#
# Author: Long Phan, JSC
# Modified: Elena Erastova, RZG, 27.08.2015
#
#-------------------------------------------------------------------------------
EUDATCreateAVU(*Key, *Value, *Path) {
logDebug("[EUDATCreateAVU] Adding AVU: *Key = *Value to metadata of *Path");
msiAddKeyVal(*Keyval, *Key, *Value);
......@@ -529,7 +519,6 @@ EUDATCreateAVU(*Key, *Value, *Path) {
msiSetKeyValuePairsToObj(*Keyval, *Path, *objType);
}
#-----------------------------------------------------------------------------
# get the single value of a specific metadata in ICAT
#
# Parameters:
......@@ -548,7 +537,6 @@ EUDATgetLastAVU(*Path, *Key, *Value)
}
}
#-----------------------------------------------------------------------------
# Get all the AVUs of a collection
#
# Parameters:
......@@ -569,7 +557,6 @@ EUDATgetCollAVU(*path, *res)
logDebug("[EUDATgetCollAVU] AVUs: *res");
}
#-----------------------------------------------------------------------------
# Get all the AVUs of the objects under a collection
#
# Parameters:
......@@ -598,7 +585,6 @@ EUDATgetBulkMetadata(*path, *res)
logDebug("[EUDATgetBulkMetadata] metadata: *res");
}
#-----------------------------------------------------------------------------
# count metadata in ICAT
#
# Parameters:
......@@ -630,7 +616,7 @@ EUDATcountMetaKeys( *Path, *Key, *Value )
# *modtime [IN] The modification time of the checksum
#
# Author: Claudio Cacciari, CINECA
#-------------------------------------------------------------------------------
EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime) {
getMetaParameters(*metaConfPath,*enabled);
......@@ -658,14 +644,12 @@ EUDATStoreJSONMetadata(*path, *pid, *ror, *checksum, *modtime) {
}
}
################################################################################
# #
# Repository Packages #
# #
################################################################################
#-----------------------------------------------------------------------------
# Check if the ADMIN_Status value is set to Ready ToArchive and then kicks off ingestion
#
# Parameters:
......@@ -684,7 +668,6 @@ EUDATrp_checkMeta(*source,*AName,*AValue)
}
}
#-----------------------------------------------------------------------------
# Manage the ingestion in B2SAFE
# Check the checksum
# Create PID
......@@ -733,7 +716,6 @@ EUDATrp_ingestObject( *source )
}
}
#-----------------------------------------------------------------------------
# Process executed when a transfer has been initiated
# (this process is triggered by the iputPreProc hook)
#
......@@ -749,7 +731,6 @@ EUDATrp_transferInitiated( *source )
EUDATCreateAVU("INFO_TimeOfStart", *TimeNow, *source);
}
#-----------------------------------------------------------------------------
# Process executed after a transfer is finished
# (this process is triggered by the iputPostProc hook)
#
......
......@@ -9,11 +9,11 @@
# Arguments:
# *euLogLevel [OUT] the debug level for the EUDAT specific rules [0 | 1 | 2]
# 0:ERROR, 1:INFO, 2:DEBUG
#-------------------------------------------------------------------------------
getEUDATLoggerLevel(*euLogLevel) {
*euLogLevel=2
}
#
# Provides parameters for the connection with the EPIC service
#
# Arguments:
......@@ -25,7 +25,7 @@ getEUDATLoggerLevel(*euLogLevel) {
# *epicDebug [OUT] the debug level for the EPIC client scripts
#
# Author: Willem Elbers (MPI-PL)
#
#-------------------------------------------------------------------------------
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) {
*credStoreType="os";
*credStorePath="/srv/irods/current/modules/B2SAFE/cmd/credentials_test";
......@@ -39,7 +39,6 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD
}
}
# -----------------------------------------------------------------------------
# Parse the credentials to connect to an EPIC server. A file called
# "credentials" MUST contain all the connection details in the home folder of
# the user running this rule.
......@@ -50,8 +49,7 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD
# *password [OUT] password
#
# Author: Javier Quinteros, RZG
# -----------------------------------------------------------------------------
#
#-------------------------------------------------------------------------------
parseCredentials (*baseuri, *username, *prefix, *password) {
*baseuri = 'https://epic3.storage.surfsara.nl/v2_test/handles/'
......@@ -72,7 +70,7 @@ parseCredentials (*baseuri, *username, *prefix, *password) {
# containing the authorization assertions.
#
# Author: Claudio Cacciari (Cineca)
#
#-------------------------------------------------------------------------------
getAuthZParameters(*authZMapPath) {
*authZMapPath="/srv/irods/current/modules/B2SAFE/cmd/authz.map.json";
}
......@@ -83,7 +81,7 @@ getAuthZParameters(*authZMapPath) {
# *logConfPath [OUT] the file path to the logging configuration.
#
# Author: Claudio Cacciari (Cineca)
#
#-------------------------------------------------------------------------------
getLogParameters(*logConfPath) {
*logConfPath="/srv/irods/current/modules/B2SAFE/cmd/log.manager.conf";
}
......@@ -97,7 +95,7 @@ getLogParameters(*logConfPath) {
# json metadata writing
#
# Author: Claudio Cacciari (Cineca)
# -------------------------------------------------------------------------------
getMetaParameters(*metaConfPath, *enabled) {
*metaConfPath="/srv/irods/current/modules/B2SAFE/cmd/metadataManager.conf";
*enabled=bool("false");
......@@ -111,13 +109,13 @@ getMetaParameters(*metaConfPath, *enabled) {
# the usage of the messaging system
#
# Author: Claudio Cacciari (Cineca)
#
#-------------------------------------------------------------------------------
getMessageParameters(*msgConfPath, *enabled) {
*msgLogPath="/opt/eudat/b2safe/conf/msgManager.conf";
*enabled=bool("false");
}
#Provides parameters for some B2SAFE configurations.
# Provides parameters for some B2SAFE configurations.
# The plugin msifree_microservice_out is a bug fixing to avoid memory leak
# in case of collection replication involving thousands of files and the
# parameter msiFreeEnabled enables it.
......@@ -130,7 +128,7 @@ getMessageParameters(*msgConfPath, *enabled) {
# defined in the file retrieved by getAuthZParameters.
#
# Author: Claudio Cacciari (Cineca)
#
#-------------------------------------------------------------------------------
getConfParameters(*msiFreeEnabled, *msiCurlEnabled, *authzEnabled) {
*msiFreeEnabled=bool("true");
*authzEnabled=bool("true");
......@@ -143,15 +141,14 @@ getConfParameters(*msiFreeEnabled, *msiCurlEnabled, *authzEnabled) {
# *version [OUT] the B2SAFE version.
#
# Author: Claudio Cacciari (Cineca)
#
#-------------------------------------------------------------------------------
getB2SAFEVersion(*version) {
*major_version = "3";
*minor_version = "9";
*sub_version = "1";
*major_version = "4";
*minor_version = "0";
*sub_version = "0";
*version = *major_version ++ "." ++ *minor_version ++ "-" ++ *sub_version;
}
#
# This function is used to set up some parameters for the site in case you are
# going to use the EUDAT repository packages procedure to ingest data.
#
......@@ -160,7 +157,7 @@ getB2SAFEVersion(*version) {
# *archiveOwner [OUT] This is the iRods user owning the archive
#
# Author: S Coutin (CINES)
#
#-------------------------------------------------------------------------------
rp_getRpIngestParameters(*protectArchive, *archiveOwner) {
*protectArchive = false;
*archiveOwner = "rodsA";
......
......@@ -18,11 +18,10 @@
# EUDATSearchPIDchecksum(*path, *existing_pid)
# EUDATUpdatePIDWithNewChild(*parentPID, *childPID)
# EUDATGeteValPid(*pid, *key)
# EUDATGeteRorPid(*pid, *ror)
# EUDATeiChecksumMgmt(*path, *PID)
# EUDATiPIDcreate(*path, *PID)
# EUDATiFieldVALUEretrieve(*path, *FNAME, *FVALUE)
# EUDATePIDcreate(*path, *extraType, *PID, *ifchecksum)
# EUDATePIDcreate(*path, *extraType, *PID)
# EUDATePIDsearch(*field, *value, *PID)
# EUDATeCHECKSUMupdate(*PID, *path)
# EUDATeURLupdate(*PID, *newURL)
......@@ -31,12 +30,12 @@
# EUDATeiChecksumMgmtColl(*sourceColl)
# EUDATiRORupdate(*source, *pid)
# EUDATeRORupdate(*pid,*newRor)
# EUDATeFIOupdate(*pid, *newFio)
# EUDATPidsForColl(*collPath)
# EUDATePIDcreateCurl(*path, *extraType, *PID, *ifchecksum)
# EUDATSearchPIDCurl(*path, *existing_pid)
#
# Generate a new PID for a digital object.
# Fields stored in the PID record: URL, ROR and CHECKSUM
# adds a ROR field if (*ror != "None")
......@@ -51,7 +50,7 @@
#
# Author: Willem Elbers, MPI-TLA
# Edited by Elena Erastova, RZG; Long Phan, JSC; Robert Verkerk, SURFsara, Javier Quinteros, GFZ
#
#-------------------------------------------------------------------------------
EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) {
logInfo("[EUDATCreatePID] create pid for *path");
......@@ -169,7 +168,6 @@ EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) {
}
}
# -----------------------------------------------------------------------------
# Searching for a PID using URL replacing "#", "%" and "&" with "*"
# Parameters:
# *path [IN] the path of the replica
......@@ -186,14 +184,13 @@ EUDATSearchPID(*path, *existing_pid) {
*status;
}
#
# Searching fo a PID using CHECKSUM
# Parameters:
# *path [IN] the path of the replica
# *existing_pid [OUT] existing PID
#
# Author: Elena Erastova, RZG
#
#-------------------------------------------------------------------------------
EUDATSearchPIDchecksum(*path, *existing_pid) {
logInfo("search pid for *path");
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
......@@ -224,7 +221,6 @@ EUDATSearchPIDchecksum(*path, *existing_pid) {
}
}
#
# Update a PID record with a new child.
#
# Parameters:
......@@ -233,7 +229,7 @@ EUDATSearchPIDchecksum(*path, *existing_pid) {
#
# Author: Willem Elbers, MPI-TLA
# Modified by: Claudio Cacciari, CINECA
#
#-------------------------------------------------------------------------------
EUDATUpdatePIDWithNewChild(*parentPID, *childPID) {
*replicaNew = "None"
logInfo("[EUDATUpdatePIDWithNewChild] update parent pid (*parentPID) with new child (*childPID)");
......@@ -258,7 +254,6 @@ EUDATUpdatePIDWithNewChild(*parentPID, *childPID) {
*replicaNew
}
#-------------------------------------------------------------------------------
# get the KEY entry for a PID
#
# Parameters:
......@@ -283,17 +278,15 @@ EUDATGeteValPid(*pid, *key) {
*val
}
#
# This function creates a PID and stores its value and the checksum in the iCAT if it does not exist.
# Otherwhise the function modifies the PID.
#
# Arguments:
# *path [IN] Path of the source file
# *PID [OUT] PID of the source file