Commit 846a0413 authored by Robert Verkerk's avatar Robert Verkerk
Browse files

Merge remote-tracking branch 'upstream/devel' into devel

parents e7cfacb4 f29889ef
#!/usr/bin/env python
#
# * use 4 spaces!!! not tabs
# * See PEP-8 Python style guide http://www.python.org/dev/peps/pep-0008/
# * use pylint
#
import json
user_map_file="/etc/irods/user_map.json"
def parseUserMap():
"""Parse the user map file"""
with open(user_map_file, 'r') as f:
user_map = json.load(f)
jstr = json.dumps(user_map, ensure_ascii=True, indent=4)
print(jstr)
if __name__ == "__main__":
parseUserMap()
......@@ -7,12 +7,14 @@
# List of the functions:
#
# ---- logging ---
# logVerbose(*msg)
# logInfo(*msg)
# logDebug(*msg)
# logError(*msg)
# logWithLevel(*level, *msg)
#---- authorization ---
# EUDATAuthZ(*user, *action, *target, *response)
# EUDATGetPAMusers(*json_map)
#---- utility ---
# EUDATObjExist(*path, *response)
# EUDATPushMetadata(*path, *queue)
......@@ -747,6 +749,21 @@ EUDATcountMetaKeys( *Path, *Key, *Value ) {
logVerbose("[EUDATcountMetaKeys] got count = *Value");
}
# get the content of the user map file for OAuth2 authentication in json format
#
# Parameters:
# *json_map [OUT] a string representing the user mapping
#
# Author: Claudio Cacciari, CINECA
# -----------------------------------------------------------------------------
EUDATGetPAMusers(*json_map) {
logVerbose("[EUDATGetPAMusers] checking authorization for $userNameClient to read users");
EUDATAuthZ($userNameClient, "read", "users", *response);
msiExecCmd("pam_user_reader.py", "null", "null", "null", "null", *outUsersJson);
msiGetStdoutInExecCmdOut(*outUsersJson, *json_map);
logVerbose("[EUDATGetPAMusers] json user map = *json_map");
}
################################################################################
# #
# Repository Packages #
......
......@@ -39,6 +39,20 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD
}
}
# Provides parameters for the connection with the EPIC service
#
# Arguments:
# *serverApireg [OUT] the endpoint of the HTTP API interface for data with PIDs
# *serverApipub [OUT] the endpoint of the HTTP API interface for public data
#
# Author: Claudio Cacciari (Cineca)
#-------------------------------------------------------------------------------
getHttpApiParameters(*serverApireg, *serverApipub) {
*serverApireg="https://<hostnameWithFullDomain>/api/registered";
*serverApipub="https://<hostnameWithFullDomain>/api/public";
}
# Parse the credentials to connect to an EPIC server. A file called
# "credentials" MUST contain all the connection details in the home folder of
# the user running this rule.
......@@ -124,8 +138,8 @@ getConfParameters(*authzEnabled) {
#-------------------------------------------------------------------------------
getB2SAFEVersion(*version) {
*major_version = "4";
*minor_version = "1";
*sub_version = "1";
*minor_version = "2";
*sub_version = "0";
*version = *major_version ++ "." ++ *minor_version ++ "-" ++ *sub_version;
}
......
......@@ -23,7 +23,7 @@
# EUDATePIDsearch(*field, *value, *PID)
# EUDATeCHECKSUMupdate(*PID, *path)
# EUDATeURLupdate(*PID, *newURL)
# EUDATeURLsearch(*PID, *URL)
# EUDATeURLupdateColl(*PID, *newURL)
# EUDATePIDremove(*path, *force)
# EUDATiRORupdate(*source, *pid)
# EUDATeRORupdate(*pid,*newRor)
......@@ -148,8 +148,12 @@ EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) {
EUDATSearchPID(*path, *existing_pid) {
logDebug("[EUDATSearchPID] search pid for *path");
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
getHttpApiParameters(*serverApireg, *serverApipub);
EUDATReplaceHash(*path, *path1);
*status = EUDATePIDsearch("URL", "*serverID*path1", *existing_pid);
*status = EUDATePIDsearch("URL", "*serverApireg*path1", *existing_pid);
if (!(*status)) {
*status = EUDATePIDsearch("URL", "*serverApipub*path1", *existing_pid);
}
logDebug("[EUDATSearchPID] pid = *existing_pid");
*status;
}
......@@ -163,13 +167,13 @@ EUDATSearchPID(*path, *existing_pid) {
#-------------------------------------------------------------------------------
EUDATSearchPIDchecksum(*path, *existing_pid, *existing_url) {
logDebug("[EUDATSearchPIDchecksum] search pid for *path");
logDebug("[EUDATSearchPIDchecksum] searching checksum for *path");
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
*resource = "";
EUDATiCHECKSUMget(*path, *checksum, *modtime, *resource)
EUDATePIDsearch("EUDAT/CHECKSUM", *checksum, *existing_pid);
EUDATeURLsearch(*existing_pid, *existing_url);
*existing_url = EUDATGeteValPid(*existing_pid, "URL");
logDebug("[EUDATSearchPIDchecksum] PID = *existing_pid, URL = *existing_url");
}
......@@ -264,8 +268,36 @@ EUDATiPIDcreate(*path, *PID) {
EUDATePIDcreate(*path, *extraType, *PID) {
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) ;
getHttpApiParameters(*serverApireg, *serverApipub);
*url="*serverID*path";
*access = "private";
# Verify that source input path is a collection
msiGetObjType(*path, *type);
if (*type != '-c') {
msiSplitPath(*path, *parent, *child);
foreach ( *R in SELECT USER_NAME WHERE DATA_NAME = '*child' AND COLL_NAME = '*parent') {
if (*R.USER_NAME == "anonymous") {
*access = "public";
break;
}
}
}
else {
foreach ( *R in SELECT COLL_OWNER_NAME WHERE COLL_NAME = '*path' ) {
if (*R.COLL_OWNER_NAME == "anonymous") {
*access = "public";
break;
}
}
}
if (*access == "public") {
*url = "*serverApipub*path";
}
else {
*url = "*serverApireg*path";
}
*resource = "";
EUDATiCHECKSUMget(*path, *checksum, *modtime, *resource);
if (*checksum != "") {
......@@ -380,8 +412,8 @@ EUDATeURLupdate(*PID, *newURL) {
# and all its sub-collections and objects.
#
# Arguments:
# *PID [IN] The PID associated to $collName
# *newURL [IN] The new URL to be associated to the PID of $collName
# *PID [IN] The PID associated to the collection
# *newURL [IN] The new URL to be associated to the PID of the collection
#
# Author: Claudio Cacciari, CINECA
#-------------------------------------------------------------------------------
......@@ -389,10 +421,17 @@ EUDATeURLupdateColl(*PID, *newURL) {
logDebug("[EUDATeURLupdateColl] updating collection *PID with URL *newURL and all its content");
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
EUDATeURLsearch(*PID, *oldURL);
msiStrlen(*serverID,*serverIDLength);
msiSubstr(*oldURL, str(int(*serverIDLength)), "null", *sourcePath);
msiSubstr(*newURL, str(int(*serverIDLength)), "null", *targetPath);
getHttpApiParameters(*serverApireg, *serverApipub);
*oldURL = EUDATGeteValPid(*PID, "URL");
if (*oldURL like "*serverApireg\*" ) {
*serverApi = *serverApireg
}
else {
*serverApi = *serverApipub
}
msiStrlen(*serverApi,*serverApiLength);
msiSubstr(*oldURL, str(int(*serverApiLength)), "null", *sourcePath);
msiSubstr(*newURL, str(int(*serverApiLength)), "null", *targetPath);
msiStrlen(*targetPath, *pathLength);
logDebug("[EUDATeURLupdateColl] loop over the sub-collections of *targetPath");
......@@ -401,7 +440,7 @@ EUDATeURLupdateColl(*PID, *newURL) {
msiSubstr(*Row.COLL_NAME, str(int(*pathLength)), "null", *subCollection);
*sourceColl = "*sourcePath*subCollection"
if (EUDATSearchPID(*sourceColl, *existing_pid)) {
*targetColl = *serverID ++ *Row.COLL_NAME;
*targetColl = *serverApi ++ *Row.COLL_NAME;
EUDATeURLupdate(*existing_pid, *targetColl);
}
}
......@@ -413,29 +452,12 @@ EUDATeURLupdateColl(*PID, *newURL) {
msiSubstr(*objPath, str(int(*pathLength)), "null", *subObjPath);
*sourceObj = "*sourcePath*subObjPath";
if (EUDATSearchPID(*sourceObj, *existing_pid)) {
*targetObj = *serverID ++ *objPath;
*targetObj = *serverApi ++ *objPath;
EUDATeURLupdate(*existing_pid, *targetObj);
}
}
}
# This function search the URL field of the PID
#
# Arguments:
# *PID [IN] The PID associated to $objPath
# *newURL [IN] The new URL to be associated to the PID
#
# Author: Giacomo Mariani, CINECA
#-------------------------------------------------------------------------------
EUDATeURLsearch(*PID, *URL) {
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
logDebug("[EUDATeURLsearch] search URL in PID *PID");
msiExecCmd("epicclient.py","*credStoreType *credStorePath read *PID --key URL ",
"null", "null", "null", *outEUS);
msiGetStdoutInExecCmdOut(*outEUS, *URL);
logDebug("[EUDATeURLsearch] response = *URL");
}
# This function remove an ePID... even if its EUDAT/REPLICA field is not empty!
# To be improved.
#
......@@ -445,7 +467,7 @@ EUDATeURLsearch(*PID, *URL) {
# Author: Giacomo Mariani, CINECA
#-------------------------------------------------------------------------------
EUDATePIDremove(*path, *force) {
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug)
getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
logDebug("[EUDATePIDremove] removing PID associated to: $userNameClient, *path");
if (EUDATSearchPID(*path, *pid)) {
......@@ -547,7 +569,7 @@ EUDATeFIOupdate(*pid, *newFio) {
#-----------------------------------------------------------------------------
EUDATPidsForColl(*collPath, *fixed) {
logInfo("[EUDATPidsForColl] Creating PIDs for collection *collPath");
logInfo("[EUDATPidsForColl] Creating PIDs for collection *collPath and fixed = *fixed");
# Verify that source input path is a collection
msiGetObjType(*collPath, *type);
......
......@@ -10,15 +10,20 @@
# List of the functions:
#
# EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause)
# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled,*notification,*response)
# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive)
# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled, *notification, *response)
# EUDATCheckIntegrity(*source, *destination, *logEnabled, *notification, *response)
# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response)
# EUDATReplication(*source, *destination, *registered, *recursive, *response)
# EUDATTransferUsingFailLog(*buffer_length, *stats)
# EUDATRegDataRepl(*source, *destination, *dest_res, *recursive, *response)
# EUDATRegDataRepl(*source, *destination, *recursive, *response)
# EUDATPIDRegistration(*source, *destination, *notification, *registration_response)
# EUDATSearchAndCreatePID(*path, *pid)
# EUDATSearchAndDefineField(*path, *pid, *key)
# EUDATCheckIntegrityColl(*sCollPesult*source_res, *dCollPath, *dest_res, *logEnabled, *response)
# EUDATCheckIntegrityColl(*sCollPath, *source_res, *dCollPath, *dest_res, *logEnabled, *response)
# EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *check_response)
# EUDATCheckIntegrityDO(*source, *source_res, *destination, *dest_res, *logEnabled, *response)
# EUDATCheckIntegrityDO(*source, *destination, *logEnabled, *response)
# Update the logging files specific for EUDAT B2SAFE
......
......@@ -21,6 +21,8 @@ fi
# Define test file name
testFileName="test_data.txt"
testFileNameRemote="test_data2.txt"
testCollName="test_coll_root"
testCollNameSub="test_coll_sub"
# Get user name and zone name from ienv
irods_user_name=`ienv | grep irods_user_name | cut -d '-' -f 2 | tr -d '[[:space:]]'`
......@@ -72,6 +74,35 @@ echo "############ Data Object ############"
ils -l ${testFileName}
echo ""
# Define collection
collPath="${irods_home}/${testCollName}"
# If exists, ask whether replace or exit
exists=`ils ${irods_home} | grep ${testCollName}`
if [ ! -z $exists ]
then
echo "The collection $collPath already exists. Remove it before continuing (y or n)? Otherwise, script will exit."
read shouldRemove
echo "You entered $shouldRemove"
if [ $shouldRemove == "y" ]
then
echo "Will be removed"
irm -rf $collPath
else
echo "Exiting..."
exit 1
fi
fi
imkdir ${testCollName}
imkdir "${testCollName}/${testCollNameSub}"
echo "Hello World!" > ${testFileName}
iput ${testFileName} ${testCollName}
iput ${testFileName} "${testCollName}/${testCollNameSub}"
rm ${testFileName}
echo "############ Collection ############"
ils -rl ${testCollName}
echo ""
createPID () {
rule="{EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID)}"
......@@ -105,8 +136,12 @@ createPID () {
replication () {
echo ""
echo "############ REPLICATION ############"
# destPath="${irods_home}/test_data2.txt"
destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
if [ "${REMOTE_ZONE}" == "${irods_zone_name}" ]
then
destPath="${irods_home}/${testFileNameRemote}"
else
destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
fi
echo "Replica path: ${destPath}"
rule="{*status = EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response);
if (*status) {
......@@ -161,10 +196,53 @@ replication () {
irm ${destPath}
}
moveCollection () {
rule="{EUDATPidsForColl(*collPath, *fixed)}"
input="*collPath=${collPath}%*fixed=true"
echo ""
echo "############ Collection PID creation ############"
echo "Rule: irule ${rule} ${input} null"
irule "${rule}" "${input}" null
rule="{EUDATSearchPID(*path, *existing_pid)}"
input="*path=${collPath}"
output="*existing_pid"
pid_raw=`irule "${rule}" "${input}" "${output}"`
pid=`echo ${pid_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
echo "ROOT Collection PID: ${pid}"
root_coll_pid=${pid}
root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
root_coll_url=`echo ${root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
echo "ROOT Collection URL: ${root_coll_url}"
new_collPath="${irods_home}/${testCollName}_new"
echo ""
echo "############ Moving collection ############"
imv ${collPath} ${new_collPath}
echo ""
echo "############ Updating collection PID ############"
http_url_raw=`irule "{getHttpApiParameters(*serverApireg, *serverApipub)}" null "*serverApireg"`
http_url=`echo ${http_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
newURL="${http_url}${new_collPath}"
irule "{EUDATeURLupdateColl(*pid, *newURL)}" "*pid=${pid}%*newURL=${newURL}" null
new_root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
new_root_coll_url=`echo ${new_root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
echo "NEW ROOT Collection URL: ${new_root_coll_url}"
irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}%*force=true" null
irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testFileName}%*force=true" null
irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}%*force=true" null
irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}/${testFileName}%*force=true" null
irm -rf ${new_collPath}
}
createPID
if [ -n "$REMOTE_ZONE" ]; then
replication
fi
moveCollection
irule "{EUDATePIDremove(*path, *force)}" "*path=${sourcePath}%*force=true" null
irm ${sourcePath}
#irm -rf ${collPath}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment