Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Robert Verkerk
B2SAFE Core
Commits
846a0413
Commit
846a0413
authored
Nov 21, 2018
by
Robert Verkerk
Browse files
Merge remote-tracking branch 'upstream/devel' into devel
parents
e7cfacb4
f29889ef
Changes
6
Hide whitespace changes
Inline
Side-by-side
cmd/pam_user_reader.py
0 → 100755
View file @
846a0413
#!/usr/bin/env python
#
# * use 4 spaces!!! not tabs
# * See PEP-8 Python style guide http://www.python.org/dev/peps/pep-0008/
# * use pylint
#
import
json
user_map_file
=
"/etc/irods/user_map.json"
def
parseUserMap
():
"""Parse the user map file"""
with
open
(
user_map_file
,
'r'
)
as
f
:
user_map
=
json
.
load
(
f
)
jstr
=
json
.
dumps
(
user_map
,
ensure_ascii
=
True
,
indent
=
4
)
print
(
jstr
)
if
__name__
==
"__main__"
:
parseUserMap
()
rulebase/eudat.re
View file @
846a0413
...
...
@@ -7,12 +7,14 @@
#
List
of
the
functions
:
#
#
----
logging
---
#
logVerbose
(
*
msg
)
#
logInfo
(
*
msg
)
#
logDebug
(
*
msg
)
#
logError
(
*
msg
)
#
logWithLevel
(
*
level
,
*
msg
)
#----
authorization
---
#
EUDATAuthZ
(
*
user
,
*
action
,
*
target
,
*
response
)
#
EUDATGetPAMusers
(
*
json_map
)
#----
utility
---
#
EUDATObjExist
(
*
path
,
*
response
)
#
EUDATPushMetadata
(
*
path
,
*
queue
)
...
...
@@ -747,6 +749,21 @@ EUDATcountMetaKeys( *Path, *Key, *Value ) {
logVerbose
(
"[EUDATcountMetaKeys] got count = *Value"
);
}
#
get
the
content
of
the
user
map
file
for
OAuth2
authentication
in
json
format
#
#
Parameters
:
#
*
json_map
[
OUT
]
a
string
representing
the
user
mapping
#
#
Author
:
Claudio
Cacciari
,
CINECA
#
-----------------------------------------------------------------------------
EUDATGetPAMusers
(
*
json_map
)
{
logVerbose
(
"[EUDATGetPAMusers] checking authorization for $userNameClient to read users"
);
EUDATAuthZ
(
$
userNameClient
,
"read"
,
"users"
,
*
response
);
msiExecCmd
(
"pam_user_reader.py"
,
"null"
,
"null"
,
"null"
,
"null"
,
*
outUsersJson
);
msiGetStdoutInExecCmdOut
(
*
outUsersJson
,
*
json_map
);
logVerbose
(
"[EUDATGetPAMusers] json user map = *json_map"
);
}
################################################################################
#
#
#
Repository
Packages
#
...
...
rulebase/local.re
View file @
846a0413
...
...
@@ -39,6 +39,20 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD
}
}
#
Provides
parameters
for
the
connection
with
the
EPIC
service
#
#
Arguments
:
#
*
serverApireg
[
OUT
]
the
endpoint
of
the
HTTP
API
interface
for
data
with
PIDs
#
*
serverApipub
[
OUT
]
the
endpoint
of
the
HTTP
API
interface
for
public
data
#
#
Author
:
Claudio
Cacciari
(
Cineca
)
#-------------------------------------------------------------------------------
getHttpApiParameters
(
*
serverApireg
,
*
serverApipub
)
{
*
serverApireg
=
"https://<hostnameWithFullDomain>/api/registered"
;
*
serverApipub
=
"https://<hostnameWithFullDomain>/api/public"
;
}
#
Parse
the
credentials
to
connect
to
an
EPIC
server
.
A
file
called
#
"credentials"
MUST
contain
all
the
connection
details
in
the
home
folder
of
#
the
user
running
this
rule
.
...
...
@@ -124,8 +138,8 @@ getConfParameters(*authzEnabled) {
#-------------------------------------------------------------------------------
getB2SAFEVersion
(
*
version
)
{
*
major_version
=
"4"
;
*
minor_version
=
"
1
"
;
*
sub_version
=
"
1
"
;
*
minor_version
=
"
2
"
;
*
sub_version
=
"
0
"
;
*
version
=
*
major_version
++
"."
++
*
minor_version
++
"-"
++
*
sub_version
;
}
...
...
rulebase/pid-service.re
View file @
846a0413
...
...
@@ -23,7 +23,7 @@
#
EUDATePIDsearch
(
*
field
,
*
value
,
*
PID
)
#
EUDATeCHECKSUMupdate
(
*
PID
,
*
path
)
#
EUDATeURLupdate
(
*
PID
,
*
newURL
)
#
EUDATeURL
search
(
*
PID
,
*
URL
)
#
EUDATeURL
updateColl
(
*
PID
,
*
new
URL
)
#
EUDATePIDremove
(
*
path
,
*
force
)
#
EUDATiRORupdate
(
*
source
,
*
pid
)
#
EUDATeRORupdate
(
*
pid
,*
newRor
)
...
...
@@ -148,8 +148,12 @@ EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) {
EUDATSearchPID
(
*
path
,
*
existing_pid
)
{
logDebug
(
"[EUDATSearchPID] search pid for *path"
);
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
);
getHttpApiParameters
(
*
serverApireg
,
*
serverApipub
);
EUDATReplaceHash
(
*
path
,
*
path1
);
*
status
=
EUDATePIDsearch
(
"URL"
,
"*serverID*path1"
,
*
existing_pid
);
*
status
=
EUDATePIDsearch
(
"URL"
,
"*serverApireg*path1"
,
*
existing_pid
);
if
(
!
(
*
status
))
{
*
status
=
EUDATePIDsearch
(
"URL"
,
"*serverApipub*path1"
,
*
existing_pid
);
}
logDebug
(
"[EUDATSearchPID] pid = *existing_pid"
);
*
status
;
}
...
...
@@ -163,13 +167,13 @@ EUDATSearchPID(*path, *existing_pid) {
#-------------------------------------------------------------------------------
EUDATSearchPIDchecksum
(
*
path
,
*
existing_pid
,
*
existing_url
)
{
logDebug
(
"[EUDATSearchPIDchecksum] search
pid
for *path"
);
logDebug
(
"[EUDATSearchPIDchecksum] search
ing checksum
for *path"
);
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
);
*
resource
=
""
;
EUDATiCHECKSUMget
(
*
path
,
*
checksum
,
*
modtime
,
*
resource
)
EUDATePIDsearch
(
"EUDAT/CHECKSUM"
,
*
checksum
,
*
existing_pid
);
EUDATeURLsearch
(
*
existing_pid
,
*
existing_url
);
*
existing_url
=
EUDATGeteValPid
(
*
existing_pid
,
"URL"
);
logDebug
(
"[EUDATSearchPIDchecksum] PID = *existing_pid, URL = *existing_url"
);
}
...
...
@@ -264,8 +268,36 @@ EUDATiPIDcreate(*path, *PID) {
EUDATePIDcreate
(
*
path
,
*
extraType
,
*
PID
)
{
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
)
;
getHttpApiParameters
(
*
serverApireg
,
*
serverApipub
);
*
url
=
"*serverID*path"
;
*
access
=
"private"
;
#
Verify
that
source
input
path
is
a
collection
msiGetObjType
(
*
path
,
*
type
);
if
(
*
type
!=
'
-
c'
)
{
msiSplitPath
(
*
path
,
*
parent
,
*
child
);
foreach
(
*
R
in
SELECT
USER_NAME
WHERE
DATA_NAME
=
'
*
child'
AND
COLL_NAME
=
'
*
parent'
)
{
if
(
*
R
.
USER_NAME
==
"anonymous"
)
{
*
access
=
"public"
;
break
;
}
}
}
else
{
foreach
(
*
R
in
SELECT
COLL_OWNER_NAME
WHERE
COLL_NAME
=
'
*
path'
)
{
if
(
*
R
.
COLL_OWNER_NAME
==
"anonymous"
)
{
*
access
=
"public"
;
break
;
}
}
}
if
(
*
access
==
"public"
)
{
*
url
=
"*serverApipub*path"
;
}
else
{
*
url
=
"*serverApireg*path"
;
}
*
resource
=
""
;
EUDATiCHECKSUMget
(
*
path
,
*
checksum
,
*
modtime
,
*
resource
);
if
(
*
checksum
!=
""
)
{
...
...
@@ -380,8 +412,8 @@ EUDATeURLupdate(*PID, *newURL) {
#
and
all
its
sub
-
collections
and
objects
.
#
#
Arguments
:
#
*
PID
[
IN
]
The
PID
associated
to
$
collName
#
*
newURL
[
IN
]
The
new
URL
to
be
associated
to
the
PID
of
$
collName
#
*
PID
[
IN
]
The
PID
associated
to
the
collection
#
*
newURL
[
IN
]
The
new
URL
to
be
associated
to
the
PID
of
the
collection
#
#
Author
:
Claudio
Cacciari
,
CINECA
#-------------------------------------------------------------------------------
...
...
@@ -389,10 +421,17 @@ EUDATeURLupdateColl(*PID, *newURL) {
logDebug
(
"[EUDATeURLupdateColl] updating collection *PID with URL *newURL and all its content"
);
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
);
EUDATeURLsearch
(
*
PID
,
*
oldURL
);
msiStrlen
(
*
serverID
,*
serverIDLength
);
msiSubstr
(
*
oldURL
,
str
(
int
(
*
serverIDLength
))
,
"null"
,
*
sourcePath
);
msiSubstr
(
*
newURL
,
str
(
int
(
*
serverIDLength
))
,
"null"
,
*
targetPath
);
getHttpApiParameters
(
*
serverApireg
,
*
serverApipub
);
*
oldURL
=
EUDATGeteValPid
(
*
PID
,
"URL"
);
if
(
*
oldURL
like
"*serverApireg
\
*"
)
{
*
serverApi
=
*
serverApireg
}
else
{
*
serverApi
=
*
serverApipub
}
msiStrlen
(
*
serverApi
,*
serverApiLength
);
msiSubstr
(
*
oldURL
,
str
(
int
(
*
serverApiLength
))
,
"null"
,
*
sourcePath
);
msiSubstr
(
*
newURL
,
str
(
int
(
*
serverApiLength
))
,
"null"
,
*
targetPath
);
msiStrlen
(
*
targetPath
,
*
pathLength
);
logDebug
(
"[EUDATeURLupdateColl] loop over the sub-collections of *targetPath"
);
...
...
@@ -401,7 +440,7 @@ EUDATeURLupdateColl(*PID, *newURL) {
msiSubstr
(
*
Row
.
COLL_NAME
,
str
(
int
(
*
pathLength
))
,
"null"
,
*
subCollection
);
*
sourceColl
=
"*sourcePath*subCollection"
if
(
EUDATSearchPID
(
*
sourceColl
,
*
existing_pid
))
{
*
targetColl
=
*
server
ID
++
*
Row
.
COLL_NAME
;
*
targetColl
=
*
server
Api
++
*
Row
.
COLL_NAME
;
EUDATeURLupdate
(
*
existing_pid
,
*
targetColl
);
}
}
...
...
@@ -413,29 +452,12 @@ EUDATeURLupdateColl(*PID, *newURL) {
msiSubstr
(
*
objPath
,
str
(
int
(
*
pathLength
))
,
"null"
,
*
subObjPath
);
*
sourceObj
=
"*sourcePath*subObjPath"
;
if
(
EUDATSearchPID
(
*
sourceObj
,
*
existing_pid
))
{
*
targetObj
=
*
server
ID
++
*
objPath
;
*
targetObj
=
*
server
Api
++
*
objPath
;
EUDATeURLupdate
(
*
existing_pid
,
*
targetObj
);
}
}
}
#
This
function
search
the
URL
field
of
the
PID
#
#
Arguments
:
#
*
PID
[
IN
]
The
PID
associated
to
$
objPath
#
*
newURL
[
IN
]
The
new
URL
to
be
associated
to
the
PID
#
#
Author
:
Giacomo
Mariani
,
CINECA
#-------------------------------------------------------------------------------
EUDATeURLsearch
(
*
PID
,
*
URL
)
{
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
);
logDebug
(
"[EUDATeURLsearch] search URL in PID *PID"
);
msiExecCmd
(
"epicclient.py"
,
"*credStoreType *credStorePath read *PID --key URL "
,
"null"
,
"null"
,
"null"
,
*
outEUS
);
msiGetStdoutInExecCmdOut
(
*
outEUS
,
*
URL
);
logDebug
(
"[EUDATeURLsearch] response = *URL"
);
}
#
This
function
remove
an
ePID
...
even
if
its
EUDAT
/
REPLICA
field
is
not
empty
!
#
To
be
improved
.
#
...
...
@@ -445,7 +467,7 @@ EUDATeURLsearch(*PID, *URL) {
#
Author
:
Giacomo
Mariani
,
CINECA
#-------------------------------------------------------------------------------
EUDATePIDremove
(
*
path
,
*
force
)
{
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
)
getEpicApiParameters
(
*
credStoreType
,
*
credStorePath
,
*
epicApi
,
*
serverID
,
*
epicDebug
)
;
logDebug
(
"[EUDATePIDremove] removing PID associated to: $userNameClient, *path"
);
if
(
EUDATSearchPID
(
*
path
,
*
pid
))
{
...
...
@@ -547,7 +569,7 @@ EUDATeFIOupdate(*pid, *newFio) {
#-----------------------------------------------------------------------------
EUDATPidsForColl
(
*
collPath
,
*
fixed
)
{
logInfo
(
"[EUDATPidsForColl] Creating PIDs for collection *collPath"
);
logInfo
(
"[EUDATPidsForColl] Creating PIDs for collection *collPath
and fixed = *fixed
"
);
#
Verify
that
source
input
path
is
a
collection
msiGetObjType
(
*
collPath
,
*
type
);
...
...
rulebase/replication.re
View file @
846a0413
...
...
@@ -10,15 +10,20 @@
#
List
of
the
functions
:
#
#
EUDATUpdateLogging
(
*
status_transfer_success
,
*
source
,
*
destination
,
*
cause
)
#
EUDATCheckIntegrity
(
*
source
,
*
source_res
,
*
destination
,
*
dest_res
,
*
logEnabled
,*
notification
,*
response
)
#
EUDATReplication
(
*
source
,
*
destination
,
*
dest_res
,
*
registered
,
*
recursive
)
#
EUDATCheckIntegrity
(
*
source
,
*
source_res
,
*
destination
,
*
dest_res
,
*
logEnabled
,
*
notification
,
*
response
)
#
EUDATCheckIntegrity
(
*
source
,
*
destination
,
*
logEnabled
,
*
notification
,
*
response
)
#
EUDATReplication
(
*
source
,
*
destination
,
*
dest_res
,
*
registered
,
*
recursive
,
*
response
)
#
EUDATReplication
(
*
source
,
*
destination
,
*
registered
,
*
recursive
,
*
response
)
#
EUDATTransferUsingFailLog
(
*
buffer_length
,
*
stats
)
#
EUDATRegDataRepl
(
*
source
,
*
destination
,
*
dest_res
,
*
recursive
,
*
response
)
#
EUDATRegDataRepl
(
*
source
,
*
destination
,
*
recursive
,
*
response
)
#
EUDATPIDRegistration
(
*
source
,
*
destination
,
*
notification
,
*
registration_response
)
#
EUDATSearchAndCreatePID
(
*
path
,
*
pid
)
#
EUDATSearchAndDefineField
(
*
path
,
*
pid
,
*
key
)
#
EUDATCheckIntegrityColl
(
*
sCollPesult
*
source_res
,
*
dCollPath
,
*
dest_res
,
*
logEnabled
,
*
response
)
#
EUDATCheckIntegrityColl
(
*
sCollPath
,
*
source_res
,
*
dCollPath
,
*
dest_res
,
*
logEnabled
,
*
response
)
#
EUDATCheckIntegrityColl
(
*
sCollPath
,
*
dCollPath
,
*
logEnabled
,
*
check_response
)
#
EUDATCheckIntegrityDO
(
*
source
,
*
source_res
,
*
destination
,
*
dest_res
,
*
logEnabled
,
*
response
)
#
EUDATCheckIntegrityDO
(
*
source
,
*
destination
,
*
logEnabled
,
*
response
)
#
Update
the
logging
files
specific
for
EUDAT
B2SAFE
...
...
scripts/tests/test_suite.sh
View file @
846a0413
...
...
@@ -21,6 +21,8 @@ fi
# Define test file name
testFileName
=
"test_data.txt"
testFileNameRemote
=
"test_data2.txt"
testCollName
=
"test_coll_root"
testCollNameSub
=
"test_coll_sub"
# Get user name and zone name from ienv
irods_user_name
=
`
ienv |
grep
irods_user_name |
cut
-d
'-'
-f
2 |
tr
-d
'[[:space:]]'
`
...
...
@@ -72,6 +74,35 @@ echo "############ Data Object ############"
ils
-l
${
testFileName
}
echo
""
# Define collection
collPath
=
"
${
irods_home
}
/
${
testCollName
}
"
# If exists, ask whether replace or exit
exists
=
`
ils
${
irods_home
}
|
grep
${
testCollName
}
`
if
[
!
-z
$exists
]
then
echo
"The collection
$collPath
already exists. Remove it before continuing (y or n)? Otherwise, script will exit."
read
shouldRemove
echo
"You entered
$shouldRemove
"
if
[
$shouldRemove
==
"y"
]
then
echo
"Will be removed"
irm
-rf
$collPath
else
echo
"Exiting..."
exit
1
fi
fi
imkdir
${
testCollName
}
imkdir
"
${
testCollName
}
/
${
testCollNameSub
}
"
echo
"Hello World!"
>
${
testFileName
}
iput
${
testFileName
}
${
testCollName
}
iput
${
testFileName
}
"
${
testCollName
}
/
${
testCollNameSub
}
"
rm
${
testFileName
}
echo
"############ Collection ############"
ils
-rl
${
testCollName
}
echo
""
createPID
()
{
rule
=
"{EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID)}"
...
...
@@ -105,8 +136,12 @@ createPID () {
replication
()
{
echo
""
echo
"############ REPLICATION ############"
# destPath="${irods_home}/test_data2.txt"
destPath
=
"/
${
REMOTE_ZONE
}
/home/
${
irods_user_name
}
#
${
irods_zone_name
}
/
${
testFileNameRemote
}
"
if
[
"
${
REMOTE_ZONE
}
"
==
"
${
irods_zone_name
}
"
]
then
destPath
=
"
${
irods_home
}
/
${
testFileNameRemote
}
"
else
destPath
=
"/
${
REMOTE_ZONE
}
/home/
${
irods_user_name
}
#
${
irods_zone_name
}
/
${
testFileNameRemote
}
"
fi
echo
"Replica path:
${
destPath
}
"
rule
=
"{*status = EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response);
if (*status) {
...
...
@@ -161,10 +196,53 @@ replication () {
irm
${
destPath
}
}
moveCollection
()
{
rule
=
"{EUDATPidsForColl(*collPath, *fixed)}"
input
=
"*collPath=
${
collPath
}
%*fixed=true"
echo
""
echo
"############ Collection PID creation ############"
echo
"Rule: irule
${
rule
}
${
input
}
null"
irule
"
${
rule
}
"
"
${
input
}
"
null
rule
=
"{EUDATSearchPID(*path, *existing_pid)}"
input
=
"*path=
${
collPath
}
"
output
=
"*existing_pid"
pid_raw
=
`
irule
"
${
rule
}
"
"
${
input
}
"
"
${
output
}
"
`
pid
=
`
echo
${
pid_raw
}
|
cut
-d
'='
-f
2 |
tr
-d
'[[:space:]]'
`
echo
"ROOT Collection PID:
${
pid
}
"
root_coll_pid
=
${
pid
}
root_coll_url_raw
=
`
irule
"{EUDATGeteValPid(*pid, *key)}"
"*pid=
${
root_coll_pid
}
%*key='URL'"
ruleExecOut
`
root_coll_url
=
`
echo
${
root_coll_url_raw
}
|
cut
-d
'='
-f
2 |
tr
-d
'[[:space:]]'
`
echo
"ROOT Collection URL:
${
root_coll_url
}
"
new_collPath
=
"
${
irods_home
}
/
${
testCollName
}
_new"
echo
""
echo
"############ Moving collection ############"
imv
${
collPath
}
${
new_collPath
}
echo
""
echo
"############ Updating collection PID ############"
http_url_raw
=
`
irule
"{getHttpApiParameters(*serverApireg, *serverApipub)}"
null
"*serverApireg"
`
http_url
=
`
echo
${
http_url_raw
}
|
cut
-d
'='
-f
2 |
tr
-d
'[[:space:]]'
`
newURL
=
"
${
http_url
}${
new_collPath
}
"
irule
"{EUDATeURLupdateColl(*pid, *newURL)}"
"*pid=
${
pid
}
%*newURL=
${
newURL
}
"
null
new_root_coll_url_raw
=
`
irule
"{EUDATGeteValPid(*pid, *key)}"
"*pid=
${
root_coll_pid
}
%*key='URL'"
ruleExecOut
`
new_root_coll_url
=
`
echo
${
new_root_coll_url_raw
}
|
cut
-d
'='
-f
2 |
tr
-d
'[[:space:]]'
`
echo
"NEW ROOT Collection URL:
${
new_root_coll_url
}
"
irule
"{EUDATePIDremove(*path, *force)}"
"*path=
${
new_collPath
}
%*force=true"
null
irule
"{EUDATePIDremove(*path, *force)}"
"*path=
${
new_collPath
}
/
${
testFileName
}
%*force=true"
null
irule
"{EUDATePIDremove(*path, *force)}"
"*path=
${
new_collPath
}
/
${
testCollNameSub
}
%*force=true"
null
irule
"{EUDATePIDremove(*path, *force)}"
"*path=
${
new_collPath
}
/
${
testCollNameSub
}
/
${
testFileName
}
%*force=true"
null
irm
-rf
${
new_collPath
}
}
createPID
if
[
-n
"
$REMOTE_ZONE
"
]
;
then
replication
fi
moveCollection
irule
"{EUDATePIDremove(*path, *force)}"
"*path=
${
sourcePath
}
%*force=true"
null
irm
${
sourcePath
}
#irm -rf ${collPath}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment