Unverified Commit 3e5a6a5b authored by ccacciari's avatar ccacciari Committed by GitHub
Browse files

Merge pull request #108 from EUDAT-B2SAFE/prace-integration

Added scripts for PRACE-B2ACCESS and for B2ACCESS-B2STAGE synchronization
parents 85903d4e dab0e6f9
# section containing the common options
[common]
#path to the log file
logfile=/var/log/irods_user_sync.log
# log level, possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
loglevel=DEBUG
#logging format
logformat=%(asctime)s %(levelname)s in %(module)s: %(message)s
#Max size of the log file
logmaxbytes=4194304
#number of log files to rotate
logbackupcnt=10
#temp directory to store info on expiring user authorisation data
expiration_tempdir=/tmp/irods_user_sync_exp
#period of time in which the authorisation data is assumed as valid (in seconds))
#longer period => less connections to B2ACCESS and iCAT and better performance
#shorter period => security information more up to date
expiration_period_sec=36
# configuration of B2ACCESS API bind
[B2ACCESS]
#base URL to access the API
base_url=https://b2access.eudat.eu/rest-admin/v1/
#API user name
username=PRACE_proxy
#file with API user password
password_file=~/.unity.pwd
#path to file or directory containing certificates used to verify the B2ACCESS server
#it is not mandatory, but recommended for security
cert_verify=/etc/ssl/certs/b2access.eudat.eu.pem
# configuration for parsing subject (DN) of the certificate issued by B2ACCESS
[usercert]
#pattern the DN shall match
dn_pattern=^CN=(.*),CN=(([a-f]|[0-9]|-)*),OU=B2ACCESS Intrgration,O=EUDAT,C=EU$
#number of regex match subgroup containing entity id
id_match=2
#identity type of the above id
id_type=persistent
#configuration of iRODS
[iRods]
#iRODS host
host=localhost
#iRODS port
port=1247
#iRODS admin user
rods_user=rods
#file with admin user password
rods_password_file=~/.irods.pwd
#zone of the admin user
rods_zone=mainZone
#zone of the common users
user_zone=mainZone
#type of the common user
user_type=rodsuser
#prefix of iRods account to be mapped to the user
#it will be concatenated with identity value
account_prefix=eudat_
#Unity identity type to be used for the concatenation
# -allowed values are: entityId, persistent
account_identity_type=persistent
#In case DN is mapped to an account named different than expected (computed according to the above rules),
#if this parameter is nonzero the script will replace the old mapping by the new one
replace_mapping=1
#B2ACCESS to iRods group mapping
#syntax:
#<unity_group>=<comma separated list of iRods groups>
#Note, that ':' in <unity_group> must be replaced by '//' while using Python 2.x
[groupmap]
/PRACE=EUDAT,prace
/eudat//b2safe=EUDAT,b2safe
/bitusejf=bitusejf
/A/B/C/de=abcde,EUDAT, zocha, krycha
# section containing the common options
[common]
#path to the log file
logfile=log/prace_eudat_users_sync.log
# log level, possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
loglevel=DEBUG
#logging format
logformat=%(asctime)s %(levelname)s in %(module)s: %(message)s
#Max size of the log file
logmaxbytes=4194304
#number of log files to rotate
logbackupcnt=10
# configuration of B2ACCESS API bind
[B2ACCESS]
#base URL to access the API
base_url=https://b2access.eudat.eu/rest-admin/v1/
#API user name
username=PRACE_proxy
#file with API user password
password_file=~/.unity.pwd
#path to file or directory containing certificates used to verify the B2ACCESS server
#it is not mandatory, but recommended for security
cert_verify=/etc/ssl/certs/b2access.eudat.eu.pem
[PRACE]
#Host with PRACE LDAP
host=ldap://ldap.prace.eu:389
#Bind DN to LDAP
binddn=cn=admin,dc=deisa,dc=org
#file with password to LDAP
password_file=~/.prace.pwd
#search base
searchbase=dc=deisa,dc=org
#attribute containing DN from user's certificate
usercertdn=deisaSubjectDN
#attribure contaning user id
userid=uid
#attribute containing user name
username=cn
#MAP SECTIONS
# The sections names must be MAP*
# They are analysed in alphabetical order.
# A single section maps users from prace_searchbase LDAP subtree,
# filtered by prace_userfilter to possibly many eudat_groups
# Note, that a member of /X/Y EUDAT subgrop must be also a member of /X group
[MAP_1]
eudat_groups=/PRACE
prace_searchbase=dc=deisa,dc=org
prace_userfilter=(deisaUserProfile=eudat)
[MAP_2]
eudat_groups=/PRACE/inactive
prace_searchbase=dc=deisa,dc=org
prace_userfilter=(&(!(praceAccountStatus=active))(deisaUserProfile=eudat))
[MAP_3]
eudat_groups=/PRACE/psnc2 /PRACE/psnc
prace_searchbase=ou=psnc.pl,ou=ua,dc=deisa,dc=org
prace_userfilter=(deisaUserProfile=eudat2)
[MAP_4]
eudat_groups=/PRACE/active
prace_searchbase=dc=deisa,dc=org
prace_userfilter=(&(praceAccountStatus=active)(deisaUserProfile=eudat))
#!/usr/bin/env python
# -*- python -*-
##################################
# Michal Jankowski PSNC
# EUDAT-PRACE integration
# 03.2017
##################################
import sys
import traceback
import argparse
import ConfigParser
import logging
import re
from utilities.drivers.unity_api import *
from utilities.drivers.irods_api_facade import *
from utilities.conf_logger import *
from utilities.expiration_guard import *
logger = logging.getLogger(__name__)
class IrodsUserSync:
def __init__(self):
"""
Initialize the object.
Anything fails here, the script must return an error.
The error is printed as logging not configured yet.
"""
try:
"""
Parse arguments --------------------------------------------------
"""
argParser = argparse.ArgumentParser(description='Synchronizing single user between B2ACCESS and iRods')
argParser.add_argument('-c', '--config', dest='conf', default='conf/irods_user_sync.conf', help='path to the configuration file')
argParser.add_argument('-d', '--dn', dest='dn', required=True, help='DN from user certificate')
argParser.add_argument('-dr', '--dry-run', dest='dryRunFlag', action='store_true', help='do not modify iRODS if set, only print output')
arguments = argParser.parse_args()
self.dn = arguments.dn
self.dryRun = arguments.dryRunFlag
"""
Read and parse config file ---------------------------------------
"""
self.config = ConfigParser.ConfigParser()
self.config.optionxform=str
self.config.readfp(open(arguments.conf))
"""
Configure and start logging
"""
self.logger = logger
configureLogger(self.logger, self.config, 'common')
self.logger.info('Script started #################################################################')
self.logger.info('User '+self.dn)
except IOError, e:
print 'Cannot read config file.'
print e.message
sys.exit(1)
except ConfigParser.Error, e:
print 'Configuration error.'
print e.message
sys.exit(1)
except Exception, e:
tbck = traceback.format_exc()
print tbck
print e.message
sys.exit(1)
def main(self):
"""
Synchronizing user between B2SAFE and IRods
"""
try:
#check if the user authorization info is up to date, if so, there is no need to go on
guard=ExpirationGuard(self.dn,
self.config.get('common','expiration_tempdir'),
self.config.get('common','expiration_period_sec'),
parentLogger=self.logger)
if guard.expired() :
#get Unity entity and groups related to the user
unityEntity,unityEntityGroups = self.__getUnityData()
#process group mapping
irodsGroupsMember, irodsGroupsNonMember = self.__processGroupMaps(unityEntity,unityEntityGroups)
#process iRODS user
#the user is authorized if member of any iRods group: bool(irodsGroupsMember)
self.__processIrodsUser(self.dn, bool(irodsGroupsMember),irodsGroupsMember, irodsGroupsNonMember, unityEntity)
#user info is refreshed, refresh the guard
guard.refresh()
except ConfigParser.Error, e:
self.logger.error(e.message)
print e.message
print 'Configuration error.'
sys.exit(1)
except UnityApiException, e:
self.logger.error(e.message)
print e.message
print 'Unity API error.'
sys.exit(1)
except IrodsApiException, e:
self.logger.error(e.message)
print e.message
print 'iRODS API error.'
sys.exit(1)
except Exception, e:
self.logger.debug(traceback.format_exc())
self.logger.error(e.message)
print e.message
print 'Error.'
sys.exit(1)
print 'Success.'
self.logger.info('Finished with success')
sys.exit(0)
def __getUnityData(self):
"""
Get data from B2ACCESS -if the user exists in the service and belongs to at least one group..
Handle exceptions -in case of error assume the user is not authorized and try to communicate it to iRODS
Returns:
Info on Unity entity
List of Unity groups the user is member of
"""
"""
Change the format of dn suitable for Unity API
"""
unityDn = ','.join(reversed(self.dn.split('/')))[:-1]
self.logger.debug('DN suitable for Unity : '+ unityDn )
"""
Determine if the certificate issuer is B2ACCESS
Then set unityId and unityIdType depending on the issuer
"""
dnPattern=self.config.get('usercert','dn_pattern')
self.logger.debug('Checking DN '+unityDn+' against pattern '+dnPattern)
matchObj = re.match(dnPattern, unityDn, re.I)
if matchObj:
self.logger.info('The certificate is issued by B2ACCESS.')
unityId=matchObj.group(int(self.config.get('usercert','id_match')))
unityIdType=self.config.get('usercert','id_type')
self.logger.debug('User '+unityIdType+' = '+unityId)
else:
self.logger.info('The certificate is NOT issued by B2ACCESS.')
unityId=unityDn
unityIdType='x500Name'
"""
Get data from B2ACCESS
"""
# init connection
unityApi = UnityApi(self.config, confSection='B2ACCESS', parentLogger=self.logger)
# get entity
unityEntity = unityApi.getEntity(unityId, unityIdType)
if unityEntity is None or unityEntity['entityInformation']['state'] != 'valid':
self.logger.info('User not authorized by B2ACCESS.')
return None, []
# get entity groups
unityEntityGroups = unityApi.getEntityGroups(unityId, unityIdType)
self.logger.debug(' User belongs to B2ACCESS groups : '+str(unityEntityGroups))
return unityEntity, unityEntityGroups
def __getExpectedIrodsUname(self, unityEntity):
"""
Compute irods expected user name.
Args :
unityEntity: info on Unity entity
Returns:
expected user name
"""
if unityEntity is None :
return None
try:
expectedUname = self.config.get('iRods','account_prefix')
if self.config.get('iRods','account_identity_type') == 'entityId' :
expectedUname += str(unityEntity['entityInformation']['entityId'])
elif self.config.get('iRods','account_identity_type') == 'persistent':
persistentId=None
for identity in unityEntity['identities']:
if identity['typeId'] == 'persistent':
persistentId = identity['value']
break
if persistentId is None :
raise Exception('No persistent identity for user '+self.dn)
expectedUname += persistentId
else:
raise Exception('"account_identity" incorrectly configured')
self.logger.debug('Expected iRods user name is '+expectedUname)
return expectedUname
except Exception, e:
self.logger.debug(traceback.format_exc())
self.logger.error(e.message)
print e.message
raise Exception('Cannot determine expected iRODS user name.')
def __processGroupMaps(self,unityEntity,unityEntityGroups):
"""
Process group maps by comparing Unity groups the user belongs with groupmap configuration.
Args:
unityEntity: info on Unity entity
unityEntityGroups: list of Unity groups the user is member
Returns:
List of iRODS groups the user must be a member of (groupmap intersect unityEntityGroups)
iRODS groups the user must not be a member of (groupmap diff unityEntityGroups diff irodsGroupsMember)
"""
irodsGroupsMember=set()
irodsGroupsNonMember=set()
if unityEntity is None or not unityEntityGroups:
for unityGroup,irodsGroupsStr in self.config.items('groupmap') :
irodsGroupsNonMember|=set([group.strip() for group in irodsGroupsStr.split(',')])
else:
for unityGroup,irodsGroupsStr in self.config.items('groupmap') :
irodsGroups=[group.strip() for group in irodsGroupsStr.split(',')]
if unityGroup.replace('//',':') in unityEntityGroups : #workaround for Python 2.x where ':' (often used in Unity group names) is a hardcoded delimiter
irodsGroupsMember|=set(irodsGroups)
else:
irodsGroupsNonMember|=set(irodsGroups)
irodsGroupsNonMember-=irodsGroupsMember
self.logger.debug(' User belongs to iRODS groups : '+str(irodsGroupsMember))
self.logger.debug(' User does not belong to iRODS groups : '+str(irodsGroupsNonMember))
return irodsGroupsMember, irodsGroupsNonMember
def __processIrodsUser(self, dn, isAuthorized, irodsGroupsMember, irodsGroupsNonMember, unityEntity):
"""
Use existing iRods user and mapping or create them, providing the user is authorized
return iRODS user name and info if the user has existed before
Note: do not throw an exception on iRods operation failure -instead try to continue and
perform as many authorization related operations as possible. TODO: this could be configurable
Args:
dn: user's distinguished name'
isAuthorized: true if the user shall be authorized
expectedIrodsUname: expected iRODS user name
"""
#initialize iRODS interface
irodsFacade=IrodsApiFacade(self.config, confSection='iRods', parentLogger=self.logger)
irodsUname=irodsFacade.getUserName(dn)
irodsFacade.userExists(irodsUname)
if not isAuthorized and irodsUname is None:
self.logger.info("DN "+dn+" not authorized by B2ACCESS and not found in iRODS.")
return
#get user's groups
irodsGroupsAlreadyMember = irodsFacade.getUserGroups(irodsUname)
if not isAuthorized :
self.logger.info("DN "+dn+" not authorized by B2ACCESS, but mapping to "+irodsUname+" found in iRODS.")
irodsFacade.removeUserAuth(irodsUname,dn)
for irodsGname in irodsGroupsNonMember:
if irodsGname in irodsGroupsAlreadyMember : irodsFacade.removeUserFromGroup(irodsUname,irodsGname)
return
#compute expected iRODS user name
expectedIrodsUname=self.__getExpectedIrodsUname(unityEntity)
if irodsUname is None:
#DN mapping and possibly iRODS account are missing
irodsUname = expectedIrodsUname
if not irodsFacade.userExists(irodsUname) : irodsFacade.createUser(irodsUname)
irodsFacade.addUserAuth(irodsUname, dn)
elif expectedIrodsUname != irodsUname:
#DN is mapped to an account named different than expected
if self.config.get('iRods','replace_mapping') != 0 :
self.logger.warning("DN " +dn+ " already mapped to " +str(irodsUname) +', while expected to ' +expectedIrodsUname + ', use the new mapping.')
irodsFacade.removeUserAuth(irodsUname,dn)
irodsFacade.addUserAuth(expectedIrodsUname,dn)
irodsUname=expectedIrodsUname
else:
self.logger.warning("DN " +dn+ " already mapped to " +str(irodsUname) +', while expected to ' +expectedIrodsUname + ', use the old mapping.')
else:
#existing DN mapping is ok
self.logger.info("DN " +dn+ " already mapped to " +irodsUname)
for irodsGname in irodsGroupsNonMember:
if irodsGname in irodsGroupsAlreadyMember: irodsFacade.removeUserFromGroup(irodsUname,irodsGname)
for irodsGname in irodsGroupsMember:
if irodsGname not in irodsGroupsAlreadyMember: irodsFacade.addUserToGroup(irodsUname,irodsGname)
return
if __name__ == '__main__':
IrodsUserSync().main()
#!/usr/bin/env python
# -*- python -*-
##################################
# Michal Jankowski PSNC
# EUDAT-PRACE integration
# 03.2017
##################################
import sys
import traceback
import argparse
import ConfigParser
import logging
import logging.handlers
from pprint import pformat
from utilities.drivers.unity_api import *
from utilities.drivers.prace_ldap import *
from utilities.conf_logger import *
logger = logging.getLogger(__name__)
class PraceEudatUsersSync:
def __init__(self):
"""initialize the object"""
self.logger = logger
def main(self):
"""
Synchronizing users between PRACE and EUDAT
"""
try:
""" Parse arguments """
argParser = argparse.ArgumentParser(description='Synchronizing users between PRACE and EUDAT')
argParser.add_argument('-c', '--config', dest='conf', default='conf/prace_eudat_users_sync.conf', help='path to the configuration file')
arguments = argParser.parse_args()
""" read and parse config file """
self.config = ConfigParser.ConfigParser()
self.config.readfp(open(arguments.conf))
self.maps={}
for section in self.config.sections():
if section.startswith('MAP_') :
self.maps[section] = ({k:v for k,v in self.config.items(section)})
"""
Configure and start logging
"""
self.logger = logger
configureLogger(self.logger, self.config, 'common')
self.logger.info('Script started #################################################################')
""" init connection to B2ACCESS """
unityApi = UnityApi(self.config, confSection='B2ACCESS', parentLogger=self.logger)
""" init connection to PRACE LDAP """
praceLdap = PraceLdap(self.config, confSection='PRACE', parentLogger=self.logger)
""" synchronize """
for mapName in sorted(self.maps) :
self._syncOneMap(mapName, praceLdap, self.maps[mapName]['prace_searchbase'], self.maps[mapName]['prace_userfilter'], unityApi, self.maps[mapName]['eudat_groups'].split())
print 'Success.'
self.logger.info('Finished with success')
sys.exit(0)
except ConfigParser.Error, e:
tbck = traceback.format_exc()
self.logger.error(tbck)
self.logger.error(e.message)
print tbck
print 'Configuration error.'
print e.message
except Exception, e:
tbck = traceback.format_exc()
self.logger.error(tbck)
self.logger.error(e.message)
print tbck
print e.message
print 'Failure.'
sys.exit(1)
def _syncOneMap(self, mapName, praceLdap, praceSearchBase, praceUserFilter, unityApi, groupNames):
"""
performs synchronization for single map section
"""
""" get users from PRACE """
usersInPrace = praceLdap.getUsers(praceSearchBase, praceUserFilter)
for groupName in groupNames :
""" get eudat group from B2ACCESS """
self.logger.info("Performing synchronization for section "+mapName+" and group "+groupName)
group = unityApi.getGroup(groupName)
if group is None :
""" create the group in B2ACCESS if necessary """
self.logger.info('Adding group ' + groupName +'.')
unityApi.createGroup(groupName)
group = {'members': [], 'subGroups': []}
groupMembers = group['members']
groupMemberIds = [member['entityId'] for member in groupMembers]
self.logger.debug('PRACE users ids in B2ACCESS group ' + groupName + ": " + pformat(groupMemberIds))
""" loop over selected PRACE users """
for userInPrace in usersInPrace :
""" look for the user in B2ACCESS """
userInEudat = unityApi.getEntity(userInPrace['certdn'], 'x500Name')
if userInEudat is None :