Added scripts for PRACE-B2ACCESS synchronization and for B2ACCESS-B2STAGE synchronization.

scripts/authN_and_authZ/conf/irods_user_sync.conf_example

+# section containing the common options
+[common]
+
+#path to the log file
+logfile=/var/log/irods_user_sync.log
+# log level, possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
+loglevel=DEBUG
+#logging format
+logformat=%(asctime)s %(levelname)s in %(module)s: %(message)s
+#Max size of the log file
+logmaxbytes=4194304
+#number of log files to rotate
+logbackupcnt=10
+#temp directory to store info on expiring user authorisation data
+expiration_tempdir=/tmp/irods_user_sync_exp
+#period of time in which the authorisation data is assumed as valid (in seconds))
+#longer period => less connections to B2ACCESS and iCAT and better performance
+#shorter period => security information more up to date
+expiration_period_sec=36
+
+# configuration of B2ACCESS API bind
+[B2ACCESS]
+
+#base URL to access the API
+base_url=https://b2access.eudat.eu/rest-admin/v1/
+#API user name
+username=PRACE_proxy
+#file with API user password
+password_file=~/.unity.pwd
+#path to file or directory containing certificates used to verify the B2ACCESS server
+#it is not mandatory, but recommended for security
+cert_verify=/etc/ssl/certs/b2access.eudat.eu.pem
+
+# configuration for parsing subject (DN) of the certificate issued by B2ACCESS
+[usercert]
+
+#pattern the DN shall match
+dn_pattern=^CN=(.*),CN=(([a-f]|[0-9]|-)*),OU=B2ACCESS Intrgration,O=EUDAT,C=EU$
+#number of regex match subgroup containing entity id
+id_match=2
+#identity type of the above id
+id_type=persistent
+
+#configuration of iRODS
+[iRods]
+
+#iRODS host
+host=localhost
+#iRODS port
+port=1247
+#iRODS admin user
+rods_user=rods
+#file with admin user password
+rods_password_file=~/.irods.pwd
+#zone of the admin user
+rods_zone=mainZone
+#zone of the common users
+user_zone=mainZone
+#type of the common user
+user_type=rodsuser
+#prefix of iRods account to be mapped to the user
+#it will be concatenated with identity value
+account_prefix=eudat_
+#Unity identity type to be used for the concatenation
+# -allowed values are: entityId, persistent
+account_identity_type=persistent
+#In case DN is mapped to an account named different than expected (computed according to the above rules),
+#if this parameter is nonzero the script will replace the old mapping by the new one
+replace_mapping=1
+
+#B2ACCESS to iRods group mapping
+#syntax:
+#<unity_group>=<comma separated list of iRods groups>
+#Note, that ':' in <unity_group> must be replaced by '//' while using Python 2.x
+[groupmap]
+/PRACE=EUDAT,prace
+/eudat//b2safe=EUDAT,b2safe
+/bitusejf=bitusejf
+/A/B/C/de=abcde,EUDAT,     zocha,		krycha

scripts/authN_and_authZ/conf/prace_eudat_users_sync.conf_example

+# section containing the common options
+[common]
+
+#path to the log file
+logfile=log/prace_eudat_users_sync.log
+# log level, possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
+loglevel=DEBUG
+#logging format
+logformat=%(asctime)s %(levelname)s in %(module)s: %(message)s
+#Max size of the log file
+logmaxbytes=4194304
+#number of log files to rotate
+logbackupcnt=10
+
+# configuration of B2ACCESS API bind
+[B2ACCESS]
+
+#base URL to access the API
+base_url=https://b2access.eudat.eu/rest-admin/v1/
+#API user name
+username=PRACE_proxy
+#file with API user password
+password_file=~/.unity.pwd
+#path to file or directory containing certificates used to verify the B2ACCESS server
+#it is not mandatory, but recommended for security
+cert_verify=/etc/ssl/certs/b2access.eudat.eu.pem
+
+[PRACE]
+#Host with PRACE LDAP
+host=ldap://ldap.prace.eu:389
+#Bind DN to LDAP
+binddn=cn=admin,dc=deisa,dc=org
+#file with password to LDAP
+password_file=~/.prace.pwd
+#search base
+searchbase=dc=deisa,dc=org
+#attribute containing DN from user's certificate
+usercertdn=deisaSubjectDN
+#attribure contaning user id
+userid=uid
+#attribute containing user name
+username=cn
+
+#MAP SECTIONS
+# The sections names must be MAP*
+# They are analysed in alphabetical order.
+# A single section maps users from prace_searchbase LDAP subtree,
+# filtered by prace_userfilter to possibly many eudat_groups
+# Note, that a member of /X/Y EUDAT subgrop must be also a member of /X group
+[MAP_1]
+eudat_groups=/PRACE
+prace_searchbase=dc=deisa,dc=org
+prace_userfilter=(deisaUserProfile=eudat)
+
+[MAP_2]
+eudat_groups=/PRACE/inactive
+prace_searchbase=dc=deisa,dc=org
+prace_userfilter=(&(!(praceAccountStatus=active))(deisaUserProfile=eudat))
+
+[MAP_3]
+eudat_groups=/PRACE/psnc2 /PRACE/psnc
+prace_searchbase=ou=psnc.pl,ou=ua,dc=deisa,dc=org
+prace_userfilter=(deisaUserProfile=eudat2)
+
+[MAP_4]
+eudat_groups=/PRACE/active
+prace_searchbase=dc=deisa,dc=org
+prace_userfilter=(&(praceAccountStatus=active)(deisaUserProfile=eudat))

scripts/authN_and_authZ/irods_user_sync.py

+#!/usr/bin/env python
+# -*- python -*-
+
+##################################
+# Michal Jankowski PSNC
+# EUDAT-PRACE integration
+# 03.2017
+##################################
+
+import sys
+import traceback
+
+import argparse
+import ConfigParser
+import logging
+import re
+
+from utilities.drivers.unity_api import *
+from utilities.drivers.irods_api_facade import *
+from utilities.conf_logger import *
+from utilities.expiration_guard import *
+
+
+logger = logging.getLogger(__name__) 
+
+
+class IrodsUserSync:
+	
+	def __init__(self):
+		"""
+		Initialize the object.
+		Anything fails here, the script must return an error.
+		The error is printed as logging not configured yet.
+		"""
+		
+		try:
+			""" 
+			Parse arguments --------------------------------------------------
+			"""
+			argParser = argparse.ArgumentParser(description='Synchronizing single user between B2ACCESS and iRods')
+			argParser.add_argument('-c', '--config', dest='conf',	default='conf/irods_user_sync.conf', help='path to the configuration file')
+			argParser.add_argument('-d', '--dn', dest='dn',	required=True, help='DN from user certificate')
+			argParser.add_argument('-dr', '--dry-run', dest='dryRunFlag',	action='store_true', help='do not modify iRODS if set, only print output')
+			arguments = argParser.parse_args()
+			
+			self.dn = arguments.dn
+			self.dryRun = arguments.dryRunFlag
+
+			"""
+			Read and parse config file ---------------------------------------
+			"""
+			self.config = ConfigParser.ConfigParser()
+			self.config.optionxform=str
+			self.config.readfp(open(arguments.conf))
+			
+			""" 
+			Configure and start logging
+			"""
+			self.logger = logger
+			configureLogger(self.logger, self.config, 'common')
+			self.logger.info('Script started #################################################################')
+			self.logger.info('User '+self.dn)
+			
+		except IOError, e:
+			print 'Cannot read config file.'
+			print e.message
+			sys.exit(1)
+			
+		except ConfigParser.Error, e:
+			print 'Configuration error.'
+			print e.message
+			sys.exit(1)
+		
+		except Exception, e:
+			tbck = traceback.format_exc()
+			print tbck
+			print e.message
+			sys.exit(1)
+
+
+	def main(self):
+		"""
+		Synchronizing user between B2SAFE and IRods
+		"""
+	
+		try:
+			#check if the user authorization info is up to date, if so, there is no need to go on
+			guard=ExpirationGuard(self.dn, 
+														self.config.get('common','expiration_tempdir'), 
+														self.config.get('common','expiration_period_sec'),
+														parentLogger=self.logger)
+			if guard.expired() :
+		
+				#get Unity entity and groups related to the user
+				unityEntity,unityEntityGroups = self.__getUnityData()
+				
+				#process group mapping
+				irodsGroupsMember, irodsGroupsNonMember = self.__processGroupMaps(unityEntity,unityEntityGroups)
+				
+				#process iRODS user
+				#the user is authorized if member of any iRods group: bool(irodsGroupsMember)
+				self.__processIrodsUser(self.dn, bool(irodsGroupsMember),irodsGroupsMember, irodsGroupsNonMember, unityEntity)
+				
+				#user info is refreshed, refresh the guard
+				guard.refresh()
+			
+		except ConfigParser.Error, e:
+			self.logger.error(e.message)
+			print e.message
+			print 'Configuration error.'
+			sys.exit(1)
+			
+		except UnityApiException, e:
+			self.logger.error(e.message)
+			print e.message
+			print 'Unity API error.'
+			sys.exit(1)
+			
+		except IrodsApiException, e:
+			self.logger.error(e.message)
+			print e.message
+			print 'iRODS API error.'
+			sys.exit(1)
+			
+		except Exception, e:
+			self.logger.debug(traceback.format_exc())
+			self.logger.error(e.message)
+			print e.message
+			print 'Error.'
+			sys.exit(1)
+		
+		print 'Success.'
+		self.logger.info('Finished with success')
+		sys.exit(0)
+	
+	
+		
+	def __getUnityData(self):
+		"""
+		Get data from B2ACCESS -if the user exists in the service and belongs to at least one group..
+		Handle exceptions -in case of error assume the user is not authorized and try to communicate it to iRODS
+		
+		Returns:
+			Info on Unity entity
+			List of Unity groups the user is member of
+		"""
+		
+		"""
+		Change the format of dn suitable for Unity API
+		"""
+		unityDn = ','.join(reversed(self.dn.split('/')))[:-1]
+		self.logger.debug('DN suitable for Unity : '+ unityDn )
+		
+		"""
+		Determine if the certificate issuer is B2ACCESS
+		Then set unityId and unityIdType depending on the issuer
+		"""
+		dnPattern=self.config.get('usercert','dn_pattern')
+		self.logger.debug('Checking DN '+unityDn+' against pattern '+dnPattern)
+		matchObj = re.match(dnPattern, unityDn, re.I)
+		if matchObj:
+			self.logger.info('The certificate is issued by B2ACCESS.')
+			unityId=matchObj.group(int(self.config.get('usercert','id_match')))
+			unityIdType=self.config.get('usercert','id_type')
+			self.logger.debug('User '+unityIdType+' = '+unityId)
+		else:
+			self.logger.info('The certificate is NOT issued by B2ACCESS.')
+			unityId=unityDn
+			unityIdType='x500Name'
+		
+		"""
+		Get data from B2ACCESS
+		"""
+		# init connection
+		unityApi = UnityApi(self.config, confSection='B2ACCESS', parentLogger=self.logger)
+		# get entity
+		unityEntity = unityApi.getEntity(unityId, unityIdType)
+		if unityEntity is None or unityEntity['entityInformation']['state'] != 'valid':
+			self.logger.info('User not authorized by B2ACCESS.')
+			return None, []
+			
+		# get entity groups
+		unityEntityGroups = unityApi.getEntityGroups(unityId, unityIdType)
+		self.logger.debug(' User belongs to B2ACCESS groups : '+str(unityEntityGroups))
+		
+		return unityEntity, unityEntityGroups
+
+
+	
+	def __getExpectedIrodsUname(self, unityEntity):
+		""" 
+		Compute irods expected user name.
+		
+		Args :
+			unityEntity: info on Unity entity
+			
+		Returns:
+			expected user name
+		"""
+		if unityEntity is None :
+			return None
+			
+		try:
+			expectedUname = self.config.get('iRods','account_prefix')
+			if self.config.get('iRods','account_identity_type') == 'entityId' :
+				expectedUname += str(unityEntity['entityInformation']['entityId'])
+			elif self.config.get('iRods','account_identity_type') == 'persistent':
+				persistentId=None
+				for identity in unityEntity['identities']:	
+					if identity['typeId'] == 'persistent':
+						persistentId = identity['value']
+						break
+				if persistentId is None :
+					raise Exception('No persistent identity for user '+self.dn)
+				expectedUname +=	persistentId
+			else:
+				raise Exception('"account_identity" incorrectly configured')
+				
+			self.logger.debug('Expected iRods user name is '+expectedUname)
+			
+			return expectedUname
+			
+			
+		except Exception, e:
+			self.logger.debug(traceback.format_exc())
+			self.logger.error(e.message)
+			print e.message
+			raise Exception('Cannot determine expected iRODS user name.')
+
+	
+	def __processGroupMaps(self,unityEntity,unityEntityGroups):
+		"""
+		Process group maps by comparing Unity groups the user belongs with groupmap configuration.
+		
+		Args:
+			unityEntity: info on Unity entity
+			unityEntityGroups: list of Unity groups the user is member
+			
+		Returns:
+			List of iRODS groups the user must be a member of (groupmap intersect unityEntityGroups)
+			iRODS groups the user must not be a member of (groupmap diff unityEntityGroups diff irodsGroupsMember) 
+		"""
+		
+		irodsGroupsMember=set()
+		irodsGroupsNonMember=set()
+		
+		if unityEntity is None or not unityEntityGroups:
+			for unityGroup,irodsGroupsStr in self.config.items('groupmap') :
+				irodsGroupsNonMember|=set([group.strip() for group in irodsGroupsStr.split(',')])
+		else:
+			for unityGroup,irodsGroupsStr in self.config.items('groupmap') :
+				irodsGroups=[group.strip() for group in irodsGroupsStr.split(',')] 
+				if unityGroup.replace('//',':') in unityEntityGroups : #workaround for Python 2.x where ':' (often used in Unity group names) is a hardcoded delimiter
+					irodsGroupsMember|=set(irodsGroups)
+				else:
+					irodsGroupsNonMember|=set(irodsGroups)
+			irodsGroupsNonMember-=irodsGroupsMember
+			
+		self.logger.debug(' User belongs to iRODS groups : '+str(irodsGroupsMember))
+		self.logger.debug(' User does not belong to iRODS groups : '+str(irodsGroupsNonMember))
+			
+		return irodsGroupsMember, irodsGroupsNonMember
+
+	def __processIrodsUser(self, dn, isAuthorized, irodsGroupsMember, irodsGroupsNonMember, unityEntity):
+		"""
+		Use existing iRods user and mapping or create them, providing the user is authorized
+		return iRODS user name and info if the user has existed before 
+		Note: do not throw an exception on iRods operation failure -instead try to continue and 
+		perform as many authorization related operations as possible. TODO: this could be configurable
+		
+		Args:
+			dn: user's distinguished name'
+			isAuthorized: true if the user shall be authorized
+			expectedIrodsUname: expected iRODS user name
+		"""
+		
+		#initialize iRODS interface
+		irodsFacade=IrodsApiFacade(self.config, confSection='iRods', parentLogger=self.logger)
+		
+		irodsUname=irodsFacade.getUserName(dn)
+		irodsFacade.userExists(irodsUname)
+		
+		if not isAuthorized and irodsUname is None:
+			self.logger.info("DN "+dn+" not authorized by B2ACCESS and not found in iRODS.")
+			return
+		
+		#get user's groups
+		irodsGroupsAlreadyMember = irodsFacade.getUserGroups(irodsUname)
+
+		if not isAuthorized :
+			self.logger.info("DN "+dn+" not authorized by B2ACCESS, but mapping to "+irodsUname+" found in iRODS.")
+			irodsFacade.removeUserAuth(irodsUname,dn)
+			for irodsGname in irodsGroupsNonMember:
+				if irodsGname in irodsGroupsAlreadyMember : irodsFacade.removeUserFromGroup(irodsUname,irodsGname)
+			return
+		
+		#compute expected iRODS user name
+		expectedIrodsUname=self.__getExpectedIrodsUname(unityEntity)
+		
+		if irodsUname is None:
+			#DN mapping and possibly iRODS account are missing
+			irodsUname = expectedIrodsUname
+			if not irodsFacade.userExists(irodsUname) : irodsFacade.createUser(irodsUname)
+			irodsFacade.addUserAuth(irodsUname, dn)
+		elif expectedIrodsUname != irodsUname:
+			#DN is mapped to an account named different than expected
+			if self.config.get('iRods','replace_mapping') != 0 :
+				self.logger.warning("DN " +dn+ " already mapped to " +str(irodsUname) +', while expected to ' +expectedIrodsUname + ', use the new mapping.')
+				irodsFacade.removeUserAuth(irodsUname,dn)
+				irodsFacade.addUserAuth(expectedIrodsUname,dn)
+				irodsUname=expectedIrodsUname
+			else:
+				self.logger.warning("DN " +dn+ " already mapped to " +str(irodsUname) +', while expected to ' +expectedIrodsUname + ', use the old mapping.')
+		else:
+			#existing DN mapping is ok
+			self.logger.info("DN " +dn+ " already mapped to " +irodsUname)
+		
+		for irodsGname in irodsGroupsNonMember: 
+			if irodsGname in irodsGroupsAlreadyMember: irodsFacade.removeUserFromGroup(irodsUname,irodsGname)
+		for irodsGname in irodsGroupsMember:
+			if irodsGname not in irodsGroupsAlreadyMember: irodsFacade.addUserToGroup(irodsUname,irodsGname)
+	
+		return
+
+
+if __name__ == '__main__':
+	IrodsUserSync().main()

scripts/authN_and_authZ/prace_eudat_users_sync.py

+#!/usr/bin/env python
+# -*- python -*-
+
+##################################
+# Michal Jankowski PSNC
+# EUDAT-PRACE integration
+# 03.2017
+##################################
+
+import sys
+
+import traceback
+
+import argparse
+import ConfigParser
+import logging
+import logging.handlers 
+from pprint import pformat
+
+from utilities.drivers.unity_api import *
+from utilities.drivers.prace_ldap import *
+from utilities.conf_logger import *
+
+logger = logging.getLogger(__name__) 
+
+
+
+class PraceEudatUsersSync:
+	
+	def __init__(self):
+		"""initialize the object"""
+
+		self.logger = logger
+
+
+	def main(self):
+		"""
+		Synchronizing users between PRACE and EUDAT
+		"""
+		
+		try:
+			""" Parse arguments """
+			argParser = argparse.ArgumentParser(description='Synchronizing users between PRACE and EUDAT')
+			argParser.add_argument('-c', '--config', dest='conf',  default='conf/prace_eudat_users_sync.conf', help='path to the configuration file')
+			arguments = argParser.parse_args()
+
+			""" read and parse config file """
+			self.config = ConfigParser.ConfigParser()
+			self.config.readfp(open(arguments.conf))
+			self.maps={}
+			for section in self.config.sections():
+				if section.startswith('MAP_') :
+					self.maps[section] = ({k:v for k,v in self.config.items(section)})
+			
+
+			""" 
+			Configure and start logging
+			"""
+			self.logger = logger
+			configureLogger(self.logger, self.config, 'common')
+			self.logger.info('Script started #################################################################')
+			
+			""" init connection to B2ACCESS """
+			unityApi = UnityApi(self.config, confSection='B2ACCESS', parentLogger=self.logger)
+
+			""" init connection to PRACE LDAP """
+			praceLdap = PraceLdap(self.config, confSection='PRACE', parentLogger=self.logger)
+
+			""" synchronize """
+			for mapName in sorted(self.maps) :
+				self._syncOneMap(mapName, praceLdap, self.maps[mapName]['prace_searchbase'], self.maps[mapName]['prace_userfilter'], unityApi, self.maps[mapName]['eudat_groups'].split())
+
+			print 'Success.'
+			self.logger.info('Finished with success')
+			sys.exit(0)
+		
+		except ConfigParser.Error, e:
+			tbck = traceback.format_exc()
+			self.logger.error(tbck)
+			self.logger.error(e.message)
+			print tbck
+			print 'Configuration error.'
+			print e.message
+			
+		except Exception, e:
+			tbck = traceback.format_exc()
+			self.logger.error(tbck)
+			self.logger.error(e.message)
+			print tbck
+			print e.message
+	
+		print 'Failure.'
+		sys.exit(1)
+
+
+
+	def _syncOneMap(self, mapName, praceLdap, praceSearchBase, praceUserFilter, unityApi, groupNames):
+		"""
+		performs synchronization for single map section
+		"""
+
+		""" get users from PRACE """
+		usersInPrace = praceLdap.getUsers(praceSearchBase, praceUserFilter)
+
+		for groupName in groupNames :
+			""" get eudat group from B2ACCESS """
+			self.logger.info("Performing synchronization for section "+mapName+" and group "+groupName)
+			group = unityApi.getGroup(groupName)
+
+			if group is None :
+				""" create the group in B2ACCESS if necessary """
+				self.logger.info('Adding group ' + groupName +'.')
+				unityApi.createGroup(groupName)
+				group = {'members': [], 'subGroups': []}
+
+			groupMembers = group['members']
+			groupMemberIds = [member['entityId'] for member in groupMembers]
+			self.logger.debug('PRACE users ids in B2ACCESS group ' + groupName + ": " + pformat(groupMemberIds))
+
+			""" loop over selected PRACE users """
+			for userInPrace in usersInPrace :
+				""" look for the user in B2ACCESS """
+				userInEudat = unityApi.getEntity(userInPrace['certdn'], 'x500Name')
+				if userInEudat is None :
+					""" add the user to B2ACCESS and add him to all relevant groups"""