Skip to content

GitLab

Unverified Commit 966e2f30 authored by michec81's avatar michec81 Committed by GitHub
Browse files

Merge pull request #145 from cookie33/python3 

Implement python3 for EUDAT B2SAFE python scripts and automated testing.
parents f71a206b cf172f14
Hide whitespace changes
Inline Side-by-side
cmd/authZmanager.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
#
# authZ.manager.py
#
......@@ -36,7 +36,7 @@ class AuthZClient(object):
def __init__(self, mapFilePath):
"""Initialize object with connection parameters."""
self.mapfile = mapFilePath
self.mapfile = mapFilePath
def _debugMsg(self, method, msg):
"""Internal: Print a debug message if debug is enabled."""
......
cmd/epicclient2.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
#
# epicclient.py
#
......
cmd/logmanager.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
#
# authZ.manager.py
#
......@@ -48,7 +48,7 @@ class LogManager(object):
"""Internal: Print a debug message if debug is enabled."""
if self.debug:
print "[", method, "]", msg
print("[", method, "]", msg)
def _parseConf(self):
"""Internal: Parse the configuration file.
......@@ -59,8 +59,8 @@ class LogManager(object):
tmp = eval(filehandle.read())
filehandle.close()
except (OSError, IOError) as e:
print "problem while reading configuration file %s" % self.file
print "Error:", e
print("problem while reading configuration file %s" % self.file)
print("Error:", e)
self.log_level_value = self.log_level[tmp['log_level']]
self.log_dir = tmp['log_dir']
......@@ -72,7 +72,7 @@ class LogManager(object):
self.logger.setLevel(self.log_level_value)
self.logfile = self.log_dir+'/b2safe.log'
rfh = logging.handlers.RotatingFileHandler(self.logfile,
maxBytes=2097152,
maxBytes=2097152,
backupCount=9)
formatter = logging.Formatter('%(asctime)s %(levelname)s:%(message)s')
rfh.setFormatter(formatter)
......@@ -83,7 +83,7 @@ class LogManager(object):
queuedir = self.log_dir+'/b2safe.queue'
self.queue = FifoDiskQueue(queuedir)
def getLogger(self):
"""get the logger instance."""
......@@ -91,12 +91,12 @@ class LogManager(object):
def getLogFile(self):
"""get the log file."""
return self.logfile
def getQueue(self):
"""get the queue instance."""
return self.queue
......@@ -128,17 +128,17 @@ def log(args):
def push(args):
'''perform push action'''
logManager = LogManager(args.conffilepath, args.debug)
logManager.initializeQueue()
queue = logManager.getQueue()
queue.push(' '.join(args.message))
queue.push(' '.join(args.message).encode())
queue.close()
def pop(args):
'''perform pop action'''
logManager = LogManager(args.conffilepath, args.debug)
logManager.initializeQueue()
queue = logManager.getQueue()
......@@ -147,66 +147,66 @@ def pop(args):
i = 0
messages = []
while i < int(args.number):
message = queue.pop()
message = queue.pop().decode()
if message is not None:
messages.append(message)
else:
break
i += 1
print messages
print(messages)
else:
message = queue.pop()
print message
message = queue.pop().decode()
print(message)
queue.close()
def queuesize(args):
'''get the current size of the queue'''
logManager = LogManager(args.conffilepath, args.debug)
logManager.initializeQueue()
queue = logManager.getQueue()
length = str(len(queue))
print length
print(length)
queue.close()
def test(args):
'''do a serie of tests'''
def read_log_last_line(logger, logfile):
'''local helper func: read the last line of the log file'''
logger.info('test log message')
try:
filehandle = open(logfile, "r")
linelist = filehandle.readlines()
filehandle.close()
except (OSError, IOError) as er:
print "problem while reading file %s" % logfile
print "Error:", er
print("problem while reading file %s" % logfile)
print("Error:", er)
return False
lastline = linelist[len(linelist)-1]
if 'test log message' in lastline:
return True
return False
def test_result(result, testval):
"""local helper func: print OK/FAIL for test result
Returns 0 on OK, 1 on failure"""
if type(result) != type(testval):
print "FAIL (wrong type; test is bad)"
print("FAIL (wrong type; test is bad)")
return 1
if result == testval:
print "OK"
print("OK")
return 0
else:
print "FAIL"
print("FAIL")
return 1
try:
......@@ -215,55 +215,55 @@ def test(args):
logger = logManager.getLogger()
logfile = logManager.getLogFile()
except IOError as er:
print "IOError, directory 'log' might not exist"
print "Error: ", er
print("IOError, directory 'log' might not exist")
print("Error: ", er)
return False
logManager.initializeQueue()
queue = logManager.getQueue()
fail = 0
print
print "logging info to the file " + logfile
print "Test: log message"
print()
print("logging info to the file " + logfile)
print("Test: log message")
fail += test_result(read_log_last_line(logger, logfile), True)
print
print "Test: queue size before push"
print("Test: queue size before push")
fail += test_result(len(queue), 0)
print
print "Test: push info to the queue"
queue.push('test message')
print "OK"
print()
print("Test: push info to the queue")
queue.push('test message'.encode())
print("OK")
print
print "Test: queue size after push "
print()
print("Test: queue size after push ")
fail += test_result(len(queue), 1)
print
print "Test: pop info from queue"
fail += test_result('test message', queue.pop())
print
print "Test: check queue size, again (should be 0)"
print()
print("Test: pop info from queue")
fail += test_result('test message', queue.pop().decode())
print()
print("Test: check queue size, again (should be 0)")
fail += test_result(len(queue), 0)
queue.close()
if fail == 0:
print
print "All tests passed OK"
print()
print("All tests passed OK")
else:
print
print "%d tests failed" % fail
print()
print("%d tests failed" % fail)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='EUDAT log manager. ')
parser.add_argument("conffilepath", default="NULL",
help="path to the configuration file")
parser.add_argument("-d", "--debug", help="Show debug output",
parser.add_argument("-d", "--debug", help="Show debug output",
action="store_true")
subparsers = parser.add_subparsers(title='Actions',
......@@ -273,12 +273,12 @@ if __name__ == "__main__":
parser_log = subparsers.add_parser('log', help='log a message')
parser_log.add_argument("level", help='the value of the log level')
parser_log.add_argument("message", nargs='+',
parser_log.add_argument("message", nargs='+',
help='the message to be logged')
parser_log.set_defaults(func=log)
parser_push = subparsers.add_parser('push', help='push a message')
parser_push.add_argument("message", nargs='+',
parser_push.add_argument("message", nargs='+',
help='the message to be queued')
parser_push.set_defaults(func=push)
......
cmd/messageManager.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
import dweepy
import argparse
import sys
import logging
import logging.handlers
import json
import dweepy
logger = logging.getLogger('messageManager')
......@@ -31,11 +31,11 @@ def getlast(args):
try:
jsonPayLoad = dweepy.get_latest_dweet_for(args.queue)
except dweepy.DweepyError as e:
logger.exception('Failed to get the last message from queue %s',
args.queue)
logger.exception('Failed to get the last message from queue %s',
args.queue)
sys.exit(1)
print json.dumps({jsonPayLoad[0]['created']:
jsonPayLoad[0]['content']['message']})
print(json.dumps({jsonPayLoad[0]['created']:
jsonPayLoad[0]['content']['message']}))
def getall(args):
"""get all the messages from the queue"""
......@@ -43,13 +43,13 @@ def getall(args):
try:
jsonPayLoad = dweepy.get_dweets_for(args.queue)
except dweepy.DweepyError as e:
logger.exception('Failed to get all the message from queue %s',
args.queue)
logger.exception('Failed to get all the message from queue %s',
args.queue)
sys.exit(1)
payLoad = {}
for message in jsonPayLoad:
payLoad[message['created']] = message['content']['message']
print json.dumps(payLoad)
print(json.dumps(payLoad))
def _initializeLogger(args):
"""initialize the logger instance."""
......@@ -60,7 +60,7 @@ def _initializeLogger(args):
logger.setLevel(logging.INFO)
if (args.log):
han = logging.handlers.RotatingFileHandler(args.log,
maxBytes=6000000,
maxBytes=6000000,
backupCount=9)
else:
han = logging.StreamHandler()
......@@ -74,7 +74,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description='EUDAT message manager.')
parser.add_argument("-l", "--log", help="Path to the log file")
parser.add_argument("-d", "--debug", help="Show debug output",
parser.add_argument("-d", "--debug", help="Show debug output",
action="store_true")
subparsers = parser.add_subparsers(title='Actions',
......
cmd/pam_user_reader.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
#
# * use 4 spaces!!! not tabs
# * See PEP-8 Python style guide http://www.python.org/dev/peps/pep-0008/
......@@ -7,7 +7,7 @@
import json
user_map_file="/etc/irods/user_map.json"
user_map_file = "/etc/irods/user_map.json"
def parseUserMap():
"""Parse the user map file"""
......@@ -19,5 +19,5 @@ def parseUserMap():
if __name__ == "__main__":
parseUserMap()
cmd/regex_search_string.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
import re
import sys
......@@ -11,11 +11,11 @@ def search_pattern_in_string(pattern, string):
"""
searchObj = re.search(pattern, string, re.U|re.I)
if searchObj:
print searchObj.group()
print(searchObj.group())
else:
print "no match found!"
print("no match found!")
if __name__ == "__main__":
search_pattern_in_string(sys.argv[1], sys.argv[2])
search_pattern_in_string(sys.argv[1], sys.argv[2])
cmd/timeconvert.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
import calendar
import datetime as DT
......@@ -11,23 +11,23 @@ def epoch_to_iso8601(timestamp):
'2015-12-03T13:02:42Z'
"""
iso_time = DT.datetime.utcfromtimestamp(timestamp).isoformat()+"Z"
print iso_time
print(iso_time)
def iso8601_to_epoch(dateTime_string):
"""
iso8601_to_epoch - convert a iso8601 formatted date into the unix epoch time
iso8601_to_epoch - convert a iso8601 formatted date into the unix epoch time
>>> iso8601_to_epoch('2015-12-03T13:02:42Z')
1449147762
"""
epoch_time = calendar.timegm(DT.datetime.strptime(dateTime_string, "%Y-%m-%dT%H:%M:%SZ").timetuple())
print epoch_time
epoch_time = calendar.timegm(DT.datetime.strptime(dateTime_string, "%Y-%m-%dT%H:%M:%SZ").timetuple())
print(epoch_time)
if __name__ == "__main__":
if sys.argv[1] == 'epoch_to_iso8601':
epoch_to_iso8601(float(sys.argv[2]))
elif sys.argv[1] == 'iso8601_to_epoch':
iso8601_to_epoch(sys.argv[2])
else:
print "Error, not valid option"
if sys.argv[1] == 'epoch_to_iso8601':
epoch_to_iso8601(float(sys.argv[2]))
elif sys.argv[1] == 'iso8601_to_epoch':
iso8601_to_epoch(sys.argv[2])
else:
print("Error, not valid option")
install_centos7.md
View file @ 966e2f30
......@@ -28,7 +28,7 @@ EOF
```
Configure the yum repository for iRODS 4.2.6
Configure the yum repository for iRODS 4.2.7
---------------------------------------------
configure the repo as root:
......@@ -36,7 +36,7 @@ configure the repo as root:
cat > /etc/yum.repos.d/surfsara-irods.repo <<EOF
[surfsara-irods]
name=Surfsara iRODS repo
baseurl=https://software.irodspoc-sara.surf-hosted.nl/CentOS/7/irods-4.2.6
baseurl=https://software.irodspoc-sara.surf-hosted.nl/CentOS/7/irods-4.2.7
sslverify=0
gpgcheck=0
EOF
......@@ -59,7 +59,7 @@ To install/configure it in iRODS do following as the user who runs iRODS:
As the user who runs iRODS do following:
```bash
cd /opt/eudat/b2safe/cmd
pip install --user -r requirements.txt
pip3 install --user -r requirements.txt
```
### convert install.conf to install.json if needed
......
packaging/install.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
#
# epicclient.py
# install.py
#
# * use 4 spaces!!! not tabs
# * set tabstop=4
......@@ -107,13 +107,13 @@ def check_user(json_config):
if line.rstrip() == 'IRODS_SERVICE_ACCOUNT_NAME='+username:
user_match = True
except IOError:
print "Error, Unable to open file: %s" % service_account_config_file
print("Error, Unable to open file: %s" % service_account_config_file)
exit(1)
if user_match:
print "The iRODS user matches the user who is running this script"
print("The iRODS user matches the user who is running this script")
else:
print "The iRODS user does not match the user who is running this script"
print("The iRODS user does not match the user who is running this script")
exit(1)
def check_missing_parameters(json_config):
......@@ -128,9 +128,9 @@ def check_missing_parameters(json_config):
missing_parameters.append(parameter)
if not parameters_match:
print "Not all parameters are present. Please add following parameter(s)"
print("Not all parameters are present. Please add following parameter(s)")
for item in missing_parameters:
print "The missing parameter = %s" % item
print("The missing parameter = %s" % item)
exit(1)
def create_b2safe_symbolic_links(json_config):
......@@ -144,7 +144,7 @@ def create_b2safe_symbolic_links(json_config):
try:
os.unlink(symlink_file)
except IOError:
print "Error deleting symbolic link: %s" % symlink_file
print("Error deleting symbolic link: %s" % symlink_file)
exit(1)
# add new symlinks
......@@ -153,7 +153,7 @@ def create_b2safe_symbolic_links(json_config):
os.symlink(symlink_file, json_config["irods_conf_dir"]+'/eudat'+str(count)+'.re')
json_config["re_rulebase_set"].append('eudat'+str(count))
except IOError:
print "Error creating symbolic link: %s" % symlink_file
print("Error creating symbolic link: %s" % symlink_file)
exit(1)
count = count + 1
......@@ -171,10 +171,10 @@ def install_python_b2safe_scripts(json_config):
# add new links
os.symlink(symlink_file, destination_path)
except IOError:
print "Error creating symbolic link: %s" % symlink_file
print("Error creating symbolic link: %s" % symlink_file)
exit(1)
except OSError:
print "Error creating symbolic link: " + symlink_file + ", " + destination_path
print("Error creating symbolic link: " + symlink_file + ", " + destination_path)
exit(1)
def read_json_config(json_config_file):
......@@ -184,7 +184,7 @@ def read_json_config(json_config_file):
with open(json_config_file) as config_file:
json_config = json.load(config_file)
except IOError:
print "Error, Unable to open file: %s" % json_config_file
print("Error, Unable to open file: %s" % json_config_file)
exit(1)
return dict(json_config)
......@@ -196,10 +196,10 @@ def secure_file(chmod_file):
try:
os.chmod(chmod_file, 0o600)
except IOError:
print "Error, Unable to protect file: %s" % chmod_file
print("Error, Unable to protect file: %s" % chmod_file)
exit(1)
else:
print "Error, Unable to protect file: %s as it not present" % chmod_file
print("Error, Unable to protect file: %s as it not present" % chmod_file)
exit(1)
def save_config_file(save_file):
......@@ -213,7 +213,7 @@ def save_config_file(save_file):
shutil.copy2(save_file, save_file_copy)
secure_file(save_file_copy)
except IOError:
print "Error, Unable to copy file: %s" % save_file
print("Error, Unable to copy file: %s" % save_file)
exit(1)
def update_authz_map(json_config):
......@@ -227,7 +227,7 @@ def update_authz_map(json_config):
# read authz file
authz_map_config = read_json_config(authz_map_file)
print "TODO: Update %s not yet implemented !! " % authz_map_file
print("TODO: Update %s not yet implemented !! " % authz_map_file)
### write authz config
##write_json_config(authz_map_config, authz_map_file)
......@@ -275,7 +275,7 @@ def update_flat_file_parameter(modify_file, mod_key, mod_value, irods_file=False
line = line.rstrip()
lines.append(line)
except IOError:
print "Error, Unable to open file: %s" % modify_file
print("Error, Unable to open file: %s" % modify_file)
exit(1)
for idx, line in enumerate(lines):
......@@ -293,10 +293,10 @@ def update_flat_file_parameter(modify_file, mod_key, mod_value, irods_file=False
with open(modify_file, "w+") as config_file:
config_file.write("\n".join(lines))
except IOError:
print "Error, Unable to open file: %s" % modify_file
print("Error, Unable to open file: %s" % modify_file)
exit(1)
else:
print "Parameter %s not found in file %s !!!" % (mod_key, modify_file)
print("Parameter %s not found in file %s !!!" % (mod_key, modify_file))
def update_irods_server_config(json_config):
''' add the eudat rules to the iRODS server config '''
......@@ -476,7 +476,7 @@ def update_pid_uservice_config(json_config):
if json_config["server_id"] == json_config["server_api_reg"] == json_config["server_api_pub"]:
pid_uservice_config["lookup"]["value"] = "{IRODS_URL_PREFIX}{OBJECT}"
# print json.dumps(pid_uservice_config, indent=2, sort_keys=True)
# print(json.dumps(pid_uservice_config, indent=2, sort_keys=True))
# write pid uService config
write_json_config(pid_uservice_config, pid_uservice_conf_file)
......@@ -489,7 +489,7 @@ def write_json_config(json_dict, json_config_file):
with open(json_config_file, "w+") as config_file:
config_file.write(json.dumps(json_dict, indent=4, sort_keys=True))
except IOError:
print "Error, Unable to open file: %s" % json_config_file
print("Error, Unable to open file: %s" % json_config_file)
exit(1)
......
scripts/tests/testB2SafeCmd.py
View file @ 966e2f30
#!/usr/bin/env python
#!/usr/bin/env python3
import argparse
import unittest
import sys
from testB2SafeCmd.testLogManager import LogManagerTestCase
from testB2SafeCmd.testAuthzManager import AuthzManagerTestCase
from testB2SafeCmd.irodsintgtest import IrodsIntegrationTests
from testB2SafeCmd.irodsb2safetest import IrodsB2safeIntegrationTests
from testB2SafeCmd.epic2intgtest import EpicClient2IntegrationTests
from testB2SafeCmd.epic2intgtest import EpicClient2IntegrationTests
from testB2SafeCmd.msipidintgtest import MsiPidIntegrationTests
import argparse
import unittest
import sys
__author__ = 'lphan'
......@@ -28,51 +29,51 @@ if __name__ == '__main__':
if param.script == "epic2":
# test cases for B2safe-epicclient2#
print "Test Epicclient2 Script"
print("Test Epicclient2 Script")
epic2_suite = unittest.TestLoader().loadTestsFromTestCase(
EpicClient2IntegrationTests)
EpicClient2IntegrationTests)
ret = unittest.TextTestRunner(descriptions=2, verbosity=2).run(epic2_suite)
elif param.script == "msipid":