Skip to content

GitLab

Commit f791e3ed authored by Antoine de Torcy's avatar Antoine de Torcy
Browse files

Modify test suite to use Quantum's Lattus S3 storage 

Updated for iRODS 4.2
parent 6d9def44
Hide whitespace changes
Inline Side-by-side
packaging/test_irods_resource_plugin_s3.py
View file @ f791e3ed
#import boto3
import commands
import datetime
import os
import re
import shutil
import subprocess
import platform
import datetime
import boto3
import random
import re
import shutil
import string
import json
import hashlib
import time
import subprocess
import sys
if sys.version_info >= (2,7):
import unittest
else:
import unittest2 as unittest
import lib
from resource_suite import ResourceSuite
from test_chunkydevtest import ChunkyDevTest
from .. import lib
from . import session
from ..configuration import IrodsConfig
from .resource_suite import ResourceSuite
from .test_chunkydevtest import ChunkyDevTest
import logging
logging.basicConfig(filename='/tmp/pydevtestlog.txt', level=logging.INFO)
@unittest.skip("AWS Only")
class Test_Compound_With_S3_Resource(ResourceSuite, ChunkyDevTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair'
self.archive_naming_policy='decoupled'
self.s3stsdate=''
self.s3region='us-east-1'
self.s3endPoint='s3.amazonaws.com'
self.s3signature_version=2
super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs)
def setUp(self):
# skip ssl tests on ub12
distro_str = ''.join(platform.linux_distribution()[:2]).replace(' ','')
if self._testMethodName.startswith('test_ssl') and distro_str.lower().startswith('ubuntu12'):
self.skipTest("skipping ssl tests on ubuntu 12")
# set up aws configuration
self.set_up_aws_config_dir()
# set up s3 bucket
s3 = boto3.resource('s3', region_name=self.s3region)
self.s3bucketname = 'irods-ci-' + distro_str + datetime.datetime.utcnow().strftime('-%Y-%m-%d.%H-%M-%S-%f-')
self.s3bucketname += ''.join(random.choice(string.letters) for i in xrange(10))
self.s3bucketname = self.s3bucketname[:63].lower() # bucket names can be no more than 63 characters long
if self.s3region == 'us-east-1':
self.bucket = s3.create_bucket(Bucket=self.s3bucketname)
else:
self.bucket = s3.create_bucket(Bucket=self.s3bucketname,
CreateBucketConfiguration={'LocationConstraint': self.s3region})
# set up resources
hostname = lib.get_hostname()
s3params=( 'S3_DEFAULT_HOSTNAME=' + self.s3endPoint +
';S3_AUTH_FILE=' + self.keypairfile +
';S3_RETRY_COUNT=15;S3_WAIT_TIME_SEC=1;S3_PROTO=HTTPS;S3_MPU_CHUNK=10;S3_MPU_THREADS=4;S3_STSDATE=' + self.s3stsdate +
';S3_REGIONNAME=' + self.s3region +
';S3_SIGNATURE_VERSION=' + str(self.s3signature_version) +
';ARCHIVE_NAMING_POLICY=' + self.archive_naming_policy +
';S3_ENABLE_MD5=1'
)
s3params=os.environ.get('S3PARAMS', s3params);
with lib.make_session_for_existing_admin() as admin_session:
admin_session.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', stdin_string='yes\n')
admin_session.assert_icommand("iadmin mkresc demoResc compound", 'STDOUT_SINGLELINE', 'compound')
admin_session.assert_icommand("iadmin mkresc cacheResc 'unixfilesystem' "+hostname+":/var/lib/irods/cacheRescVault", 'STDOUT_SINGLELINE', 'cacheResc')
admin_session.assert_icommand('iadmin mkresc archiveResc s3 '+hostname+':/'+self.s3bucketname+'/irods/Vault "'+s3params+'"', 'STDOUT_SINGLELINE', 'archiveResc')
admin_session.assert_icommand("iadmin addchildtoresc demoResc cacheResc cache")
admin_session.assert_icommand("iadmin addchildtoresc demoResc archiveResc archive")
super(Test_Compound_With_S3_Resource, self).setUp()
def tearDown(self):
super(Test_Compound_With_S3_Resource, self).tearDown()
# delete s3 bucket
for obj in self.bucket.objects.all():
obj.delete()
self.bucket.delete()
# tear down resources
with lib.make_session_for_existing_admin() as admin_session:
admin_session.assert_icommand("iadmin rmchildfromresc demoResc archiveResc")
admin_session.assert_icommand("iadmin rmchildfromresc demoResc cacheResc")
admin_session.assert_icommand("iadmin rmresc archiveResc")
admin_session.assert_icommand("iadmin rmresc cacheResc")
admin_session.assert_icommand("iadmin rmresc demoResc")
admin_session.assert_icommand("iadmin modresc origResc name demoResc", 'STDOUT_SINGLELINE', 'rename', stdin_string='yes\n')
shutil.rmtree(lib.get_irods_top_level_dir() + "/cacheRescVault", ignore_errors=True)
def set_up_aws_config_dir(self):
# read access keys from keypair file
with open(self.keypairfile) as f:
aws_access_key_id = f.readline().rstrip()
aws_secret_access_key = f.readline().rstrip()
# make config dir
aws_cfg_dir_path = os.path.join(os.path.expanduser('~'), '.aws')
try:
os.makedirs(aws_cfg_dir_path)
except OSError:
if not os.path.isdir(aws_cfg_dir_path):
raise
# make config file
with open(os.path.join(aws_cfg_dir_path, 'config'), 'w') as cfg_file:
cfg_file.write('[default]\n')
cfg_file.write('region=' + self.s3region + '\n')
# make credentials file
with open(os.path.join(aws_cfg_dir_path, 'credentials'), 'w') as cred_file:
cred_file.write('[default]\n')
cred_file.write('aws_access_key_id = ' + aws_access_key_id + '\n')
cred_file.write('aws_secret_access_key = ' + aws_secret_access_key + '\n')
def test_irm_specific_replica(self):
self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed
self.admin.assert_icommand("irepl -R "+self.testresc+" "+self.testfile) # creates replica
self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',self.testfile) # should be listed twice
self.admin.assert_icommand("irm -n 0 "+self.testfile) # remove original from cacheResc only
self.admin.assert_icommand("ils -L "+self.testfile,'STDOUT_SINGLELINE',["2 "+self.testresc,self.testfile]) # replica 2 should still be there
self.admin.assert_icommand_fail("ils -L "+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica 0 should be gone
trashpath = self.admin.session_collection_trash
self.admin.assert_icommand_fail("ils -L "+trashpath+"/"+self.testfile,'STDOUT_SINGLELINE',["0 "+self.admin.default_resource,self.testfile]) # replica should not be in trash
@unittest.skip("--wlock has possible race condition due to Compound/Replication PDMO")
def test_local_iput_collision_with_wlock(self):
pass
@unittest.skip("NOTSURE / FIXME ... -K not supported, perhaps")
def test_local_iput_checksum(self):
pass
@unittest.skip("EMPTY_RESC_PATH - no vault path for coordinating resources")
def test_ireg_as_rodsuser_in_vault(self):
pass
@unittest.skip("No Vault for S3 archive resource")
def test_iput_overwrite_others_file__ticket_2086(self):
pass
def test_local_iput_with_force_and_destination_resource__ticket_1706(self):
# local setup
filename = "iputwithforceanddestination.txt"
filepath = lib.create_local_testfile(filename)
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) #
self.admin.assert_icommand("iput -f -R %s %s %s" % (self.testresc, doublefile, filename) ) # overwrite test repl with different data
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+filename]) # default resource cache should have dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+filename]) # default resource archive should have dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," "+filename]) # default resource cache should not have doublesize file
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," "+filename]) # default resource archive should not have doublesize file
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," "+doublesize+" ","& "+filename]) # targeted resource should have new double clean copy
# local cleanup
os.remove(filepath)
os.remove(doublefile)
###################
# irepl
###################
def test_irepl_update_replicas(self):
# local setup
filename = "updatereplicasfile.txt"
filepath = lib.create_local_testfile(filename)
hostname = lib.get_hostname()
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource
self.admin.assert_icommand("iadmin mkresc fourthresc unixfilesystem %s:/tmp/fourthrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create fourth resource
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource
self.admin.assert_icommand("irepl -R fourthresc "+filename) # replicate to fourth resource
self.admin.assert_icommand("iput -f -R "+self.testresc+" "+doublefile+" "+filename) # repave overtop test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("irepl -U "+filename) # update last replica
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a dirty copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("irepl -aU "+filename) # update all replicas
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should have a clean copy
self.admin.assert_icommand("irm -f "+filename) # cleanup file
self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource
self.admin.assert_icommand("iadmin rmresc fourthresc") # remove third resource
# local cleanup
os.remove(filepath)
os.remove(doublefile)
def test_irepl_over_existing_second_replica__ticket_1705(self):
# local setup
filename = "secondreplicatest.txt"
filepath = lib.create_local_testfile(filename)
# assertions
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput -R "+self.testresc+" "+filename) # put file
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl "+filename) # replicate to default resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3
# local cleanup
os.remove(filepath)
def test_irepl_over_existing_third_replica__ticket_1705(self):
# local setup
filename = "thirdreplicatest.txt"
filepath = lib.create_local_testfile(filename)
hostname = lib.get_hostname()
# assertions
self.admin.assert_icommand("iadmin mkresc thirdresc unixfilesystem %s:/tmp/thirdrescVault" % hostname, 'STDOUT_SINGLELINE', "Creating") # create third resource
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate to third resource
self.admin.assert_icommand("irepl "+filename) # replicate overtop default resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate overtop test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl -R thirdresc "+filename) # replicate overtop third resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 4 "," & "+filename]) # should not have a replica 4
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 5 "," & "+filename]) # should not have a replica 5
self.admin.assert_icommand("irm -f "+filename) # cleanup file
self.admin.assert_icommand("iadmin rmresc thirdresc") # remove third resource
# local cleanup
os.remove(filepath)
def test_irepl_over_existing_bad_replica__ticket_1705(self):
# local setup
filename = "reploverwritebad.txt"
filepath = lib.create_local_testfile(filename)
doublefile = "doublefile.txt"
os.system("cat %s %s > %s" % (filename, filename, doublefile))
doublesize = str(os.stat(doublefile).st_size)
# assertions
self.admin.assert_icommand("ils -L "+filename,'STDERR_SINGLELINE',"does not exist") # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate to test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',filename) # for debugging
self.admin.assert_icommand("iput -f %s %s" % (doublefile, filename) ) # overwrite default repl with different data
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," & "+filename]) # default resource cache should have clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 "," "+doublesize+" "," & "+filename]) # default resource cache should have new double clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," & "+filename]) # default resource archive should have clean copy
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 "," "+doublesize+" "," & "+filename]) # default resource archive should have new double clean copy
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," "+filename]) # test resource should not have doublesize file
self.admin.assert_icommand("irepl -R "+self.testresc+" "+filename) # replicate back onto test resource
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 "+self.testresc," "+doublesize+" "," & "+filename]) # test resource should have new clean doublesize file
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 3 "," & "+filename]) # should not have a replica 3
# local cleanup
os.remove(filepath)
os.remove(doublefile)
def test_iput_with_purgec(self):
# local setup
filename = "purgecfile.txt"
filepath = os.path.abspath(filename)
f = open(filepath,'wb')
f.write("TESTFILE -- ["+filepath+"]")
f.close()
# assertions
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed
self.admin.assert_icommand("iput --purgec "+filename) # put file
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed)
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once - replica 1
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed only once
# local cleanup
output = commands.getstatusoutput( 'rm '+filepath )
def test_iget_with_purgec(self):
# local setup
filename = "purgecgetfile.txt"
filepath = os.path.abspath(filename)
f = open(filepath,'wb')
f.write("TESTFILE -- ["+filepath+"]")
f.close()
# assertions
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("iget -f --purgec "+filename) # get file and purge 'cached' replica
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed)
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed once
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should not be listed
# local cleanup
output = commands.getstatusoutput( 'rm '+filepath )
def test_irepl_with_purgec(self):
# local setup
filename = "purgecreplfile.txt"
filepath = os.path.abspath(filename)
f = open(filepath,'wb')
f.write("TESTFILE -- ["+filepath+"]")
f.close()
# assertions
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed
self.admin.assert_icommand("iput "+filename) # put file
self.admin.assert_icommand("irepl -R "+self.testresc+" --purgec "+filename) # replicate to test resource
self.admin.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',[" 0 ",filename]) # should not be listed (trimmed)
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 1 ",filename]) # should be listed twice - 2 of 3
self.admin.assert_icommand("ils -L "+filename,'STDOUT_SINGLELINE',[" 2 ",filename]) # should be listed twice - 1 of 3
# local cleanup
output = commands.getstatusoutput( 'rm '+filepath )
def test_decoupled_naming_policy(self):
if self.archive_naming_policy != 'decoupled':
self.skipTest("Archive naming policy is not set to 'decoupled'")
# local setup
filename = "testfile.txt"
filepath = os.path.abspath(filename)
f = open(filepath,'wb')
f.write("TESTFILE -- ["+filepath+"]")
f.close()
# run as regular user
session = self.user0
collection = session.session_collection
# iquest to get the object id of the replica on the S3 archive
id_query = ( "select DATA_ID where COLL_NAME =" + "'" + collection + "'" +
" and DATA_NAME =" + "'" + filename + "'" +
" and DATA_REPL_NUM ='1'" )
# iquest to get the pysical path of the replica on the S3 archive
path_query = ( "select DATA_PATH where COLL_NAME =" + "'" + collection + "'" +
" and DATA_NAME =" + "'" + filename + "'" +
" and DATA_REPL_NUM ='1'" )
# assertions
session.assert_icommand_fail("ils -L "+filename,'STDOUT_SINGLELINE',filename) # should not be listed
session.assert_icommand("iput "+filename) # put file
# get object id
object_id = session.run_icommand('iquest "%s" ' + '"' + id_query + '"')[1].strip()
# physical path we expect to see: /{bucket_name}/{reversed_id}/{obj_name}
target_path = '/' + self.s3bucketname + '/' + object_id[::-1] + '/' + filename
# get object path
physical_path = session.run_icommand('iquest "%s" ' + '"' + path_query + '"')[1].strip()
# verify object path
self.assertEqual(target_path, physical_path)
# cleanup
session.run_icommand('irm -f ' + filename)
os.unlink(filepath)
@unittest.skip("AWS Only")
class Test_Compound_With_S3_Resource_EU_Central_1(Test_Compound_With_S3_Resource):
'''
This also tests signature V4 with the x-amz-date header.
'''
def __init__(self, *args, **kwargs):
self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair'
self.archive_naming_policy='decoupled'
self.s3stsdate=''
self.s3region='eu-central-1'
self.s3endPoint='s3.eu-central-1.amazonaws.com'
self.s3signature_version=4
super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs)
@unittest.skip("AWS Only")
class Test_Compound_With_S3_Resource_STSDate_Header(Test_Compound_With_S3_Resource):
def __init__(self, *args, **kwargs):
self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair'
self.archive_naming_policy='decoupled'
self.s3stsdate='date'
self.s3region='us-east-1'
self.s3endPoint='s3.amazonaws.com'
self.s3signature_version=2
super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs)
@unittest.skip("AWS Only")
class Test_Compound_With_S3_Resource_STSDate_Header_V4(Test_Compound_With_S3_Resource):
def __init__(self, *args, **kwargs):
self.keypairfile='/projects/irods/vsphere-testing/externals/amazon_web_services-CI.keypair'
self.archive_naming_policy='decoupled'
self.s3stsdate='date'
self.s3region='us-east-1'
self.s3endPoint='s3.amazonaws.com'
self.s3signature_version=4
super(Test_Compound_With_S3_Resource, self).__init__(*args, **kwargs)
class Test_Compound_With_S3_Resource_OSCS(ResourceSuite, ChunkyDevTest, unittest.TestCase):
class Test_Compound_With_S3_Resource_Quantum(ResourceSuite, ChunkyDevTest, unittest.TestCase):
def __init__(self, *args, **kwargs):
self.keypairfile='/projects/irods/vsphere-testing/externals/oracle.keypair'
self.s3endPoint='uscom-central-1.storage.oraclecloud.com'
super(Test_Compound_With_S3_Resource_OSCS, self).__init__(*args, **kwargs)
self.keypairfile='/tmp/quantum_test.keypair'
self.s3endPoint='demo.lattusdemo.com'
super(Test_Compound_With_S3_Resource_Quantum, self).__init__(*args, **kwargs)
def setUp(self):
# skip ssl tests on ub12
......@@ -444,7 +39,7 @@ class Test_Compound_With_S3_Resource_OSCS(ResourceSuite, ChunkyDevTest, unittest
if self._testMethodName.startswith('test_ssl') and distro_str.lower().startswith('ubuntu12'):
self.skipTest("skipping ssl tests on ubuntu 12")
self.s3bucketname = 'my_bucket'
self.s3bucketname = 'lattuss3bucket295622'
self.archivebucketname = 'test_archive' # deep archive
# set up resources
......@@ -457,29 +52,30 @@ class Test_Compound_With_S3_Resource_OSCS(ResourceSuite, ChunkyDevTest, unittest
)
s3params=os.environ.get('S3PARAMS', s3params);
with lib.make_session_for_existing_admin() as admin_session:
admin_session.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', stdin_string='yes\n')
with session.make_session_for_existing_admin() as admin_session:
irods_config = IrodsConfig()
admin_session.assert_icommand("iadmin modresc demoResc name origResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n')
admin_session.assert_icommand("iadmin mkresc demoResc compound", 'STDOUT_SINGLELINE', 'compound')
admin_session.assert_icommand("iadmin mkresc cacheResc 'unixfilesystem' "+hostname+":/var/lib/irods/cacheRescVault", 'STDOUT_SINGLELINE', 'cacheResc')
admin_session.assert_icommand("iadmin mkresc cacheResc 'unixfilesystem' " + hostname + ":" + irods_config.irods_directory + "/cacheRescVault", 'STDOUT_SINGLELINE', 'cacheResc')
admin_session.assert_icommand('iadmin mkresc archiveResc s3 '+hostname+':/'+self.s3bucketname+'/irods/Vault "'+s3params+'"', 'STDOUT_SINGLELINE', 'archiveResc')
admin_session.assert_icommand("iadmin addchildtoresc demoResc cacheResc cache")
admin_session.assert_icommand("iadmin addchildtoresc demoResc archiveResc archive")
super(Test_Compound_With_S3_Resource_OSCS, self).setUp()
super(Test_Compound_With_S3_Resource_Quantum, self).setUp()
def tearDown(self):
super(Test_Compound_With_S3_Resource_OSCS, self).tearDown()
super(Test_Compound_With_S3_Resource_Quantum, self).tearDown()
# tear down resources
with lib.make_session_for_existing_admin() as admin_session:
with session.make_session_for_existing_admin() as admin_session:
admin_session.assert_icommand("iadmin rmchildfromresc demoResc archiveResc")
admin_session.assert_icommand("iadmin rmchildfromresc demoResc cacheResc")
admin_session.assert_icommand("iadmin rmresc archiveResc")
admin_session.assert_icommand("iadmin rmresc cacheResc")
admin_session.assert_icommand("iadmin rmresc demoResc")
admin_session.assert_icommand("iadmin modresc origResc name demoResc", 'STDOUT_SINGLELINE', 'rename', stdin_string='yes\n')
admin_session.assert_icommand("iadmin modresc origResc name demoResc", 'STDOUT_SINGLELINE', 'rename', input='yes\n')
shutil.rmtree(lib.get_irods_top_level_dir() + "/cacheRescVault", ignore_errors=True)
shutil.rmtree(IrodsConfig().irods_directory + "/cacheRescVault", ignore_errors=True)
@staticmethod
......@@ -494,70 +90,9 @@ class Test_Compound_With_S3_Resource_OSCS(ResourceSuite, ChunkyDevTest, unittest
return(hasher.hexdigest())
@unittest.skip("OSCS")
def test_deep_archive(self):
# make test file
filename = "DA_test_file.txt"
filepath = os.path.join('/tmp', filename)
with open(filepath,'w') as f:
f.write("TESTFILE -- ["+filepath+"]")
checksum = self.md5(filepath)
# upload test file to s3 resource
self.user0.assert_icommand("iput -f "+filepath)
# get object info for DA rule
resc_hier = self.user0.run_icommand('''iquest "%s" "select DATA_RESC_HIER where DATA_NAME ='{0}' and DATA_REPL_NUM ='1'"'''.format(filename))[1].strip()
file_path = self.user0.run_icommand('''iquest "%s" "select DATA_PATH where DATA_NAME ='{0}' and DATA_REPL_NUM ='1'"'''.format(filename))[1].strip()
obj_path = self.user0.run_icommand('''iquest "%s/%s" "select COLL_NAME, DATA_NAME where DATA_NAME ='{0}' and DATA_REPL_NUM ='1'"'''.format(filename))[1].strip()
# call rule to move object to deep archive
self.user0.assert_icommand('''irule -F move2DA.r "'{resc_hier}'" "'{obj_path}'" "'{file_path}'"'''.format(**locals()))
# trim replica on cache
self.user0.assert_icommand("itrim -n 0 -N 1 "+filename)
# remove local file
os.remove(filepath)
# fetch object from DA
while True:
iget_status, iget_out, iget_err = self.user0.run_icommand("iget -V {filename} {filepath}".format(**locals()))
if not iget_err:
break
logging.info('------')
logging.info('status')
logging.info(iget_status)
logging.info('------')
logging.info('out')
# we want only 'Level 0: { ... }', minus the 'Level 0: ' prefix
out_str = iget_out.splitlines()[0].split(': ', 1)[1]
res = json.loads(out_str)
logging.info("completed: {0}%".format(res['completedPercentage']))
logging.info('------')
logging.info('err')
logging.info(iget_err)
logging.info('========')
time.sleep(60)
# verify checksum of newly retrieved file
self.assertEqual(checksum, self.md5(filepath))
# cleanup
self.user0.assert_icommand("irm -f "+filename)
try: