From b5217b9eed334f42e2805bd5b44bf5c269f3e67b Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Sat, 26 Mar 2016 23:08:24 -0400 Subject: [PATCH] More tests, change file to filename --- resync/client.py | 60 ++++++++++++++-------------- tests/test_client.py | 93 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 32 deletions(-) diff --git a/resync/client.py b/resync/client.py index a083c9f..6d0cb7c 100644 --- a/resync/client.py +++ b/resync/client.py @@ -194,18 +194,18 @@ class Client(object): num_deleted=0 for resource in created: uri = resource.uri - file = self.mapper.src_to_dst(uri) - self.logger.info("created: %s -> %s" % (uri,file)) - num_created+=self.update_resource(resource,file,'created') + filename = self.mapper.src_to_dst(uri) + self.logger.info("created: %s -> %s" % (uri,filename)) + num_created+=self.update_resource(resource,filename,'created') for resource in updated: uri = resource.uri - file = self.mapper.src_to_dst(uri) - self.logger.info("updated: %s -> %s" % (uri,file)) - num_updated+=self.update_resource(resource,file,'updated') + filename = self.mapper.src_to_dst(uri) + self.logger.info("updated: %s -> %s" % (uri,filename)) + num_updated+=self.update_resource(resource,filename,'updated') for resource in deleted: uri = resource.uri - file = self.mapper.src_to_dst(uri) - num_deleted+=self.delete_resource(resource,file,allow_deletion) + filename = self.mapper.src_to_dst(uri) + num_deleted+=self.delete_resource(resource,filename,allow_deletion) ### 6. Store last timestamp to allow incremental sync if (not audit_only and self.last_timestamp>0): ClientState().set_state(self.sitemap,self.last_timestamp) @@ -316,17 +316,17 @@ class Client(object): num_created = 0 for resource in src_change_list: uri = resource.uri - file = self.mapper.src_to_dst(uri) + filename = self.mapper.src_to_dst(uri) if (resource.change == 'updated'): - self.logger.info("updated: %s -> %s" % (uri,file)) - self.update_resource(resource,file,'updated') + self.logger.info("updated: %s -> %s" % (uri,filename)) + self.update_resource(resource,filename,'updated') num_updated+=1 elif (resource.change == 'created'): - self.logger.info("created: %s -> %s" % (uri,file)) - self.update_resource(resource,file,'created') + self.logger.info("created: %s -> %s" % (uri,filename)) + self.update_resource(resource,filename,'created') num_created+=1 elif (resource.change == 'deleted'): - num_deleted+=self.delete_resource(resource,file,allow_deletion) + num_deleted+=self.delete_resource(resource,filename,allow_deletion) else: raise ClientError("Unknown change type %s" % (resource.change) ) ### 7. Report status and planned actions @@ -339,8 +339,8 @@ class Client(object): ### 9. Done self.logger.debug("Completed incremental sync") - def update_resource(self, resource, file, change=None): - """Update resource from uri to file on local system. + def update_resource(self, resource, filename, change=None): + """Update resource from uri to filename on local system. Update means three things: 1. GET resources @@ -355,15 +355,15 @@ class Client(object): Returns the number of resources updated/created (0 or 1) """ - path = os.path.dirname(file) + path = os.path.dirname(filename) distutils.dir_util.mkpath(path) num_updated=0 if (self.dryrun): - self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,file)) + self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,filename)) else: # 1. GET try: - urlretrieve(resource.uri,file) + urlretrieve(resource.uri,filename) num_updated+=1 except IOError as e: msg = "Failed to GET %s -- %s" % (resource.uri,str(e)) @@ -375,22 +375,22 @@ class Client(object): # 2. set timestamp if we have one if (resource.timestamp is not None): unixtime = int(resource.timestamp) #no fractional - os.utime(file,(unixtime,unixtime)) + os.utime(filename,(unixtime,unixtime)) if (resource.timestamp > self.last_timestamp): self.last_timestamp = resource.timestamp self.log_event(Resource(resource=resource, change=change)) # 3. sanity check - length = os.stat(file).st_size - if (resource.length != length): + length = os.stat(filename).st_size + if (resource.length is not None and resource.length != length): self.logger.info("Downloaded size for %s of %d bytes does not match expected %d bytes" % (resource.uri,length,resource.length)) if (self.checksum and resource.md5 is not None): - file_md5 = compute_md5_for_file(file) + file_md5 = compute_md5_for_file(filename) if (resource.md5 != file_md5): self.logger.info("MD5 mismatch for %s, got %s but expected %s bytes" % (resource.uri,file_md5,resource.md5)) return(num_updated) - def delete_resource(self, resource, file, allow_deletion=False): - """Delete copy of resource in file on local system. + def delete_resource(self, resource, filename, allow_deletion=False): + """Delete copy of resource in filename on local system. Will only actually do the deletion if allow_deletion is True. Regardless of whether the deletion occurs, self.last_timestamp will be updated @@ -405,20 +405,20 @@ class Client(object): self.last_timestamp = resource.timestamp if (allow_deletion): if (self.dryrun): - self.logger.info("dryrun: would delete %s -> %s" % (uri,file)) + self.logger.info("dryrun: would delete %s -> %s" % (uri,filename)) else: try: - os.unlink(file) + os.unlink(filename) num_deleted+=1 + self.logger.info("deleted: %s -> %s" % (uri,filename)) + self.log_event(Resource(resource=resource, change="deleted")) except OSError as e: - msg = "Failed to DELETE %s -> %s : %s" % (uri,file,str(e)) + msg = "Failed to DELETE %s -> %s : %s" % (uri,filename,str(e)) #if (self.ignore_failures): self.logger.warning(msg) # return #else: # raise ClientFatalError(msg) - self.logger.info("deleted: %s -> %s" % (uri,file)) - self.log_event(Resource(resource=resource, change="deleted")) else: self.logger.info("nodelete: would delete %s (--delete to enable)" % uri) return(num_deleted) diff --git a/tests/test_client.py b/tests/test_client.py index cb1b1d1..5a02fe1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -9,6 +9,7 @@ import sys import os.path from resync.client import Client, ClientFatalError +from resync.resource import Resource from resync.resource_list import ResourceList from resync.change_list import ChangeList @@ -34,6 +35,78 @@ class TestClient(TestCase): self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' ) self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' ) + def test18_update_resource(self): + c = Client() + resource = Resource(uri='http://example.org/dir/2') + filename = os.path.join(self.tmpdir,'dir/resource2') + # dryrun + with LogCapture() as lc: + c.dryrun = True + c.logger = logging.getLogger('resync.client') + n = c.update_resource( resource, filename ) + self.assertEqual( n, 0 ) + self.assertTrue( lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ') ) + c.dryrun = False + # get from file uri that does not exist + resource = Resource(uri='tests/testdata/i_do_not_exist') + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + self.assertRaises( ClientFatalError, c.update_resource, resource, filename ) + # get from file uri + resource = Resource(uri='tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + n = c.update_resource( resource, filename ) + self.assertEqual( n, 1 ) + self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) + + def test19_delete_resource(self): + c = Client() + resource = Resource(uri='http://example.org/1') + filename = os.path.join(self.tmpdir,'resource1') + c.last_timestamp = 5 + # no delete, no timestamp update + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + n = c.delete_resource( resource, filename ) + self.assertEqual( n, 0 ) + self.assertEqual( lc.records[-1].msg, + 'nodelete: would delete http://example.org/1 (--delete to enable)' ) + self.assertEqual( c.last_timestamp, 5 ) + # no delete but timestamp update + resource.timestamp = 10 + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + n = c.delete_resource( resource, filename ) + self.assertEqual( n, 0 ) + self.assertEqual( lc.records[-1].msg, + 'nodelete: would delete http://example.org/1 (--delete to enable)' ) + self.assertEqual( c.last_timestamp, 10 ) + # allow delete but dryrun + with LogCapture() as lc: + c.dryrun = True + c.logger = logging.getLogger('resync.client') + n = c.delete_resource( resource, filename, allow_deletion=True ) + self.assertEqual( n, 0 ) + self.assertTrue( lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1') ) + c.dryrun = False + # allow delete but no resource present + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + n = c.delete_resource( resource, filename, allow_deletion=True ) + self.assertEqual( n, 0 ) + self.assertTrue( lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1') ) + # successful deletion, first make file... + with open(filename, 'w') as fh: + fh.write('delete me') + fh.close() + with LogCapture() as lc: + c.logger = logging.getLogger('resync.client') + n = c.delete_resource( resource, filename, allow_deletion=True ) + self.assertEqual( n, 1 ) + self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) + self.assertTrue( lc.records[-2].msg.startswith('deleted: http://example.org/1 ->') ) + def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type @@ -54,6 +127,18 @@ class TestClient(TestCase): c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) ) + # Document that doesn't exist + c.sitemap_name='/does_not_exist' + self.assertRaises( ClientFatalError, c.parse_document ) + # and verbose with truncation... + with capture_stdout() as capturer: + c.verbose = True + c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' + c.max_sitemap_entries = 1 + c.parse_document() + self.assertTrue( re.search(r'Showing first 1 entries', capturer.result ) ) + self.assertTrue( re.search(r'\[0\] ', capturer.result ) ) + self.assertFalse( re.search(r'\[1\] ', capturer.result ) ) def test40_write_resource_list_mappings(self): c = Client() @@ -70,13 +155,17 @@ class TestClient(TestCase): def test41_write_resource_list_path(self): c = Client() c.set_mappings( ['http://example.org/','tests/testdata/'] ) + links=[{'rel':'uri_c','href':'uri_d'}] # with an explicit paths setting only the specified paths will be included with capture_stdout() as capturer: - c.write_resource_list(paths='tests/testdata/dir1') + c.write_resource_list(paths='tests/testdata/dir1', links=links) self.assertTrue( re.search(r'http://example.org/dir1/file_a', capturer.result ) ) self.assertTrue( re.search(r'http://example.org/dir1/file_b', capturer.result ) ) self.assertFalse( re.search(r'http://example.org/dir2/file_x', capturer.result ) ) + # check link present + self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) ) + self.assertTrue( re.search(r'href="uri_d"', capturer.result ) ) # Travis CI does not preserve timestamps from github so test here for the file # size but not the datestamp #self.assertTrue( re.search(r'http://example.org/dir1/file_a[\w\-:]+', capturer.result ) ) @@ -91,7 +180,7 @@ class TestClient(TestCase): self.assertFalse( os.path.exists(outfile) ) c.write_resource_list(paths='tests/testdata/dir1', dump=True) self.assertTrue( os.path.getsize(outfile)>100 ) - # (specific fuile) + # (specific file) outbase = os.path.join(self.tmpdir,'rl_out_dump') outfile = outbase+'00000.zip' self.assertFalse( os.path.exists(outfile) )