More tests, change file to filename
This commit is contained in:
Simeon Warner
parent
750ec878f2
commit
b5217b9eed
@ -194,18 +194,18 @@ class Client(object):
|
||||
num_deleted=0
|
||||
for resource in created:
|
||||
uri = resource.uri
|
||||
file = self.mapper.src_to_dst(uri)
|
||||
self.logger.info("created: %s -> %s" % (uri,file))
|
||||
num_created+=self.update_resource(resource,file,'created')
|
||||
filename = self.mapper.src_to_dst(uri)
|
||||
self.logger.info("created: %s -> %s" % (uri,filename))
|
||||
num_created+=self.update_resource(resource,filename,'created')
|
||||
for resource in updated:
|
||||
uri = resource.uri
|
||||
file = self.mapper.src_to_dst(uri)
|
||||
self.logger.info("updated: %s -> %s" % (uri,file))
|
||||
num_updated+=self.update_resource(resource,file,'updated')
|
||||
filename = self.mapper.src_to_dst(uri)
|
||||
self.logger.info("updated: %s -> %s" % (uri,filename))
|
||||
num_updated+=self.update_resource(resource,filename,'updated')
|
||||
for resource in deleted:
|
||||
uri = resource.uri
|
||||
file = self.mapper.src_to_dst(uri)
|
||||
num_deleted+=self.delete_resource(resource,file,allow_deletion)
|
||||
filename = self.mapper.src_to_dst(uri)
|
||||
num_deleted+=self.delete_resource(resource,filename,allow_deletion)
|
||||
### 6. Store last timestamp to allow incremental sync
|
||||
if (not audit_only and self.last_timestamp>0):
|
||||
ClientState().set_state(self.sitemap,self.last_timestamp)
|
||||
@ -316,17 +316,17 @@ class Client(object):
|
||||
num_created = 0
|
||||
for resource in src_change_list:
|
||||
uri = resource.uri
|
||||
file = self.mapper.src_to_dst(uri)
|
||||
filename = self.mapper.src_to_dst(uri)
|
||||
if (resource.change == 'updated'):
|
||||
self.logger.info("updated: %s -> %s" % (uri,file))
|
||||
self.update_resource(resource,file,'updated')
|
||||
self.logger.info("updated: %s -> %s" % (uri,filename))
|
||||
self.update_resource(resource,filename,'updated')
|
||||
num_updated+=1
|
||||
elif (resource.change == 'created'):
|
||||
self.logger.info("created: %s -> %s" % (uri,file))
|
||||
self.update_resource(resource,file,'created')
|
||||
self.logger.info("created: %s -> %s" % (uri,filename))
|
||||
self.update_resource(resource,filename,'created')
|
||||
num_created+=1
|
||||
elif (resource.change == 'deleted'):
|
||||
num_deleted+=self.delete_resource(resource,file,allow_deletion)
|
||||
num_deleted+=self.delete_resource(resource,filename,allow_deletion)
|
||||
else:
|
||||
raise ClientError("Unknown change type %s" % (resource.change) )
|
||||
### 7. Report status and planned actions
|
||||
@ -339,8 +339,8 @@ class Client(object):
|
||||
### 9. Done
|
||||
self.logger.debug("Completed incremental sync")
|
||||
|
||||
def update_resource(self, resource, file, change=None):
|
||||
"""Update resource from uri to file on local system.
|
||||
def update_resource(self, resource, filename, change=None):
|
||||
"""Update resource from uri to filename on local system.
|
||||
|
||||
Update means three things:
|
||||
1. GET resources
|
||||
@ -355,15 +355,15 @@ class Client(object):
|
||||
|
||||
Returns the number of resources updated/created (0 or 1)
|
||||
"""
|
||||
path = os.path.dirname(file)
|
||||
path = os.path.dirname(filename)
|
||||
distutils.dir_util.mkpath(path)
|
||||
num_updated=0
|
||||
if (self.dryrun):
|
||||
self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,file))
|
||||
self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,filename))
|
||||
else:
|
||||
# 1. GET
|
||||
try:
|
||||
urlretrieve(resource.uri,file)
|
||||
urlretrieve(resource.uri,filename)
|
||||
num_updated+=1
|
||||
except IOError as e:
|
||||
msg = "Failed to GET %s -- %s" % (resource.uri,str(e))
|
||||
@ -375,22 +375,22 @@ class Client(object):
|
||||
# 2. set timestamp if we have one
|
||||
if (resource.timestamp is not None):
|
||||
unixtime = int(resource.timestamp) #no fractional
|
||||
os.utime(file,(unixtime,unixtime))
|
||||
os.utime(filename,(unixtime,unixtime))
|
||||
if (resource.timestamp > self.last_timestamp):
|
||||
self.last_timestamp = resource.timestamp
|
||||
self.log_event(Resource(resource=resource, change=change))
|
||||
# 3. sanity check
|
||||
length = os.stat(file).st_size
|
||||
if (resource.length != length):
|
||||
length = os.stat(filename).st_size
|
||||
if (resource.length is not None and resource.length != length):
|
||||
self.logger.info("Downloaded size for %s of %d bytes does not match expected %d bytes" % (resource.uri,length,resource.length))
|
||||
if (self.checksum and resource.md5 is not None):
|
||||
file_md5 = compute_md5_for_file(file)
|
||||
file_md5 = compute_md5_for_file(filename)
|
||||
if (resource.md5 != file_md5):
|
||||
self.logger.info("MD5 mismatch for %s, got %s but expected %s bytes" % (resource.uri,file_md5,resource.md5))
|
||||
return(num_updated)
|
||||
|
||||
def delete_resource(self, resource, file, allow_deletion=False):
|
||||
"""Delete copy of resource in file on local system.
|
||||
def delete_resource(self, resource, filename, allow_deletion=False):
|
||||
"""Delete copy of resource in filename on local system.
|
||||
|
||||
Will only actually do the deletion if allow_deletion is True. Regardless
|
||||
of whether the deletion occurs, self.last_timestamp will be updated
|
||||
@ -405,20 +405,20 @@ class Client(object):
|
||||
self.last_timestamp = resource.timestamp
|
||||
if (allow_deletion):
|
||||
if (self.dryrun):
|
||||
self.logger.info("dryrun: would delete %s -> %s" % (uri,file))
|
||||
self.logger.info("dryrun: would delete %s -> %s" % (uri,filename))
|
||||
else:
|
||||
try:
|
||||
os.unlink(file)
|
||||
os.unlink(filename)
|
||||
num_deleted+=1
|
||||
self.logger.info("deleted: %s -> %s" % (uri,filename))
|
||||
self.log_event(Resource(resource=resource, change="deleted"))
|
||||
except OSError as e:
|
||||
msg = "Failed to DELETE %s -> %s : %s" % (uri,file,str(e))
|
||||
msg = "Failed to DELETE %s -> %s : %s" % (uri,filename,str(e))
|
||||
#if (self.ignore_failures):
|
||||
self.logger.warning(msg)
|
||||
# return
|
||||
#else:
|
||||
# raise ClientFatalError(msg)
|
||||
self.logger.info("deleted: %s -> %s" % (uri,file))
|
||||
self.log_event(Resource(resource=resource, change="deleted"))
|
||||
else:
|
||||
self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
|
||||
return(num_deleted)
|
||||
|
||||
@ -9,6 +9,7 @@ import sys
|
||||
import os.path
|
||||
|
||||
from resync.client import Client, ClientFatalError
|
||||
from resync.resource import Resource
|
||||
from resync.resource_list import ResourceList
|
||||
from resync.change_list import ChangeList
|
||||
|
||||
@ -34,6 +35,78 @@ class TestClient(TestCase):
|
||||
self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' )
|
||||
self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )
|
||||
|
||||
def test18_update_resource(self):
|
||||
c = Client()
|
||||
resource = Resource(uri='http://example.org/dir/2')
|
||||
filename = os.path.join(self.tmpdir,'dir/resource2')
|
||||
# dryrun
|
||||
with LogCapture() as lc:
|
||||
c.dryrun = True
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.update_resource( resource, filename )
|
||||
self.assertEqual( n, 0 )
|
||||
self.assertTrue( lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ') )
|
||||
c.dryrun = False
|
||||
# get from file uri that does not exist
|
||||
resource = Resource(uri='tests/testdata/i_do_not_exist')
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
self.assertRaises( ClientFatalError, c.update_resource, resource, filename )
|
||||
# get from file uri
|
||||
resource = Resource(uri='tests/testdata/examples_from_spec/resourcesync_ex_1.xml')
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.update_resource( resource, filename )
|
||||
self.assertEqual( n, 1 )
|
||||
self.assertTrue( lc.records[-1].msg.startswith('Event: {') )
|
||||
|
||||
def test19_delete_resource(self):
|
||||
c = Client()
|
||||
resource = Resource(uri='http://example.org/1')
|
||||
filename = os.path.join(self.tmpdir,'resource1')
|
||||
c.last_timestamp = 5
|
||||
# no delete, no timestamp update
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.delete_resource( resource, filename )
|
||||
self.assertEqual( n, 0 )
|
||||
self.assertEqual( lc.records[-1].msg,
|
||||
'nodelete: would delete http://example.org/1 (--delete to enable)' )
|
||||
self.assertEqual( c.last_timestamp, 5 )
|
||||
# no delete but timestamp update
|
||||
resource.timestamp = 10
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.delete_resource( resource, filename )
|
||||
self.assertEqual( n, 0 )
|
||||
self.assertEqual( lc.records[-1].msg,
|
||||
'nodelete: would delete http://example.org/1 (--delete to enable)' )
|
||||
self.assertEqual( c.last_timestamp, 10 )
|
||||
# allow delete but dryrun
|
||||
with LogCapture() as lc:
|
||||
c.dryrun = True
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.delete_resource( resource, filename, allow_deletion=True )
|
||||
self.assertEqual( n, 0 )
|
||||
self.assertTrue( lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1') )
|
||||
c.dryrun = False
|
||||
# allow delete but no resource present
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.delete_resource( resource, filename, allow_deletion=True )
|
||||
self.assertEqual( n, 0 )
|
||||
self.assertTrue( lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1') )
|
||||
# successful deletion, first make file...
|
||||
with open(filename, 'w') as fh:
|
||||
fh.write('delete me')
|
||||
fh.close()
|
||||
with LogCapture() as lc:
|
||||
c.logger = logging.getLogger('resync.client')
|
||||
n = c.delete_resource( resource, filename, allow_deletion=True )
|
||||
self.assertEqual( n, 1 )
|
||||
self.assertTrue( lc.records[-1].msg.startswith('Event: {') )
|
||||
self.assertTrue( lc.records[-2].msg.startswith('deleted: http://example.org/1 ->') )
|
||||
|
||||
def test20_parse_document(self):
|
||||
# Key property of the parse_document() method is that it parses the
|
||||
# document and identifies its type
|
||||
@ -54,6 +127,18 @@ class TestClient(TestCase):
|
||||
c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_22.xml'
|
||||
c.parse_document()
|
||||
self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) )
|
||||
# Document that doesn't exist
|
||||
c.sitemap_name='/does_not_exist'
|
||||
self.assertRaises( ClientFatalError, c.parse_document )
|
||||
# and verbose with truncation...
|
||||
with capture_stdout() as capturer:
|
||||
c.verbose = True
|
||||
c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
|
||||
c.max_sitemap_entries = 1
|
||||
c.parse_document()
|
||||
self.assertTrue( re.search(r'Showing first 1 entries', capturer.result ) )
|
||||
self.assertTrue( re.search(r'\[0\] ', capturer.result ) )
|
||||
self.assertFalse( re.search(r'\[1\] ', capturer.result ) )
|
||||
|
||||
def test40_write_resource_list_mappings(self):
|
||||
c = Client()
|
||||
@ -70,13 +155,17 @@ class TestClient(TestCase):
|
||||
def test41_write_resource_list_path(self):
|
||||
c = Client()
|
||||
c.set_mappings( ['http://example.org/','tests/testdata/'] )
|
||||
links=[{'rel':'uri_c','href':'uri_d'}]
|
||||
# with an explicit paths setting only the specified paths will be included
|
||||
with capture_stdout() as capturer:
|
||||
c.write_resource_list(paths='tests/testdata/dir1')
|
||||
c.write_resource_list(paths='tests/testdata/dir1', links=links)
|
||||
self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
|
||||
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
|
||||
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
|
||||
self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
|
||||
# check link present
|
||||
self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) )
|
||||
self.assertTrue( re.search(r'href="uri_d"', capturer.result ) )
|
||||
# Travis CI does not preserve timestamps from github so test here for the file
|
||||
# size but not the datestamp
|
||||
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
|
||||
@ -91,7 +180,7 @@ class TestClient(TestCase):
|
||||
self.assertFalse( os.path.exists(outfile) )
|
||||
c.write_resource_list(paths='tests/testdata/dir1', dump=True)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
# (specific fuile)
|
||||
# (specific file)
|
||||
outbase = os.path.join(self.tmpdir,'rl_out_dump')
|
||||
outfile = outbase+'00000.zip'
|
||||
self.assertFalse( os.path.exists(outfile) )
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
