More tests, change file to filename

resync/client.py

@@ -194,18 +194,18 @@ class Client(object):
         num_deleted=0
         for resource in created:
             uri = resource.uri
-            file = self.mapper.src_to_dst(uri)
+            filename = self.mapper.src_to_dst(uri)
-            self.logger.info("created: %s -> %s" % (uri,file))
+            self.logger.info("created: %s -> %s" % (uri,filename))
-            num_created+=self.update_resource(resource,file,'created')
+            num_created+=self.update_resource(resource,filename,'created')
         for resource in updated:
             uri = resource.uri
-            file = self.mapper.src_to_dst(uri)
+            filename = self.mapper.src_to_dst(uri)
-            self.logger.info("updated: %s -> %s" % (uri,file))
+            self.logger.info("updated: %s -> %s" % (uri,filename))
-            num_updated+=self.update_resource(resource,file,'updated')
+            num_updated+=self.update_resource(resource,filename,'updated')
         for resource in deleted:
             uri = resource.uri
-            file = self.mapper.src_to_dst(uri)
+            filename = self.mapper.src_to_dst(uri)
-            num_deleted+=self.delete_resource(resource,file,allow_deletion)
+            num_deleted+=self.delete_resource(resource,filename,allow_deletion)
         ### 6. Store last timestamp to allow incremental sync
         if (not audit_only and self.last_timestamp>0):
             ClientState().set_state(self.sitemap,self.last_timestamp)
@@ -316,17 +316,17 @@ class Client(object):
         num_created = 0
         for resource in src_change_list:
             uri = resource.uri
-            file = self.mapper.src_to_dst(uri)
+            filename = self.mapper.src_to_dst(uri)
             if (resource.change == 'updated'):
-                self.logger.info("updated: %s -> %s" % (uri,file))
+                self.logger.info("updated: %s -> %s" % (uri,filename))
-                self.update_resource(resource,file,'updated')
+                self.update_resource(resource,filename,'updated')
                 num_updated+=1
             elif (resource.change == 'created'):
-                self.logger.info("created: %s -> %s" % (uri,file))
+                self.logger.info("created: %s -> %s" % (uri,filename))
-                self.update_resource(resource,file,'created')
+                self.update_resource(resource,filename,'created')
                 num_created+=1
             elif (resource.change == 'deleted'):
-                num_deleted+=self.delete_resource(resource,file,allow_deletion)
+                num_deleted+=self.delete_resource(resource,filename,allow_deletion)
             else:
                 raise ClientError("Unknown change type %s" % (resource.change) )
         ### 7. Report status and planned actions
@@ -339,8 +339,8 @@ class Client(object):
         ### 9. Done
         self.logger.debug("Completed incremental sync")
 
-    def update_resource(self, resource, file, change=None):
+    def update_resource(self, resource, filename, change=None):
-        """Update resource from uri to file on local system.
+        """Update resource from uri to filename on local system.
 
         Update means three things:
         1. GET resources
@@ -355,15 +355,15 @@ class Client(object):
 
         Returns the number of resources updated/created (0 or 1)
         """
-        path = os.path.dirname(file)
+        path = os.path.dirname(filename)
         distutils.dir_util.mkpath(path)
         num_updated=0
         if (self.dryrun):
-            self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,file))
+            self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,filename))
         else:
             # 1. GET
             try:
-                urlretrieve(resource.uri,file)
+                urlretrieve(resource.uri,filename)
                 num_updated+=1
             except IOError as e:
                 msg = "Failed to GET %s -- %s" % (resource.uri,str(e))
@@ -375,22 +375,22 @@ class Client(object):
             # 2. set timestamp if we have one
             if (resource.timestamp is not None):
                 unixtime = int(resource.timestamp) #no fractional
-                os.utime(file,(unixtime,unixtime))
+                os.utime(filename,(unixtime,unixtime))
                 if (resource.timestamp > self.last_timestamp):
                     self.last_timestamp = resource.timestamp
             self.log_event(Resource(resource=resource, change=change))
             # 3. sanity check
-            length = os.stat(file).st_size
+            length = os.stat(filename).st_size
-            if (resource.length != length):
+            if (resource.length is not None and resource.length != length):
                 self.logger.info("Downloaded size for %s of %d bytes does not match expected %d bytes" % (resource.uri,length,resource.length))
             if (self.checksum and resource.md5 is not None):
-                file_md5 = compute_md5_for_file(file)
+                file_md5 = compute_md5_for_file(filename)
                 if (resource.md5 != file_md5):
                     self.logger.info("MD5 mismatch for %s, got %s but expected %s bytes" % (resource.uri,file_md5,resource.md5))
         return(num_updated)
 
-    def delete_resource(self, resource, file, allow_deletion=False):
+    def delete_resource(self, resource, filename, allow_deletion=False):
-        """Delete copy of resource in file on local system.
+        """Delete copy of resource in filename on local system.
 
         Will only actually do the deletion if allow_deletion is True. Regardless 
         of whether the deletion occurs, self.last_timestamp will be updated 
@@ -405,20 +405,20 @@ class Client(object):
             self.last_timestamp = resource.timestamp
         if (allow_deletion):
             if (self.dryrun):
-                self.logger.info("dryrun: would delete %s -> %s" % (uri,file))
+                self.logger.info("dryrun: would delete %s -> %s" % (uri,filename))
             else:
                 try:
-                    os.unlink(file)
+                    os.unlink(filename)
                     num_deleted+=1
+                    self.logger.info("deleted: %s -> %s" % (uri,filename))
+                    self.log_event(Resource(resource=resource, change="deleted"))
                 except OSError as e:
-                    msg = "Failed to DELETE %s -> %s : %s" % (uri,file,str(e))
+                    msg = "Failed to DELETE %s -> %s : %s" % (uri,filename,str(e))
                     #if (self.ignore_failures):
                     self.logger.warning(msg)
                     #    return
                     #else:
                     #    raise ClientFatalError(msg)
-                self.logger.info("deleted: %s -> %s" % (uri,file))
-                self.log_event(Resource(resource=resource, change="deleted"))
         else:
             self.logger.info("nodelete: would delete %s (--delete to enable)" % uri)
         return(num_deleted)

tests/test_client.py

@@ -9,6 +9,7 @@ import sys
 import os.path
 
 from resync.client import Client, ClientFatalError
+from resync.resource import Resource
 from resync.resource_list import ResourceList
 from resync.change_list import ChangeList
 
@@ -34,6 +35,78 @@ class TestClient(TestCase):
         self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' )
         self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )
 
+    def test18_update_resource(self):
+        c = Client()
+        resource = Resource(uri='http://example.org/dir/2')
+        filename = os.path.join(self.tmpdir,'dir/resource2')
+        # dryrun
+        with LogCapture() as lc:
+            c.dryrun = True
+            c.logger = logging.getLogger('resync.client') 
+            n = c.update_resource( resource, filename )
+            self.assertEqual( n, 0 )
+            self.assertTrue( lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ') )
+        c.dryrun = False
+        # get from file uri that does not exist
+        resource = Resource(uri='tests/testdata/i_do_not_exist')
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            self.assertRaises( ClientFatalError, c.update_resource, resource, filename )
+        # get from file uri
+        resource = Resource(uri='tests/testdata/examples_from_spec/resourcesync_ex_1.xml')
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            n = c.update_resource( resource, filename )
+            self.assertEqual( n, 1 )
+            self.assertTrue( lc.records[-1].msg.startswith('Event: {') )
+
+    def test19_delete_resource(self):
+        c = Client()
+        resource = Resource(uri='http://example.org/1')
+        filename = os.path.join(self.tmpdir,'resource1')
+        c.last_timestamp = 5
+        # no delete, no timestamp update
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            n = c.delete_resource( resource, filename )
+            self.assertEqual( n, 0 )
+            self.assertEqual( lc.records[-1].msg,
+                              'nodelete: would delete http://example.org/1 (--delete to enable)' ) 
+            self.assertEqual( c.last_timestamp, 5 )
+        # no delete but timestamp update
+        resource.timestamp = 10
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            n = c.delete_resource( resource, filename )
+            self.assertEqual( n, 0 )
+            self.assertEqual( lc.records[-1].msg,
+                              'nodelete: would delete http://example.org/1 (--delete to enable)' ) 
+            self.assertEqual( c.last_timestamp, 10 )
+        # allow delete but dryrun
+        with LogCapture() as lc:
+            c.dryrun = True
+            c.logger = logging.getLogger('resync.client') 
+            n = c.delete_resource( resource, filename, allow_deletion=True )
+            self.assertEqual( n, 0 )
+            self.assertTrue( lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1') )
+        c.dryrun = False
+        # allow delete but no resource present
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            n = c.delete_resource( resource, filename, allow_deletion=True )
+            self.assertEqual( n, 0 )
+            self.assertTrue( lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1') )
+        # successful deletion, first make file...
+        with open(filename, 'w') as fh:
+            fh.write('delete me')
+            fh.close()
+        with LogCapture() as lc:
+            c.logger = logging.getLogger('resync.client') 
+            n = c.delete_resource( resource, filename, allow_deletion=True )
+            self.assertEqual( n, 1 )
+            self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) 
+            self.assertTrue( lc.records[-2].msg.startswith('deleted: http://example.org/1 ->') ) 
+
     def test20_parse_document(self):
         # Key property of the parse_document() method is that it parses the
         # document and identifies its type
@@ -54,6 +127,18 @@ class TestClient(TestCase):
             c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_22.xml'
             c.parse_document()
         self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) )
+        # Document that doesn't exist
+        c.sitemap_name='/does_not_exist'
+        self.assertRaises( ClientFatalError, c.parse_document )
+        # and verbose with truncation...
+        with capture_stdout() as capturer:
+            c.verbose = True
+            c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
+            c.max_sitemap_entries = 1 
+            c.parse_document()
+        self.assertTrue( re.search(r'Showing first 1 entries', capturer.result ) )
+        self.assertTrue( re.search(r'\[0\] ', capturer.result ) )
+        self.assertFalse( re.search(r'\[1\] ', capturer.result ) )
 
     def test40_write_resource_list_mappings(self):
         c = Client()
@@ -70,13 +155,17 @@ class TestClient(TestCase):
     def test41_write_resource_list_path(self):
         c = Client()
         c.set_mappings( ['http://example.org/','tests/testdata/'] )
+        links=[{'rel':'uri_c','href':'uri_d'}]
         # with an explicit paths setting only the specified paths will be included
         with capture_stdout() as capturer:
-            c.write_resource_list(paths='tests/testdata/dir1')
+            c.write_resource_list(paths='tests/testdata/dir1', links=links)
         self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
         self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
         self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
         self.assertFalse( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
+        # check link present
+        self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) )
+        self.assertTrue( re.search(r'href="uri_d"', capturer.result ) )
         # Travis CI does not preserve timestamps from github so test here for the file
         # size but not the datestamp
         #self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
@@ -91,7 +180,7 @@ class TestClient(TestCase):
         self.assertFalse( os.path.exists(outfile) )
         c.write_resource_list(paths='tests/testdata/dir1', dump=True)
         self.assertTrue( os.path.getsize(outfile)>100 )
-        # (specific fuile)
+        # (specific file)
         outbase = os.path.join(self.tmpdir,'rl_out_dump')
         outfile = outbase+'00000.zip'
         self.assertFalse( os.path.exists(outfile) )