Use spec_version to correctly handle incrementals using ChangeList

docs/pypi_upload.md

@@ -17,24 +17,43 @@ resync is at <https://pypi.python.org/pypi/resync> on pypi
 8. Check client works with simulator:
 
    ```
-   simeon@RottenApple resync>resync-sync --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   # Run simulator in one window
-   Status: NOT IN SYNC (same=92, to create=0, to update=1, to delete=0)
+   resync-simulator> ./resync-simulator
-   Will GET 1 resources, and delete 0 resources
+
-   Status:      SYNCED (same=92, created=0, updated=1, deleted=0)
+   # Run client in another
-   simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   resync> rm -rf localhost_8888
-   Status:  NO CHANGES (created=0, updated=0, deleted=0)
+
-   simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   resync> ./resync-sync --baseline --delete http://localhost:8888/
-   Status:     CHANGES (created=1, updated=0, deleted=0)
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
-   simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   Status:     NOT IN SYNC (same=0, to create=1003, to update=0, to delete=0)
-   Status:  NO CHANGES (created=0, updated=0, deleted=0)
+   Will GET 1003 resources, and delete 0 resources
-   simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   Status:          SYNCED (same=0, created=1003, updated=0, deleted=0)
-   Status:     IN SYNC (same=94, to create=0, to update=0, to delete=0)
+
-   simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   resync> ./resync-sync --baseline --delete http://localhost:8888/
-   Status: NOT IN SYNC (same=94, to create=1, to update=0, to delete=0)
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
-   simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   Status:     NOT IN SYNC (same=1000, to create=2, to update=1, to delete=2)
-   Status:     CHANGES (created=1, updated=0, deleted=0)
+   Will GET 3 resources, and delete 2 resources
-   simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
+   Status:          SYNCED (same=1000, created=2, updated=1, deleted=2)
-   Status:     IN SYNC (same=95, to create=0, to update=0, to delete=0)
+
+   resync> ./resync-sync --incremental --delete http://localhost:8888/
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
+   Status: CHANGES APPLIED (created=4, updated=5, deleted=4)
+   Will apply 13 changes, and delete 4 resources
+   Status:      NO CHANGES (created=4, updated=5, deleted=4)
+
+   esync> ./resync-sync --incremental --delete http://localhost:8888/
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
+   Status: CHANGES APPLIED (created=0, updated=3, deleted=0)
+   Will apply 3 changes, and delete 0 resources
+   Status:      NO CHANGES (created=0, updated=3, deleted=0)
+
+   resync> ./resync-sync --incremental --delete http://localhost:8888/; ./resync-sync --audit http://localhost:8888/
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
+   Status: CHANGES APPLIED (created=1, updated=2, deleted=0)
+   Will apply 3 changes, and delete 0 resources
+   Status:      NO CHANGES (created=1, updated=2, deleted=0)
+   Using URI mapping: http://localhost:8888/ -> localhost_8888
+   Status:         IN SYNC (same=1001, to create=0, to update=0, to delete=0)
    ```
 
 9. If all checks out OK, tag and push the new version to github:

resync-build

@@ -127,7 +127,8 @@ def main():
 
     process_shared_misc_options(args)
 
-    c = Client(hashes=args.hash,
+    c = Client(spec_version=args.spec_version,
+               hashes=args.hash,
                verbose=args.verbose,
                dryrun=args.dryrun)
 

resync-sync

@@ -111,7 +111,8 @@ def main():
 
     process_shared_misc_options(args, include_remote=True)
 
-    c = Client(hashes=args.hash,
+    c = Client(spec_version=args.spec_version,
+               hashes=args.hash,
                verbose=args.verbose,
                dryrun=args.dryrun)
 

resync/change_list.py

@@ -63,3 +63,24 @@ class ChangeList(ListBaseWithIndex):
         for resource in resources:
             rc = Resource(resource=resource, change=change)
             self.add(rc)
+
+    def prune_updates_before(self, timestamp, spec_version='1.1'):
+        """Remove all resource updates earlier than the given timestamp.
+
+        Returns the number of entries removed. Will raise an excpetion
+        if there are any entries without a datetime (1.1) or
+        timestamp (1.0).
+        """
+        n = 0
+        pruned = []
+        use_timestamp = (spec_version == '1.0')  # Else use datetime
+        for r in self.resources:
+            ts = r.timestamp if use_timestamp else r.ts_datetime
+            if (ts is None):
+                raise Exception("Entry %s has no update datetime/timestamp" % (r.uri))
+            elif (ts >= timestamp):
+                pruned.append(r)
+            else:
+                n += 1
+        self.resources = pruned
+        return(n)

resync/client.py

@@ -41,8 +41,9 @@ class Client(object):
       debug   - very verbose for automated analysis
     """
 
-    def __init__(self, hashes=None, verbose=False, dryrun=False):
+    def __init__(self, spec_version='1.1', hashes=None, verbose=False, dryrun=False):
         """Initialize Client object with default parameters."""
+        self.spec_version = spec_version
         self.hashes = set(hashes) if hashes else set()
         self.verbose = verbose
         self.dryrun = dryrun
@@ -382,12 +383,14 @@ class Client(object):
         # Check all changes have timestamp and record last
         self.last_timestamp = 0
         for resource in src_change_list:
-            if (resource.timestamp is None):
+            if resource.timestamp is None and resource.ts_datetime is None:
                 raise ClientFatalError(
-                    "Aborting - missing timestamp for change in %s" %
+                    "Aborting - no datetime or lastmod for change in %s" %
-                    (uri))
+                    (resource.uri))
-            if (resource.timestamp > self.last_timestamp):
+            # Work with 1.0 or 1.1 -- use datetime if given, else lastmod
-                self.last_timestamp = resource.timestamp
+            ts = resource.ts_datetime if resource.ts_datetime is not None else resource.timestamp
+            if (ts > self.last_timestamp):
+                self.last_timestamp = ts
         # 4. Check that the change list has authority over URIs listed
         # FIXME - What does authority mean for change list? Here use both the
         # change list URI and, if we used it, the sitemap URI
@@ -403,7 +406,7 @@ class Client(object):
                             (change_list, resource.uri))
         # 5. Prune entries before starting timestamp and dupe changes for a
         # resource
-        num_skipped = src_change_list.prune_before(from_timestamp)
+        num_skipped = src_change_list.prune_updates_before(from_timestamp, spec_version=self.spec_version)
         if (num_skipped > 0):
             self.logger.info(
                 "Skipped %d changes before %s" %

resync/client_utils.py

@@ -199,6 +199,11 @@ def add_shared_misc_options(opt, default_logfile, include_remote=False):
                           "to %s unless specified with --logfile" % (default_logfile))
     opt.add_argument('--logfile', type=str, action='store',
                      help="create detailed log of client actions")
+    opt.add_argument('--spec-version', default='1.1', choices=('1.0', '1.1'),
+                     help="follow given ResourceSync specification version. The key difference is that v1.0 "
+                          "used lastmod for the time of a change (often also the resource Last-Modification "
+                          "time but not always). In v1.1 the rs:md datetime attribute in a ChangeList "
+                          "indicates the time of the change, and use of lastmod is entirely optional")
     opt.add_argument('--verbose', '-v', action='store_true',
                      help="verbose, show additional informational messages")
 

resync/resource_container.py

@@ -202,24 +202,6 @@ class ResourceContainer(object):
             uris.append(r.uri)
         return(uris)
 
-    def prune_before(self, timestamp):
-        """Remove all resources with timestamp earlier than that given.
-
-        Returns the number of entries removed. Will raise an excpetion
-        if there are any entries without a timestamp.
-        """
-        n = 0
-        pruned = []
-        for r in self.resources:
-            if (r.timestamp is None):
-                raise Exception("Entry %s has no timestamp" % (r.uri))
-            elif (r.timestamp >= timestamp):
-                pruned.append(r)
-            else:
-                n += 1
-        self.resources = pruned
-        return(n)
-
     def prune_dupes(self):
         """Remove all but the last entry for a given resource URI.
 

tests/test_change_list.py

@@ -146,3 +146,49 @@ class TestChangeList(unittest.TestCase):
 </urlset>'
         cl = ChangeList()
         self.assertRaises(SitemapParseError, cl.parse, fh=io.StringIO(xml))
+
+    def test11_prune_updates_before(self):
+        """Test prune_updates_before method."""
+        # v1.0
+        cl = ChangeList()
+        cl.resources.append(Resource('a', timestamp=1))
+        cl.resources.append(Resource('b', timestamp=2))
+        cl.resources.append(Resource('c', timestamp=3))
+        cl.resources.append(Resource('d', timestamp=4))
+        cl.prune_updates_before(3, spec_version='1.0')
+        self.assertEqual(len(cl.resources), 2)
+        i = iter(cl)
+        self.assertEqual(next(i).uri, 'c')
+        self.assertEqual(next(i).uri, 'd')
+        # put some more back out of order
+        cl.resources.append(Resource('a', timestamp=1))
+        cl.resources.append(Resource('b', timestamp=2))
+        cl.resources.append(Resource('e', timestamp=1000))
+        cl.prune_updates_before(3.5, spec_version='1.0')
+        self.assertEqual(len(cl.resources), 2)
+        i = iter(cl)
+        self.assertEqual(next(i).uri, 'd')
+        self.assertEqual(next(i).uri, 'e')
+        # without a timestamp
+        cl.resources.append(Resource('nt_1_0', ts_datetime=123))
+        self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.0')
+        # v1.1
+        cl = ChangeList()
+        cl.resources.append(Resource('aa', ts_datetime=1))
+        cl.resources.append(Resource('bb', ts_datetime=2))
+        cl.resources.append(Resource('cc', ts_datetime=3))
+        cl.prune_updates_before(3, spec_version='1.1')
+        self.assertEqual(len(cl.resources), 1)
+        i = iter(cl)
+        self.assertEqual(next(i).uri, 'cc')
+        # put some more back out of order
+        cl.resources.append(Resource('aa', ts_datetime=1))
+        cl.resources.append(Resource('bb', ts_datetime=2))
+        cl.resources.append(Resource('ee', ts_datetime=1000))
+        cl.prune_updates_before(3.5, spec_version='1.1')
+        self.assertEqual(len(cl.resources), 1)
+        i = iter(cl)
+        self.assertEqual(next(i).uri, 'ee')
+        # without a ts_datetime
+        cl.resources.append(Resource('nt_1_1', timestamp=456))
+        self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.1')

tests/test_resource_container.py

@@ -25,27 +25,6 @@ class TestResourceContainer(unittest.TestCase):
         self.assertEqual(resources[0].uri, 'a')
         self.assertEqual(resources[3].uri, 'd')
 
-    def test03_prune_before(self):
-        rc = ResourceContainer()
-        rc.resources.append(Resource('a', timestamp=1))
-        rc.resources.append(Resource('b', timestamp=2))
-        rc.resources.append(Resource('c', timestamp=3))
-        rc.resources.append(Resource('d', timestamp=4))
-        rc.prune_before(3)
-        self.assertEqual(len(rc.resources), 2)
-        i = iter(rc)
-        self.assertEqual(next(i).uri, 'c')
-        self.assertEqual(next(i).uri, 'd')
-        # put some more back out of order
-        rc.resources.append(Resource('a', timestamp=1))
-        rc.resources.append(Resource('b', timestamp=2))
-        rc.resources.append(Resource('e', timestamp=1000))
-        rc.prune_before(3.5)
-        self.assertEqual(len(rc.resources), 2)
-        i = iter(rc)
-        self.assertEqual(next(i).uri, 'd')
-        self.assertEqual(next(i).uri, 'e')
-
     def test04_prune_dupes(self):
         rc = ResourceContainer()
         rc.resources.append(Resource('a', timestamp=1, change='created'))