From b5c3142435a21a1edc6ebafddfaaf2a7f0741e5f Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Wed, 16 Dec 2020 12:30:04 -0500 Subject: [PATCH] Use spec_version to correctly handle incrementals using ChangeList --- docs/pypi_upload.md | 55 +++++++++++++++++++++----------- resync-build | 3 +- resync-sync | 3 +- resync/change_list.py | 21 ++++++++++++ resync/client.py | 17 ++++++---- resync/client_utils.py | 5 +++ resync/resource_container.py | 18 ----------- tests/test_change_list.py | 46 ++++++++++++++++++++++++++ tests/test_resource_container.py | 21 ------------ 9 files changed, 123 insertions(+), 66 deletions(-) diff --git a/docs/pypi_upload.md b/docs/pypi_upload.md index 76bbe9e..70b4363 100644 --- a/docs/pypi_upload.md +++ b/docs/pypi_upload.md @@ -17,24 +17,43 @@ resync is at on pypi 8. Check client works with simulator: ``` - simeon@RottenApple resync>resync-sync --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: NOT IN SYNC (same=92, to create=0, to update=1, to delete=0) - Will GET 1 resources, and delete 0 resources - Status: SYNCED (same=92, created=0, updated=1, deleted=0) - simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: NO CHANGES (created=0, updated=0, deleted=0) - simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: CHANGES (created=1, updated=0, deleted=0) - simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: NO CHANGES (created=0, updated=0, deleted=0) - simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: IN SYNC (same=94, to create=0, to update=0, to delete=0) - simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: NOT IN SYNC (same=94, to create=1, to update=0, to delete=0) - simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: CHANGES (created=1, updated=0, deleted=0) - simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100 - Status: IN SYNC (same=95, to create=0, to update=0, to delete=0) + # Run simulator in one window + resync-simulator> ./resync-simulator + + # Run client in another + resync> rm -rf localhost_8888 + + resync> ./resync-sync --baseline --delete http://localhost:8888/ + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: NOT IN SYNC (same=0, to create=1003, to update=0, to delete=0) + Will GET 1003 resources, and delete 0 resources + Status: SYNCED (same=0, created=1003, updated=0, deleted=0) + + resync> ./resync-sync --baseline --delete http://localhost:8888/ + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: NOT IN SYNC (same=1000, to create=2, to update=1, to delete=2) + Will GET 3 resources, and delete 2 resources + Status: SYNCED (same=1000, created=2, updated=1, deleted=2) + + resync> ./resync-sync --incremental --delete http://localhost:8888/ + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: CHANGES APPLIED (created=4, updated=5, deleted=4) + Will apply 13 changes, and delete 4 resources + Status: NO CHANGES (created=4, updated=5, deleted=4) + + esync> ./resync-sync --incremental --delete http://localhost:8888/ + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: CHANGES APPLIED (created=0, updated=3, deleted=0) + Will apply 3 changes, and delete 0 resources + Status: NO CHANGES (created=0, updated=3, deleted=0) + + resync> ./resync-sync --incremental --delete http://localhost:8888/; ./resync-sync --audit http://localhost:8888/ + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: CHANGES APPLIED (created=1, updated=2, deleted=0) + Will apply 3 changes, and delete 0 resources + Status: NO CHANGES (created=1, updated=2, deleted=0) + Using URI mapping: http://localhost:8888/ -> localhost_8888 + Status: IN SYNC (same=1001, to create=0, to update=0, to delete=0) ``` 9. If all checks out OK, tag and push the new version to github: diff --git a/resync-build b/resync-build index f0e2b04..1be0d96 100755 --- a/resync-build +++ b/resync-build @@ -127,7 +127,8 @@ def main(): process_shared_misc_options(args) - c = Client(hashes=args.hash, + c = Client(spec_version=args.spec_version, + hashes=args.hash, verbose=args.verbose, dryrun=args.dryrun) diff --git a/resync-sync b/resync-sync index 41b8a71..10fd6ed 100755 --- a/resync-sync +++ b/resync-sync @@ -111,7 +111,8 @@ def main(): process_shared_misc_options(args, include_remote=True) - c = Client(hashes=args.hash, + c = Client(spec_version=args.spec_version, + hashes=args.hash, verbose=args.verbose, dryrun=args.dryrun) diff --git a/resync/change_list.py b/resync/change_list.py index ce9ada0..04cb6a1 100644 --- a/resync/change_list.py +++ b/resync/change_list.py @@ -63,3 +63,24 @@ class ChangeList(ListBaseWithIndex): for resource in resources: rc = Resource(resource=resource, change=change) self.add(rc) + + def prune_updates_before(self, timestamp, spec_version='1.1'): + """Remove all resource updates earlier than the given timestamp. + + Returns the number of entries removed. Will raise an excpetion + if there are any entries without a datetime (1.1) or + timestamp (1.0). + """ + n = 0 + pruned = [] + use_timestamp = (spec_version == '1.0') # Else use datetime + for r in self.resources: + ts = r.timestamp if use_timestamp else r.ts_datetime + if (ts is None): + raise Exception("Entry %s has no update datetime/timestamp" % (r.uri)) + elif (ts >= timestamp): + pruned.append(r) + else: + n += 1 + self.resources = pruned + return(n) diff --git a/resync/client.py b/resync/client.py index a1aa17e..741ce81 100644 --- a/resync/client.py +++ b/resync/client.py @@ -41,8 +41,9 @@ class Client(object): debug - very verbose for automated analysis """ - def __init__(self, hashes=None, verbose=False, dryrun=False): + def __init__(self, spec_version='1.1', hashes=None, verbose=False, dryrun=False): """Initialize Client object with default parameters.""" + self.spec_version = spec_version self.hashes = set(hashes) if hashes else set() self.verbose = verbose self.dryrun = dryrun @@ -382,12 +383,14 @@ class Client(object): # Check all changes have timestamp and record last self.last_timestamp = 0 for resource in src_change_list: - if (resource.timestamp is None): + if resource.timestamp is None and resource.ts_datetime is None: raise ClientFatalError( - "Aborting - missing timestamp for change in %s" % - (uri)) - if (resource.timestamp > self.last_timestamp): - self.last_timestamp = resource.timestamp + "Aborting - no datetime or lastmod for change in %s" % + (resource.uri)) + # Work with 1.0 or 1.1 -- use datetime if given, else lastmod + ts = resource.ts_datetime if resource.ts_datetime is not None else resource.timestamp + if (ts > self.last_timestamp): + self.last_timestamp = ts # 4. Check that the change list has authority over URIs listed # FIXME - What does authority mean for change list? Here use both the # change list URI and, if we used it, the sitemap URI @@ -403,7 +406,7 @@ class Client(object): (change_list, resource.uri)) # 5. Prune entries before starting timestamp and dupe changes for a # resource - num_skipped = src_change_list.prune_before(from_timestamp) + num_skipped = src_change_list.prune_updates_before(from_timestamp, spec_version=self.spec_version) if (num_skipped > 0): self.logger.info( "Skipped %d changes before %s" % diff --git a/resync/client_utils.py b/resync/client_utils.py index 19c5aae..003c0c2 100644 --- a/resync/client_utils.py +++ b/resync/client_utils.py @@ -199,6 +199,11 @@ def add_shared_misc_options(opt, default_logfile, include_remote=False): "to %s unless specified with --logfile" % (default_logfile)) opt.add_argument('--logfile', type=str, action='store', help="create detailed log of client actions") + opt.add_argument('--spec-version', default='1.1', choices=('1.0', '1.1'), + help="follow given ResourceSync specification version. The key difference is that v1.0 " + "used lastmod for the time of a change (often also the resource Last-Modification " + "time but not always). In v1.1 the rs:md datetime attribute in a ChangeList " + "indicates the time of the change, and use of lastmod is entirely optional") opt.add_argument('--verbose', '-v', action='store_true', help="verbose, show additional informational messages") diff --git a/resync/resource_container.py b/resync/resource_container.py index 6496c87..d2ed00f 100644 --- a/resync/resource_container.py +++ b/resync/resource_container.py @@ -202,24 +202,6 @@ class ResourceContainer(object): uris.append(r.uri) return(uris) - def prune_before(self, timestamp): - """Remove all resources with timestamp earlier than that given. - - Returns the number of entries removed. Will raise an excpetion - if there are any entries without a timestamp. - """ - n = 0 - pruned = [] - for r in self.resources: - if (r.timestamp is None): - raise Exception("Entry %s has no timestamp" % (r.uri)) - elif (r.timestamp >= timestamp): - pruned.append(r) - else: - n += 1 - self.resources = pruned - return(n) - def prune_dupes(self): """Remove all but the last entry for a given resource URI. diff --git a/tests/test_change_list.py b/tests/test_change_list.py index 8b87d48..3a79eba 100644 --- a/tests/test_change_list.py +++ b/tests/test_change_list.py @@ -146,3 +146,49 @@ class TestChangeList(unittest.TestCase): ' cl = ChangeList() self.assertRaises(SitemapParseError, cl.parse, fh=io.StringIO(xml)) + + def test11_prune_updates_before(self): + """Test prune_updates_before method.""" + # v1.0 + cl = ChangeList() + cl.resources.append(Resource('a', timestamp=1)) + cl.resources.append(Resource('b', timestamp=2)) + cl.resources.append(Resource('c', timestamp=3)) + cl.resources.append(Resource('d', timestamp=4)) + cl.prune_updates_before(3, spec_version='1.0') + self.assertEqual(len(cl.resources), 2) + i = iter(cl) + self.assertEqual(next(i).uri, 'c') + self.assertEqual(next(i).uri, 'd') + # put some more back out of order + cl.resources.append(Resource('a', timestamp=1)) + cl.resources.append(Resource('b', timestamp=2)) + cl.resources.append(Resource('e', timestamp=1000)) + cl.prune_updates_before(3.5, spec_version='1.0') + self.assertEqual(len(cl.resources), 2) + i = iter(cl) + self.assertEqual(next(i).uri, 'd') + self.assertEqual(next(i).uri, 'e') + # without a timestamp + cl.resources.append(Resource('nt_1_0', ts_datetime=123)) + self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.0') + # v1.1 + cl = ChangeList() + cl.resources.append(Resource('aa', ts_datetime=1)) + cl.resources.append(Resource('bb', ts_datetime=2)) + cl.resources.append(Resource('cc', ts_datetime=3)) + cl.prune_updates_before(3, spec_version='1.1') + self.assertEqual(len(cl.resources), 1) + i = iter(cl) + self.assertEqual(next(i).uri, 'cc') + # put some more back out of order + cl.resources.append(Resource('aa', ts_datetime=1)) + cl.resources.append(Resource('bb', ts_datetime=2)) + cl.resources.append(Resource('ee', ts_datetime=1000)) + cl.prune_updates_before(3.5, spec_version='1.1') + self.assertEqual(len(cl.resources), 1) + i = iter(cl) + self.assertEqual(next(i).uri, 'ee') + # without a ts_datetime + cl.resources.append(Resource('nt_1_1', timestamp=456)) + self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.1') diff --git a/tests/test_resource_container.py b/tests/test_resource_container.py index a5403c5..d60031f 100644 --- a/tests/test_resource_container.py +++ b/tests/test_resource_container.py @@ -25,27 +25,6 @@ class TestResourceContainer(unittest.TestCase): self.assertEqual(resources[0].uri, 'a') self.assertEqual(resources[3].uri, 'd') - def test03_prune_before(self): - rc = ResourceContainer() - rc.resources.append(Resource('a', timestamp=1)) - rc.resources.append(Resource('b', timestamp=2)) - rc.resources.append(Resource('c', timestamp=3)) - rc.resources.append(Resource('d', timestamp=4)) - rc.prune_before(3) - self.assertEqual(len(rc.resources), 2) - i = iter(rc) - self.assertEqual(next(i).uri, 'c') - self.assertEqual(next(i).uri, 'd') - # put some more back out of order - rc.resources.append(Resource('a', timestamp=1)) - rc.resources.append(Resource('b', timestamp=2)) - rc.resources.append(Resource('e', timestamp=1000)) - rc.prune_before(3.5) - self.assertEqual(len(rc.resources), 2) - i = iter(rc) - self.assertEqual(next(i).uri, 'd') - self.assertEqual(next(i).uri, 'e') - def test04_prune_dupes(self): rc = ResourceContainer() rc.resources.append(Resource('a', timestamp=1, change='created'))