gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use spec_version to correctly handle incrementals using ChangeList

Browse Source
This commit is contained in:
Simeon Warner 2020-12-16 12:30:04 -05:00
parent 9f26daaf74
commit b5c3142435
9 changed files with 123 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
docs/pypi_upload.md
View File

@ -17,24 +17,43 @@ resync is at <https://pypi.python.org/pypi/resync> on pypi
8. Check client works with simulator:
```
simeon@RottenApple resync>resync-sync --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: NOT IN SYNC (same=92, to create=0, to update=1, to delete=0)
Will GET 1 resources, and delete 0 resources
Status: SYNCED (same=92, created=0, updated=1, deleted=0)
simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: NO CHANGES (created=0, updated=0, deleted=0)
simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: CHANGES (created=1, updated=0, deleted=0)
simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: NO CHANGES (created=0, updated=0, deleted=0)
simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: IN SYNC (same=94, to create=0, to update=0, to delete=0)
simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: NOT IN SYNC (same=94, to create=1, to update=0, to delete=0)
simeon@RottenApple resync>resync-sync -i --delete --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: CHANGES (created=1, updated=0, deleted=0)
simeon@RottenApple resync>resync-sync -a --capabilitylist=http://resync.library.cornell.edu/sim100/capabilitylist.xml http://resync.library.cornell.edu/sim100
Status: IN SYNC (same=95, to create=0, to update=0, to delete=0)
# Run simulator in one window
resync-simulator> ./resync-simulator
# Run client in another
resync> rm -rf localhost_8888
resync> ./resync-sync --baseline --delete http://localhost:8888/
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: NOT IN SYNC (same=0, to create=1003, to update=0, to delete=0)
Will GET 1003 resources, and delete 0 resources
Status: SYNCED (same=0, created=1003, updated=0, deleted=0)
resync> ./resync-sync --baseline --delete http://localhost:8888/
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: NOT IN SYNC (same=1000, to create=2, to update=1, to delete=2)
Will GET 3 resources, and delete 2 resources
Status: SYNCED (same=1000, created=2, updated=1, deleted=2)
resync> ./resync-sync --incremental --delete http://localhost:8888/
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: CHANGES APPLIED (created=4, updated=5, deleted=4)
Will apply 13 changes, and delete 4 resources
Status: NO CHANGES (created=4, updated=5, deleted=4)
esync> ./resync-sync --incremental --delete http://localhost:8888/
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: CHANGES APPLIED (created=0, updated=3, deleted=0)
Will apply 3 changes, and delete 0 resources
Status: NO CHANGES (created=0, updated=3, deleted=0)
resync> ./resync-sync --incremental --delete http://localhost:8888/; ./resync-sync --audit http://localhost:8888/
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: CHANGES APPLIED (created=1, updated=2, deleted=0)
Will apply 3 changes, and delete 0 resources
Status: NO CHANGES (created=1, updated=2, deleted=0)
Using URI mapping: http://localhost:8888/ -> localhost_8888
Status: IN SYNC (same=1001, to create=0, to update=0, to delete=0)
```
9. If all checks out OK, tag and push the new version to github:

3
resync-build
View File

@ -127,7 +127,8 @@ def main():
process_shared_misc_options(args)
c = Client(hashes=args.hash,
c = Client(spec_version=args.spec_version,
hashes=args.hash,
verbose=args.verbose,
dryrun=args.dryrun)

3
resync-sync
View File

@ -111,7 +111,8 @@ def main():
process_shared_misc_options(args, include_remote=True)
c = Client(hashes=args.hash,
c = Client(spec_version=args.spec_version,
hashes=args.hash,
verbose=args.verbose,
dryrun=args.dryrun)

21
resync/change_list.py
View File

@ -63,3 +63,24 @@ class ChangeList(ListBaseWithIndex):
for resource in resources:
rc = Resource(resource=resource, change=change)
self.add(rc)
def prune_updates_before(self, timestamp, spec_version='1.1'):
"""Remove all resource updates earlier than the given timestamp.
Returns the number of entries removed. Will raise an excpetion
if there are any entries without a datetime (1.1) or
timestamp (1.0).
"""
n = 0
pruned = []
use_timestamp = (spec_version == '1.0') # Else use datetime
for r in self.resources:
ts = r.timestamp if use_timestamp else r.ts_datetime
if (ts is None):
raise Exception("Entry %s has no update datetime/timestamp" % (r.uri))
elif (ts >= timestamp):
pruned.append(r)
else:
n += 1
self.resources = pruned
return(n)

17
resync/client.py
View File

@ -41,8 +41,9 @@ class Client(object):
debug - very verbose for automated analysis
"""
def __init__(self, hashes=None, verbose=False, dryrun=False):
def __init__(self, spec_version='1.1', hashes=None, verbose=False, dryrun=False):
"""Initialize Client object with default parameters."""
self.spec_version = spec_version
self.hashes = set(hashes) if hashes else set()
self.verbose = verbose
self.dryrun = dryrun
@ -382,12 +383,14 @@ class Client(object):
# Check all changes have timestamp and record last
self.last_timestamp = 0
for resource in src_change_list:
if (resource.timestamp is None):
if resource.timestamp is None and resource.ts_datetime is None:
raise ClientFatalError(
"Aborting - missing timestamp for change in %s" %
(uri))
if (resource.timestamp > self.last_timestamp):
self.last_timestamp = resource.timestamp
"Aborting - no datetime or lastmod for change in %s" %
(resource.uri))
# Work with 1.0 or 1.1 -- use datetime if given, else lastmod
ts = resource.ts_datetime if resource.ts_datetime is not None else resource.timestamp
if (ts > self.last_timestamp):
self.last_timestamp = ts
# 4. Check that the change list has authority over URIs listed
# FIXME - What does authority mean for change list? Here use both the
# change list URI and, if we used it, the sitemap URI
@ -403,7 +406,7 @@ class Client(object):
(change_list, resource.uri))
# 5. Prune entries before starting timestamp and dupe changes for a
# resource
num_skipped = src_change_list.prune_before(from_timestamp)
num_skipped = src_change_list.prune_updates_before(from_timestamp, spec_version=self.spec_version)
if (num_skipped > 0):
self.logger.info(
"Skipped %d changes before %s" %

5
resync/client_utils.py
View File

@ -199,6 +199,11 @@ def add_shared_misc_options(opt, default_logfile, include_remote=False):
"to %s unless specified with --logfile" % (default_logfile))
opt.add_argument('--logfile', type=str, action='store',
help="create detailed log of client actions")
opt.add_argument('--spec-version', default='1.1', choices=('1.0', '1.1'),
help="follow given ResourceSync specification version. The key difference is that v1.0 "
"used lastmod for the time of a change (often also the resource Last-Modification "
"time but not always). In v1.1 the rs:md datetime attribute in a ChangeList "
"indicates the time of the change, and use of lastmod is entirely optional")
opt.add_argument('--verbose', '-v', action='store_true',
help="verbose, show additional informational messages")

18
resync/resource_container.py
View File

@ -202,24 +202,6 @@ class ResourceContainer(object):
uris.append(r.uri)
return(uris)
def prune_before(self, timestamp):
"""Remove all resources with timestamp earlier than that given.
Returns the number of entries removed. Will raise an excpetion
if there are any entries without a timestamp.
"""
n = 0
pruned = []
for r in self.resources:
if (r.timestamp is None):
raise Exception("Entry %s has no timestamp" % (r.uri))
elif (r.timestamp >= timestamp):
pruned.append(r)
else:
n += 1
self.resources = pruned
return(n)
def prune_dupes(self):
"""Remove all but the last entry for a given resource URI.

46
tests/test_change_list.py
View File

@ -146,3 +146,49 @@ class TestChangeList(unittest.TestCase):
</urlset>'
cl = ChangeList()
self.assertRaises(SitemapParseError, cl.parse, fh=io.StringIO(xml))
def test11_prune_updates_before(self):
"""Test prune_updates_before method."""
# v1.0
cl = ChangeList()
cl.resources.append(Resource('a', timestamp=1))
cl.resources.append(Resource('b', timestamp=2))
cl.resources.append(Resource('c', timestamp=3))
cl.resources.append(Resource('d', timestamp=4))
cl.prune_updates_before(3, spec_version='1.0')
self.assertEqual(len(cl.resources), 2)
i = iter(cl)
self.assertEqual(next(i).uri, 'c')
self.assertEqual(next(i).uri, 'd')
# put some more back out of order
cl.resources.append(Resource('a', timestamp=1))
cl.resources.append(Resource('b', timestamp=2))
cl.resources.append(Resource('e', timestamp=1000))
cl.prune_updates_before(3.5, spec_version='1.0')
self.assertEqual(len(cl.resources), 2)
i = iter(cl)
self.assertEqual(next(i).uri, 'd')
self.assertEqual(next(i).uri, 'e')
# without a timestamp
cl.resources.append(Resource('nt_1_0', ts_datetime=123))
self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.0')
# v1.1
cl = ChangeList()
cl.resources.append(Resource('aa', ts_datetime=1))
cl.resources.append(Resource('bb', ts_datetime=2))
cl.resources.append(Resource('cc', ts_datetime=3))
cl.prune_updates_before(3, spec_version='1.1')
self.assertEqual(len(cl.resources), 1)
i = iter(cl)
self.assertEqual(next(i).uri, 'cc')
# put some more back out of order
cl.resources.append(Resource('aa', ts_datetime=1))
cl.resources.append(Resource('bb', ts_datetime=2))
cl.resources.append(Resource('ee', ts_datetime=1000))
cl.prune_updates_before(3.5, spec_version='1.1')
self.assertEqual(len(cl.resources), 1)
i = iter(cl)
self.assertEqual(next(i).uri, 'ee')
# without a ts_datetime
cl.resources.append(Resource('nt_1_1', timestamp=456))
self.assertRaises(Exception, cl.prune_updates_before, 3.5, spec_version='1.1')

21
tests/test_resource_container.py
View File

@ -25,27 +25,6 @@ class TestResourceContainer(unittest.TestCase):
self.assertEqual(resources[0].uri, 'a')
self.assertEqual(resources[3].uri, 'd')
def test03_prune_before(self):
rc = ResourceContainer()
rc.resources.append(Resource('a', timestamp=1))
rc.resources.append(Resource('b', timestamp=2))
rc.resources.append(Resource('c', timestamp=3))
rc.resources.append(Resource('d', timestamp=4))
rc.prune_before(3)
self.assertEqual(len(rc.resources), 2)
i = iter(rc)
self.assertEqual(next(i).uri, 'c')
self.assertEqual(next(i).uri, 'd')
# put some more back out of order
rc.resources.append(Resource('a', timestamp=1))
rc.resources.append(Resource('b', timestamp=2))
rc.resources.append(Resource('e', timestamp=1000))
rc.prune_before(3.5)
self.assertEqual(len(rc.resources), 2)
i = iter(rc)
self.assertEqual(next(i).uri, 'd')
self.assertEqual(next(i).uri, 'e')
def test04_prune_dupes(self):
rc = ResourceContainer()
rc.resources.append(Resource('a', timestamp=1, change='created'))