diff --git a/CHANGES.md b/CHANGES.md index 7c90c97..bc35e13 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ core specification version. Versions 1.0.x implement the v1.0 ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014 . +v1.0.8 2017-??-?? + * Change argument names for local modes to have `--write-` prefix, e.g. --write-resourcelist + * Improve modes to detect capabilities + v1.0.7 2017-03-27 * Fixed silly error in `resync-explorer` introduced in 1.0.6 diff --git a/bin/resync b/bin/resync index 9be3d2d..998f2da 100755 --- a/bin/resync +++ b/bin/resync @@ -54,32 +54,32 @@ def main(): # b. modes based solely on files on local disk loc = p.add_option_group('LOCAL MODES', 'These modes act on files on the local disk') - loc.add_option('--resourcelist', '--resource-list', '-r', action='store_true', + loc.add_option('--write-resourcelist', '--write-resource-list', action='store_true', help="write a resource list based on files on disk using uri=path mappings " "in reverse to calculate URIs from the local paths. Scans local disk " "based either on explicit --paths setting, else starting from all local " "paths specified in the mappings. Writes to STDOUT by default, override " "with --outfile") - loc.add_option('--changelist', '--change-list', '-c', action='store_true', + loc.add_option('--write-changelist', '--write-change-list', action='store_true', help="write a change list based on comparison of a reference sitemap " "(specify file with --reference) and either files on disk (using " "the mapping provided) or a second sitemap (specify file with " - "--newreference). Otherwise follows --resourcelist options. Also accepts " + "--newreference). Otherwise follows --write-resourcelist options. Also accepts " "the --empty option (with no mapping) to write and empty changelist.") - loc.add_option('--capabilitylist', '--capability-list', type=str, action='store', + loc.add_option('--write-capabilitylist', '--write-capability-list', type=str, action='store', help="write a capability list based on the set of capabilities and " "URIs supplied in cap_name=URI,cap_name=URI format. Otherwise " - "follows --resourcelist options.") - loc.add_option('--sourcedescription', '--source-description', type=str, action='store', + "follows --write-resourcelist options.") + loc.add_option('--write-sourcedescription', '--write-source-description', type=str, action='store', help="write a Source Description document based on the set of capability " "list URIs supplied as a comma separated list. Otherwise " - "follows --resourcelist options.") - loc.add_option('--resourcedump', '--resource-dump', '-d', action='store_true', + "follows --write-resourcelist options.") + loc.add_option('--write-resourcedump', '--write-resource-dump', '-d', action='store_true', help="write a Resource Dump. Specify output file with --outfile and use other " - "options as for --resourcelist") - loc.add_option('--changedump', '--change-dump', action='store_true', + "options as for --write-resourcelist") + loc.add_option('--write-changedump', '--write-change-dump', action='store_true', help="write a Resource Dump. Specify output file with --outfile and use other " - "options as for --changelist") + "options as for --write-changelist") # Specification of map between remote URI and local file paths, and remote # sitemap @@ -92,10 +92,13 @@ def main(): nam.add_option('--sitemap', type=str, action='store', help="explicitly set sitemap name, overriding default sitemap.xml " "appended to first source URI specified in the mappings") + nam.add_option('--capabilitylist', '--capability-list', type=str, action='store', + help="explicitly set capability list URI to search for instead of " + "looking for the source description") nam.add_option('--reference', type=str, action='store', - help="reference sitemap name for --changelist calculation") + help="reference sitemap name for --write-changelist calculation") nam.add_option('--newreference', type=str, action='store', - help="updated reference sitemap name for --changelist calculation") + help="updated reference sitemap name for --write-changelist calculation") nam.add_option('--changelist-uri', '--change-list-uri', type=str, action='store', help="explicitly set the changelist URI that will be use in --inc mode, " "overrides process of getting this from the sitemap") @@ -176,18 +179,21 @@ def main(): # Implement exclusive arguments and default --baseline (support for exclusive # groups in argparse is incomplete is python2.6) if (not args.baseline and not args.incremental and not args.audit and - not args.parse and not args.resourcelist and not args.changelist and - not args.capabilitylist and not args.sourcedescription and - not args.resourcedump and not args.changedump): + not args.parse and not args.write_resourcelist and not args.write_changelist and + not args.write_capabilitylist and not args.write_sourcedescription and + not args.write_resourcedump and not args.write_changedump): if (len(map) == 0): p.error("No arguments specified (use -h for help)") return else: args.baseline = True elif (count_true_args(args.baseline, args.incremental, args.audit, args.parse, - args.resourcelist, args.changelist, args.capabilitylist, - args.sourcedescription, args.resourcedump, args.changedump) > 1): - p.error("Only one of --baseline, --incremental, --audit, --parse, --resourcelist, --changelist, --capabilitylist, --sourcedescription, --resourcedump, --changedump modes allowed") + args.write_resourcelist, args.write_changelist, + args.write_capabilitylist, args.write_sourcedescription, + args.write_resourcedump, args.write_changedump) > 1): + p.error("Only one of --baseline, --incremental, --audit, --parse, --write-resourcelist, " + "--write-changelist, --write-capabilitylist, --write-sourcedescription, " + "--write-resourcedump, --write-changedump modes allowed") # Configure logging module and create logger instance init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, @@ -205,6 +211,8 @@ def main(): c.set_mappings(map) if (args.sitemap): c.sitemap_name = args.sitemap + if (args.capabilitylist): + c.capability_list_uri = args.capabilitylist if (args.warc): c.dump_format = 'warc' if (args.exclude): @@ -224,12 +232,12 @@ def main(): links = parse_links(args.link) # Add specific links is appropriate cases if (args.capabilitylist_link and - not args.capabilitylist and - not args.sourcedescription): + not args.write_capabilitylist and + not args.write_sourcedescription): # rel="up" to Capability List in all but Capability List # and Source Description links.insert(0, {'rel': 'up', 'href': args.capabilitylist_link}) - if (args.sourcedescription_link and args.capabilitylist): + if (args.sourcedescription_link and args.write_capabilitylist): # rel="up" to Source Description from Capability List links.insert(0, {'rel': 'up', 'href': args.sourcedescription_link}) if (args.describedby_link): @@ -246,12 +254,12 @@ def main(): from_datetime=args.from_datetime) elif (args.parse): c.parse_document() - elif (args.resourcelist or args.resourcedump): + elif (args.write_resourcelist or args.write_resourcedump): c.write_resource_list(paths=args.paths, outfile=args.outfile, links=links, - dump=args.resourcedump) - elif (args.changelist or args.changedump): + dump=args.write_resourcedump) + elif (args.write_changelist or args.write_changedump): if (not args.reference and not args.empty): p.error( "Must supply --reference sitemap for --changelist, or --empty") @@ -262,16 +270,16 @@ def main(): paths=args.paths, outfile=args.outfile, links=links, - dump=args.changedump) - elif (args.capabilitylist): + dump=args.write_changedump) + elif (args.write_capabilitylist): c.write_capability_list( - capabilities=parse_capabilities(args.capabilitylist), + capabilities=parse_capabilities(args.write_capabilitylist), outfile=args.outfile, links=links) - elif (args.sourcedescription): + elif (args.write_sourcedescription): c.write_source_description( capability_lists=parse_capability_lists( - args.sourcedescription), + args.write_sourcedescription), outfile=args.outfile, links=links) else: diff --git a/resync/_version.py b/resync/_version.py index 0f1c79c..1a698ed 100644 --- a/resync/_version.py +++ b/resync/_version.py @@ -3,4 +3,4 @@ # Format: x.y.z where # x.y is spec version, see http://www.openarchives.org/rs/x.y/ # z is incremented for revisions within that version, 1... -__version__ = '1.0.7' +__version__ = '1.0.8' diff --git a/resync/client.py b/resync/client.py index d796d21..8930403 100644 --- a/resync/client.py +++ b/resync/client.py @@ -54,6 +54,7 @@ class Client(object): self.dump_format = None self.exclude_patterns = [] self.sitemap_name = None + self.capability_list_uri = None self.allow_multifile = True self.noauth = False self.strictauth = False @@ -122,11 +123,20 @@ class Client(object): "Source description %s has no sources" % (uri)) elif (len(sd) > 1): raise ClientFatalError( - "Source description %s has multiple sources" % (uri)) + "Source description %s has multiple sources, specify one " + "with --capabilitylist" % (uri)) self.logger.info("Finished reading source description") - # Now read capability list cluri = sd.resources.first().uri uri = urljoin(uri, cluri) # FIXME - Should relative URI handling be elsewhere? + return(self.find_resource_list_from_capability_list(uri)) + + def find_resource_list_from_capability_list(self, uri): + """Read capability list to find resource list. + + Raises a ClientError in cases where the client might look for a + capability list in another location, but a ClientFatalError if + a capability list is found but there is some problem using it. + """ self.logger.info("Reading capability list %s" % (uri)) try: cl = CapabilityList() @@ -146,18 +156,23 @@ class Client(object): 1. Use explicitly specified self.sitemap_name (and fail if that doesn't work) - 2. Look for base_url/.well-known/resourcesync (then look + 2. Use explicitly specified self.capability_list_uri (and + fail is that doesn't work) + 3. Look for base_url/.well-known/resourcesync (then look for capability, look for resourcelist) - 3. Look for host/.well-known/resourcesync (then look + 4. Look for host/.well-known/resourcesync (then look for capability, look for resourcelist) - 4. Look for base_url/resourcelist.xml - 5. Look for base_url/sitemap.xml - 6. Look for host/sitemap.xml + 5. Look for base_url/resourcelist.xml + 6. Look for base_url/sitemap.xml + 7. Look for host/sitemap.xml """ - # 1 + # 1 & 2 if (self.sitemap_name is not None): return(self.read_resource_list(self.sitemap_name)) - # 2 & 3 + if (self.capability_list_uri is not None): + rluri = self.find_resource_list_from_capability_list(self.capability_list_uri) + return(self.read_resource_list(rluri)) + # 3 & 4 parts = urlsplit(self.sitemap) uri_host = urlunsplit([parts[0], parts[1], '', '', '']) errors = [] @@ -169,7 +184,7 @@ class Client(object): return(self.read_resource_list(rluri)) except ClientError as e: errors.append(str(e)) - # 4, 5 & 6 + # 5, 6 & 7 for uri in [urljoin(self.sitemap, 'resourcelist.xml'), urljoin(self.sitemap, 'sitemap.xml'), urljoin(uri_host, 'sitemap.xml')]: diff --git a/tests/test_client_link_options.py b/tests/test_client_link_options.py index 7eb40f3..eb9678f 100644 --- a/tests/test_client_link_options.py +++ b/tests/test_client_link_options.py @@ -25,7 +25,7 @@ def run_resync(args): class TestClientLinkOptions(unittest.TestCase): def test01_no_links(self): - xml = run_resync(['--resourcelist', + xml = run_resync(['--write-resourcelist', 'http://example.org/t', 'tests/testdata/dir1']) rl = ResourceList() rl.parse(fh=io.BytesIO(xml)) @@ -33,7 +33,7 @@ class TestClientLinkOptions(unittest.TestCase): self.assertEqual(rl.link('describedby'), None) def test02_resource_list_links(self): - xml = run_resync(['--resourcelist', + xml = run_resync(['--write-resourcelist', '--describedby-link=a', '--sourcedescription-link=b', # will be ignored '--capabilitylist-link=c', @@ -47,7 +47,7 @@ class TestClientLinkOptions(unittest.TestCase): self.assertEqual(rl.link('up')['href'], 'c') def test03_capability_list_links(self): - xml = run_resync(['--capabilitylist=resourcelist=rl,changedump=cd', + xml = run_resync(['--write-capabilitylist=resourcelist=rl,changedump=cd', '--describedby-link=a', '--sourcedescription-link=b', '--capabilitylist-link=c']) # will be ignored