diff --git a/.travis.yml b/.travis.yml index 10a4250..869670f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,12 @@ python: install: - pip install requests - pip install python-dateutil - - pip install coveralls pep257 restructuredtext_lint testfixtures + - pip install coveralls pep8 pep257 restructuredtext_lint testfixtures - python setup.py install # command to run tests script: - python setup.py test + - pep8 --ignore=E501 resync tests bin/resync bin/resync-explorer - pep257 resync - rst-lint README - coverage run --source=resync setup.py test diff --git a/bin/resync b/bin/resync index 5a96b86..a65fa82 100755 --- a/bin/resync +++ b/bin/resync @@ -22,26 +22,27 @@ import sys from resync import __version__ from resync.client import Client, ClientFatalError -from resync.client_utils import init_logging,count_true_args,parse_links,parse_capabilities,parse_capability_lists +from resync.client_utils import init_logging, count_true_args, parse_links, parse_capabilities, parse_capability_lists DEFAULT_LOGFILE = 'resync-client.log' + def main(): - if (sys.version_info < (2,6)): + if (sys.version_info < (2, 6)): sys.exit("This program requires python version 2.6 or later") - + # Options and arguments p = optparse.OptionParser(description='ResourceSync command line client', usage='usage: %prog [options] uri_path local_path (-h for help)', - version='%prog '+__version__ ) + version='%prog ' + __version__) # Modes # a. modes using remote sitemap/resources rem = p.add_option_group('REMOTE MODES', - 'These modes use a remote source that is specified in a set of uri=path mappings ' - 'and potentially also using an explicit --sitemap location. The default mode is ' - '--baseline. See also: resync-explorer for an interactive client.') + 'These modes use a remote source that is specified in a set of uri=path mappings ' + 'and potentially also using an explicit --sitemap location. The default mode is ' + '--baseline. See also: resync-explorer for an interactive client.') rem.add_option('--baseline', '-b', action='store_true', help='baseline sync of resources from remote source (src) to local filesystem (dst)') rem.add_option('--incremental', '--inc', '-i', action='store_true', @@ -52,7 +53,7 @@ def main(): help="parse a remote sitemap/sitemapindex (from mapping or explicit --sitemap) and show summary information including document type and number of entries") # b. modes based solely on files on local disk loc = p.add_option_group('LOCAL MODES', - 'These modes act on files on the local disk') + 'These modes act on files on the local disk') loc.add_option('--resourcelist', '--resource-list', '-r', action='store_true', help="write a resource list based on files on disk using uri=path mappings " "in reverse to calculate URIs from the local paths. Scans local disk " @@ -80,7 +81,8 @@ def main(): help="write a Resource Dump. Specify output file with --outfile and use other " "options as for --changelist") - # Specification of map between remote URI and local file paths, and remote sitemap + # Specification of map between remote URI and local file paths, and remote + # sitemap nam = p.add_option_group('FILE/URI NAMING OPTIONS') nam.add_option('--outfile', type=str, action='store', help="write output to specified file rather than STDOUT or default") @@ -94,7 +96,7 @@ def main(): help="reference sitemap name for --changelist calculation") nam.add_option('--newreference', type=str, action='store', help="updated reference sitemap name for --changelist calculation") - nam.add_option('--changelist-uri','--change-list-uri', type=str, action='store', + nam.add_option('--changelist-uri', '--change-list-uri', type=str, action='store', help="explicitly set the changelist URI that will be use in --inc mode, " "overrides process of getting this from the sitemap") @@ -105,16 +107,16 @@ def main(): "(repeat option for multiple links)") lks.add_option('--describedby-link', type=str, action='store', help="add an 1): + args.baseline = True + elif (count_true_args(args.baseline, args.incremental, args.audit, args.parse, + args.resourcelist, args.changelist, args.capabilitylist, + args.sourcedescription, args.resourcedump, args.changedump) > 1): p.error("Only one of --baseline, --incremental, --audit, --parse, --resourcelist, --changelist, --capabilitylist, --sourcedescription, --resourcedump, --changedump modes allowed") # Configure logging module and create logger instance - init_logging( to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, - verbose=args.verbose, eval_mode=args.eval ) + init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, + verbose=args.verbose, eval_mode=args.eval) - c = Client( checksum=args.checksum, - verbose=args.verbose, - dryrun=args.dryrun ) + c = Client(checksum=args.checksum, + verbose=args.verbose, + dryrun=args.dryrun) try: if (map): # Mappings apply to (almost) everything c.set_mappings(map) if (args.sitemap): - c.sitemap_name=args.sitemap + c.sitemap_name = args.sitemap if (args.warc): - c.dump_format='warc' + c.dump_format = 'warc' if (args.exclude): - c.exclude_patterns=args.exclude + c.exclude_patterns = args.exclude if (args.multifile): - c.allow_multifile=not args.multifile + c.allow_multifile = not args.multifile if (args.noauth): - c.noauth=args.noauth + c.noauth = args.noauth if (args.strictauth): - c.strictauth=args.strictauth + c.strictauth = args.strictauth if (args.max_sitemap_entries): - c.max_sitemap_entries=args.max_sitemap_entries + c.max_sitemap_entries = args.max_sitemap_entries if (args.ignore_failures): - c.ignore_failures=args.ignore_failures + c.ignore_failures = args.ignore_failures # Links apply to anything that writes sitemaps links = parse_links(args.link) # Add specific links is appropriate cases - if (args.capabilitylist_link and - not args.capabilitylist and - not args.sourcedescription): - # rel="up" to Capability List in all but Capability List + if (args.capabilitylist_link and + not args.capabilitylist and + not args.sourcedescription): + # rel="up" to Capability List in all but Capability List # and Source Description - links.insert(0,{'rel':'up','href':args.capabilitylist_link}) + links.insert(0, {'rel': 'up', 'href': args.capabilitylist_link}) if (args.sourcedescription_link and args.capabilitylist): # rel="up" to Source Description from Capability List - links.insert(0,{'rel':'up','href':args.sourcedescription_link}) + links.insert(0, {'rel': 'up', 'href': args.sourcedescription_link}) if (args.describedby_link): - links.insert(0,{'rel':'describedby','href':args.describedby_link}) - + links.insert(0, {'rel': 'describedby', + 'href': args.describedby_link}) # Finally, do something... if (args.baseline or args.audit): @@ -246,9 +248,11 @@ def main(): dump=args.resourcedump) elif (args.changelist or args.changedump): if (not args.reference and not args.empty): - p.error("Must supply --reference sitemap for --changelist, or --empty") + p.error( + "Must supply --reference sitemap for --changelist, or --empty") c.write_change_list(ref_sitemap=args.reference, - newref_sitemap=( args.newreference if (args.newreference) else None ), + newref_sitemap=(args.newreference if ( + args.newreference) else None), empty=args.empty, paths=args.paths, outfile=args.outfile, @@ -261,12 +265,13 @@ def main(): links=links) elif (args.sourcedescription): c.write_source_description( - capability_lists=parse_capability_lists(args.sourcedescription), + capability_lists=parse_capability_lists( + args.sourcedescription), outfile=args.outfile, links=links) else: p.error("Unknown mode requested") - # Any problem we expect will come as a ClientFatalError, anything else + # Any problem we expect will come as a ClientFatalError, anything else # is... an exception ;-) except ClientFatalError as e: sys.stderr.write("\nFatalError: " + str(e) + "\n") diff --git a/bin/resync-explorer b/bin/resync-explorer index 4e93c62..d604ef1 100755 --- a/bin/resync-explorer +++ b/bin/resync-explorer @@ -27,17 +27,19 @@ from resync.explorer import Explorer DEFAULT_LOGFILE = 'resync-explorer.log' + def main(): - if (sys.version_info < (2,6)): + if (sys.version_info < (2, 6)): sys.exit("This program requires python version 2.6 or later") - + # Options and arguments p = optparse.OptionParser(description='ResourceSync explorer', usage='usage: %prog [options] uri_path local_path (-h for help)', - version='%prog '+__version__ ) + version='%prog ' + __version__) - # Specification of map between remote URI and local file paths, and remote sitemap + # Specification of map between remote URI and local file paths, and remote + # sitemap nam = p.add_option_group('FILE/URI NAMING OPTIONS') nam.add_option('--outfile', type=str, action='store', help="write sitemap to specified file rather than STDOUT") @@ -53,7 +55,7 @@ def main(): help="updated reference sitemap name for --changelist calculation") nam.add_option('--dump', metavar='DUMPFILE', type=str, action='store', help="write dump to specified file for --resourcelist or --changelist") - nam.add_option('--changelist-uri','--change-list-uri', type=str, action='store', + nam.add_option('--changelist-uri', '--change-list-uri', type=str, action='store', help="explicitly set the changelist URI that will be use in --inc mode, " "overrides process of getting this from the sitemap") @@ -88,31 +90,31 @@ def main(): (args, map) = p.parse_args() - init_logging( to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, - verbose=args.verbose ) + init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, + verbose=args.verbose) print("----- ResourceSync Explorer -----") - c = Explorer( checksum=args.checksum, - verbose=args.verbose ) + c = Explorer(checksum=args.checksum, + verbose=args.verbose) try: if (map): # Mappings apply to (almost) everything c.set_mappings(map) if (args.sitemap): - c.sitemap_name=args.sitemap + c.sitemap_name = args.sitemap if (args.exclude): - c.exclude_patterns=args.exclude + c.exclude_patterns = args.exclude if (args.multifile): - c.allow_multifile=not args.multifile + c.allow_multifile = not args.multifile if (args.noauth): - c.noauth=args.noauth + c.noauth = args.noauth if (args.max_sitemap_entries): - c.max_sitemap_entries=args.max_sitemap_entries + c.max_sitemap_entries = args.max_sitemap_entries c.explore() - # Any problem we expect will come as a ClientFatalError, anything else + # Any problem we expect will come as a ClientFatalError, anything else # is... an exception ;-) except ClientFatalError as e: sys.stderr.write("\nFatalError: " + str(e) + "\n") diff --git a/resync/__init__.py b/resync/__init__.py index 4dadd1c..9174030 100644 --- a/resync/__init__.py +++ b/resync/__init__.py @@ -11,6 +11,5 @@ from resync.change_list import ChangeList from resync.resource_dump import ResourceDump from resync.resource_dump_manifest import ResourceDumpManifest from resync.change_dump import ChangeDump -from resync.archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive +from resync.archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive from resync.resource import Resource - diff --git a/resync/archives.py b/resync/archives.py index e618266..b2bf6c0 100644 --- a/resync/archives.py +++ b/resync/archives.py @@ -2,22 +2,23 @@ The ResourceSync Archives specification http://www.openarchives.org/rs/archives specifies capabilities -that provide archives of the 4 core capabilities. While some +that provide archives of the 4 core capabilities. While some optional attributes are different the basic structure is the -same for the Resource List Archive, Change List Archive, -Resource Dump Archive, and Change Dump Archive. +same for the Resource List Archive, Change List Archive, +Resource Dump Archive, and Change Dump Archive. """ import collections -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from .list_base_with_index import ListBaseWithIndex from .resource import Resource from .sitemap import Sitemap + class ResourceListArchive(ListBaseWithIndex): """Class representing an Resource List Archive.""" @@ -30,6 +31,7 @@ class ResourceListArchive(ListBaseWithIndex): super(ResourceListArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri, capability_name='resourcelist-archive') + class ChangeListArchive(ListBaseWithIndex): """Class representing an Change List Archive.""" @@ -39,6 +41,7 @@ class ChangeListArchive(ListBaseWithIndex): super(ChangeListArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri, capability_name='changelist-archive') + class ResourceDumpArchive(ListBaseWithIndex): """Class representing an Resource Dump Archive.""" @@ -49,6 +52,7 @@ class ResourceDumpArchive(ListBaseWithIndex): capability_name='resourcedump-archive', resources_class=resources_class) + class ChangeDumpArchive(ListBaseWithIndex): """Class representing an Change Dump Archive.""" diff --git a/resync/capability_list.py b/resync/capability_list.py index 8a3a4ee..1aad02c 100644 --- a/resync/capability_list.py +++ b/resync/capability_list.py @@ -1,7 +1,7 @@ """ResourceSync Capability List object. -An Capability List is a set of capabilitys with some metadata for -each capability. The Capability List object may also contain metadata +An Capability List is a set of capabilitys with some metadata for +each capability. The Capability List object may also contain metadata and links like other lists. """ @@ -12,26 +12,27 @@ from .resource_set import ResourceSet from .list_base import ListBase from .sitemap import Sitemap + class CapabilitySet(ResourceSet): """Class for storage of resources in a Capability List. - Extends the ResourceSet to add checks to ensure that there are - never two entries for the same resource, and that values are + Extends the ResourceSet to add checks to ensure that there are + never two entries for the same resource, and that values are returned in the canonical order. """ def __init__(self): """Initialize CpabilitySet.""" - self.order = [ 'resourcelist', 'resourcedump', - 'changelist', 'changedump', - 'resourcelist-archive', 'resourcedump-archive', - 'changelist-archive', 'changedump-archive' ] + self.order = ['resourcelist', 'resourcedump', + 'changelist', 'changedump', + 'resourcelist-archive', 'resourcedump-archive', + 'changelist-archive', 'changedump-archive'] def __iter__(self): """Iterator over all the resources in capability order. Deals with the case of unknown capabilities or duplicate entries - by using uri order for duplicates and adding any unknown ones + by using uri order for duplicates and adding any unknown ones at the end """ self._iter_next_list = [] @@ -40,7 +41,7 @@ class CapabilitySet(ResourceSet): for uri in self.keys(): cap = self[uri].capability if (cap not in uris): - uris[cap]=[] + uris[cap] = [] uris[cap].append(uri) # build list or uris in defined order for iterator for cap in self.order: @@ -48,7 +49,8 @@ class CapabilitySet(ResourceSet): for uri in sorted(uris[cap]): self._iter_next_list.append(uri) del uris[cap] - # add any left over capabilities we don't know about in alphabetical order + # add any left over capabilities we don't know about in alphabetical + # order for cap in uris: for uri in sorted(uris[cap]): self._iter_next_list.append(uri) @@ -56,7 +58,7 @@ class CapabilitySet(ResourceSet): return(iter(self._iter_next, None)) def _iter_next(self): - if (len(self._iter_next_list)>0): + if (len(self._iter_next_list) > 0): return(self[self._iter_next_list.pop()]) else: return(None) @@ -65,7 +67,7 @@ class CapabilitySet(ResourceSet): class CapabilityList(ListBase): """Class representing a Capability List. - An Capability List will admit only one resource with any given + An Capability List will admit only one resource with any given URI. The iterator over resources is expected to return them in canonical order of capability names as defined in main specification section 7 and archives specification section 6. @@ -88,15 +90,15 @@ class CapabilityList(ListBase): """ if isinstance(resource, collections.Iterable): for r in resource: - self.resources.add(r,replace) + self.resources.add(r, replace) else: - self.resources.add(resource,replace) + self.resources.add(resource, replace) - def add_capability(self,capability=None,uri=None,name=None): + def add_capability(self, capability=None, uri=None, name=None): """Specific add function for capabilities. - + Takes either: - - a capability object (derived from ListBase) as the first argument + - a capability object (derived from ListBase) as the first argument from which the capability name is extracted, and the URI if given - or a plain name string and @@ -105,14 +107,14 @@ class CapabilityList(ListBase): if (capability is not None): name = capability.capability_name if (capability.uri is not None): - uri=capability.uri - self.add( Resource(uri=uri,capability=name) ) + uri = capability.uri + self.add(Resource(uri=uri, capability=name)) - def has_capability(self,name=None): + def has_capability(self, name=None): """True if the Capability List includes the named capability.""" - return( self.capability_info(name) is not None ) + return(self.capability_info(name) is not None) - def capability_info(self,name=None): + def capability_info(self, name=None): """Return information about the requested capability from this list. Will return None if there is no information about the requested capability. diff --git a/resync/change_dump.py b/resync/change_dump.py index 6a48033..ba64be8 100644 --- a/resync/change_dump.py +++ b/resync/change_dump.py @@ -1,7 +1,7 @@ """ResourceSync Change Dump object. A Change Dump is a set of content dump package resources -with some metadata for each resource. +with some metadata for each resource. The Change Dump may also contain metadata and links like other ResourceSync documents. @@ -12,6 +12,7 @@ http://www.openarchives.org/rs/resourcesync#ChangeDump from .resource_list import ResourceList + class ChangeDump(ResourceList): """Class representing an Change Dump. @@ -20,7 +21,8 @@ class ChangeDump(ResourceList): and implemented as a sub-class of ResourceList. """ - def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None): + def __init__(self, resources=None, md=None, ln=None, + uri=None, allow_multifile=None, mapper=None): """Initialize ChangeDump. Simply sets capability_name to 'changedump' when @@ -28,4 +30,4 @@ class ChangeDump(ResourceList): """ super(ChangeDump, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper) - self.capability_name='changedump' + self.capability_name = 'changedump' diff --git a/resync/change_dump_manifest.py b/resync/change_dump_manifest.py index 1b97930..91c9413 100644 --- a/resync/change_dump_manifest.py +++ b/resync/change_dump_manifest.py @@ -1,10 +1,10 @@ """ResourceSync ChangeDumpManifest object. -A ChangeDumpManifest lists the set of files/resources included -within a content package that is included in a ChangeDump. +A ChangeDumpManifest lists the set of files/resources included +within a content package that is included in a ChangeDump. The ChangeeDumpManifest object will include the change type, -a path for each item (except in the case of "deleted"), and may +a path for each item (except in the case of "deleted"), and may also contain metadata and links. Described in specification at: @@ -13,6 +13,7 @@ http://www.openarchives.org/rs/resourcesync#ChangeDumpManifest from .change_list import ChangeList + class ChangeDumpManifest(ChangeList): """Class representing a Change Dump Manifest. @@ -21,11 +22,14 @@ class ChangeDumpManifest(ChangeList): and implemented as a sub-class of ChangeeList. """ - def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None): + def __init__(self, resources=None, md=None, ln=None, + uri=None, allow_multifile=None, mapper=None): """Initialize ChangeDumpManifest. Simply sets capability_name to 'changedump-manifest' when subclassing ChangeList. """ - super(ChangeDumpManifest, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper) + super(ChangeDumpManifest, self).__init__( + resources=resources, md=md, ln=ln, + uri=uri, mapper=mapper) self.capability_name = 'changedump-manifest' diff --git a/resync/change_list.py b/resync/change_list.py index e74528c..1fd2f66 100644 --- a/resync/change_list.py +++ b/resync/change_list.py @@ -8,23 +8,24 @@ are Resource objects. Different from an resource_list, a change_list may include multiple descriptions for the same resource. The change_list is ordered -from first entry to last entry. +from first entry to last entry. -Different from an resource_list, dereference by a URI yields a -ChangeList containing descriptions pertaining to that +Different from an resource_list, dereference by a URI yields a +ChangeList containing descriptions pertaining to that particular resource. """ import collections -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from .list_base_with_index import ListBaseWithIndex -from .resource import Resource,ChangeTypeError +from .resource import Resource, ChangeTypeError from .sitemap import Sitemap + class ChangeList(ListBaseWithIndex): """Class representing an Change List.""" @@ -38,14 +39,14 @@ class ChangeList(ListBaseWithIndex): def add_if_changed(self, resource): """Add resource if change is not None else ChangeTypeError.""" if (resource.change is not None): - self.resources.append(resource) + self.resources.append(resource) else: raise ChangeTypeError(resource.change) def add(self, resource): """Add a resource change or an iterable collection of them. - - Allows multiple resource_change objects for the same + + Allows multiple resource_change objects for the same resource (ie. URI) and preserves the order of addition. """ if isinstance(resource, collections.Iterable): @@ -57,9 +58,9 @@ class ChangeList(ListBaseWithIndex): def add_changed_resources(self, resources, change=None): """Add items from a ResourceContainer resources. - If change is specified then the attribute is set in the Resource + If change is specified then the attribute is set in the Resource objects created. """ for resource in resources: - rc = Resource( resource=resource, change=change ) + rc = Resource(resource=resource, change=change) self.add(rc) diff --git a/resync/client.py b/resync/client.py index b26dba1..efe05fa 100644 --- a/resync/client.py +++ b/resync/client.py @@ -1,15 +1,15 @@ """ResourceSync client implementation.""" import sys -try: #python3 +try: # python3 from urllib.request import urlretrieve from urllib.parse import urlparse, urlunparse -except ImportError: #python2 +except ImportError: # python2 from urllib import urlretrieve from urlparse import urlparse, urlunparse import os.path import datetime -import distutils.dir_util +import distutils.dir_util import re import time import logging @@ -29,7 +29,8 @@ from .utils import compute_md5_for_file from .client_state import ClientState from .client_utils import ClientFatalError, url_or_file_open from .list_base_with_index import ListBaseIndexError -from .w3c_datetime import str_to_datetime,datetime_to_str +from .w3c_datetime import str_to_datetime, datetime_to_str + class Client(object): """Implementation of a ResourceSync client. @@ -65,17 +66,16 @@ class Client(object): self.default_resource_dump = 'resourcedump.zip' self.default_change_dump = 'changedump.zip' - - def set_mappings(self,mappings): + def set_mappings(self, mappings): """Build and set Mapper object based on input mappings.""" self.mapper = Mapper(mappings, use_default_path=True) - def sitemap_uri(self,basename): + def sitemap_uri(self, basename): """Get full URI (filepath) for sitemap based on basename.""" - if (re.match(r"\w+:",basename)): + if (re.match(r"\w+:", basename)): # looks like URI return(basename) - elif (re.match(r"/",basename)): + elif (re.match(r"/", basename)): # looks like full path return(basename) else: @@ -97,21 +97,22 @@ class Client(object): paths - override paths from mappings if specified - set_path - set true to set the path information for each resource + set_path - set true to set the path information for each resource included. This is used to build a resource list as the basis for creating a dump. Return ResourceList. Uses existing self.mapper settings. """ # 0. Sanity checks, parse paths is specified - if (len(self.mapper)<1): - raise ClientFatalError("No source to destination mapping specified") + if (len(self.mapper) < 1): + raise ClientFatalError( + "No source to destination mapping specified") if (paths is not None): # Expect comma separated list of paths - paths=paths.split(',') + paths = paths.split(',') # 1. Build from disk - rlb = ResourceListBuilder(set_md5=self.checksum,mapper=self.mapper) - rlb.set_path=set_path + rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper) + rlb.set_path = set_path try: rlb.add_exclude_files(self.exclude_patterns) rl = rlb.from_disk(paths=paths) @@ -127,137 +128,171 @@ class Client(object): def log_event(self, change): """Log a Resource object as an event for automated analysis.""" - self.logger.debug( "Event: "+repr(change) ) + self.logger.debug("Event: " + repr(change)) def baseline_or_audit(self, allow_deletion=False, audit_only=False): """Baseline synchonization or audit. - + Both functions implemented in this routine because audit is a prerequisite for a baseline sync. In the case of baseline sync the last timestamp seen is recorded as client state. """ - action = ( 'audit' if (audit_only) else 'baseline sync' ) - self.logger.debug("Starting "+action) - ### 0. Sanity checks - if (len(self.mapper)<1): - raise ClientFatalError("No source to destination mapping specified") + action = ('audit' if (audit_only) else 'baseline sync') + self.logger.debug("Starting " + action) + # 0. Sanity checks + if (len(self.mapper) < 1): + raise ClientFatalError( + "No source to destination mapping specified") if (not audit_only and self.mapper.unsafe()): - raise ClientFatalError("Source to destination mappings unsafe: %s" % str(self.mapper)) - ### 1. Get inventories from both src and dst + raise ClientFatalError( + "Source to destination mappings unsafe: %s" % str( + self.mapper)) + # 1. Get inventories from both src and dst # 1.a source resource list try: self.logger.info("Reading sitemap %s" % (self.sitemap)) - src_resource_list = ResourceList(allow_multifile=self.allow_multifile, mapper=self.mapper) + src_resource_list = ResourceList( + allow_multifile=self.allow_multifile, mapper=self.mapper) src_resource_list.read(uri=self.sitemap) self.logger.debug("Finished reading sitemap") except Exception as e: - raise ClientFatalError("Can't read source resource list from %s (%s)" % (self.sitemap,str(e))) - self.logger.info("Read source resource list, %d resources listed" % (len(src_resource_list))) - if (len(src_resource_list)==0): - raise ClientFatalError("Aborting as there are no resources to sync") + raise ClientFatalError( + "Can't read source resource list from %s (%s)" % + (self.sitemap, str(e))) + self.logger.info( + "Read source resource list, %d resources listed" % + (len(src_resource_list))) + if (len(src_resource_list) == 0): + raise ClientFatalError( + "Aborting as there are no resources to sync") if (self.checksum and not src_resource_list.has_md5()): - self.checksum=False - self.logger.info("Not calculating checksums on destination as not present in source resource list") + self.checksum = False + self.logger.info( + "Not calculating checksums on destination as not present in source resource list") # 1.b destination resource list mapped back to source URIs rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper) dst_resource_list = rlb.from_disk() - ### 2. Compare these resource lists respecting any comparison options - (same,updated,deleted,created)=dst_resource_list.compare(src_resource_list) - ### 3. Report status and planned actions - self.log_status(in_sync=(len(updated)+len(deleted)+len(created)==0), - audit=True,same=len(same),created=len(created), - updated=len(updated),deleted=len(deleted)) - if (audit_only or len(created)+len(updated)+len(deleted)==0): - self.logger.debug("Completed "+action) + # 2. Compare these resource lists respecting any comparison options + (same, updated, deleted, created) = dst_resource_list.compare(src_resource_list) + # 3. Report status and planned actions + self.log_status(in_sync=(len(updated) + len(deleted) + len(created) == 0), + audit=True, same=len(same), created=len(created), + updated=len(updated), deleted=len(deleted)) + if (audit_only or len(created) + len(updated) + len(deleted) == 0): + self.logger.debug("Completed " + action) return - ### 4. Check that sitemap has authority over URIs listed + # 4. Check that sitemap has authority over URIs listed if (not self.noauth): uauth = UrlAuthority(self.sitemap, strict=self.strictauth) for resource in src_resource_list: if (not uauth.has_authority_over(resource.uri)): - raise ClientFatalError("Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (self.sitemap,resource.uri)) - ### 5. Grab files to do sync - delete_msg = (", and delete %d resources" % len(deleted)) if (allow_deletion) else '' - self.logger.warning("Will GET %d resources%s" % (len(created)+len(updated),delete_msg)) + raise ClientFatalError( + "Aborting as sitemap (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % + (self.sitemap, resource.uri)) + # 5. Grab files to do sync + delete_msg = ( + ", and delete %d resources" % + len(deleted)) if (allow_deletion) else '' + self.logger.warning( + "Will GET %d resources%s" % + (len(created) + len(updated), delete_msg)) self.last_timestamp = 0 - num_created=0 - num_updated=0 - num_deleted=0 + num_created = 0 + num_updated = 0 + num_deleted = 0 for resource in created: uri = resource.uri filename = self.mapper.src_to_dst(uri) - self.logger.info("created: %s -> %s" % (uri,filename)) - num_created+=self.update_resource(resource,filename,'created') + self.logger.info("created: %s -> %s" % (uri, filename)) + num_created += self.update_resource(resource, filename, 'created') for resource in updated: uri = resource.uri filename = self.mapper.src_to_dst(uri) - self.logger.info("updated: %s -> %s" % (uri,filename)) - num_updated+=self.update_resource(resource,filename,'updated') + self.logger.info("updated: %s -> %s" % (uri, filename)) + num_updated += self.update_resource(resource, filename, 'updated') for resource in deleted: uri = resource.uri filename = self.mapper.src_to_dst(uri) - num_deleted+=self.delete_resource(resource,filename,allow_deletion) - ### 6. Store last timestamp to allow incremental sync - if (not audit_only and self.last_timestamp>0): - ClientState().set_state(self.sitemap,self.last_timestamp) - self.logger.info("Written last timestamp %s for incremental sync" % (datetime_to_str(self.last_timestamp))) - ### 7. Done - self.log_status(in_sync=(len(updated)+len(deleted)+len(created)==0), - same=len(same),created=num_created, - updated=num_updated,deleted=num_deleted,to_delete=len(deleted)) + num_deleted += self.delete_resource(resource, + filename, allow_deletion) + # 6. Store last timestamp to allow incremental sync + if (not audit_only and self.last_timestamp > 0): + ClientState().set_state(self.sitemap, self.last_timestamp) + self.logger.info( + "Written last timestamp %s for incremental sync" % + (datetime_to_str( + self.last_timestamp))) + # 7. Done + self.log_status(in_sync=(len(updated) + len(deleted) + len(created) == 0), + same=len(same), created=num_created, + updated=num_updated, deleted=num_deleted, to_delete=len(deleted)) self.logger.debug("Completed %s" % (action)) - def incremental(self, allow_deletion=False, change_list_uri=None, from_datetime=None): + def incremental(self, allow_deletion=False, + change_list_uri=None, from_datetime=None): """Incremental synchronization. Use Change List to do incremental sync """ self.logger.debug("Starting incremental sync") - ### 0. Sanity checks - if (len(self.mapper)<1): - raise ClientFatalError("No source to destination mapping specified") + # 0. Sanity checks + if (len(self.mapper) < 1): + raise ClientFatalError( + "No source to destination mapping specified") if (self.mapper.unsafe()): - raise ClientFatalError("Source to destination mappings unsafe: %s" % str(self.mapper)) + raise ClientFatalError( + "Source to destination mappings unsafe: %s" % str( + self.mapper)) from_timestamp = None if (from_datetime is not None): try: from_timestamp = str_to_datetime(from_datetime) except ValueError: - raise ClientFatalError("Bad datetime in --from (%s)" % from_datetime) - ### 1. Work out where to start from + raise ClientFatalError( + "Bad datetime in --from (%s)" % + from_datetime) + # 1. Work out where to start from if (from_timestamp is None): - from_timestamp=ClientState().get_state(self.sitemap) + from_timestamp = ClientState().get_state(self.sitemap) if (from_timestamp is None): - raise ClientFatalError("Cannot do incremental sync. No stored timestamp for this site, and no explicit --from.") - ### 2. Get URI of change list, from sitemap or explicit + raise ClientFatalError( + "Cannot do incremental sync. No stored timestamp for this site, and no explicit --from.") + # 2. Get URI of change list, from sitemap or explicit if (change_list_uri): # Translate as necessary using maps change_list = self.sitemap_uri(change_list_uri) else: # Try default name change_list = self.sitemap_uri(self.change_list_name) - ### 3. Read change list from source + # 3. Read change list from source try: self.logger.info("Reading change list %s" % (change_list)) src_change_list = ChangeList() src_change_list.read(uri=change_list) self.logger.debug("Finished reading change list") except Exception as e: - raise ClientFatalError("Can't read source change list from %s (%s)" % (change_list,str(e))) - self.logger.info("Read source change list, %d changes listed" % (len(src_change_list))) - #if (len(src_change_list)==0): + raise ClientFatalError( + "Can't read source change list from %s (%s)" % + (change_list, str(e))) + self.logger.info( + "Read source change list, %d changes listed" % + (len(src_change_list))) + # if (len(src_change_list)==0): # raise ClientFatalError("Aborting as there are no resources to sync") if (self.checksum and not src_change_list.has_md5()): - self.checksum=False - self.logger.info("Not calculating checksums on destination as not present in source change list") + self.checksum = False + self.logger.info( + "Not calculating checksums on destination as not present in source change list") # Check all changes have timestamp and record last self.last_timestamp = 0 for resource in src_change_list: if (resource.timestamp is None): - raise ClientFatalError("Aborting - missing timestamp for change in %s" % (uri)) + raise ClientFatalError( + "Aborting - missing timestamp for change in %s" % + (uri)) if (resource.timestamp > self.last_timestamp): self.last_timestamp = resource.timestamp - ### 4. Check that the change list has authority over URIs listed + # 4. Check that the change list has authority over URIs listed # FIXME - What does authority mean for change list? Here use both the # change list URI and, if we used it, the sitemap URI if (not self.noauth): @@ -265,15 +300,20 @@ class Client(object): if (not change_list_uri): uauth_sm = UrlAuthority(self.sitemap) for resource in src_change_list: - if (not uauth_cs.has_authority_over(resource.uri) and - (change_list_uri or not uauth_sm.has_authority_over(resource.uri))): - raise ClientFatalError("Aborting as change list (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (change_list,resource.uri)) - ### 5. Prune entries before starting timestamp and dupe changes for a resource + if (not uauth_cs.has_authority_over(resource.uri) and + (change_list_uri or not uauth_sm.has_authority_over(resource.uri))): + raise ClientFatalError( + "Aborting as change list (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % + (change_list, resource.uri)) + # 5. Prune entries before starting timestamp and dupe changes for a + # resource num_skipped = src_change_list.prune_before(from_timestamp) - if (num_skipped>0): - self.logger.info("Skipped %d changes before %s" % (num_skipped,datetime_to_str(from_timestamp))) + if (num_skipped > 0): + self.logger.info( + "Skipped %d changes before %s" % + (num_skipped, datetime_to_str(from_timestamp))) num_dupes = src_change_list.prune_dupes() - if (num_dupes>0): + if (num_dupes > 0): self.logger.info("Removed %d prior changes" % (num_dupes)) # Review and log status before # FIXME - should at this stage prune the change list to pick out @@ -283,25 +323,29 @@ class Client(object): to_delete = 0 for resource in src_change_list: if (resource.change == 'updated'): - to_update+=1 + to_update += 1 elif (resource.change == 'created'): - to_create+=1 + to_create += 1 elif (resource.change == 'deleted'): - to_delete+=1 + to_delete += 1 else: - raise ClientError("Unknown change type %s" % (resource.change) ) + raise ClientError("Unknown change type %s" % (resource.change)) # Log status based on what we know from the Change List. Exit if # either there are no changes or if there are only deletions and # we don't allow deletion - in_sync = ((to_update+to_delete+to_create)==0) - self.log_status(in_sync=in_sync, incremental=True, created=to_create, + in_sync = ((to_update + to_delete + to_create) == 0) + self.log_status(in_sync=in_sync, incremental=True, created=to_create, updated=to_update, deleted=to_delete) - if (in_sync or ((to_update+to_create)==0 and not allow_deletion)): + if (in_sync or ((to_update + to_create) == 0 and not allow_deletion)): self.logger.debug("Completed incremental") return - ### 6. Apply changes at same time or after from_timestamp - delete_msg = (", and delete %d resources" % to_delete) if (allow_deletion) else '' - self.logger.warning("Will apply %d changes%s" % (len(src_change_list),delete_msg)) + # 6. Apply changes at same time or after from_timestamp + delete_msg = ( + ", and delete %d resources" % + to_delete) if (allow_deletion) else '' + self.logger.warning( + "Will apply %d changes%s" % + (len(src_change_list), delete_msg)) num_updated = 0 num_deleted = 0 num_created = 0 @@ -309,25 +353,29 @@ class Client(object): uri = resource.uri filename = self.mapper.src_to_dst(uri) if (resource.change == 'updated'): - self.logger.info("updated: %s -> %s" % (uri,filename)) - self.update_resource(resource,filename,'updated') - num_updated+=1 + self.logger.info("updated: %s -> %s" % (uri, filename)) + self.update_resource(resource, filename, 'updated') + num_updated += 1 elif (resource.change == 'created'): - self.logger.info("created: %s -> %s" % (uri,filename)) - self.update_resource(resource,filename,'created') - num_created+=1 + self.logger.info("created: %s -> %s" % (uri, filename)) + self.update_resource(resource, filename, 'created') + num_created += 1 elif (resource.change == 'deleted'): - num_deleted+=self.delete_resource(resource,filename,allow_deletion) + num_deleted += self.delete_resource( + resource, filename, allow_deletion) else: - raise ClientError("Unknown change type %s" % (resource.change) ) - ### 7. Report status and planned actions - self.log_status(incremental=True,created=num_created, updated=num_updated, - deleted=num_deleted,to_delete=to_delete) - ### 8. Record last timestamp we have seen - if (self.last_timestamp>0): - ClientState().set_state(self.sitemap,self.last_timestamp) - self.logger.info("Written last timestamp %s for incremental sync" % (datetime_to_str(self.last_timestamp))) - ### 9. Done + raise ClientError("Unknown change type %s" % (resource.change)) + # 7. Report status and planned actions + self.log_status(incremental=True, created=num_created, updated=num_updated, + deleted=num_deleted, to_delete=to_delete) + # 8. Record last timestamp we have seen + if (self.last_timestamp > 0): + ClientState().set_state(self.sitemap, self.last_timestamp) + self.logger.info( + "Written last timestamp %s for incremental sync" % + (datetime_to_str( + self.last_timestamp))) + # 9. Done self.logger.debug("Completed incremental sync") def update_resource(self, resource, filename, change=None): @@ -336,8 +384,8 @@ class Client(object): Update means three things: 1. GET resources 2. set mtime in local time to be equal to timestamp in UTC (should perhaps - or at least warn if different from LastModified from the GET response instead - but maybe warn if different (or just earlier than) the lastmod we expected + or at least warn if different from LastModified from the GET response instead + but maybe warn if different (or just earlier than) the lastmod we expected from the resource list 3. check that resource matches expected information @@ -348,16 +396,18 @@ class Client(object): """ path = os.path.dirname(filename) distutils.dir_util.mkpath(path) - num_updated=0 + num_updated = 0 if (self.dryrun): - self.logger.info("dryrun: would GET %s --> %s" % (resource.uri,filename)) + self.logger.info( + "dryrun: would GET %s --> %s" % + (resource.uri, filename)) else: # 1. GET try: - urlretrieve(resource.uri,filename) - num_updated+=1 + urlretrieve(resource.uri, filename) + num_updated += 1 except IOError as e: - msg = "Failed to GET %s -- %s" % (resource.uri,str(e)) + msg = "Failed to GET %s -- %s" % (resource.uri, str(e)) if (self.ignore_failures): self.logger.warning(msg) return(num_updated) @@ -365,62 +415,74 @@ class Client(object): raise ClientFatalError(msg) # 2. set timestamp if we have one if (resource.timestamp is not None): - unixtime = int(resource.timestamp) #no fractional - os.utime(filename,(unixtime,unixtime)) + unixtime = int(resource.timestamp) # no fractional + os.utime(filename, (unixtime, unixtime)) if (resource.timestamp > self.last_timestamp): self.last_timestamp = resource.timestamp self.log_event(Resource(resource=resource, change=change)) # 3. sanity check length = os.stat(filename).st_size if (resource.length is not None and resource.length != length): - self.logger.info("Downloaded size for %s of %d bytes does not match expected %d bytes" % (resource.uri,length,resource.length)) + self.logger.info( + "Downloaded size for %s of %d bytes does not match expected %d bytes" % + (resource.uri, length, resource.length)) if (self.checksum and resource.md5 is not None): file_md5 = compute_md5_for_file(filename) if (resource.md5 != file_md5): - self.logger.info("MD5 mismatch for %s, got %s but expected %s bytes" % (resource.uri,file_md5,resource.md5)) + self.logger.info( + "MD5 mismatch for %s, got %s but expected %s bytes" % + (resource.uri, file_md5, resource.md5)) return(num_updated) def delete_resource(self, resource, filename, allow_deletion=False): """Delete copy of resource in filename on local system. - Will only actually do the deletion if allow_deletion is True. Regardless - of whether the deletion occurs, self.last_timestamp will be updated + Will only actually do the deletion if allow_deletion is True. Regardless + of whether the deletion occurs, self.last_timestamp will be updated if the resource.timestamp is later than the current value. Returns the number of files actually deleted (0 or 1). """ - num_deleted=0 + num_deleted = 0 uri = resource.uri if (resource.timestamp is not None and - resource.timestamp > self.last_timestamp): + resource.timestamp > self.last_timestamp): self.last_timestamp = resource.timestamp if (allow_deletion): if (self.dryrun): - self.logger.info("dryrun: would delete %s -> %s" % (uri,filename)) + self.logger.info( + "dryrun: would delete %s -> %s" % + (uri, filename)) else: try: os.unlink(filename) - num_deleted+=1 - self.logger.info("deleted: %s -> %s" % (uri,filename)) - self.log_event(Resource(resource=resource, change="deleted")) + num_deleted += 1 + self.logger.info("deleted: %s -> %s" % (uri, filename)) + self.log_event( + Resource( + resource=resource, + change="deleted")) except OSError as e: - msg = "Failed to DELETE %s -> %s : %s" % (uri,filename,str(e)) - #if (self.ignore_failures): + msg = "Failed to DELETE %s -> %s : %s" % ( + uri, filename, str(e)) + # if (self.ignore_failures): self.logger.warning(msg) # return - #else: + # else: # raise ClientFatalError(msg) else: - self.logger.info("nodelete: would delete %s (--delete to enable)" % uri) + self.logger.info( + "nodelete: would delete %s (--delete to enable)" % + uri) return(num_deleted) def parse_document(self): """Parse any ResourceSync document and show information. - + Will use sitemap URI taken either from explicit self.sitemap_name or derived from the mappings supplied. """ - s=Sitemap() + s = Sitemap() self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap)) try: list = s.parse_xml(url_or_file_open(self.sitemap)) @@ -430,33 +492,36 @@ class Client(object): capability = '(unknown capability)' if ('capability' in list.md): capability = list.md['capability'] - print("Parsed %s document with %d entries" % (capability,num_entries)) + print("Parsed %s document with %d entries" % (capability, num_entries)) if (self.verbose): to_show = 100 override_str = ' (override with --max-sitemap-entries)' if (self.max_sitemap_entries): to_show = self.max_sitemap_entries override_str = '' - if (num_entries>to_show): - print("Showing first %d entries sorted by URI%s..." % (to_show,override_str)) - n=0 + if (num_entries > to_show): + print( + "Showing first %d entries sorted by URI%s..." % + (to_show, override_str)) + n = 0 for resource in list: - print('[%d] %s' % (n,str(resource))) - n+=1 - if ( n >= to_show ): + print('[%d] %s' % (n, str(resource))) + n += 1 + if (n >= to_show): break - def write_resource_list(self,paths=None,outfile=None,links=None,dump=None): + def write_resource_list( + self, paths=None, outfile=None, links=None, dump=None): """Write a Resource List or a Resource Dump for files on local disk. - Set of resources included is based on paths setting or else the mappings. + Set of resources included is based on paths setting or else the mappings. Optionally links can be added. Output will be to stdout unless outfile is specified. - + If dump is true then a Resource Dump is written instead of a Resource List. If outfile is not set then self.default_resource_dump will be used. """ - rl = self.build_resource_list(paths=paths,set_path=dump) + rl = self.build_resource_list(paths=paths, set_path=dump) if (links is not None): rl.ln = links if (dump): @@ -470,15 +535,17 @@ class Client(object): try: print(rl.as_xml()) except ListBaseIndexError as e: - raise ClientFatalError("%s. Use --output option to specify base name for output files." % str(e)) + raise ClientFatalError( + "%s. Use --output option to specify base name for output files." % + str(e)) else: rl.write(basename=outfile) - def write_change_list(self,paths=None,outfile=None,ref_sitemap=None,newref_sitemap=None, - empty=None,links=None,dump=None): + def write_change_list(self, paths=None, outfile=None, ref_sitemap=None, newref_sitemap=None, + empty=None, links=None, dump=None): """Write a change list. - - Unless the both ref_sitemap and newref_sitemap are specified then the Change + + Unless the both ref_sitemap and newref_sitemap are specified then the Change List is calculated between the reference an the current state of files on disk. The files on disk are scanned based either on the paths setting or else on the mappings. @@ -487,18 +554,19 @@ class Client(object): if (not empty): # 1. Get and parse reference sitemap old_rl = self.read_reference_resource_list(ref_sitemap) - # 2. Depending on whether a newref_sitemap was specified, either read that + # 2. Depending on whether a newref_sitemap was specified, either read that # or build resource list from files on disk if (newref_sitemap is None): # Get resource list from disk - new_rl = self.build_resource_list(paths=paths,set_path=dump) + new_rl = self.build_resource_list(paths=paths, set_path=dump) else: - new_rl = self.read_reference_resource_list(newref_sitemap,name='new reference') + new_rl = self.read_reference_resource_list( + newref_sitemap, name='new reference') # 3. Calculate change list - (same,updated,deleted,created)=old_rl.compare(new_rl) - cl.add_changed_resources( updated, change='updated' ) - cl.add_changed_resources( deleted, change='deleted' ) - cl.add_changed_resources( created, change='created' ) + (same, updated, deleted, created) = old_rl.compare(new_rl) + cl.add_changed_resources(updated, change='updated') + cl.add_changed_resources(deleted, change='deleted') + cl.add_changed_resources(created, change='created') # 4. Write out change list cl.mapper = self.mapper cl.pretty_xml = self.pretty_xml @@ -508,9 +576,10 @@ class Client(object): print(cl.as_xml()) else: cl.write(basename=outfile) - self.write_dump_if_requested(cl,dump) + self.write_dump_if_requested(cl, dump) - def write_capability_list(self,capabilities=None,outfile=None,links=None): + def write_capability_list(self, capabilities=None, + outfile=None, links=None): """Write a Capability List to outfile or STDOUT.""" capl = CapabilityList(ln=links) capl.pretty_xml = self.pretty_xml @@ -522,7 +591,8 @@ class Client(object): else: capl.write(basename=outfile) - def write_source_description(self,capability_lists=None,outfile=None,links=None): + def write_source_description( + self, capability_lists=None, outfile=None, links=None): """Write a ResourceSync Description document to outfile or STDOUT.""" rsd = SourceDescription(ln=links) rsd.pretty_xml = self.pretty_xml @@ -534,70 +604,76 @@ class Client(object): else: rsd.write(basename=outfile) - def write_dump_if_requested(self,resource_list,dump): + def write_dump_if_requested(self, resource_list, dump): """Write a dump to the file dump.""" if (dump is None): return print("OOPS - FIXME - Wrinting dump to %s not yet implemented" % (dump)) return(1) - def read_reference_resource_list(self,ref_sitemap,name='reference'): + def read_reference_resource_list(self, ref_sitemap, name='reference'): """Read reference resource list and return the ResourceList object. The name parameter is used just in output messages to say what type of resource list is being read. """ rl = ResourceList() - self.logger.info("Reading %s resource list from %s ..." % (name,ref_sitemap)) - rl.mapper=self.mapper - rl.read(uri=ref_sitemap,index_only=(not self.allow_multifile)) + self.logger.info( + "Reading %s resource list from %s ..." % + (name, ref_sitemap)) + rl.mapper = self.mapper + rl.read(uri=ref_sitemap, index_only=(not self.allow_multifile)) num_entries = len(rl.resources) - self.logger.info("Read %s resource list with %d entries in %d sitemaps" % (name,num_entries,rl.num_files)) + self.logger.info( + "Read %s resource list with %d entries in %d sitemaps" % + (name, num_entries, rl.num_files)) if (self.verbose): to_show = 100 override_str = ' (override with --max-sitemap-entries)' if (self.max_sitemap_entries): to_show = self.max_sitemap_entries override_str = '' - if (num_entries>to_show): - print("Showing first %d entries sorted by URI%s..." % (to_show,override_str)) - n=0 + if (num_entries > to_show): + print( + "Showing first %d entries sorted by URI%s..." % + (to_show, override_str)) + n = 0 for r in rl.resources: print(r) - n+=1 - if ( n >= to_show ): + n += 1 + if (n >= to_show): break return(rl) def log_status(self, in_sync=True, incremental=False, audit=False, same=None, created=0, updated=0, deleted=0, to_delete=0): """Write log message regarding status in standard form. - + Split this off so all messages from baseline/audit/incremental are written in a consistent form. """ if (audit): - words = { 'created': 'to create', - 'updated': 'to update', - 'deleted': 'to delete' } + words = {'created': 'to create', + 'updated': 'to update', + 'deleted': 'to delete'} else: - words = { 'created': 'created', - 'updated': 'updated', - 'deleted': 'deleted' } + words = {'created': 'created', + 'updated': 'updated', + 'deleted': 'deleted'} if in_sync: # status rather than action - status = "NO CHANGES" if incremental else "IN SYNC" + status = "NO CHANGES" if incremental else "IN SYNC" else: if audit: status = "NOT IN SYNC" - elif (to_delete>deleted): - #will need --delete + elif (to_delete > deleted): + # will need --delete status = "PART APPLIED" if incremental else"PART SYNCED" - words['deleted']='to delete (--delete)' - deleted=to_delete - else: + words['deleted'] = 'to delete (--delete)' + deleted = to_delete + else: status = "CHANGES APPLIED" if incremental else "SYNCED" - same = "" if (same is None) else ("same=%d, " % same) - self.logger.warning("Status: %15s (%s%s=%d, %s=%d, %s=%d)" %\ - (status, same, words['created'], created, - words['updated'], updated, words['deleted'], deleted)) + same = "" if (same is None) else ("same=%d, " % same) + self.logger.warning("Status: %15s (%s%s=%d, %s=%d, %s=%d)" % + (status, same, words['created'], created, + words['updated'], updated, words['deleted'], deleted)) diff --git a/resync/client_state.py b/resync/client_state.py index 588fab8..ada410e 100644 --- a/resync/client_state.py +++ b/resync/client_state.py @@ -8,13 +8,13 @@ of the last change seen. import sys import os.path import datetime -import distutils.dir_util +import distutils.dir_util import re import time import logging -try: #python3 +try: # python3 from configparser import ConfigParser, NoSectionError, NoOptionError -except ImportError: #python2 +except ImportError: # python2 from ConfigParser import SafeConfigParser as ConfigParser, NoSectionError, NoOptionError @@ -25,9 +25,9 @@ class ClientState(object): """Initialize ClientState object with default status file name.""" self.status_file = '.resync-client-status.cfg' - def set_state(self,site,timestamp=None): + def set_state(self, site, timestamp=None): """Write status dict to client status file. - + FIXME - should have some file lock to avoid race """ parser = ConfigParser() @@ -36,21 +36,29 @@ class ClientState(object): if (not parser.has_section(status_section)): parser.add_section(status_section) if (timestamp is None): - parser.remove_option(status_section, self.config_site_to_name(site)) + parser.remove_option( + status_section, + self.config_site_to_name(site)) else: - parser.set(status_section, self.config_site_to_name(site), str(timestamp)) + parser.set( + status_section, + self.config_site_to_name(site), + str(timestamp)) with open(self.status_file, 'w') as configfile: parser.write(configfile) configfile.close() - def get_state(self,site): + def get_state(self, site): """Read client status file and return dict.""" parser = ConfigParser() status_section = 'incremental' parser.read(self.status_file) timestamp = None try: - timestamp = float(parser.get(status_section,self.config_site_to_name(site))) + timestamp = float( + parser.get( + status_section, + self.config_site_to_name(site))) except NoSectionError as e: pass except NoOptionError as e: @@ -64,4 +72,4 @@ class ClientState(object): lead to multiple site names mapping to one config name but probably not too likely. """ - return( re.sub(r"[^\w]",'_',name) ) + return(re.sub(r"[^\w]", '_', name)) diff --git a/resync/client_utils.py b/resync/client_utils.py index 89dbfe6..0bd316b 100644 --- a/resync/client_utils.py +++ b/resync/client_utils.py @@ -18,20 +18,22 @@ Copyright 2012-2016 Simeon Warner limitations under the License """ -try: #python3 +try: # python3 from urllib.request import urlopen -except ImportError: #python2 +except ImportError: # python2 from urllib import urlopen import logging import logging.config from datetime import datetime import re + class ClientFatalError(Exception): """Non-recoverable error in client, should include message to user.""" pass + class UTCFormatter(logging.Formatter): """Format datetime values as ISO8601 UTC Z form. @@ -43,8 +45,9 @@ class UTCFormatter(logging.Formatter): timestamp = record.created return datetime.utcfromtimestamp(timestamp).isoformat() + 'Z' + def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log', - human=True, verbose=False, eval_mode=False, + human=True, verbose=False, eval_mode=False, default_logger='client', extra_loggers=None): """Initialize logging. @@ -53,116 +56,126 @@ def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log', INFO - verbose, only seen by users if they ask for it (-v) WARNING - messages output messages to console - Logging to a file: If to_file is True then output will be written to + Logging to a file: If to_file is True then output will be written to a file. This will be logfile if set, else default_logfile (which may also be overridden). """ fmt = '%(asctime)s | %(name)s | %(levelname)s | %(message)s' formatter = UTCFormatter(fmt) - + if human: # Create a special handler designed just for human readable output hh = logging.StreamHandler() - hh.setLevel( logging.INFO if (verbose) else logging.WARNING ) + hh.setLevel(logging.INFO if (verbose) else logging.WARNING) hh.setFormatter(logging.Formatter(fmt='%(message)s')) if to_file: if (logfile is None): logfile = default_logfile fh = logging.FileHandler(filename=logfile, mode='a') fh.setFormatter(formatter) - fh.setLevel( logging.DEBUG if (eval_mode) else logging.INFO ) + fh.setLevel(logging.DEBUG if (eval_mode) else logging.INFO) - loggers = [default_logger,'resync'] + loggers = [default_logger, 'resync'] if (extra_loggers is not None): for logger in extra_loggers: loggers.append(logger) for logger in loggers: log = logging.getLogger(logger) - log.setLevel(logging.DEBUG) #control at handler instead + log.setLevel(logging.DEBUG) # control at handler instead if human: log.addHandler(hh) if to_file: log.addHandler(fh) - log=logging.getLogger(default_logger) + log = logging.getLogger(default_logger) if (to_file): log.info("Writing detailed log to %s" % (logfile)) def count_true_args(*args): """Count number of list of arguments that evaluate True.""" - count=0 + count = 0 for arg in args: if (arg): - count+=1 + count += 1 return(count) + def parse_links(args_link): """Parse --link options. Uses parse_link() to parse each option. """ - links=[] + links = [] if (args_link is not None): for link_str in args_link: try: links.append(parse_link(link_str)) except ClientFatalError as e: - raise ClientFatalError("Bad --link option '%s' (%s)" % (link_str,str(e))) + raise ClientFatalError( + "Bad --link option '%s' (%s)" % + (link_str, str(e))) return(links) + def parse_link(link_str): """Parse one --link option to add to links. Input string of the form: rel,href,att1=val1,att2=val2 """ - atts={} - help_str = "--link option '%s' (format rel,href,att1=val1...)"%(link_str) + atts = {} + help_str = "--link option '%s' (format rel,href,att1=val1...)" % (link_str) try: segs = link_str.split(',') # First segments are relation and subject atts['rel'] = segs.pop(0) atts['href'] = segs.pop(0) - if (atts['href']==''): + if (atts['href'] == ''): raise ClientFatalError("Missing uri in " + help_str) # Remaining segments are attributes for term in segs: - (k,v)=term.split('=') - if (k=='' or v==''): - raise ClientFatalError("Bad attribute (%s) in " % (term) + help_str) - atts[k]=v + (k, v) = term.split('=') + if (k == '' or v == ''): + raise ClientFatalError( + "Bad attribute (%s) in " % + (term) + help_str) + atts[k] = v except ValueError as e: raise ClientFatalError("Bad component of " + help_str) except IndexError as e: raise ClientFatalError("Incomplete component of " + help_str) - + return(atts) + def parse_capabilities(caps_str): """Parse list of capabilities in --capabilitylist option. Input string of the form: cap_name=uri,cap_name=uri """ - capabilities={} + capabilities = {} try: segs = caps_str.split(',') for term in segs: - (k,v)=term.split('=') - capabilities[k]=v + (k, v) = term.split('=') + capabilities[k] = v except ValueError as e: - raise ClientFatalError("Bad component of --capabilitylist option '%s' (%s)"%(caps_str,str(e))) + raise ClientFatalError( + "Bad component of --capabilitylist option '%s' (%s)" % + (caps_str, str(e))) return(capabilities) + def parse_capability_lists(cls_str): """Parse list of capability lists in --capabilitylistindex option. Input string of the form: uri,uri """ return(cls_str.split(',')) - + + def url_or_file_open(uri): """Wrapper around urlopen() to prepend file: if no scheme provided.""" - if (not re.match(r'''\w+:''',uri)): - uri = 'file:'+uri + if (not re.match(r'''\w+:''', uri)): + uri = 'file:' + uri return(urlopen(uri)) - diff --git a/resync/dump.py b/resync/dump.py index 04b64e3..2679061 100644 --- a/resync/dump.py +++ b/resync/dump.py @@ -5,6 +5,7 @@ import os.path from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from resync.resource_dump_manifest import ResourceDumpManifest + class Dump(object): """Dump of content for a Resource Dump or Change Dump. @@ -25,31 +26,33 @@ class Dump(object): self.resources = resources self.format = ('zip' if (format is None) else format) self.compress = compress - self.manifest_class = ResourceDumpManifest #FIXME - self.max_size = 100*1024*1024 #100MB + self.manifest_class = ResourceDumpManifest # FIXME + self.max_size = 100 * 1024 * 1024 # 100MB self.max_files = 50000 self.path_prefix = None self.logger = logging.getLogger('resync.dump') - + def write(self, basename=None, write_separate_manifests=True): """Write one or more dump files to complete this dump. Returns the number of dump/archive files written. """ self.check_files() - n=0 + n = 0 for manifest in self.partition_dumps(): - dumpbase="%s%05d" % (basename,n) - dumpfile="%s.%s" % (dumpbase,self.format) + dumpbase = "%s%05d" % (basename, n) + dumpfile = "%s.%s" % (dumpbase, self.format) if (write_separate_manifests): - manifest.write(basename=dumpbase+'.xml') + manifest.write(basename=dumpbase + '.xml') if (self.format == 'zip'): - self.write_zip(manifest.resources,dumpfile) + self.write_zip(manifest.resources, dumpfile) elif (self.format == 'warc'): - self.write_warc(manifest.resources,dumpfile) + self.write_warc(manifest.resources, dumpfile) else: - raise DumpError("Unknown dump format requested (%s)" % (self.format)) - n+=1 + raise DumpError( + "Unknown dump format requested (%s)" % + (self.format)) + n += 1 self.logger.info("Wrote %d dump files" % (n)) return(n) @@ -60,8 +63,12 @@ class Dump(object): a manifest file manifest.xml (written first). No checks on the size of files or total size are performed, this is expected to have been done beforehand. """ - compression = ( ZIP_DEFLATED if self.compress else ZIP_STORED ) - zf = ZipFile(dumpfile, mode="w", compression=compression, allowZip64=True) + compression = (ZIP_DEFLATED if self.compress else ZIP_STORED) + zf = ZipFile( + dumpfile, + mode="w", + compression=compression, + allowZip64=True) # Write resources first rdm = ResourceDumpManifest(resources=resources) real_path = {} @@ -69,24 +76,26 @@ class Dump(object): archive_path = self.archive_path(resource.path) real_path[archive_path] = resource.path resource.path = archive_path - zf.writestr('manifest.xml',rdm.as_xml()) + zf.writestr('manifest.xml', rdm.as_xml()) # Add all files in the resources for resource in resources: zf.write(real_path[resource.path], arcname=resource.path) zf.close() zipsize = os.path.getsize(dumpfile) - self.logger.info("Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize)) - + self.logger.info( + "Wrote ZIP file dump %s with size %d bytes" % + (dumpfile, zipsize)) + def write_warc(self, resources=None, dumpfile=None): """Write a WARC dump file. WARC support is not part of ResourceSync v1.0 (Z39.99 2014) but is left in this library for experimentation. """ - # Load library late as we want to be able to run rest of code + # Load library late as we want to be able to run rest of code # without this installed try: - from warc import WARCFile,WARCHeader,WARCRecord + from warc import WARCFile, WARCHeader, WARCRecord except: raise DumpError("Failed to load WARC library") wf = WARCFile(dumpfile, mode="w", compress=self.compress) @@ -100,46 +109,56 @@ class Dump(object): wh.result_code = 200 wh.checksum = 'aabbcc' wh.location = self.archive_path(resource.path) - wf.write_record( WARCRecord( header=wh, payload=resource.path ) ) + wf.write_record(WARCRecord(header=wh, payload=resource.path)) wf.close() warcsize = os.path.getsize(dumpfile) - self.logging.info("Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize)) - - def check_files(self,set_length=True,check_length=True): + self.logging.info( + "Wrote WARC file dump %s with size %d bytes" % + (dumpfile, warcsize)) + + def check_files(self, set_length=True, check_length=True): """Check all files in self.resources, find longest common prefix. Go though and check all files in self.resources, add up size, and find - longest common path that can be used when writing the dump file. Saved in + longest common path that can be used when writing the dump file. Saved in self.path_prefix. Parameters set_length and check_length control control whether then set_length - attribute should be set from the file size if not specified, and whether any + attribute should be set from the file size if not specified, and whether any length specified should be checked. By default both are True. In any event, the total size calculated is the size of files on disk. """ - total_size = 0 #total size of all files in bytes + total_size = 0 # total size of all files in bytes path_prefix = None for resource in self.resources: if (resource.path is None): - #explicit test because exception raised by getsize otherwise confusing - raise DumpError("No file path defined for resource %s" % resource.uri) + # explicit test because exception raised by getsize otherwise + # confusing + raise DumpError( + "No file path defined for resource %s" % + resource.uri) if (path_prefix is None): path_prefix = os.path.dirname(resource.path) else: - path_prefix = os.path.commonprefix( [ path_prefix, os.path.dirname(resource.path) ]) + path_prefix = os.path.commonprefix( + [path_prefix, os.path.dirname(resource.path)]) size = os.path.getsize(resource.path) if (resource.length is not None): - if (check_length and resource.length!=size): + if (check_length and resource.length != size): raise DumpError("Size of resource %s is %d on disk, not %d as specified" % - (resource.uri, size, resource.length) ) + (resource.uri, size, resource.length)) elif (set_length): resource.length = size if (size > self.max_size): - raise DumpError("Size of file (%s, %d) exceeds maximum (%d) dump size" % (resource.path,size,self.max_size)) + raise DumpError( + "Size of file (%s, %d) exceeds maximum (%d) dump size" % + (resource.path, size, self.max_size)) total_size += size self.path_prefix = path_prefix self.total_size = total_size - self.logger.info("Total size of files to include in dump %d bytes" % (total_size)) + self.logger.info( + "Total size of files to include in dump %d bytes" % + (total_size)) return True def partition_dumps(self): @@ -147,27 +166,26 @@ class Dump(object): Simply adds resources/files to a manifest until their are either the the correct number of files or the size limit is exceeded, then yields - that manifest. + that manifest. """ - manifest=self.manifest_class() - manifest_size=0 - manifest_files=0 + manifest = self.manifest_class() + manifest_size = 0 + manifest_files = 0 for resource in self.resources: manifest.add(resource) - manifest_size+=resource.length - manifest_files+=1 - if (manifest_size>=self.max_size or - manifest_files>=self.max_files): + manifest_size += resource.length + manifest_files += 1 + if (manifest_size >= self.max_size or + manifest_files >= self.max_files): yield(manifest) # Need to start a new manifest - manifest=self.manifest_class() - manifest_size=0 - manifest_files=0 - if (manifest_files>0): + manifest = self.manifest_class() + manifest_size = 0 + manifest_files = 0 + if (manifest_files > 0): yield(manifest) - - def archive_path(self,real_path): + def archive_path(self, real_path): """Return the archive path for file with real_path. Mapping is based on removal of self.path_prefix which is determined @@ -176,10 +194,10 @@ class Dump(object): if (not self.path_prefix): return(real_path) else: - return( os.path.relpath(real_path,self.path_prefix) ) + return(os.path.relpath(real_path, self.path_prefix)) class DumpError(Exception): """Error class used by Dump() objects.""" - + pass diff --git a/resync/explorer.py b/resync/explorer.py index df07f27..a83e799 100644 --- a/resync/explorer.py +++ b/resync/explorer.py @@ -1,19 +1,19 @@ """ResourceSync explorer. This is the guts of a client designed to 'explore' the ResourceSync -facilities offered by a source. Will use standard practices to +facilities offered by a source. Will use standard practices to look for and interpret capabilities. """ import sys -try: #python3 +try: # python3 from urllib.parse import urlparse, urlunparse, urljoin -except ImportError: #python2 +except ImportError: # python2 from urlparse import urlparse, urlunparse, urljoin input = raw_input import os.path import datetime -import distutils.dir_util +import distutils.dir_util import re import time import logging @@ -21,23 +21,24 @@ import requests from .mapper import Mapper from .sitemap import Sitemap -from .client import Client,ClientFatalError +from .client import Client, ClientFatalError from .client_state import ClientState from .client_utils import ClientFatalError, url_or_file_open from .resource import Resource -from .w3c_datetime import str_to_datetime,datetime_to_str +from .w3c_datetime import str_to_datetime, datetime_to_str + class Explorer(Client): """Extension of the client code to explore a ResourceSync source. Designed to support a text-based command-line client that starts from - a given URI, server, or local file, and then allows interactive + a given URI, server, or local file, and then allows interactive exploration of ResourceSync capabilities offered by the source. """ def explore(self): """INTERACTIVE exploration source capabilities. - + Will use sitemap URI taken either from explicit self.sitemap_name or derived from the mappings supplied. """ @@ -47,57 +48,67 @@ class Explorer(Client): starts = [] if (self.sitemap_name is not None): print("Starting from explicit --sitemap %s" % (self.sitemap_name)) - starts.append( XResource(self.sitemap_name) ) - elif (len(self.mapper)>0): + starts.append(XResource(self.sitemap_name)) + elif (len(self.mapper) > 0): uri = self.mapper.default_src_uri() (scheme, netloc, path, params, query, fragment) = urlparse(uri) if (not scheme and not netloc): if (os.path.isdir(path)): # have a dir, look for 'likely' file names - print("Looking for capability documents in local directory %s" % (path)) - for name in ['resourcesync','capabilities.xml', - 'resourcelist.xml','changelist.xml']: - file = os.path.join(path,name) + print( + "Looking for capability documents in local directory %s" % + (path)) + for name in ['resourcesync', 'capabilities.xml', + 'resourcelist.xml', 'changelist.xml']: + file = os.path.join(path, name) if (os.path.isfile(file)): - starts.append( XResource(file) ) - if (len(starts)==0): - raise ClientFatalError( "No likely capability files found in local directory %s" % - (path) ) + starts.append(XResource(file)) + if (len(starts) == 0): + raise ClientFatalError("No likely capability files found in local directory %s" % + (path)) else: # local file, might be anything (or not exist) print("Starting from local file %s" % (path)) - starts.append( XResource(path) ) + starts.append(XResource(path)) else: - # remote, can't tell whether we have a sitemap or a server name or something + # remote, can't tell whether we have a sitemap or a server name or something # else, build list of options depending on whether there is a path and whether # there is an extension/name - well_known = urlunparse( [ scheme,netloc,'/.well-known/resourcesync','','','' ] ) + well_known = urlunparse( + [scheme, netloc, '/.well-known/resourcesync', '', '', '']) if (not path): # root, just look for .well-known - starts.append( XResource(well_known, ['capabilitylist','capabilitylistindex']) ) + starts.append( + XResource( + well_known, [ + 'capabilitylist', 'capabilitylistindex'])) else: - starts.append( XResource(uri) ) - starts.append( XResource(well_known, ['capabilitylist','capabilitylistindex']) ) + starts.append(XResource(uri)) + starts.append( + XResource( + well_known, [ + 'capabilitylist', 'capabilitylistindex'])) print("Looking for discovery information based on mappings") else: - raise ClientFatalError("No source information (server base uri or capability uri) specified, use -h for help") - # + raise ClientFatalError( + "No source information (server base uri or capability uri) specified, use -h for help") + # # Have list of one or more possible starting point, try them in turn try: for start in starts: # For each starting point we create a fresh history - history = [ start ] + history = [start] input = None - while (len(history)>0): + while (len(history) > 0): print() xr = history.pop() - new_xr = self.explore_uri(xr,len(history)>0) + new_xr = self.explore_uri(xr, len(history) > 0) if (new_xr): # Add current and new to history - history.append( xr ) - history.append( new_xr ) + history.append(xr) + history.append(new_xr) except ExplorerQuit: - pass # expected way to exit + pass # expected way to exit print("\nresync-explorer done, bye...\n") def explore_uri(self, explorer_resource, show_back=True): @@ -109,31 +120,32 @@ class Explorer(Client): caps = explorer_resource.acceptable_capabilities checks = explorer_resource.checks print("Reading %s" % (uri)) - options={} - capability=None + options = {} + capability = None try: - if (caps=='resource'): + if (caps == 'resource'): # Not expecting a capability document - self.explore_show_head(uri,check_headers=checks) - else: - s=Sitemap() + self.explore_show_head(uri, check_headers=checks) + else: + s = Sitemap() list = s.parse_xml(url_or_file_open(uri)) - (options,capability)=self.explore_show_summary(list,s.parsed_index,caps,context=uri) + (options, capability) = self.explore_show_summary( + list, s.parsed_index, caps, context=uri) except IOError as e: - print("Cannot read %s (%s)" % (uri,str(e))) + print("Cannot read %s (%s)" % (uri, str(e))) except Exception as e: - print("Cannot parse %s (%s)" % (uri,str(e))) + print("Cannot parse %s (%s)" % (uri, str(e))) # # Loop until we have some valid input # while (True): # don't offer number option for no resources/capabilities - num_prompt = '' if (len(options)==0) else 'number, ' + num_prompt = '' if (len(options) == 0) else 'number, ' up_prompt = 'b(ack), ' if (show_back) else '' if (self.fake_input): inp = self.fake_input else: - inp = input( "Follow [%s%sq(uit)]?" % (num_prompt,up_prompt) ) + inp = input("Follow [%s%sq(uit)]?" % (num_prompt, up_prompt)) if (inp in options.keys()): break if (inp == 'q'): @@ -145,9 +157,9 @@ class Explorer(Client): # checks = {} r = options[inp] - if ( r.capability is None ): - if (capability in ['resourcelist','changelist', - 'resourcedump','changedump']): + if (r.capability is None): + if (capability in ['resourcelist', 'changelist', + 'resourcedump', 'changedump']): caps = 'resource' else: caps = self.allowed_entries(capability) @@ -157,21 +169,22 @@ class Explorer(Client): caps = [r.capability] # Record anything we know about the resource to check if (r.length is not None): - checks['content-length']=r.length + checks['content-length'] = r.length if (r.lastmod is not None): - checks['last-modified']=r.lastmod + checks['last-modified'] = r.lastmod if (r.mime_type is not None): - checks['content-type']=r.mime_type + checks['content-type'] = r.mime_type # FIXME - could add fixity checks here too - return( XResource(r.uri, caps, checks) ) + return(XResource(r.uri, caps, checks)) - def explore_show_summary(self, list, index=False, expected=None, context=None): + def explore_show_summary(self, list, index=False, + expected=None, context=None): """Show summary of one capability document. - Given a capability document or index (in list, index True if it is an - index), write out a simply textual summary of the document with all - related documents shown as numbered options (of the form - [#] ...description...) which will then form a menu for the next + Given a capability document or index (in list, index True if it is an + index), write out a simply textual summary of the document with all + related documents shown as numbered options (of the form + [#] ...description...) which will then form a menu for the next exploration. If expected is not None then it should be a list of expected document @@ -186,75 +199,79 @@ class Explorer(Client): capability = list.md['capability'] if (index): capability += 'index' - print("Parsed %s document with %d entries:" % (capability,num_entries)) + print( + "Parsed %s document with %d entries:" % + (capability, num_entries)) if (expected is not None and capability not in expected): print("WARNING - expected a %s document" % (','.join(expected))) - if (capability not in ['description','descriptionoindex','capabilitylist', - 'resourcelist','resourcelistindex','changelist','changelistindex', - 'resourcedump','resourcedumpindex','changedump','changedumpindex', + if (capability not in ['description', 'descriptionoindex', 'capabilitylist', + 'resourcelist', 'resourcelistindex', 'changelist', 'changelistindex', + 'resourcedump', 'resourcedumpindex', 'changedump', 'changedumpindex', 'resourcelist-archive', 'resourcedump-archive', 'changelist-archive', 'changedump-archive']): print("WARNING - unknown %s document type" % (capability)) to_show = num_entries - if (num_entries>21): + if (num_entries > 21): to_show = 20 - # What capability entries are allowed/expected? - entry_caps = self.allowed_entries(capability); + # What capability entries are allowed/expected? + entry_caps = self.allowed_entries(capability) options = {} - n=0 + n = 0 # Look for elements in this document ln_describedby = list.link('describedby') if (ln_describedby): if ('href' in ln_describedby): uri = ln_describedby['href'] - print("[%s] rel='describedby' link to %s" % ('d',uri)) - uri = self.expand_relative_uri(context,uri) - options['d']=Resource(uri,capability='resource') + print("[%s] rel='describedby' link to %s" % ('d', uri)) + uri = self.expand_relative_uri(context, uri) + options['d'] = Resource(uri, capability='resource') else: print("WARNING - describedby link with no href, ignoring") ln_up = list.link('up') if (ln_up): if ('href' in ln_up): uri = ln_up['href'] - print("[%s] rel='up' link to %s" % ('u',uri)) - uri = self.expand_relative_uri(context,uri) - options['u']=Resource(uri) + print("[%s] rel='up' link to %s" % ('u', uri)) + uri = self.expand_relative_uri(context, uri) + options['u'] = Resource(uri) else: print("WARNING - up link with no href, ignoring") ln_index = list.link('index') if (ln_index): if ('href' in ln_index): uri = ln_index['href'] - print("[%s] rel='index' link to %s" % ('i',uri)) - uri = self.expand_relative_uri(context,uri) - options['i']=Resource(uri) + print("[%s] rel='index' link to %s" % ('i', uri)) + uri = self.expand_relative_uri(context, uri) + options['i'] = Resource(uri) else: print("WARNING - index link with no href, ignoring") # Show listed resources as numbered options for r in list.resources: - if (n>=to_show): - print("(not showing remaining %d entries)" % (num_entries-n)) + if (n >= to_show): + print("(not showing remaining %d entries)" % (num_entries - n)) break - n+=1 - options[str(n)]=r - print("[%d] %s" % (n,r.uri)) + n += 1 + options[str(n)] = r + print("[%d] %s" % (n, r.uri)) if (self.verbose): print(" " + str(r)) - r.uri = self.expand_relative_uri(context,r.uri) + r.uri = self.expand_relative_uri(context, r.uri) if (r.capability is not None): warning = '' if (r.capability not in entry_caps): warning = " (EXPECTED %s)" % (' or '.join(entry_caps)) - print(" %s%s" % (r.capability,warning)) - elif (len(entry_caps)==1): - r.capability=entry_caps[0] - print(" capability not specified, should be %s" % (r.capability)) - return(options,capability) + print(" %s%s" % (r.capability, warning)) + elif (len(entry_caps) == 1): + r.capability = entry_caps[0] + print( + " capability not specified, should be %s" % + (r.capability)) + return(options, capability) - def explore_show_head(self,uri,check_headers=None): + def explore_show_head(self, uri, check_headers=None): """Do HEAD on uri and show infomation. - Will also check headers against any values specified in + Will also check headers against any values specified in check_headers. """ print("HEAD %s" % (uri)) @@ -265,44 +282,49 @@ class Explorer(Client): # Mock up response if we have a local file response = self.head_on_file(uri) print(" status: %s" % (response.status_code)) - if (response.status_code=='200'): + if (response.status_code == '200'): # print some of the headers - for header in ['content-length','last-modified','lastmod','content-type','etag']: + for header in ['content-length', 'last-modified', + 'lastmod', 'content-type', 'etag']: if header in response.headers: - check_str='' + check_str = '' if (check_headers is not None and - header in check_headers): + header in check_headers): if (response.headers[header] == check_headers[header]): - check_str=' MATCHES EXPECTED VALUE' + check_str = ' MATCHES EXPECTED VALUE' else: - check_str=' EXPECTED %s' % (check_headers[header]) - print(" %s: %s%s" % (header, response.headers[header], check_str)) + check_str = ' EXPECTED %s' % ( + check_headers[header]) + print( + " %s: %s%s" % + (header, response.headers[header], check_str)) - def head_on_file(self,file): + def head_on_file(self, file): """Mock up requests.head(..) response on local file.""" response = HeadResponse() if (not os.path.isfile(file)): - response.status_code='404' + response.status_code = '404' else: - response.status_code='200' - response.headers['last-modified']=datetime_to_str(os.path.getmtime(file)) - response.headers['content-length']=os.path.getsize(file) + response.status_code = '200' + response.headers[ + 'last-modified'] = datetime_to_str(os.path.getmtime(file)) + response.headers['content-length'] = os.path.getsize(file) return(response) - def allowed_entries(self,capability): + def allowed_entries(self, capability): """Return list of allowed entries for given capability document. - Includes handling of capability = *index where the only acceptable + Includes handling of capability = *index where the only acceptable entries are *. """ - index = re.match(r'(.+)index$',capability) - archive = re.match(r'(.+)\-archive$',capability) + index = re.match(r'(.+)index$', capability) + archive = re.match(r'(.+)\-archive$', capability) if (capability == 'capabilitylistindex'): - return([]) #not allowed so no valid references + return([]) # not allowed so no valid references elif (index): - return([index.group(1)]) #name without index ending + return([index.group(1)]) # name without index ending elif (archive): - return([archive.group(1)]) #name without -archive ending + return([archive.group(1)]) # name without -archive ending elif (capability == 'description'): return(['capabilitylist']) elif (capability == 'capabilitylist'): @@ -312,12 +334,12 @@ class Explorer(Client): 'changelist-archive', 'changedump-archive']) return([]) - def expand_relative_uri(self,context,uri): + def expand_relative_uri(self, context, uri): """If uri is relative then expand in context. - + Prints warning if expansion happens. """ - full_uri = urljoin(context,uri) + full_uri = urljoin(context, uri) if (full_uri != uri): print(" WARNING - expanded relative URI to %s" % (full_uri)) uri = full_uri @@ -328,20 +350,21 @@ class XResource(object): """Information about a resource for the explorer. Must have a uri but may also store: - - acceptable_capabilities - None for any acceptable, 'resource' if a - resource rather than a capability document is expected, else + + acceptable_capabilities - None for any acceptable, 'resource' if a + resource rather than a capability document is expected, else a list of capability names checks - a set of information to check when then XResource is inspected - base_uri - on creation interpret any relative URI specified in the + base_uri - on creation interpret any relative URI specified in the context of base_uri, store the resulting full URI """ - def __init__(self, uri, acceptable_capabilities=None, checks=None, context=None): + def __init__(self, uri, acceptable_capabilities=None, + checks=None, context=None): """Initialize XResource object.""" - self.uri=urljoin(context,uri) - self.acceptable_capabilities=acceptable_capabilities - self.checks=checks + self.uri = urljoin(context, uri) + self.acceptable_capabilities = acceptable_capabilities + self.checks = checks class HeadResponse(object): @@ -349,8 +372,8 @@ class HeadResponse(object): def __init__(self): """Initialize with no status_code and no headers.""" - self.status_code=None - self.headers={} + self.status_code = None + self.headers = {} class ExplorerQuit(Exception): diff --git a/resync/list_base.py b/resync/list_base.py index 0b16bcf..abb6915 100644 --- a/resync/list_base.py +++ b/resync/list_base.py @@ -10,38 +10,39 @@ import os from datetime import datetime import re import sys -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io import logging -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from .resource_container import ResourceContainer from .sitemap import Sitemap + class ListBase(ResourceContainer): """Class that adds Sitemap based IO to ResourceContainer. resources - an iterable of resources - count - add optional explicit setting of the number of items in + count - add optional explicit setting of the number of items in resources which is useful when this is an iterator/generator. Is used instead of trying len(resources) md - metadata information for the list () - + ln - link information for the list () sitemapindex - defaults to False, set True if this is an index object """ - def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, + def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, capability_name='unknown'): """Initialize ListBase.""" super(ListBase, self).__init__(resources=resources, md=md, ln=ln, uri=uri, @@ -73,9 +74,9 @@ class ListBase(ResourceContainer): return(self.count) return(len(self.resources)) - ##### INPUT ##### + # INPUT - def read(self,uri=None): + def read(self, uri=None): """Default case is just to parse document at this URI. Intention is that the read() method may be overridden to support reading @@ -83,10 +84,10 @@ class ListBase(ResourceContainer): """ self.parse(uri=uri) - def parse(self,uri=None,fh=None,str_data=None,**kwargs): + def parse(self, uri=None, fh=None, str_data=None, **kwargs): """Parse a single XML document for this list. - Accepts either a uri (uri or default if parameter not specified), + Accepts either a uri (uri or default if parameter not specified), or a filehandle (fh) or a string (str_data). Note that this method does not handle the case of a sitemapindex+sitemaps. @@ -97,45 +98,52 @@ class ListBase(ResourceContainer): try: fh = URLopener().open(uri) except IOError as e: - raise Exception("Failed to load sitemap/sitemapindex from %s (%s)" % (uri,str(e))) + raise Exception( + "Failed to load sitemap/sitemapindex from %s (%s)" % + (uri, str(e))) elif (str_data is not None): - fh=io.StringIO(str_data) + fh = io.StringIO(str_data) elif ('str' in kwargs): # Legacy support for str argument, see # https://github.com/resync/resync/pull/21 # One test for this in tests/test_list_base.py - self.logger.warn("Legacy parse(str=...), use parse(str_data=...) instead") + self.logger.warn( + "Legacy parse(str=...), use parse(str_data=...) instead") fh = io.StringIO(kwargs['str']) if (fh is None): raise Exception("Nothing to parse") s = self.new_sitemap() - s.parse_xml(fh=fh,resources=self,capability=self.capability_name,sitemapindex=False) + s.parse_xml( + fh=fh, + resources=self, + capability=self.capability_name, + sitemapindex=False) self.parsed_index = s.parsed_index - - ##### OUTPUT ##### + + # OUTPUT def as_xml(self): """Return XML serialization of this list. - This code does not support the case where the list is too big for + This code does not support the case where the list is too big for a single XML document. """ self.default_capability() s = self.new_sitemap() - return s.resources_as_xml(self,sitemapindex=self.sitemapindex) + return s.resources_as_xml(self, sitemapindex=self.sitemapindex) - def write(self,basename="/tmp/resynclist.xml"): + def write(self, basename="/tmp/resynclist.xml"): """Write a single sitemap or sitemapindex XML document. Must be overridden to support multi-file lists. """ self.default_capability() - fh = open(basename,'w') + fh = open(basename, 'w') s = self.new_sitemap() - s.resources_as_xml(self,fh=fh,sitemapindex=self.sitemapindex) + s.resources_as_xml(self, fh=fh, sitemapindex=self.sitemapindex) fh.close() - ##### UTILITY ##### + # UTILITY def new_sitemap(self): """Create new Sitemap object with default settings.""" diff --git a/resync/list_base_with_index.py b/resync/list_base_with_index.py index 8c6dc55..c88e29a 100644 --- a/resync/list_base_with_index.py +++ b/resync/list_base_with_index.py @@ -5,15 +5,15 @@ sitemapindexes. Extends ListBase to add the support for sitemapindexes. """ import collections -import math +import math import os from datetime import datetime import re import sys import itertools -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from .list_base import ListBase @@ -23,6 +23,7 @@ from .mapper import Mapper, MapperError from .url_authority import UrlAuthority from .utils import compute_md5_for_file + class ListBaseWithIndex(ListBase): """Class that add handling of sitemapindexes to ListBase. @@ -49,25 +50,27 @@ class ListBaseWithIndex(ListBase): to those that the component sitemap files will be exposed as """ - def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, + def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, capability_name='unknown', allow_multifile=None, mapper=None, resources_class=None): """Initialize ListBaseWithIndex.""" self.resources_class = list if resources_class is None else resources_class if (resources is None): resources = self.resources_class() - super(ListBaseWithIndex, self).__init__(resources=resources, count=count, md=md, ln=ln, + super(ListBaseWithIndex, self).__init__(resources=resources, count=count, md=md, ln=ln, uri=uri, capability_name=capability_name) # specific to lists with indexes - self.max_sitemap_entries=50000 + self.max_sitemap_entries = 50000 self.mapper = mapper - self.allow_multifile = (True if (allow_multifile is None) else allow_multifile) + self.allow_multifile = ( + True if ( + allow_multifile is None) else allow_multifile) self.check_url_authority = False self.content_length = 0 self.num_files = 0 # Number of files read self.bytes_read = 0 # Aggregate of content_length values - ##### INPUT ##### + # INPUT def read(self, uri=None, resources=None, index_only=False): """Read sitemap from a URI including handling sitemapindexes. @@ -76,7 +79,7 @@ class ListBaseWithIndex(ListBase): will not be read. This will result in no resources being returned and is useful only to read the metadata and links listed in the sitemapindex. - Includes the subtlety that if the input URI is a local file and is a + Includes the subtlety that if the input URI is a local file and is a sitemapindex which contains URIs for the individual sitemaps, then these are mapped to the filesystem also. """ @@ -84,63 +87,81 @@ class ListBaseWithIndex(ListBase): fh = URLopener().open(uri) self.num_files += 1 except IOError as e: - raise IOError("Failed to load sitemap/sitemapindex from %s (%s)" % (uri,str(e))) + raise IOError( + "Failed to load sitemap/sitemapindex from %s (%s)" % + (uri, str(e))) # Get the Content-Length if we can (works fine for local files) try: self.content_length = int(fh.info()['Content-Length']) self.bytes_read += self.content_length - self.logger.debug( "Read %d bytes from %s" % (self.content_length,uri) ) + self.logger.debug( + "Read %d bytes from %s" % + (self.content_length, uri)) except KeyError: # If we don't get a length then c'est la vie - self.logger.debug( "Read ????? bytes from %s" % (uri) ) + self.logger.debug("Read ????? bytes from %s" % (uri)) pass - self.logger.info( "Read sitemap/sitemapindex from %s" % (uri) ) + self.logger.info("Read sitemap/sitemapindex from %s" % (uri)) s = self.new_sitemap() - s.parse_xml(fh=fh,resources=self,capability=self.capability_name) + s.parse_xml(fh=fh, resources=self, capability=self.capability_name) # what did we read? sitemap or sitemapindex? if (s.parsed_index): # sitemapindex if (not self.allow_multifile): - raise ListBaseIndexError("Got sitemapindex from %s but support for sitemapindex disabled" % (uri)) - self.logger.info( "Parsed as sitemapindex, %d sitemaps" % (len(self.resources)) ) + raise ListBaseIndexError( + "Got sitemapindex from %s but support for sitemapindex disabled" % + (uri)) + self.logger.info( + "Parsed as sitemapindex, %d sitemaps" % + (len( + self.resources))) sitemapindex_is_file = self.is_file_uri(uri) if (index_only): # don't read the component sitemaps self.sitemapindex = True return - # now loop over all entries to read each sitemap and add to resources + # now loop over all entries to read each sitemap and add to + # resources sitemaps = self.resources self.resources = self.resources_class() - self.logger.info( "Now reading %d sitemaps" % len(sitemaps.uris()) ) + self.logger.info("Now reading %d sitemaps" % len(sitemaps.uris())) for sitemap_uri in sorted(sitemaps.uris()): - self.read_component_sitemap(uri,sitemap_uri,s,sitemapindex_is_file) + self.read_component_sitemap( + uri, sitemap_uri, s, sitemapindex_is_file) else: # sitemap - self.logger.info( "Parsed as sitemap, %d resources" % (len(self.resources)) ) + self.logger.info("Parsed as sitemap, %d resources" % + (len(self.resources))) - - def read_component_sitemap(self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file): + def read_component_sitemap( + self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file): """Read a component sitemap of a Resource List with index. - Each component must be a sitemap with the + Each component must be a sitemap with the """ if (sitemapindex_is_file): if (not self.is_file_uri(sitemap_uri)): # Attempt to map URI to local file remote_uri = sitemap_uri sitemap_uri = self.mapper.src_to_dst(remote_uri) - self.logger.info("Mapped %s to local file %s" % (remote_uri, sitemap_uri)) + self.logger.info( + "Mapped %s to local file %s" % + (remote_uri, sitemap_uri)) else: # The individual sitemaps should be at a URL (scheme/server/path) # that the sitemapindex URL can speak authoritatively about if (self.check_url_authority and - not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)): - raise ListBaseIndexError("The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" % (sitemapindex_uri,sitemap_uri)) + not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)): + raise ListBaseIndexError( + "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" % + (sitemapindex_uri, sitemap_uri)) try: fh = URLopener().open(sitemap_uri) self.num_files += 1 except IOError as e: - raise ListBaseIndexError("Failed to load sitemap from %s listed in sitemap index %s (%s)" % (sitemap_uri,sitemapindex_uri,str(e))) + raise ListBaseIndexError( + "Failed to load sitemap from %s listed in sitemap index %s (%s)" % + (sitemap_uri, sitemapindex_uri, str(e))) # Get the Content-Length if we can (works fine for local files) try: self.content_length = int(fh.info()['Content-Length']) @@ -148,34 +169,36 @@ class ListBaseWithIndex(ListBase): except KeyError: # If we don't get a length then c'est la vie pass - self.logger.info( "Reading sitemap from %s (%d bytes)" % (sitemap_uri,self.content_length) ) - component = sitemap.parse_xml( fh=fh, sitemapindex=False ) + self.logger.info( + "Reading sitemap from %s (%d bytes)" % + (sitemap_uri, self.content_length)) + component = sitemap.parse_xml(fh=fh, sitemapindex=False) # Copy resources into self, check any metadata for r in component: self.resources.add(r) # FIXME - if rel="up" check it goes to correct place # FIXME - check capability - ##### OUTPUT ##### + # OUTPUT def requires_multifile(self): """Return False or the number of component sitemaps required. - + In the case that no len() is available for self.resources then then self.count must be set beforehand to avoid an exception. """ if (self.max_sitemap_entries is None or - len(self)<=self.max_sitemap_entries): + len(self) <= self.max_sitemap_entries): return(False) - return( int( math.ceil( len(self) / float(self.max_sitemap_entries) ) ) ) + return(int(math.ceil(len(self) / float(self.max_sitemap_entries)))) def as_xml(self, allow_multifile=False, basename="/tmp/sitemap.xml"): """Return XML serialization of this list. - If this list can be serialized as a single sitemap then the + If this list can be serialized as a single sitemap then the superclass method is used. - If there is no single XML serialization (in the case that the + If there is no single XML serialization (in the case that the number of list resources is more than is allowed in a single sitemap) then raise an exception unless allow_multifile is set True. If allow_multifile is set True then will return the sitemapindex @@ -186,55 +209,65 @@ class ListBaseWithIndex(ListBase): elif (allow_multifile): return self.as_xml_index(basename) else: - raise ListBaseIndexError("Attempt to write single XML string for list with %d entries when max_sitemap_entries is set to %d" % (len(self),self.max_sitemap_entries)) + raise ListBaseIndexError( + "Attempt to write single XML string for list with %d entries when max_sitemap_entries is set to %d" % + (len(self), self.max_sitemap_entries)) def as_xml_index(self, basename="/tmp/sitemap.xml"): """Return a string of the index for a large list that is split. - + All we need to do is determine the number of component sitemaps will be is and generate their URIs based on a pattern. Q - should there be a flag to select generation of each component sitemap in order to calculate the md5sum? - + Q - what timestamp should be used? """ num_parts = self.requires_multifile() if (not num_parts): - raise ListBaseIndexError("Request for sitemapindex for list with only %d entries when max_sitemap_entries is set to %s" % (len(self),str(self.max_sitemap_entries))) - index=ListBase() - index.sitemapindex=True + raise ListBaseIndexError( + "Request for sitemapindex for list with only %d entries when max_sitemap_entries is set to %s" % + (len(self), str( + self.max_sitemap_entries))) + index = ListBase() + index.sitemapindex = True index.capability_name = self.capability_name index.default_capability() for n in range(num_parts): - r = Resource( uri = self.part_name(basename,n) ) + r = Resource(uri=self.part_name(basename, n)) index.add(r) - return( index.as_xml() ) + return(index.as_xml()) def as_xml_part(self, basename="/tmp/sitemap.xml", part_number=0): """Return a string of component sitemap number part_number. Used in the case of a large list that is split into component sitemaps. - + basename is used to create "index" links to the sitemapindex - + Q - what timestamp should be used? """ if (not self.requires_multifile()): - raise ListBaseIndexError("Request for component sitemap for list with only %d entries when max_sitemap_entries is set to %s" % (len(self),str(self.max_sitemap_entries))) + raise ListBaseIndexError( + "Request for component sitemap for list with only %d entries when max_sitemap_entries is set to %s" % + (len(self), str( + self.max_sitemap_entries))) start = part_number * self.max_sitemap_entries - if (start>len(self)): - raise ListBaseIndexError("Request for component sitemap with part_number too high, would start at entry %d yet the list has only %d entries" % (start,len(self))) + if (start > len(self)): + raise ListBaseIndexError( + "Request for component sitemap with part_number too high, would start at entry %d yet the list has only %d entries" % + (start, len(self))) stop = start + self.max_sitemap_entries - if (stop>len(self)): - stop=len(self) - part = ListBase( itertools.islice(self.resources,start,stop) ) + if (stop > len(self)): + stop = len(self) + part = ListBase(itertools.islice(self.resources, start, stop)) part.capability_name = self.capability_name part.default_capability() part.index = basename s = self.new_sitemap() - return( s.resources_as_xml(part) ) + return(s.resources_as_xml(part)) def write(self, basename='/tmp/sitemap.xml'): """Write one or a set of sitemap files to disk. @@ -243,43 +276,47 @@ class ListBaseWithIndex(ListBase): a ChangeList. This may be a generator so data is read as needed and length is determined at the end. - basename is used as the name of the single sitemap file or the + basename is used as the name of the single sitemap file or the sitemapindex for a set of sitemap files. - Uses self.max_sitemap_entries to determine whether the resource_list can - be written as one sitemap. If there are more entries and - self.allow_multifile is set True then a set of sitemap files, + Uses self.max_sitemap_entries to determine whether the resource_list can + be written as one sitemap. If there are more entries and + self.allow_multifile is set True then a set of sitemap files, with an sitemapindex, will be written. """ # Access resources through iterator only resources_iter = iter(self.resources) - ( chunk, nxt ) = self.get_resources_chunk(resources_iter) + (chunk, nxt) = self.get_resources_chunk(resources_iter) s = self.new_sitemap() if (nxt is not None): # Have more than self.max_sitemap_entries => sitemapindex if (not self.allow_multifile): - raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled") + raise ListBaseIndexError( + "Too many entries for a single sitemap but multifile disabled") # Work out URI of sitemapindex so that we can link up to # it from the individual sitemap files try: index_uri = self.mapper.dst_to_src(basename) except MapperError as e: - raise ListBaseIndexError("Cannot map sitemapindex filename to URI (%s)" % str(e)) + raise ListBaseIndexError( + "Cannot map sitemapindex filename to URI (%s)" % + str(e)) # Use iterator over all resources and count off sets of # max_sitemap_entries to go into each sitemap, store the # names of the sitemaps as we go. Copy md from self into # the index and use this for all chunks also - index=ListBase(md=self.md.copy(), ln=list(self.ln)) + index = ListBase(md=self.md.copy(), ln=list(self.ln)) index.capability_name = self.capability_name index.default_capability() - while (len(chunk)>0): - file = self.part_name(basename,len(index)) + while (len(chunk) > 0): + file = self.part_name(basename, len(index)) # Check that we can map the filename of this sitemap into # URI space for the sitemapindex try: uri = self.mapper.dst_to_src(file) except MapperError as e: - raise ListBaseIndexError("Cannot map sitemap filename to URI (%s)" % str(e)) + raise ListBaseIndexError( + "Cannot map sitemap filename to URI (%s)" % str(e)) self.logger.info("Writing sitemap %s..." % (file)) f = open(file, 'w') chunk.index = index_uri @@ -287,16 +324,16 @@ class ListBaseWithIndex(ListBase): s.resources_as_xml(chunk, fh=f) f.close() # Record information about this sitemap for index - r = Resource( uri = uri, - timestamp = os.stat(file).st_mtime, - md5 = compute_md5_for_file(file) ) + r = Resource(uri=uri, + timestamp=os.stat(file).st_mtime, + md5=compute_md5_for_file(file)) index.add(r) # Get next chunk - ( chunk, nxt ) = self.get_resources_chunk(resources_iter,nxt) + (chunk, nxt) = self.get_resources_chunk(resources_iter, nxt) self.logger.info("Wrote %d sitemaps" % (len(index))) f = open(basename, 'w') self.logger.info("Writing sitemapindex %s..." % (basename)) - s.resources_as_xml(index,sitemapindex=True,fh=f) + s.resources_as_xml(index, sitemapindex=True, fh=f) f.close() self.logger.info("Wrote sitemapindex %s" % (basename)) else: @@ -310,62 +347,63 @@ class ListBaseWithIndex(ListBase): """XML serialization of this list taken to be sitemapindex entries.""" self.default_capability() s = self.new_sitemap() - return s.resources_as_xml(self,sitemapindex=True) + return s.resources_as_xml(self, sitemapindex=True) - ##### Utility ##### + # Utility def get_resources_chunk(self, resource_iter, first=None): """Return next chunk of resources from resource_iter, and next item. - + If first parameter is specified then this will be prepended to the list. - The chunk will contain self.max_sitemap_entries if the iterator + The chunk will contain self.max_sitemap_entries if the iterator returns that many. next will have the value of the next value from - the iterator, providing indication of whether more is available. + the iterator, providing indication of whether more is available. Use this as first when asking for the following chunk. """ - chunk = ListBase( md=self.md.copy(), ln=list(self.ln) ) + chunk = ListBase(md=self.md.copy(), ln=list(self.ln)) chunk.capability_name = self.capability_name chunk.default_capability() if (first is not None): chunk.add(first) for r in resource_iter: chunk.add(r) - if (len(chunk)>=self.max_sitemap_entries): + if (len(chunk) >= self.max_sitemap_entries): break # Get next to see whether there are more resources try: nxt = next(resource_iter) except StopIteration: nxt = None - return(chunk,nxt) + return(chunk, nxt) def part_name(self, basename='/tmp/sitemap.xml', part_number=0): """Name (file or URI) for one component sitemap. - + Works for both filenames and URIs because manipulates only the end of the string. - + Abstracting this into a function that starts from the basename to get prefix and suffix each time seems a bit wasteful but perhaps not worth - worrying about. Allows same code to be used for the write() and + worrying about. Allows same code to be used for the write() and as_xml_index() cases. """ - # Work out how to name the sitemaps, attempt to add %05d before ".xml$", else append + # Work out how to name the sitemaps, attempt to add %05d before + # ".xml$", else append sitemap_prefix = basename sitemap_suffix = '.xml' if (basename[-4:] == '.xml'): sitemap_prefix = basename[:-4] - return( sitemap_prefix + ( "%05d" % (part_number) ) + sitemap_suffix ) - + return(sitemap_prefix + ("%05d" % (part_number)) + sitemap_suffix) + def is_file_uri(self, uri): """Return true if uri looks like a local file URI, false otherwise. - + Test is to see whether have either an explicit file: URI or whether there is no scheme name. """ - return(re.match('file:',uri) or not re.match('\w{3,4}:',uri)) + return(re.match('file:', uri) or not re.match('\w{3,4}:', uri)) class ListBaseIndexError(Exception): diff --git a/resync/resource.py b/resync/resource.py index 2f37dd2..9695cdd 100644 --- a/resync/resource.py +++ b/resync/resource.py @@ -250,11 +250,11 @@ class Resource(object): """ hashvals = [] if (self.md5 is not None): - hashvals.append('md5:'+self.md5) + hashvals.append('md5:' + self.md5) if (self.sha1 is not None): - hashvals.append('sha-1:'+self.sha1) + hashvals.append('sha-1:' + self.sha1) if (self.sha256 is not None): - hashvals.append('sha-256:'+self.sha256) + hashvals.append('sha-256:' + self.sha256) if (len(hashvals) > 0): return(' '.join(hashvals)) return(None) @@ -424,7 +424,7 @@ class Resource(object): # not equal if only one timestamp specified if (self.timestamp is None or other.timestamp is None or - abs(self.timestamp-other.timestamp) >= delta): + abs(self.timestamp - other.timestamp) >= delta): return(False) if ((self.md5 is not None and other.md5 is not None) and self.md5 != other.md5): diff --git a/resync/resource_container.py b/resync/resource_container.py index fc3b52c..9c7629e 100644 --- a/resync/resource_container.py +++ b/resync/resource_container.py @@ -2,7 +2,7 @@ All documents in ResourceSync have the same types of information: they have some top-level metadata and links, and then a list of -resources, each of which may also have metadata and links. This +resources, each of which may also have metadata and links. This class implements this model. This is a base class for the ListBase class which is in turn the @@ -13,6 +13,7 @@ adds IO. import collections from .w3c_datetime import datetime_to_str + class ResourceContainer(object): """Class containing resource-like objects. @@ -30,13 +31,14 @@ class ResourceContainer(object): should use only the core functionality. """ - def __init__(self, resources=None, md=None, ln=None, uri=None, capability_name=None): + def __init__(self, resources=None, md=None, ln=None, + uri=None, capability_name=None): """Initialize ResourceContainer.""" - self.resources=(resources if (resources is not None) else []) - self.md=(md if (md is not None) else {}) - self.ln=(ln if (ln is not None) else []) - self.uri=uri - self.capability_name=capability_name + self.resources = (resources if (resources is not None) else []) + self.md = (md if (md is not None) else {}) + self.ln = (ln if (ln is not None) else []) + self.uri = uri + self.capability_name = capability_name def __iter__(self): """Iterator over all the resources in this resource list. @@ -45,7 +47,7 @@ class ResourceContainer(object): """ return(iter(self.resources)) - def __getitem__(self,index): + def __getitem__(self, index): """Feed through for __getitem__ of resources property.""" return(self.resources[index]) @@ -58,8 +60,8 @@ class ResourceContainer(object): return(None) @capability.setter - def capability(self,capability): - self.md['capability']=capability + def capability(self, capability): + self.md['capability'] = capability @property def md_from(self): @@ -70,8 +72,8 @@ class ResourceContainer(object): return(None) @md_from.setter - def md_from(self,md_from): - self.md['md_from']=self._str_datetime_now(md_from) + def md_from(self, md_from): + self.md['md_from'] = self._str_datetime_now(md_from) @property def md_until(self): @@ -82,8 +84,8 @@ class ResourceContainer(object): return(None) @md_until.setter - def md_until(self,md_until): - self.md['md_until']=self._str_datetime_now(md_until) + def md_until(self, md_until): + self.md['md_until'] = self._str_datetime_now(md_until) @property def md_at(self): @@ -94,8 +96,8 @@ class ResourceContainer(object): return(None) @md_at.setter - def md_at(self,md_at): - self.md['md_at']=self._str_datetime_now(md_at) + def md_at(self, md_at): + self.md['md_at'] = self._str_datetime_now(md_at) @property def md_completed(self): @@ -106,29 +108,29 @@ class ResourceContainer(object): return(None) @md_completed.setter - def md_completed(self,md_completed): - self.md['md_completed']=self._str_datetime_now(md_completed) + def md_completed(self, md_completed): + self.md['md_completed'] = self._str_datetime_now(md_completed) - def link(self,rel): + def link(self, rel): """Look for link with specified rel, return else None.""" for link in self.ln: if ('rel' in link and - link['rel']==rel): + link['rel'] == rel): return(link) return(None) - def link_href(self,rel): + def link_href(self, rel): """Look for link with specified rel, return href from it or None.""" link = self.link(rel) if (link is not None): link = link['href'] return(link) - def link_set(self,rel,href,**atts): + def link_set(self, rel, href, **atts): """Set/create link with specified rel, set href and any other attributes. Any link element must have both rel and href values, the specification - also defines the type attributes and others are permitted also. See + also defines the type attributes and others are permitted also. See description of allowed formats in http://www.openarchives.org/rs/resourcesync.html#DocumentFormats @@ -139,7 +141,7 @@ class ResourceContainer(object): link['href'] = href else: # create new link - link = {'rel':rel,'href':href} + link = {'rel': rel, 'href': href} self.ln.append(link) for k in atts: link[k] = atts[k] @@ -150,9 +152,9 @@ class ResourceContainer(object): return(self.link_href('describedby')) @describedby.setter - def describedby(self,uri): + def describedby(self, uri): """Set ResourceSync Description link to given URI.""" - self.link_set('describedby',uri) + self.link_set('describedby', uri) @property def up(self): @@ -160,9 +162,9 @@ class ResourceContainer(object): return(self.link_href('up')) @up.setter - def up(self,uri): + def up(self, uri): """Set ResourceSync rel="up" link to given URI.""" - self.link_set('up',uri) + self.link_set('up', uri) @property def index(self): @@ -170,9 +172,9 @@ class ResourceContainer(object): return(self.link_href('index')) @index.setter - def index(self,uri): + def index(self, uri): """Set index link to given URI.""" - self.link_set('index',uri) + self.link_set('index', uri) def default_capability(self): """Set capability name in md. @@ -181,7 +183,7 @@ class ResourceContainer(object): capability attributes. """ if ('capability' not in self.md and self.capability_name is not None): - self.md['capability']=self.capability_name + self.md['capability'] = self.capability_name def add(self, resource): """Add a resource or an iterable collection of resources to this container. @@ -203,7 +205,7 @@ class ResourceContainer(object): def prune_before(self, timestamp): """Remove all resources with timestamp earlier than that given. - + Returns the number of entries removed. Will raise an excpetion if there are any entries without a timestamp. """ @@ -221,8 +223,8 @@ class ResourceContainer(object): def prune_dupes(self): """Remove all but the last entry for a given resource URI. - - Returns the number of entries removed. Also removes all entries for a + + Returns the number of entries removed. Also removes all entries for a given URI where the first entry is a create and the last entry is a delete. """ @@ -234,12 +236,12 @@ class ResourceContainer(object): if (r.uri in seen): n += 1 if (r.uri in deletes): - deletes[r.uri]=r.change + deletes[r.uri] = r.change else: pruned1.append(r) seen.add(r.uri) if (r.change == 'deleted'): - deletes[r.uri]=r.change + deletes[r.uri] = r.change # go through all deletes and prune if first was create pruned2 = [] for r in reversed(pruned1): @@ -247,7 +249,7 @@ class ResourceContainer(object): n += 1 else: pruned2.append(r) - self.resources=pruned2 + self.resources = pruned2 return(n) def __str__(self): @@ -259,7 +261,7 @@ class ResourceContainer(object): def _str_datetime_now(self, x=None): """Return datetime string for use with time attributes. - + Handling depends on input: 'now' - returns datetime for now number - assume datetime values, generate string @@ -267,7 +269,7 @@ class ResourceContainer(object): """ if (x == 'now'): # Now, this is wht datetime_to_str() with no arg gives - return( datetime_to_str() ) + return(datetime_to_str()) try: # Test for number junk = x + 0.0 diff --git a/resync/resource_dump_manifest.py b/resync/resource_dump_manifest.py index 756c063..a8a95db 100644 --- a/resync/resource_dump_manifest.py +++ b/resync/resource_dump_manifest.py @@ -1,6 +1,6 @@ """ResourceSync ResourceDumpManifest object. -A ResourceDumpManifest lists the set of files/resources included +A ResourceDumpManifest lists the set of files/resources included within a content package that is included in a ResourceDump. The ResourceDumpManifest object may also contain metadata and links. @@ -11,6 +11,7 @@ http://www.openarchives.org/rs/resourcesync#ResourceDumpManifest from .resource_list import ResourceList + class ResourceDumpManifest(ResourceList): """Class representing a Resource Dump Manifest. @@ -19,7 +20,15 @@ class ResourceDumpManifest(ResourceList): and implemented as a sub-class of ResourceList. """ - def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None): + def __init__(self, resources=None, md=None, ln=None, + uri=None, allow_multifile=None, mapper=None): """Initialize ResourceDumpManifest.""" - super(ResourceDumpManifest, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper) + super( + ResourceDumpManifest, + self).__init__( + resources=resources, + md=md, + ln=ln, + uri=uri, + mapper=mapper) self.capability_name = 'resourcedump-manifest' diff --git a/resync/resource_list.py b/resync/resource_list.py index fec973d..4f9599c 100644 --- a/resync/resource_list.py +++ b/resync/resource_list.py @@ -1,16 +1,16 @@ """ResourceSync Resource List object. -A Resource List is a set of resources with some metadata for -each resource. Comparison of resource lists from a source and +A Resource List is a set of resources with some metadata for +each resource. Comparison of resource lists from a source and a destination allows understanding of whether the two are in sync or whether some resources need to be updated at the destination. There may also be metadata about the Resource List, and links -to other ResourceSync documents. Metadata include the -timestamp of the Resource List (md_at) and, optionally, the +to other ResourceSync documents. Metadata include the +timestamp of the Resource List (md_at) and, optionally, the timestamp when creation of the Resource List was completed -(md_completed).at the top level. These include a creation timestamp +(md_completed).at the top level. These include a creation timestamp (from) and links to the Capability List. Described in specification at: @@ -22,9 +22,9 @@ import os from datetime import datetime import re import sys -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from .list_base_with_index import ListBaseWithIndex @@ -48,7 +48,7 @@ class ResourceListDict(dict): return(iter(self._iter_next, None)) def _iter_next(self): - if (len(self._iter_next_list)>0): + if (len(self._iter_next_list) > 0): return(self[self._iter_next_list.pop()]) else: return(None) @@ -61,13 +61,15 @@ class ResourceListDict(dict): """Add just a single resource.""" uri = resource.uri if (uri in self and not replace): - raise ResourceListDupeError("Attempt to add resource already in resource_list") - self[uri]=resource + raise ResourceListDupeError( + "Attempt to add resource already in resource_list") + self[uri] = resource + class ResourceListOrdered(list): """Alternative implementation of class to store resources in ResourceList. - FIXME - This is a rather inefficient implementation which involves + FIXME - This is a rather inefficient implementation which involves scanning all resources to check for duplicates. Designed just to enable re-creation of examples in the spec. Something dictionary based would likely be better. Might be best to use OrderedDict but that is available @@ -91,18 +93,21 @@ class ResourceListOrdered(list): for r in self: if (uri == r.uri): if (replace): - r=resource + r = resource return else: - raise ResourceListDupeError("Attempt to add resource already in resource_list") + raise ResourceListDupeError( + "Attempt to add resource already in resource_list") # didn't find it in list, add to end self.append(resource) + class ResourceListDupeError(Exception): """Exception in case of duplicate resource.""" pass + class ResourceList(ListBaseWithIndex): """Class representing a ResourceList. @@ -114,22 +119,22 @@ class ResourceList(ListBaseWithIndex): A ResourceList will admit only one resource with any given URI. Typical usage for a small ResourceList is: - + rl = ResourceList() rl.add( Resource(...) ) rl.add( Resource(...) ) print rl.as_xml() - The default storage is unordered but the iterator imposes a canonical - order which is alphabetical by URI. If it is desired to have + The default storage is unordered but the iterator imposes a canonical + order which is alphabetical by URI. If it is desired to have resources listed in the order they are added then the ResourceDictOrdered class may be specified on creation: rl = ResourceList( resources_class=ResourceDictOrdered ) - + In normal use it is expected that any Resource List Index will be created automatically when writing out a large Resource List in - multiple sitemap files. However, should it be necessary to + multiple sitemap files. However, should it be necessary to explicitly create an index then this may be specified with: rli = ResourceList( resources_class=ResourceDictOrdered ) @@ -138,11 +143,11 @@ class ResourceList(ListBaseWithIndex): See additional descriptions in ListBaseWithIndex and ListBase. """ - def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, + def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None, resources_class=ResourceListDict): """Initialize ResourceList.""" super(ResourceList, self).__init__(resources=resources, count=count, md=md, ln=ln, uri=uri, - capability_name = 'resourcelist', + capability_name='resourcelist', allow_multifile=allow_multifile, mapper=mapper, resources_class=resources_class) @@ -154,15 +159,15 @@ class ResourceList(ListBaseWithIndex): """ if isinstance(resource, collections.Iterable): for r in resource: - self.resources.add(r,replace) + self.resources.add(r, replace) else: - self.resources.add(resource,replace) + self.resources.add(resource, replace) - def compare(self,src): + def compare(self, src): """Compare this ResourceList object with that specified as src. The parameter src must also be a ResourceList object, it is assumed - to be the source, and the current object is the destination. This + to be the source, and the current object is the destination. This written to work for any objects in self and sc, provided that the == operator can be used to compare them. @@ -171,38 +176,38 @@ class ResourceList(ListBaseWithIndex): """ dst_iter = iter(self.resources) src_iter = iter(src.resources) - same=ResourceList() - updated=ResourceList() - deleted=ResourceList() - created=ResourceList() - dst_cur=next(dst_iter,None) - src_cur=next(src_iter,None) + same = ResourceList() + updated = ResourceList() + deleted = ResourceList() + created = ResourceList() + dst_cur = next(dst_iter, None) + src_cur = next(src_iter, None) while ((dst_cur is not None) and (src_cur is not None)): - #print 'dst='+dst_cur+' src='+src_cur + # print 'dst='+dst_cur+' src='+src_cur if (dst_cur.uri == src_cur.uri): - if (dst_cur==src_cur): + if (dst_cur == src_cur): same.add(dst_cur) else: updated.add(src_cur) - dst_cur=next(dst_iter,None) - src_cur=next(src_iter,None) + dst_cur = next(dst_iter, None) + src_cur = next(src_iter, None) elif (not src_cur or dst_cur.uri < src_cur.uri): deleted.add(dst_cur) - dst_cur=next(dst_iter,None) + dst_cur = next(dst_iter, None) elif (not dst_cur or dst_cur.uri > src_cur.uri): created.add(src_cur) - src_cur=next(src_iter,None) + src_cur = next(src_iter, None) else: raise Exception("this should not be possible") # what do we have leftover in src or dst lists? while (dst_cur is not None): deleted.add(dst_cur) - dst_cur=next(dst_iter,None) + dst_cur = next(dst_iter, None) while (src_cur is not None): created.add(src_cur) - src_cur=next(src_iter,None) + src_cur = next(src_iter, None) # have now gone through both lists - return(same,updated,deleted,created) + return(same, updated, deleted, created) def has_md5(self): """Return true if at least one contained resource-like object has md5 data.""" diff --git a/resync/resource_list_builder.py b/resync/resource_list_builder.py index 77c6a83..126d809 100644 --- a/resync/resource_list_builder.py +++ b/resync/resource_list_builder.py @@ -5,9 +5,9 @@ import os.path import re import time import logging -try: #python3 +try: # python3 from urllib.request import URLopener -except ImportError: #python2 +except ImportError: # python2 from urllib import URLopener from defusedxml.ElementTree import parse @@ -17,10 +17,11 @@ from .sitemap import Sitemap from .utils import compute_md5_for_file from .w3c_datetime import datetime_to_str + class ResourceListBuilder(): """ResourceListBuilder to create ResourceList objects. - Currently implements build from files on disk only. + Currently implements build from files on disk only. Attributes: - set_path set true to add path attribute for each resource @@ -33,16 +34,17 @@ class ResourceListBuilder(): alternatives to md5. """ - def __init__(self, mapper=None, set_md5=False, set_length=True, set_path=False): + def __init__(self, mapper=None, set_md5=False, + set_length=True, set_path=False): """Create ResourceListBuilder object, optionally set options. The mapper attribute must be set before a call to from_disk() in order to map between local filenames and URIs. - The following attributes may be set to determine information added to + The following attributes may be set to determine information added to each Resource object based on the disk scan: - - set_md5 - True to add md5 digests for each resource. This may add - significant time to the scan process as each file has to be read to + - set_md5 - True to add md5 digests for each resource. This may add + significant time to the scan process as each file has to be read to compute the sum - set_length - False to not add length for each resources - set_path - True to add local path information for each file/resource @@ -52,7 +54,7 @@ class ResourceListBuilder(): self.set_md5 = set_md5 self.set_length = set_length self.exclude_files = ['sitemap\d{0,5}.xml'] - self.exclude_dirs = ['CVS','.git'] + self.exclude_dirs = ['CVS', '.git'] self.include_symlinks = False # Used internally only: self.logger = logging.getLogger('resync.resource_list_builder') @@ -70,7 +72,8 @@ class ResourceListBuilder(): try: self.compiled_exclude_files.append(re.compile(pattern)) except re.error as e: - raise ValueError("Bad python regex in exclude '%s': %s" % (pattern,str(e))) + raise ValueError( + "Bad python regex in exclude '%s': %s" % (pattern, str(e))) def exclude_file(self, file): """True if file should be exclude based on name pattern.""" @@ -82,18 +85,18 @@ class ResourceListBuilder(): def from_disk(self, resource_list=None, paths=None): """Create or extend resource_list with resources from disk scan. - Assumes very simple disk path to URL mapping (in self.mapping): chop + Assumes very simple disk path to URL mapping (in self.mapping): chop path and replace with url_path. Returns the new or extended ResourceList object. If a resource_list is specified then items are added to that rather than creating a new one. - If paths is specified then these are used instead of the set + If paths is specified then these are used instead of the set of local paths in self.mapping. Example usage with mapping start paths: - + mapper=Mapper('http://example.org/path','/path/to/files') rlb = ResourceListBuilder(mapper=mapper) m = rlb.from_disk() @@ -104,7 +107,7 @@ class ResourceListBuilder(): rlb = ResourceListBuilder(mapper=mapper) m = rlb.from_disk(paths=['/path/to/files/a','/path/to/files/b']) """ - num=0 + num = 0 # Either use resource_list passed in or make a new one if (resource_list is None): resource_list = ResourceList() @@ -112,10 +115,11 @@ class ResourceListBuilder(): self.compile_excludes() # Work out start paths from map if not explicitly specified if (paths is None): - paths=[] + paths = [] for map in self.mapper.mappings: paths.append(map.dst_path) - # Set start time unless already set (perhaps because building in chunks) + # Set start time unless already set (perhaps because building in + # chunks) if (resource_list.md_at is None): resource_list.md_at = datetime_to_str() # Run for each map in the mappings @@ -131,16 +135,18 @@ class ResourceListBuilder(): # sanity if (path is None or resource_list is None or self.mapper is None): raise ValueError("Must specify path, resource_list and mapper") - # is path a directory or a file? for each file: create Resource object, + # is path a directory or a file? for each file: create Resource object, # add, increment counter if os.path.isdir(path): - num_files=0 - for dirpath, dirs, files in os.walk(path,topdown=True): + num_files = 0 + for dirpath, dirs, files in os.walk(path, topdown=True): for file_in_dirpath in files: - num_files+=1 - if (num_files%50000 == 0): - self.logger.info("ResourceListBuilder.from_disk_add_path: %d files..." % (num_files)) - self.add_file(resource_list=resource_list,dir=dirpath,file=file_in_dirpath) + num_files += 1 + if (num_files % 50000 == 0): + self.logger.info( + "ResourceListBuilder.from_disk_add_path: %d files..." % (num_files)) + self.add_file(resource_list=resource_list, + dir=dirpath, file=file_in_dirpath) # prune list of dirs based on self.exclude_dirs for exclude in self.exclude_dirs: if exclude in dirs: @@ -148,11 +154,11 @@ class ResourceListBuilder(): dirs.remove(exclude) else: # single file - self.add_file(resource_list=resource_list,file=path) + self.add_file(resource_list=resource_list, file=path) def add_file(self, resource_list=None, dir=None, file=None): """Add a single file to resource_list. - + Follows object settings of set_path, set_md5 and set_length. """ try: @@ -161,25 +167,26 @@ class ResourceListBuilder(): return # get abs filename and also URL if (dir is not None): - file = os.path.join(dir,file) - if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): + file = os.path.join(dir, file) + if (not os.path.isfile(file) or not ( + self.include_symlinks or not os.path.islink(file))): return uri = self.mapper.dst_to_src(file) if (uri is None): raise Exception("Internal error, mapping failed") - file_stat=os.stat(file) + file_stat = os.stat(file) except OSError as e: - sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e))) + sys.stderr.write("Ignoring file %s (error: %s)" % (file, str(e))) return - timestamp = file_stat.st_mtime #UTC - r = Resource(uri=uri,timestamp=timestamp) + timestamp = file_stat.st_mtime # UTC + r = Resource(uri=uri, timestamp=timestamp) if (self.set_path): # add full local path - r.path=file + r.path = file if (self.set_md5): # add md5 - r.md5=compute_md5_for_file(file) + r.md5 = compute_md5_for_file(file) if (self.set_length): # add length - r.length=file_stat.st_size + r.length = file_stat.st_size resource_list.add(r) diff --git a/resync/resource_set.py b/resync/resource_set.py index e3532ca..b03e818 100644 --- a/resync/resource_set.py +++ b/resync/resource_set.py @@ -1,13 +1,14 @@ """Base class representing a set of Resource objects.""" + class ResourceSet(dict): """Base class representing a set of Resource objects. - The ResourceSet class is used for Capability List Indexes and + The ResourceSet class is used for Capability List Indexes and ResourceSync Description documents. Ordering of Resources is currently alphanumeric (using sorted(..)) - on the uri which is the key. + on the uri which is the key. Key properties of this class are: - has add(resource) method @@ -21,7 +22,7 @@ class ResourceSet(dict): return(iter(self._iter_next, None)) def _iter_next(self): - if (len(self._iter_next_list)>0): + if (len(self._iter_next_list) > 0): return(self[self._iter_next_list.pop()]) else: return(None) @@ -30,8 +31,10 @@ class ResourceSet(dict): """Add just a single resource.""" uri = resource.uri if (uri in self and not replace): - raise ResourceSetDupeError("Attempt to add resource already in this set") - self[uri]=resource + raise ResourceSetDupeError( + "Attempt to add resource already in this set") + self[uri] = resource + class ResourceSetDupeError(Exception): """Exception for case of attempt to add duplicate resource.""" diff --git a/resync/sitemap.py b/resync/sitemap.py index dfac7c7..426e4a8 100644 --- a/resync/sitemap.py +++ b/resync/sitemap.py @@ -6,11 +6,11 @@ import sys import logging from defusedxml.ElementTree import parse from xml.etree.ElementTree import ElementTree, Element, tostring -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io from .resource import Resource @@ -27,6 +27,7 @@ XML_ATT_NAME = { 'md_until': 'until' } + class SitemapIndexError(Exception): """Exception if sitemapindex read instead of sitemap or vice-versa. @@ -43,21 +44,24 @@ class SitemapIndexError(Exception): """Return just the message attribute.""" return(self.message) + class SitemapParseError(Exception): """Exception for sitemap parsing structural error.""" pass + class SitemapDupeError(Exception): """Exception for case of duplicate resources in sitemap.""" pass + class Sitemap(object): """Read and write sitemaps. - Implemented as a separate class that uses ResourceContainer - (ResourceList or ChangeList) and Resource classes as data objects. + Implemented as a separate class that uses ResourceContainer + (ResourceList or ChangeList) and Resource classes as data objects. Reads and write sitemaps, including multiple file sitemaps. This class does not automatically handle the reading or writing @@ -69,19 +73,18 @@ class Sitemap(object): def __init__(self, pretty_xml=False): """Initialize Sitemap object.""" self.logger = logging.getLogger('resync.sitemap') - self.pretty_xml=pretty_xml + self.pretty_xml = pretty_xml # Classes used when parsing - self.resource_class=Resource + self.resource_class = Resource # Information recorded for logging - self.resources_created=0 # Set during parsing sitemap - self.parsed_index=None # Set True for sitemapindex, False for sitemap + self.resources_created = 0 # Set during parsing sitemap + self.parsed_index = None # Set True for sitemapindex, False for sitemap - - ##### Write the XML for a sitemap or sitemapindex ##### + # Write the XML for a sitemap or sitemapindex def resources_as_xml(self, resources, sitemapindex=False, fh=None): """Write or return XML for a set of resources in sitemap format. - + Arguments: - resources - either an iterable or iterator of Resource objects; if there an md attribute this will go to @@ -90,51 +93,59 @@ class Sitemap(object): - fh - write to filehandle fh instead of returning string """ # element names depending on sitemapindex or not - root_element = ( 'sitemapindex' if (sitemapindex) else 'urlset' ) - item_element = ( 'sitemap' if (sitemapindex) else 'url') + root_element = ('sitemapindex' if (sitemapindex) else 'urlset') + item_element = ('sitemap' if (sitemapindex) else 'url') # namespaces and other settings - namespaces = { 'xmlns': SITEMAP_NS, 'xmlns:rs': RS_NS } + namespaces = {'xmlns': SITEMAP_NS, 'xmlns:rs': RS_NS} root = Element(root_element, namespaces) if (self.pretty_xml): - root.text="\n" + root.text = "\n" # - if (hasattr(resources,'ln')): + if (hasattr(resources, 'ln')): for ln in resources.ln: - self.add_element_with_atts_to_etree(root,'rs:ln',ln) + self.add_element_with_atts_to_etree(root, 'rs:ln', ln) # - if (hasattr(resources,'md')): - self.add_element_with_atts_to_etree(root,'rs:md',resources.md) + if (hasattr(resources, 'md')): + self.add_element_with_atts_to_etree(root, 'rs:md', resources.md) # entries from either an iterable or an iterator for r in resources: - e=self.resource_etree_element(r, element_name=item_element) + e = self.resource_etree_element(r, element_name=item_element) root.append(e) # have tree, now serialize - tree = ElementTree(root); - xml_buf=None + tree = ElementTree(root) + xml_buf = None if (fh is None): - xml_buf=io.StringIO() - fh=xml_buf - if (sys.version_info < (2,7)): - tree.write(fh,encoding='UTF-8') - elif (sys.version_info < (3,0)): - tree.write(fh,encoding='UTF-8',xml_declaration=True,method='xml') + xml_buf = io.StringIO() + fh = xml_buf + if (sys.version_info < (2, 7)): + tree.write(fh, encoding='UTF-8') + elif (sys.version_info < (3, 0)): + tree.write( + fh, + encoding='UTF-8', + xml_declaration=True, + method='xml') else: - tree.write(fh,encoding="unicode",xml_declaration=True,method='xml') + tree.write( + fh, + encoding="unicode", + xml_declaration=True, + method='xml') if (xml_buf is not None): return(xml_buf.getvalue()) + # Read/parse an XML sitemap or sitemapindex - ##### Read/parse an XML sitemap or sitemapindex ##### - - def parse_xml(self, fh=None, etree=None, resources=None, capability=None, sitemapindex=None): + def parse_xml(self, fh=None, etree=None, resources=None, + capability=None, sitemapindex=None): """Parse XML Sitemap and add to resources object. - Reads from fh or etree and adds resources to a resorces object + Reads from fh or etree and adds resources to a resorces object (which must support the add method). Returns the resources object. - Also sets self.resources_created to be the number of resources created. - We adopt a very lax approach here. The parsing is properly namespace - aware but we search just for the elements wanted and leave everything + Also sets self.resources_created to be the number of resources created. + We adopt a very lax approach here. The parsing is properly namespace + aware but we search just for the elements wanted and leave everything else alone. This method will read either sitemap or sitemapindex documents. Behavior @@ -143,34 +154,38 @@ class Sitemap(object): - False - SitemapIndexError exception if sitemapindex detected - True - SitemapIndexError exception if sitemap detected - Will set self.parsed_index based on whether a sitemap or sitemapindex + Will set self.parsed_index based on whether a sitemap or sitemapindex document was read: - False - sitemap - True - sitemapindex """ if (resources is None): - resources=ResourceContainer() + resources = ResourceContainer() if (fh is not None): - etree=parse(fh) + etree = parse(fh) elif (etree is None): raise ValueError("Neither fh or etree set") # check root element: urlset (for sitemap), sitemapindex or bad root_tag = etree.getroot().tag - resource_tag = None # will be or depending on type + resource_tag = None # will be or depending on type self.parsed_index = None - if (root_tag == '{'+SITEMAP_NS+"}urlset"): + if (root_tag == '{' + SITEMAP_NS + "}urlset"): self.parsed_index = False if (sitemapindex is not None and sitemapindex): - raise SitemapIndexError("Got sitemap when expecting sitemapindex",etree) - resource_tag = '{'+SITEMAP_NS+"}url" - elif (root_tag == '{'+SITEMAP_NS+"}sitemapindex"): + raise SitemapIndexError( + "Got sitemap when expecting sitemapindex", etree) + resource_tag = '{' + SITEMAP_NS + "}url" + elif (root_tag == '{' + SITEMAP_NS + "}sitemapindex"): self.parsed_index = True if (sitemapindex is not None and not sitemapindex): - raise SitemapIndexError("Got sitemapindex when expecting sitemap",etree) - resource_tag = '{'+SITEMAP_NS+"}sitemap" + raise SitemapIndexError( + "Got sitemapindex when expecting sitemap", etree) + resource_tag = '{' + SITEMAP_NS + "}sitemap" else: - raise SitemapParseError("XML is not sitemap or sitemapindex (root element is <%s>)" % root_tag) - + raise SitemapParseError( + "XML is not sitemap or sitemapindex (root element is <%s>)" % + root_tag) + # have what we expect, read it in_preamble = True self.resources_created = 0 @@ -179,27 +194,32 @@ class Sitemap(object): # look for and , first ends # then look for resources in blocks if (e.tag == resource_tag): - in_preamble = False #any later rs:md or rs:ln is error + in_preamble = False # any later rs:md or rs:ln is error r = self.resource_from_etree(e, self.resource_class) try: - resources.add( r ) + resources.add(r) except SitemapDupeError: - self.logger.warning("dupe of: %s (lastmod=%s)" % (r.uri,r.lastmod)) - self.resources_created+=1 - elif (e.tag == "{"+RS_NS+"}md"): + self.logger.warning( + "dupe of: %s (lastmod=%s)" % + (r.uri, r.lastmod)) + self.resources_created += 1 + elif (e.tag == "{" + RS_NS + "}md"): if (in_preamble): if (seen_top_level_md): - raise SitemapParseError("Multiple at top level of sitemap") + raise SitemapParseError( + "Multiple at top level of sitemap") else: - resources.md = self.md_from_etree(e,'preamble') + resources.md = self.md_from_etree(e, 'preamble') seen_top_level_md = True else: - raise SitemapParseError("Found after first in sitemap") - elif (e.tag == "{"+RS_NS+"}ln"): + raise SitemapParseError( + "Found after first in sitemap") + elif (e.tag == "{" + RS_NS + "}ln"): if (in_preamble): - resources.ln.append(self.ln_from_etree(e,'preamble')) + resources.ln.append(self.ln_from_etree(e, 'preamble')) else: - raise SitemapParseError("Found after first in sitemap") + raise SitemapParseError( + "Found after first in sitemap") else: # element we don't recognize, ignore pass @@ -207,71 +227,71 @@ class Sitemap(object): if (capability is not None): if ('capability' not in resources.md): if (capability == 'resourcelist'): - self.logger.warning('No capability specified in sitemap, assuming resourcelist') + self.logger.warning( + 'No capability specified in sitemap, assuming resourcelist') resources.md['capability'] = 'resourcelist' else: raise SitemapParseError("Expected to read a %s document, but no capability specified in sitemap" % - (capability)) + (capability)) if (resources.md['capability'] != capability): raise SitemapParseError("Expected to read a %s document, got %s" % - (capability,resources.md['capability'])) + (capability, resources.md['capability'])) # return the resource container object return(resources) - - ##### Resource methods ##### + # Resource methods def resource_etree_element(self, resource, element_name='url'): """Return xml.etree.ElementTree.Element representing the resource. - Returns and element for the specified resource, of the form + Returns and element for the specified resource, of the form with enclosed properties that are based on the sitemap with extensions for ResourceSync. """ e = Element(element_name) sub = Element('loc') - sub.text=resource.uri + sub.text = resource.uri e.append(sub) if (resource.timestamp is not None): # Create appriate element for timestamp sub = Element('lastmod') - sub.text = str(resource.lastmod) #W3C Datetime in UTC + sub.text = str(resource.lastmod) # W3C Datetime in UTC e.append(sub) md_atts = {} - for att in ('capability','change','hash','length','path','mime_type', - 'md_at','md_completed','md_from','md_until'): - val = getattr(resource,att,None) + for att in ('capability', 'change', 'hash', 'length', 'path', 'mime_type', + 'md_at', 'md_completed', 'md_from', 'md_until'): + val = getattr(resource, att, None) if (val is not None): md_atts[self._xml_att_name(att)] = str(val) - if (len(md_atts)>0): - md = Element('rs:md',md_atts) + if (len(md_atts) > 0): + md = Element('rs:md', md_atts) e.append(md) # add any - if (hasattr(resource,'ln') and - resource.ln is not None): + if (hasattr(resource, 'ln') and + resource.ln is not None): for ln in resource.ln: - self.add_element_with_atts_to_etree(e,'rs:ln',ln) + self.add_element_with_atts_to_etree(e, 'rs:ln', ln) if (self.pretty_xml): - e.tail="\n" + e.tail = "\n" return(e) - def resource_as_xml(self,resource): + def resource_as_xml(self, resource): """Return string for the resource as part of an XML sitemap. Returns a string with the XML snippet representing the resource, without any XML declaration. """ e = self.resource_etree_element(resource) - if (sys.version_info > (3,0)): + if (sys.version_info > (3, 0)): # python3.x return(tostring(e, encoding='unicode', method='xml')) - elif (sys.version_info >= (2,7)): + elif (sys.version_info >= (2, 7)): s = tostring(e, encoding='UTF-8', method='xml') else: - #must not specify method='xml' in python2.6 + # must not specify method='xml' in python2.6 s = tostring(e, encoding='UTF-8') # Chop off XML declaration that is added in 2.x... sigh - return(s.replace("\n",'')) + return(s.replace("\n", '')) def resource_from_etree(self, etree, resource_class): """Construct a Resource from an etree. @@ -280,56 +300,61 @@ class Sitemap(object): etree - the etree to parse resource_class - class of Resource object to create - The parsing is properly namespace aware but we search just - for the elements wanted and leave everything else alone. Will + The parsing is properly namespace aware but we search just + for the elements wanted and leave everything else alone. Will raise an error if there are multiple or multiple - elements. Otherwise, provided there is a element then will + elements. Otherwise, provided there is a element then will go ahead and extract as much as possible. All errors raised are SitemapParseError with messages intended to help debug problematic sitemap XML. """ - loc_elements = etree.findall('{'+SITEMAP_NS+"}loc") - if (len(loc_elements)>1): - raise SitemapParseError("Multiple elements while parsing in sitemap"); - elif (len(loc_elements)==0): - raise SitemapParseError("Missing element while parsing in sitemap") + loc_elements = etree.findall('{' + SITEMAP_NS + "}loc") + if (len(loc_elements) > 1): + raise SitemapParseError( + "Multiple elements while parsing in sitemap") + elif (len(loc_elements) == 0): + raise SitemapParseError( + "Missing element while parsing in sitemap") else: loc = loc_elements[0].text - if (loc is None or loc==''): - raise SitemapParseError("Bad element with no content while parsing in sitemap") + if (loc is None or loc == ''): + raise SitemapParseError( + "Bad element with no content while parsing in sitemap") # must at least have a URI, make this object - resource=resource_class(uri=loc) + resource = resource_class(uri=loc) # and hopefully a lastmod datetime (but none is OK) - lastmod_elements = etree.findall('{'+SITEMAP_NS+"}lastmod") - if (len(lastmod_elements)>1): - raise SitemapParseError("Multiple elements while parsing in sitemap"); - elif (len(lastmod_elements)==1): - resource.lastmod=lastmod_elements[0].text + lastmod_elements = etree.findall('{' + SITEMAP_NS + "}lastmod") + if (len(lastmod_elements) > 1): + raise SitemapParseError( + "Multiple elements while parsing in sitemap") + elif (len(lastmod_elements) == 1): + resource.lastmod = lastmod_elements[0].text # proceed to look for other resource attributes in an rs:md element - md_elements = etree.findall('{'+RS_NS+"}md") - if (len(md_elements)>1): - raise SitemapParseError("Found multiple (%d) elements for %s", (len(md_elements),loc)) - elif (len(md_elements)==1): + md_elements = etree.findall('{' + RS_NS + "}md") + if (len(md_elements) > 1): + raise SitemapParseError( + "Found multiple (%d) elements for %s", (len(md_elements), loc)) + elif (len(md_elements) == 1): # have on element, look at attributes - md = self.md_from_etree(md_elements[0],context=loc) + md = self.md_from_etree(md_elements[0], context=loc) # simple attributes that map directly to Resource object attributes - for att in ('capability','change','length','path','mime_type'): + for att in ('capability', 'change', 'length', 'path', 'mime_type'): if (att in md): - setattr(resource,att,md[att]) + setattr(resource, att, md[att]) # The ResourceSync beta spec lists md5, sha-1 and sha-256 fixity # digest types. Parse and warn of errors ignored. if ('hash' in md): try: resource.hash = md['hash'] except ValueError as e: - self.logger.warning("%s in for %s" % (str(e),loc)) + self.logger.warning("%s in for %s" % (str(e), loc)) # look for rs:ln elements (optional) - ln_elements = etree.findall('{'+RS_NS+"}ln") - if (len(ln_elements)>0): + ln_elements = etree.findall('{' + RS_NS + "}ln") + if (len(ln_elements) > 0): resource.ln = [] for ln_element in ln_elements: - resource.ln.append(self.ln_from_etree(ln_element,loc)) + resource.ln.append(self.ln_from_etree(ln_element, loc)) return(resource) def md_from_etree(self, md_element, context=''): @@ -341,31 +366,36 @@ class Sitemap(object): """ md = {} # grab all understood attributes into md dict - for att in ('capability','change','hash','length','path','mime_type', - 'md_at','md_completed','md_from','md_until'): + for att in ('capability', 'change', 'hash', 'length', 'path', 'mime_type', + 'md_at', 'md_completed', 'md_from', 'md_until'): xml_att = self._xml_att_name(att) - val = md_element.attrib.get(xml_att,None) + val = md_element.attrib.get(xml_att, None) if (val is not None): md[att] = val # capability. Allow this to be missing but do a very simple syntax # check on plausible values if present if ('capability' in md): if (re.match(r"^[\w\-]+$", md['capability']) is None): - raise SitemapParseError("Bad capability name '%s' in %s" % (capability,context)) + raise SitemapParseError( + "Bad capability name '%s' in %s" % + (capability, context)) # change should be one of defined values if ('change' in md): - if (md['change'] not in ['created','updated','deleted'] ): - self.logger.warning("Bad change attribute in for %s" % (context)) + if (md['change'] not in ['created', 'updated', 'deleted']): + self.logger.warning( + "Bad change attribute in for %s" % + (context)) # length should be an integer if ('length' in md): try: - md['length']=int(md['length']) + md['length'] = int(md['length']) except ValueError as e: - raise SitemapParseError("Invalid length element in for %s" % (context)) + raise SitemapParseError( + "Invalid length element in for %s" % + (context)) return(md) - - def ln_from_etree(self,ln_element,context=''): + def ln_from_etree(self, ln_element, context=''): """Parse rs:ln element from an etree, returning a dict of the data. Parameters: @@ -374,36 +404,44 @@ class Sitemap(object): """ ln = {} # grab all understood attributes into ln dict - for att in ('hash','href','length','modified','path','rel','pri','mime_type'): + for att in ('hash', 'href', 'length', 'modified', + 'path', 'rel', 'pri', 'mime_type'): xml_att = self._xml_att_name(att) - val = ln_element.attrib.get(xml_att,None) + val = ln_element.attrib.get(xml_att, None) if (val is not None): ln[att] = val # now do some checks and conversions... # href (MANDATORY) if ('href' not in ln): - raise SitemapParseError("Missing href in in %s" % (context)) + raise SitemapParseError( + "Missing href in in %s" % + (context)) # rel (MANDATORY) if ('rel' not in ln): raise SitemapParseError("Missing rel in in %s" % (context)) # length in bytes if ('length' in ln): try: - ln['length']=int(ln['length']) + ln['length'] = int(ln['length']) except ValueError as e: - raise SitemapParseError("Invalid length attribute value in for %s" % (context)) + raise SitemapParseError( + "Invalid length attribute value in for %s" % + (context)) # pri - priority, must be a number between 1 and 999999 if ('pri' in ln): try: - ln['pri']=int(ln['pri']) + ln['pri'] = int(ln['pri']) except ValueError as e: - raise SitemapParseError("Invalid pri attribute in for %s" % (context)) - if (ln['pri']<1 or ln['pri']>999999): - raise SitemapParseError("Bad pri attribute value in for %s" % (context)) + raise SitemapParseError( + "Invalid pri attribute in for %s" % + (context)) + if (ln['pri'] < 1 or ln['pri'] > 999999): + raise SitemapParseError( + "Bad pri attribute value in for %s" % + (context)) return(ln) - - ##### Metadata and link elements ##### + # Metadata and link elements def add_element_with_atts_to_etree(self, etree, name, atts): """Add element with name and atts to etree iff there are any atts. @@ -418,18 +456,18 @@ class Sitemap(object): val = atts[att] if (val is not None): xml_atts[self._xml_att_name(att)] = str(val) - if (len(xml_atts)>0): - e = Element(name,xml_atts) + if (len(xml_atts) > 0): + e = Element(name, xml_atts) if (self.pretty_xml): - e.tail="\n" + e.tail = "\n" etree.append(e) - def _xml_att_name(self,att): + def _xml_att_name(self, att): """Get XML attribute name corresponding to supplied Resource object attribute. We cannot use the XML attribute names in Python because 'from' and others conflict with Python reserved words. To be consistent all the extra timestamps - are prefixed with md_. Assumption is that attibute name is same unless + are prefixed with md_. Assumption is that attibute name is same unless specified in XML_ATT_NAME. """ - return XML_ATT_NAME.get(att,att) + return XML_ATT_NAME.get(att, att) diff --git a/resync/source_description.py b/resync/source_description.py index e8b9bbe..10d98f4 100644 --- a/resync/source_description.py +++ b/resync/source_description.py @@ -1,17 +1,17 @@ """ResourceSync Source Description object. -A ResourceSync Source Description enumerates the Capability -Lists offered by a Source. Since a Source has one Capability -List per set of resources that it distinguishes, the -ResourceSync Source Description will enumerate as many +A ResourceSync Source Description enumerates the Capability +Lists offered by a Source. Since a Source has one Capability +List per set of resources that it distinguishes, the +ResourceSync Source Description will enumerate as many Capability Lists as the Source has distinct sets of resources. The ResourceSync Source Description is based on the format. The specification allows for a very large Source Descriptions -to use and index based on a , though this is probably +to use and index based on a , though this is probably rarely necessary! -There is no meaning in the order of description of sets of resources +There is no meaning in the order of description of sets of resources in a Source Description so the default is to store these descriptions as a set (ResourceSet). @@ -27,6 +27,7 @@ from resync.resource import Resource from resync.resource_set import ResourceSet from resync.list_base_with_index import ListBaseWithIndex + class SourceDescription(ListBaseWithIndex): """Class representing the set of Capability Lists supported. @@ -42,7 +43,7 @@ class SourceDescription(ListBaseWithIndex): super(SourceDescription, self).__init__(resources=resources, md=md, ln=ln, capability_name='description', resources_class=ResourceSet) - self.md['from']=None #usually don't want a from date + self.md['from'] = None # usually don't want a from date def add(self, resource, replace=False): """Add a resource or an iterable collection of resources. @@ -52,23 +53,23 @@ class SourceDescription(ListBaseWithIndex): """ if isinstance(resource, collections.Iterable): for r in resource: - self.resources.add(r,replace) + self.resources.add(r, replace) else: - self.resources.add(resource,replace) + self.resources.add(resource, replace) - def add_capability_list(self,capability_list=None): + def add_capability_list(self, capability_list=None): """Add a capability list. Adds either a CapabiltyList object specified in capability_list or else creates a Resource with the URI given in capability_list and adds that to the Source Description """ - if (hasattr(capability_list,'uri')): - r = Resource( uri=capability_list.uri, - capability=capability_list.capability_name ) + if (hasattr(capability_list, 'uri')): + r = Resource(uri=capability_list.uri, + capability=capability_list.capability_name) if (capability_list.describedby is not None): - r.link_set( rel='describedby', href=capability_list.describedby ) + r.link_set(rel='describedby', href=capability_list.describedby) else: - r = Resource( uri=capability_list, - capability='capabilitylist') - self.add( r ) + r = Resource(uri=capability_list, + capability='capabilitylist') + self.add(r) diff --git a/resync/w3c_datetime.py b/resync/w3c_datetime.py index 56db1e7..ac6259f 100644 --- a/resync/w3c_datetime.py +++ b/resync/w3c_datetime.py @@ -113,7 +113,7 @@ def str_to_datetime(s, context='datetime'): mm = int(m.group(6)) if (hh > 23 or mm > 59): raise ValueError("Bad timezone offset (%s)" % s) - offset_seconds = hh*3600 + mm*60 + offset_seconds = hh * 3600 + mm * 60 if (m.group(4) == '-'): offset_seconds = -offset_seconds # timetuple() ignores timezone information so we have to add in diff --git a/tests/capture_stdout.py b/tests/capture_stdout.py index a589b40..6c03287 100644 --- a/tests/capture_stdout.py +++ b/tests/capture_stdout.py @@ -1,17 +1,22 @@ """Provide capture_stdout as contect manager.""" -import sys, contextlib -try: #python2 +import sys +import contextlib +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io -# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python +# From +# http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python + + class Data(object): pass + @contextlib.contextmanager def capture_stdout(): old = sys.stdout diff --git a/tests/test_capability_list.py b/tests/test_capability_list.py index d8bb1fd..2971df4 100644 --- a/tests/test_capability_list.py +++ b/tests/test_capability_list.py @@ -6,56 +6,61 @@ from resync.resource_set import ResourceSetDupeError from resync.change_list import ChangeList from resync.capability_list import CapabilityList + class TestCapabilityList(unittest.TestCase): def test01_add(self): # one caps = CapabilityList() - r1 = Resource( uri='http://example.org/r1' ) - caps.add( r1 ) - self.assertEqual( len(caps), 1 ) + r1 = Resource(uri='http://example.org/r1') + caps.add(r1) + self.assertEqual(len(caps), 1) # dupe - self.assertRaises( ResourceSetDupeError, caps.add, r1 ) - self.assertEqual( len(caps), 1 ) + self.assertRaises(ResourceSetDupeError, caps.add, r1) + self.assertEqual(len(caps), 1) # dupe with replace caps = CapabilityList() - caps.add( [r1,r1], replace=True ) - self.assertEqual( len(caps), 1 ) + caps.add([r1, r1], replace=True) + self.assertEqual(len(caps), 1) # diff caps = CapabilityList() - r2 = ChangeList( uri='http://example.org/r2' ) - caps.add( [r1,r2] ) - self.assertEqual( len(caps), 2 ) - + r2 = ChangeList(uri='http://example.org/r2') + caps.add([r1, r2]) + self.assertEqual(len(caps), 2) + def test02_resourcelist(self): rl = ResourceList() caps = CapabilityList() - caps.add_capability( rl, "http://example.org/resourcelist.xml" ) + caps.add_capability(rl, "http://example.org/resourcelist.xml") caps.md['from'] = "2013-02-07T22:39:00" - self.assertEqual( len(caps), 1 ) - self.assertEqual( caps.as_xml(), '\nhttp://example.org/resourcelist.xml' ) + self.assertEqual(len(caps), 1) + self.assertEqual(caps.as_xml(), '\nhttp://example.org/resourcelist.xml') def test03_multiple(self): caps = CapabilityList() rl = ResourceList() - caps.add_capability( rl, "rl.xml" ) + caps.add_capability(rl, "rl.xml") cl = ChangeList() - caps.add_capability( cl, "cl.xml" ) - self.assertEqual( len(caps), 2 ) + caps.add_capability(cl, "cl.xml") + self.assertEqual(len(caps), 2) xml = caps.as_xml() - self.assertTrue( re.search( r'rl.xml', xml ) ) - self.assertTrue( re.search( r'cl.xml', xml) ) + self.assertTrue( + re.search(r'rl.xml', xml)) + self.assertTrue( + re.search(r'cl.xml', xml)) def test04_parse(self): - xml='\nhttp://example.org/resourcelist.xml' - cl=CapabilityList() + xml = '\nhttp://example.org/resourcelist.xml' + cl = CapabilityList() cl.parse(str_data=xml) - self.assertEqual( cl.capability, 'capabilitylist') - self.assertEqual( len(cl.resources), 1, 'got 1 resource') + self.assertEqual(cl.capability, 'capabilitylist') + self.assertEqual(len(cl.resources), 1, 'got 1 resource') [r] = cl.resources - self.assertEqual( r.uri, 'http://example.org/resourcelist.xml', 'resourcelist uri') - self.assertEqual( r.capability, 'resourcelist') + self.assertEqual( + r.uri, 'http://example.org/resourcelist.xml', 'resourcelist uri') + self.assertEqual(r.capability, 'resourcelist') if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestCapabilityList) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + TestCapabilityList) unittest.TextTestRunner().run(suite) diff --git a/tests/test_change_dump.py b/tests/test_change_dump.py index 19ce2f7..4ea07c2 100644 --- a/tests/test_change_dump.py +++ b/tests/test_change_dump.py @@ -1,9 +1,9 @@ import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io import re @@ -11,53 +11,56 @@ from resync.resource import Resource from resync.change_dump import ChangeDump from resync.sitemap import SitemapParseError + class TestChangeDump(unittest.TestCase): def test01_as_xml(self): rd = ChangeDump() - rd.add( Resource('a.zip',timestamp=1) ) - rd.add( Resource('b.zip',timestamp=2) ) + rd.add(Resource('a.zip', timestamp=1)) + rd.add(Resource('b.zip', timestamp=2)) xml = rd.as_xml() - #print(xml) - self.assertTrue( re.search(r'a.zip1970-01-01T00:00:01Z', xml), 'XML has resource a' ) + # print(xml) + self.assertTrue( + re.search(r'a.zip1970-01-01T00:00:01Z', xml), 'XML has resource a') def test10_parse(self): - xml='\n\ + xml = '\n\ \ \ http://example.com/a.zip2012-03-14T18:37:36Z\ http://example.com/b.zip2012-03-14T18:37:36Z\ ' - rd=ChangeDump() + rd = ChangeDump() rd.parse(fh=io.StringIO(xml)) - self.assertEqual( len(rd.resources), 2, 'got 2 resource dumps') - self.assertEqual( rd.md['capability'], 'changedump', 'capability set' ) - self.assertEqual( rd.md_from, '2013-01-01' ) - self.assertTrue( 'http://example.com/a.zip' in rd.resources ) - self.assertTrue( rd.resources['http://example.com/a.zip'].length, 12345 ) - self.assertTrue( 'http://example.com/b.zip' in rd.resources ) - self.assertTrue( rd.resources['http://example.com/b.zip'].length, 56789 ) + self.assertEqual(len(rd.resources), 2, 'got 2 resource dumps') + self.assertEqual(rd.md['capability'], 'changedump', 'capability set') + self.assertEqual(rd.md_from, '2013-01-01') + self.assertTrue('http://example.com/a.zip' in rd.resources) + self.assertTrue(rd.resources['http://example.com/a.zip'].length, 12345) + self.assertTrue('http://example.com/b.zip' in rd.resources) + self.assertTrue(rd.resources['http://example.com/b.zip'].length, 56789) def test11_parse_no_capability(self): # For a resource dump this should be an error - xml='\n\ + xml = '\n\ \ \ http://example.com/a.zip2012-03-14T18:37:36Z\ ' - rd=ChangeDump() - self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) ) + rd = ChangeDump() + self.assertRaises(SitemapParseError, rd.parse, fh=io.StringIO(xml)) def test12_parse_bad_capability(self): # the \n\ + xml = '\n\ \ \ http://example.com/bad_res_12012-03-14T18:37:36Z\ ' - rd=ChangeDump() - self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) ) + rd = ChangeDump() + self.assertRaises(SitemapParseError, rd.parse, fh=io.StringIO(xml)) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestChangeDump) diff --git a/tests/test_change_list.py b/tests/test_change_list.py index fcef6a1..2fd49ca 100644 --- a/tests/test_change_list.py +++ b/tests/test_change_list.py @@ -1,9 +1,9 @@ import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io import re @@ -12,86 +12,90 @@ from resync.change_list import ChangeList, ChangeTypeError from resync.resource_list import ResourceList from resync.sitemap import SitemapParseError + class TestChangeList(unittest.TestCase): def test01_add_if_changed(self): cl = ChangeList() - cl.add_if_changed( Resource('a',timestamp=1,change='updated') ) - self.assertEqual( len(cl), 1 ) - self.assertRaises( ChangeTypeError, cl.add_if_changed, - Resource('c',timestamp=3) ) + cl.add_if_changed(Resource('a', timestamp=1, change='updated')) + self.assertEqual(len(cl), 1) + self.assertRaises(ChangeTypeError, cl.add_if_changed, + Resource('c', timestamp=3)) def test02_set_with_repeats(self): cl = ChangeList() - cl.add( Resource('a',timestamp=1,change='updated') ) - cl.add( Resource('b',timestamp=1,change='created') ) - cl.add( Resource('c',timestamp=1,change='deleted') ) - cl.add( Resource('a',timestamp=2,change='deleted') ) - cl.add( Resource('b',timestamp=2,change='updated') ) + cl.add(Resource('a', timestamp=1, change='updated')) + cl.add(Resource('b', timestamp=1, change='created')) + cl.add(Resource('c', timestamp=1, change='deleted')) + cl.add(Resource('a', timestamp=2, change='deleted')) + cl.add(Resource('b', timestamp=2, change='updated')) self.assertEqual(len(cl), 5, "5 changes in change_list") def test03_with_repeats_again(self): - r1 = Resource(uri='a',length=1,change='created') - r2 = Resource(uri='b',length=2,change='created') + r1 = Resource(uri='a', length=1, change='created') + r2 = Resource(uri='b', length=2, change='created') i = ChangeList() i.add(r1) i.add(r2) - self.assertEqual( len(i), 2 ) + self.assertEqual(len(i), 2) # Can add another Resource with same URI - r1d = Resource(uri='a',length=10,change='created') + r1d = Resource(uri='a', length=10, change='created') i.add(r1d) - self.assertEqual( len(i), 3 ) + self.assertEqual(len(i), 3) def test04_change_list(self): cl = ChangeList() - cl.add( Resource('a',timestamp=1,change='created') ) - cl.add( Resource('b',timestamp=2,change='created') ) - cl.add( Resource('c',timestamp=3,change='created') ) - cl.add( Resource('d',timestamp=4,change='created') ) - cl.add( Resource('e',timestamp=5,change='created') ) + cl.add(Resource('a', timestamp=1, change='created')) + cl.add(Resource('b', timestamp=2, change='created')) + cl.add(Resource('c', timestamp=3, change='created')) + cl.add(Resource('d', timestamp=4, change='created')) + cl.add(Resource('e', timestamp=5, change='created')) self.assertEqual(len(cl), 5, "5 things in src") def test05_iter(self): i = ChangeList() - i.add( Resource('a',timestamp=1,change='created') ) - i.add( Resource('b',timestamp=2,change='created') ) - i.add( Resource('c',timestamp=3,change='created') ) - i.add( Resource('d',timestamp=4,change='created') ) - resources=[] + i.add(Resource('a', timestamp=1, change='created')) + i.add(Resource('b', timestamp=2, change='created')) + i.add(Resource('c', timestamp=3, change='created')) + i.add(Resource('d', timestamp=4, change='created')) + resources = [] for r in i: resources.append(r) self.assertEqual(len(resources), 4) - self.assertEqual( resources[0].uri, 'a') - self.assertEqual( resources[3].uri, 'd') + self.assertEqual(resources[0].uri, 'a') + self.assertEqual(resources[3].uri, 'd') def test06_add_changed_resources(self): added = ResourceList() - added.add( Resource('a',timestamp=1,change='created') ) - added.add( Resource('d',timestamp=4,change='created') ) + added.add(Resource('a', timestamp=1, change='created')) + added.add(Resource('d', timestamp=4, change='created')) self.assertEqual(len(added), 2, "2 things in added resource_list") changes = ChangeList() - changes.add_changed_resources( added, change='created' ) + changes.add_changed_resources(added, change='created') self.assertEqual(len(changes), 2, "2 things added") i = iter(changes) first = next(i) self.assertEqual(first.uri, 'a', "changes[0].uri=a") self.assertEqual(first.timestamp, 1, "changes[0].timestamp=1") - self.assertEqual(first.change, 'created') #, "changes[0].change=createdd") + # , "changes[0].change=createdd") + self.assertEqual(first.change, 'created') second = next(i) self.assertEqual(second.timestamp, 4, "changes[1].timestamp=4") - self.assertEqual(second.change, 'created', "changes[1].change=createdd") + self.assertEqual(second.change, 'created', + "changes[1].change=createdd") # Now add some with updated (one same, one diff) updated = ResourceList() - updated.add( Resource('a',timestamp=5,change='created') ) - updated.add( Resource('b',timestamp=6,change='created') ) + updated.add(Resource('a', timestamp=5, change='created')) + updated.add(Resource('b', timestamp=6, change='created')) self.assertEqual(len(updated), 2, "2 things in updated resource_list") - changes.add_changed_resources( updated, change='updated' ) + changes.add_changed_resources(updated, change='updated') self.assertEqual(len(changes), 4, "4 = 2 old + 2 things updated") # Make new resource_list from the changes which should not have dupes dst = ResourceList() - dst.add( changes, replace=True ) + dst.add(changes, replace=True) self.assertEqual(len(dst), 3, "3 unique resources") - self.assertEqual(dst.resources['a'].timestamp, 5 ) # 5 was later in last the 1 + # 5 was later in last the 1 + self.assertEqual(dst.resources['a'].timestamp, 5) self.assertEqual(dst.resources['a'].change, 'updated') self.assertEqual(dst.resources['b'].timestamp, 6) self.assertEqual(dst.resources['b'].change, 'updated') @@ -101,44 +105,47 @@ class TestChangeList(unittest.TestCase): def test07_as_xml(self): cl = ChangeList() cl.md_from = '1970-01-01T00:00:00Z' - cl.add( Resource('a',timestamp=1,change='updated') ) - cl.add( Resource('b',timestamp=2,change='updated') ) + cl.add(Resource('a', timestamp=1, change='updated')) + cl.add(Resource('b', timestamp=2, change='updated')) xml = cl.as_xml() - self.assertTrue( re.search(r'a1970-01-01T00:00:01Z', xml), 'XML has resource a' ) + self.assertTrue( + re.search(r'a1970-01-01T00:00:01Z', xml), 'XML has resource a') def test08_parse(self): - xml='\n\ + xml = '\n\ \ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' - cl=ChangeList() + cl = ChangeList() cl.parse(fh=io.StringIO(xml)) - self.assertEqual( len(cl.resources), 2, 'got 2 resources') - self.assertEqual( cl.md['capability'], 'changelist', 'capability set' ) - self.assertEqual( cl.md['md_from'], '2013-01-01' ) + self.assertEqual(len(cl.resources), 2, 'got 2 resources') + self.assertEqual(cl.md['capability'], 'changelist', 'capability set') + self.assertEqual(cl.md['md_from'], '2013-01-01') def test09_parse_no_capability(self): # missing capability is an error for changelist - xml='\n\ + xml = '\n\ \ http://example.com/res12012-03-14T18:37:36Z\ ' - cl=ChangeList() - self.assertRaises( SitemapParseError, cl.parse, fh=io.StringIO(xml) ) + cl = ChangeList() + self.assertRaises(SitemapParseError, cl.parse, fh=io.StringIO(xml)) def test10_parse_bad_capability(self): # the \n\ + xml = '\n\ \ \ http://example.com/bad_res_12012-03-14T18:37:36Z\ ' - cl=ChangeList() - self.assertRaises( SitemapParseError, cl.parse, fh=io.StringIO(xml) ) + cl = ChangeList() + self.assertRaises(SitemapParseError, cl.parse, fh=io.StringIO(xml)) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestChangeList) diff --git a/tests/test_client.py b/tests/test_client.py index bf4e9da..6430a61 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,323 +15,366 @@ from resync.change_list import ChangeList logging.basicConfig(level=logging.INFO) + class TestClient(TestCase): def test01_make_resource_list_empty(self): c = Client() # No mapping is an error - self.assertRaises( ClientFatalError, c.build_resource_list ) + self.assertRaises(ClientFatalError, c.build_resource_list) def test02_bad_source_uri(self): c = Client() - self.assertRaises( ClientFatalError, c.baseline_or_audit ) - c.set_mappings( ['http://example.org/bbb','/tmp/this_does_not_exist'] ) - self.assertRaises( ClientFatalError, c.baseline_or_audit ) + self.assertRaises(ClientFatalError, c.baseline_or_audit) + c.set_mappings(['http://example.org/bbb', '/tmp/this_does_not_exist']) + self.assertRaises(ClientFatalError, c.baseline_or_audit) def test03_sitemap_uri(self): c = Client() - c.set_mappings( ['http://example.org/c','/tmp/not_there_at_all'] ) - self.assertEqual( c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1' ) - self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' ) - self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' ) + c.set_mappings(['http://example.org/c', '/tmp/not_there_at_all']) + self.assertEqual(c.sitemap_uri('abcd1'), 'http://example.org/c/abcd1') + self.assertEqual(c.sitemap_uri('/abcd2'), '/abcd2') + self.assertEqual(c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3') def test10_baseline_or_audit(self): - # FIXME - this is the guts of the client, tough to test, need to work through more cases... + # FIXME - this is the guts of the client, tough to test, need to work + # through more cases... c = Client() - dst = os.path.join(self.tmpdir,'dst_dir1') - c.set_mappings( ['file:tests/testdata/client/dir1',dst] ) + dst = os.path.join(self.tmpdir, 'dst_dir1') + c.set_mappings(['file:tests/testdata/client/dir1', dst]) # audit with empty dst, should say 3 to create with LogCapture() as lc: c.baseline_or_audit(audit_only=True) - self.assertTrue( re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg) ) - self.assertEqual( lc.records[-1].msg, 'Completed audit' ) + self.assertTrue( + re.match(r'Status:\s+NOT IN SYNC.*to create=3', lc.records[-2].msg)) + self.assertEqual(lc.records[-1].msg, 'Completed audit') # now do the sync with LogCapture() as lc: c.baseline_or_audit() - self.assertTrue( re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg) ) - self.assertEqual( lc.records[-1].msg, 'Completed baseline sync' ) + self.assertTrue( + re.match(r'Status:\s+SYNCED.*created=3', lc.records[-2].msg)) + self.assertEqual(lc.records[-1].msg, 'Completed baseline sync') def test18_update_resource(self): c = Client() resource = Resource(uri='http://example.org/dir/2') - filename = os.path.join(self.tmpdir,'dir/resource2') + filename = os.path.join(self.tmpdir, 'dir/resource2') # dryrun with LogCapture() as lc: c.dryrun = True - c.logger = logging.getLogger('resync.client') - n = c.update_resource( resource, filename ) - self.assertEqual( n, 0 ) - self.assertTrue( lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ') ) + c.logger = logging.getLogger('resync.client') + n = c.update_resource(resource, filename) + self.assertEqual(n, 0) + self.assertTrue( + lc.records[-1].msg.startswith('dryrun: would GET http://example.org/dir/2 ')) c.dryrun = False # get from file uri that does not exist resource = Resource(uri='file:tests/testdata/i_do_not_exist') - self.assertRaises( ClientFatalError, c.update_resource, resource, filename ) - # get from file uri that does not exist but with c.ignore_failures to log + self.assertRaises(ClientFatalError, + c.update_resource, resource, filename) + # get from file uri that does not exist but with c.ignore_failures to + # log resource = Resource(uri='file:tests/testdata/i_do_not_exist') with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') + c.logger = logging.getLogger('resync.client') c.ignore_failures = True - n = c.update_resource( resource, filename ) - self.assertEqual( n, 0 ) - self.assertTrue( lc.records[-1].msg.startswith('Failed to GET file:tests/testdata/i_do_not_exist ') ) + n = c.update_resource(resource, filename) + self.assertEqual(n, 0) + self.assertTrue( + lc.records[-1].msg.startswith('Failed to GET file:tests/testdata/i_do_not_exist ')) # get from file uri resource = Resource(uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=355, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') - n = c.update_resource( resource, filename ) - self.assertEqual( n, 1 ) - self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) + c.logger = logging.getLogger('resync.client') + n = c.update_resource(resource, filename) + self.assertEqual(n, 1) + self.assertTrue(lc.records[-1].msg.startswith('Event: {')) # get from file uri with length and md5 warnings resource = Resource(uri='file:tests/testdata/examples_from_spec/resourcesync_ex_1.xml', length=111, md5='abc', timestamp=10) c.last_timestamp = 0 with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') + c.logger = logging.getLogger('resync.client') c.checksum = True - n = c.update_resource( resource, filename ) - self.assertEqual( n, 1 ) - self.assertTrue( lc.records[-1].msg.startswith('MD5 mismatch ') ) - self.assertTrue( lc.records[-2].msg.startswith('Downloaded size for ') ) - self.assertTrue( lc.records[-3].msg.startswith('Event: {') ) + n = c.update_resource(resource, filename) + self.assertEqual(n, 1) + self.assertTrue(lc.records[-1].msg.startswith('MD5 mismatch ')) + self.assertTrue( + lc.records[-2].msg.startswith('Downloaded size for ')) + self.assertTrue(lc.records[-3].msg.startswith('Event: {')) def test19_delete_resource(self): c = Client() resource = Resource(uri='http://example.org/1') - filename = os.path.join(self.tmpdir,'resource1') + filename = os.path.join(self.tmpdir, 'resource1') c.last_timestamp = 5 # no delete, no timestamp update with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') - n = c.delete_resource( resource, filename ) - self.assertEqual( n, 0 ) - self.assertEqual( lc.records[-1].msg, - 'nodelete: would delete http://example.org/1 (--delete to enable)' ) - self.assertEqual( c.last_timestamp, 5 ) + c.logger = logging.getLogger('resync.client') + n = c.delete_resource(resource, filename) + self.assertEqual(n, 0) + self.assertEqual(lc.records[-1].msg, + 'nodelete: would delete http://example.org/1 (--delete to enable)') + self.assertEqual(c.last_timestamp, 5) # no delete but timestamp update resource.timestamp = 10 with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') - n = c.delete_resource( resource, filename ) - self.assertEqual( n, 0 ) - self.assertEqual( lc.records[-1].msg, - 'nodelete: would delete http://example.org/1 (--delete to enable)' ) - self.assertEqual( c.last_timestamp, 10 ) + c.logger = logging.getLogger('resync.client') + n = c.delete_resource(resource, filename) + self.assertEqual(n, 0) + self.assertEqual(lc.records[-1].msg, + 'nodelete: would delete http://example.org/1 (--delete to enable)') + self.assertEqual(c.last_timestamp, 10) # allow delete but dryrun with LogCapture() as lc: c.dryrun = True - c.logger = logging.getLogger('resync.client') - n = c.delete_resource( resource, filename, allow_deletion=True ) - self.assertEqual( n, 0 ) - self.assertTrue( lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1') ) + c.logger = logging.getLogger('resync.client') + n = c.delete_resource(resource, filename, allow_deletion=True) + self.assertEqual(n, 0) + self.assertTrue( + lc.records[-1].msg.startswith('dryrun: would delete http://example.org/1')) c.dryrun = False # allow delete but no resource present with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') - n = c.delete_resource( resource, filename, allow_deletion=True ) - self.assertEqual( n, 0 ) - self.assertTrue( lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1') ) + c.logger = logging.getLogger('resync.client') + n = c.delete_resource(resource, filename, allow_deletion=True) + self.assertEqual(n, 0) + self.assertTrue( + lc.records[-1].msg.startswith('Failed to DELETE http://example.org/1')) # successful deletion, first make file... with open(filename, 'w') as fh: fh.write('delete me') fh.close() with LogCapture() as lc: - c.logger = logging.getLogger('resync.client') - n = c.delete_resource( resource, filename, allow_deletion=True ) - self.assertEqual( n, 1 ) - self.assertTrue( lc.records[-1].msg.startswith('Event: {') ) - self.assertTrue( lc.records[-2].msg.startswith('deleted: http://example.org/1 ->') ) + c.logger = logging.getLogger('resync.client') + n = c.delete_resource(resource, filename, allow_deletion=True) + self.assertEqual(n, 1) + self.assertTrue(lc.records[-1].msg.startswith('Event: {')) + self.assertTrue( + lc.records[-2].msg.startswith('deleted: http://example.org/1 ->')) def test20_parse_document(self): # Key property of the parse_document() method is that it parses the # document and identifies its type c = Client() with capture_stdout() as capturer: - c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_1.xml' + c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' c.parse_document() - self.assertTrue( re.search(r'Parsed resourcelist document with 2 entries',capturer.result) ) + self.assertTrue( + re.search(r'Parsed resourcelist document with 2 entries', capturer.result)) with capture_stdout() as capturer: - c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_17.xml' + c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_17.xml' c.parse_document() - self.assertTrue( re.search(r'Parsed resourcedump document with 3 entries',capturer.result) ) + self.assertTrue( + re.search(r'Parsed resourcedump document with 3 entries', capturer.result)) with capture_stdout() as capturer: - c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_19.xml' + c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_19.xml' c.parse_document() - self.assertTrue( re.search(r'Parsed changelist document with 4 entries',capturer.result) ) + self.assertTrue( + re.search(r'Parsed changelist document with 4 entries', capturer.result)) with capture_stdout() as capturer: - c.sitemap_name='tests/testdata/examples_from_spec/resourcesync_ex_22.xml' + c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_22.xml' c.parse_document() - self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) ) + self.assertTrue( + re.search(r'Parsed changedump document with 3 entries', capturer.result)) # Document that doesn't exist - c.sitemap_name='/does_not_exist' - self.assertRaises( ClientFatalError, c.parse_document ) + c.sitemap_name = '/does_not_exist' + self.assertRaises(ClientFatalError, c.parse_document) # and verbose with truncation... with capture_stdout() as capturer: c.verbose = True c.sitemap_name = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' - c.max_sitemap_entries = 1 + c.max_sitemap_entries = 1 c.parse_document() - self.assertTrue( re.search(r'Showing first 1 entries', capturer.result ) ) - self.assertTrue( re.search(r'\[0\] ', capturer.result ) ) - self.assertFalse( re.search(r'\[1\] ', capturer.result ) ) + self.assertTrue(re.search(r'Showing first 1 entries', capturer.result)) + self.assertTrue(re.search(r'\[0\] ', capturer.result)) + self.assertFalse(re.search(r'\[1\] ', capturer.result)) def test40_write_resource_list_mappings(self): c = Client() - c.set_mappings( ['http://example.org/','tests/testdata/'] ) + c.set_mappings(['http://example.org/', 'tests/testdata/']) # with no explicit paths seting the mapping will be used with capture_stdout() as capturer: c.write_resource_list() - #sys.stderr.write(capturer.result) - self.assertTrue( re.search(r'http://example.org/dir1/file_a', capturer.result ) ) - self.assertTrue( re.search(r'http://example.org/dir1/file_b', capturer.result ) ) - self.assertTrue( re.search(r'http://example.org/dir2/file_x', capturer.result ) ) + # sys.stderr.write(capturer.result) + self.assertTrue( + re.search(r'http://example.org/dir1/file_a', capturer.result)) + self.assertTrue( + re.search(r'http://example.org/dir1/file_b', capturer.result)) + self.assertTrue( + re.search(r'http://example.org/dir2/file_x', capturer.result)) def test41_write_resource_list_path(self): c = Client() - c.set_mappings( ['http://example.org/','tests/testdata/'] ) - links=[{'rel':'uri_c','href':'uri_d'}] - # with an explicit paths setting only the specified paths will be included + c.set_mappings(['http://example.org/', 'tests/testdata/']) + links = [{'rel': 'uri_c', 'href': 'uri_d'}] + # with an explicit paths setting only the specified paths will be + # included with capture_stdout() as capturer: c.write_resource_list(paths='tests/testdata/dir1', links=links) - self.assertTrue( re.search(r'http://example.org/dir1/file_a', capturer.result ) ) - self.assertTrue( re.search(r'http://example.org/dir1/file_b', capturer.result ) ) - self.assertFalse( re.search(r'http://example.org/dir2/file_x', capturer.result ) ) + self.assertTrue( + re.search(r'http://example.org/dir1/file_a', capturer.result)) + self.assertTrue( + re.search(r'http://example.org/dir1/file_b', capturer.result)) + self.assertFalse( + re.search(r'http://example.org/dir2/file_x', capturer.result)) # check link present - self.assertTrue( re.search(r'rel="uri_c"', capturer.result ) ) - self.assertTrue( re.search(r'href="uri_d"', capturer.result ) ) + self.assertTrue(re.search(r'rel="uri_c"', capturer.result)) + self.assertTrue(re.search(r'href="uri_d"', capturer.result)) # Travis CI does not preserve timestamps from github so test here for the file # size but not the datestamp - #self.assertTrue( re.search(r'http://example.org/dir1/file_a[\w\-:]+', capturer.result ) ) - #self.assertTrue( re.search(r'http://example.org/dir1/file_b[\w\-:]+', capturer.result ) ) - # to file - outfile = os.path.join(self.tmpdir,'rl_out.xml') + # self.assertTrue( re.search(r'http://example.org/dir1/file_a[\w\-:]+', capturer.result ) ) + # self.assertTrue( re.search(r'http://example.org/dir1/file_b[\w\-:]+', capturer.result ) ) + # to file + outfile = os.path.join(self.tmpdir, 'rl_out.xml') c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile) - self.assertTrue( os.path.getsize(outfile)>100 ) + self.assertTrue(os.path.getsize(outfile) > 100) # dump instead (default file) - c.default_resource_dump = os.path.join(self.tmpdir,'rl_out_dump_def') - outfile = c.default_resource_dump+'00000.zip' - self.assertFalse( os.path.exists(outfile) ) + c.default_resource_dump = os.path.join(self.tmpdir, 'rl_out_dump_def') + outfile = c.default_resource_dump + '00000.zip' + self.assertFalse(os.path.exists(outfile)) c.write_resource_list(paths='tests/testdata/dir1', dump=True) - self.assertTrue( os.path.getsize(outfile)>100 ) + self.assertTrue(os.path.getsize(outfile) > 100) # (specific file) - outbase = os.path.join(self.tmpdir,'rl_out_dump') - outfile = outbase+'00000.zip' - self.assertFalse( os.path.exists(outfile) ) - c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase) - self.assertTrue( os.path.getsize(outfile)>100 ) + outbase = os.path.join(self.tmpdir, 'rl_out_dump') + outfile = outbase + '00000.zip' + self.assertFalse(os.path.exists(outfile)) + c.write_resource_list(paths='tests/testdata/dir1', + dump=True, outfile=outbase) + self.assertTrue(os.path.getsize(outfile) > 100) def test45_write_change_list(self): c = Client() ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml' with capture_stdout() as capturer: c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1) - self.assertTrue( re.search(r'http://example.com/res1', capturer.result) ) - # to file - outfile = os.path.join(self.tmpdir,'cl_out.xml') - c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile) - self.assertTrue( os.path.getsize(outfile)>100 ) + self.assertTrue( + re.search(r'http://example.com/res1', capturer.result)) + # to file + outfile = os.path.join(self.tmpdir, 'cl_out.xml') + c.write_change_list( + ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile) + self.assertTrue(os.path.getsize(outfile) > 100) def test46_write_capability_list(self): c = Client() - caps = { 'a':'uri_a', 'b':'uri_b' } + caps = {'a': 'uri_a', 'b': 'uri_b'} # simple case to STDOUT with capture_stdout() as capturer: - c.write_capability_list( caps ) - self.assertTrue( re.search(r'',capturer.result) ) - self.assertTrue( re.search(r'uri_auri_b', capturer.result)) + self.assertTrue( + re.search(r'uri_auri_b100 ) + self.assertTrue(os.path.getsize(outfile) > 100) def test47_write_source_description(self): c = Client() # simple case to STDOUT with capture_stdout() as capturer: - c.write_source_description( [ 'a','b','c' ] ) - self.assertTrue( re.search(r'',capturer.result) ) - self.assertTrue( re.search(r'a',capturer.result) ) - self.assertTrue( re.search(r'b',capturer.result) ) + c.write_source_description(['a', 'b', 'c']) + self.assertTrue(re.search(r'', capturer.result)) + self.assertTrue(re.search( + r'a', capturer.result)) + self.assertTrue(re.search( + r'b', capturer.result)) # more complex case to STDOUT with capture_stdout() as capturer: - c.write_source_description(capability_lists=['http://a.b/'], links=[{'rel':'c','href':'d'}]) - self.assertTrue( re.search(r'http://a.b/', capturer.result ) ) + c.write_source_description( + capability_lists=['http://a.b/'], links=[{'rel': 'c', 'href': 'd'}]) + self.assertTrue(re.search(r'http://a.b/', capturer.result)) # to file (just check that something is written) - outfile = os.path.join(self.tmpdir,'sd_out.xml') - c.write_source_description(capability_lists=['http://a.b/'], outfile=outfile, links=[{'rel':'c','href':'d'}]) - self.assertTrue( os.path.getsize(outfile)>100 ) + outfile = os.path.join(self.tmpdir, 'sd_out.xml') + c.write_source_description(capability_lists=[ + 'http://a.b/'], outfile=outfile, links=[{'rel': 'c', 'href': 'd'}]) + self.assertTrue(os.path.getsize(outfile) > 100) def test48_write_dump_if_requested(self): c = Client() # no dump file - self.assertFalse( c.write_dump_if_requested( ChangeList(), None ) ) + self.assertFalse(c.write_dump_if_requested(ChangeList(), None)) # with dump file with capture_stdout() as capturer: - c.write_dump_if_requested(ChangeList(),'/tmp/a_file') - self.assertTrue( re.search(r'FIXME', capturer.result) ) + c.write_dump_if_requested(ChangeList(), '/tmp/a_file') + self.assertTrue(re.search(r'FIXME', capturer.result)) def test49_read_reference_resource_list(self): c = Client() with capture_stdout() as capturer: - rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') - self.assertEqual( len(rl), 2 ) - self.assertEqual( '', capturer.result ) + rl = c.read_reference_resource_list( + 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual(len(rl), 2) + self.assertEqual('', capturer.result) c.verbose = True with capture_stdout() as capturer: - rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') - self.assertEqual( len(rl), 2 ) - self.assertTrue( re.search(r'http://example.com/res2', capturer.result) ) + rl = c.read_reference_resource_list( + 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual(len(rl), 2) + self.assertTrue(re.search(r'http://example.com/res2', capturer.result)) c.verbose = True c.max_sitemap_entries = 1 with capture_stdout() as capturer: - rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') - self.assertEqual( len(rl), 2 ) - self.assertTrue( re.search(r'http://example.com/res1', capturer.result) ) - self.assertTrue( re.search(r'Showing first 1 entries', capturer.result) ) - self.assertFalse( re.search(r'http://example.com/res2', capturer.result) ) + rl = c.read_reference_resource_list( + 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual(len(rl), 2) + self.assertTrue(re.search(r'http://example.com/res1', capturer.result)) + self.assertTrue(re.search(r'Showing first 1 entries', capturer.result)) + self.assertFalse( + re.search(r'http://example.com/res2', capturer.result)) def test50_log_status(self): c = Client() with LogCapture() as lc: c.logger = logging.getLogger('resync.client') c.log_status() - self.assertEqual( lc.records[-1].msg, - 'Status: IN SYNC (created=0, updated=0, deleted=0)' ) - c.log_status(created=123,updated=456,deleted=789) - self.assertEqual( lc.records[-1].msg, - 'Status: IN SYNC (created=123, updated=456, deleted=789)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: IN SYNC (created=0, updated=0, deleted=0)') + c.log_status(created=123, updated=456, deleted=789) + self.assertEqual(lc.records[-1].msg, + 'Status: IN SYNC (created=123, updated=456, deleted=789)') c.log_status(incremental=True) - self.assertEqual( lc.records[-1].msg, - 'Status: NO CHANGES (created=0, updated=0, deleted=0)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: NO CHANGES (created=0, updated=0, deleted=0)') c.log_status(audit=True) - self.assertEqual( lc.records[-1].msg, - 'Status: IN SYNC (to create=0, to update=0, to delete=0)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: IN SYNC (to create=0, to update=0, to delete=0)') c.log_status(in_sync=False, audit=True) - self.assertEqual( lc.records[-1].msg, - 'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)') c.log_status(in_sync=False, audit=False, to_delete=1) - self.assertEqual( lc.records[-1].msg, - 'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)' ) - c.log_status(in_sync=False, audit=False, to_delete=1, incremental=1) - self.assertEqual( lc.records[-1].msg, - 'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)') + c.log_status(in_sync=False, audit=False, + to_delete=1, incremental=1) + self.assertEqual(lc.records[-1].msg, + 'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)') c.log_status(in_sync=False) - self.assertEqual( lc.records[-1].msg, - 'Status: SYNCED (created=0, updated=0, deleted=0)' ) + self.assertEqual(lc.records[-1].msg, + 'Status: SYNCED (created=0, updated=0, deleted=0)') if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestClient) diff --git a/tests/test_client_link_options.py b/tests/test_client_link_options.py index fb76a8e..7eb40f3 100644 --- a/tests/test_client_link_options.py +++ b/tests/test_client_link_options.py @@ -1,14 +1,12 @@ import sys import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import BytesIO as io -except ImportError: #python3 +except ImportError: # python3 import io -sys.path.insert(0, '.') - from resync.resource import Resource from resync.resource_list import ResourceList from resync.capability_list import CapabilityList @@ -16,49 +14,51 @@ from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError import subprocess + def run_resync(args): - args.insert(0,'bin/resync') + args.insert(0, 'bin/resync') proc = subprocess.Popen(args, stdout=subprocess.PIPE) (out, err) = proc.communicate() return(out) + class TestClientLinkOptions(unittest.TestCase): def test01_no_links(self): xml = run_resync(['--resourcelist', - 'http://example.org/t','tests/testdata/dir1']) + 'http://example.org/t', 'tests/testdata/dir1']) rl = ResourceList() rl.parse(fh=io.BytesIO(xml)) - self.assertEqual( len(rl), 2 ) - self.assertEqual( rl.link('describedby'), None ) + self.assertEqual(len(rl), 2) + self.assertEqual(rl.link('describedby'), None) def test02_resource_list_links(self): xml = run_resync(['--resourcelist', '--describedby-link=a', - '--sourcedescription-link=b', #will be ignored + '--sourcedescription-link=b', # will be ignored '--capabilitylist-link=c', - 'http://example.org/t','tests/testdata/dir1']) + 'http://example.org/t', 'tests/testdata/dir1']) rl = ResourceList() rl.parse(fh=io.BytesIO(xml)) - self.assertEqual( len(rl), 2 ) - self.assertNotEqual( rl.link('describedby'), None ) - self.assertEqual( rl.link('describedby')['href'], 'a' ) - self.assertNotEqual( rl.link('up'), None ) - self.assertEqual( rl.link('up')['href'], 'c' ) + self.assertEqual(len(rl), 2) + self.assertNotEqual(rl.link('describedby'), None) + self.assertEqual(rl.link('describedby')['href'], 'a') + self.assertNotEqual(rl.link('up'), None) + self.assertEqual(rl.link('up')['href'], 'c') def test03_capability_list_links(self): xml = run_resync(['--capabilitylist=resourcelist=rl,changedump=cd', '--describedby-link=a', '--sourcedescription-link=b', - '--capabilitylist-link=c' ]) #will be ignored + '--capabilitylist-link=c']) # will be ignored capl = CapabilityList() capl.parse(fh=io.BytesIO(xml)) - self.assertEqual( len(capl), 2 ) - self.assertNotEqual( capl.link('describedby'), None ) - self.assertEqual( capl.link('describedby')['href'], 'a' ) - self.assertNotEqual( capl.link('up'), None ) - self.assertEqual( capl.link('up')['href'], 'b' ) + self.assertEqual(len(capl), 2) + self.assertNotEqual(capl.link('describedby'), None) + self.assertEqual(capl.link('describedby')['href'], 'a') + self.assertNotEqual(capl.link('up'), None) + self.assertEqual(capl.link('up')['href'], 'b') -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestClientLinkOptions) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_client_state.py b/tests/test_client_state.py index ebef66b..0acad2d 100644 --- a/tests/test_client_state.py +++ b/tests/test_client_state.py @@ -3,22 +3,23 @@ import os.path import unittest from resync.client_state import ClientState + class TestClientState(TestCase): def test01_set_and_get(self): cs = ClientState() - self.assertEqual( cs.status_file, '.resync-client-status.cfg') - cs.status_file = os.path.join(self.tmpdir,cs.status_file) + self.assertEqual(cs.status_file, '.resync-client-status.cfg') + cs.status_file = os.path.join(self.tmpdir, cs.status_file) # Get state site = 'https://this.site/' - self.assertEqual( cs.get_state(site), None ) + self.assertEqual(cs.get_state(site), None) cs.set_state(site, 123) - self.assertEqual( cs.get_state(site), 123 ) + self.assertEqual(cs.get_state(site), 123) cs.set_state(site, 456) - self.assertEqual( cs.get_state(site), 456 ) + self.assertEqual(cs.get_state(site), 456) cs.set_state(site) - self.assertEqual( cs.get_state(site), None ) - + self.assertEqual(cs.get_state(site), None) + if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestClientUtils) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_client_utils.py b/tests/test_client_utils.py index c8cb612..4405696 100644 --- a/tests/test_client_utils.py +++ b/tests/test_client_utils.py @@ -4,71 +4,77 @@ import logging import os.path import re import unittest -from resync.client_utils import init_logging,count_true_args,parse_links,parse_link,parse_capabilities,parse_capability_lists +from resync.client_utils import init_logging, count_true_args, parse_links, parse_link, parse_capabilities, parse_capability_lists from resync.client import ClientFatalError + class TestClientUtils(TestCase): def test01_init_logging(self): - #to_file=False, logfile=None, default_logfile='/tmp/resync.log', - # human=True, verbose=False, eval_mode=False, + # to_file=False, logfile=None, default_logfile='/tmp/resync.log', + # human=True, verbose=False, eval_mode=False, # default_logger='client', extra_loggers=None): - tmplog = os.path.join(self.tmpdir,'tmp.log') - init_logging( to_file=True, default_logfile=tmplog, extra_loggers=['x1','x2'] ) + tmplog = os.path.join(self.tmpdir, 'tmp.log') + init_logging(to_file=True, default_logfile=tmplog, + extra_loggers=['x1', 'x2']) # check x1 and x2 set, not x3 (can tell by level) - self.assertTrue( logging.getLogger('x1').level, logging.DEBUG ) - self.assertTrue( logging.getLogger('x2').level, logging.DEBUG ) - self.assertEqual( logging.getLogger('x3').level, 0 ) + self.assertTrue(logging.getLogger('x1').level, logging.DEBUG) + self.assertTrue(logging.getLogger('x2').level, logging.DEBUG) + self.assertEqual(logging.getLogger('x3').level, 0) # write something, check goes to file log = logging.getLogger('resync') log.warning('PIGS MIGHT FLY') - logtxt = open(tmplog,'r').read() - self.assertTrue( re.search(r'WARNING \| PIGS MIGHT FLY', logtxt) ) + logtxt = open(tmplog, 'r').read() + self.assertTrue(re.search(r'WARNING \| PIGS MIGHT FLY', logtxt)) def test02_count_true_args(self): - self.assertEqual( count_true_args(), 0 ) - self.assertEqual( count_true_args(True), 1 ) - self.assertEqual( count_true_args(False), 0 ) - self.assertEqual( count_true_args(0,1,2,3), 3 ) + self.assertEqual(count_true_args(), 0) + self.assertEqual(count_true_args(True), 1) + self.assertEqual(count_true_args(False), 0) + self.assertEqual(count_true_args(0, 1, 2, 3), 3) def test03_parse_links(self): - self.assertEqual( parse_links( [] ), [] ) - self.assertEqual( parse_links( ['u,h'] ), [{'href': 'h', 'rel': 'u'}] ) - self.assertEqual( parse_links( ['u,h','v,i'] ), [{'href': 'h', 'rel': 'u'},{'href': 'i', 'rel': 'v'}] ) - self.assertRaises( ClientFatalError, parse_links, 'xx' ) - self.assertRaises( ClientFatalError, parse_links, ['u'] ) - self.assertRaises( ClientFatalError, parse_links, ['u,h','u'] ) + self.assertEqual(parse_links([]), []) + self.assertEqual(parse_links(['u,h']), [{'href': 'h', 'rel': 'u'}]) + self.assertEqual(parse_links(['u,h', 'v,i']), [ + {'href': 'h', 'rel': 'u'}, {'href': 'i', 'rel': 'v'}]) + self.assertRaises(ClientFatalError, parse_links, 'xx') + self.assertRaises(ClientFatalError, parse_links, ['u']) + self.assertRaises(ClientFatalError, parse_links, ['u,h', 'u']) def test04_parse_link(self): # Input string of the form: rel,href,att1=val1,att2=val2 - self.assertEqual( parse_link('u,h'), {'href': 'h', 'rel': 'u'} ) - self.assertEqual( parse_link('u,h,a=b'), {'a': 'b', 'href': 'h', 'rel': 'u'} ) - self.assertEqual( parse_link('u,h,a=b,c=d'), {'a': 'b', 'c': 'd', 'href': 'h', 'rel': 'u'} ) - self.assertEqual( parse_link('u,h,a=b,a=d'), {'a': 'd', 'href': 'h', 'rel': 'u'} ) # desired?? - self.assertRaises( ClientFatalError, parse_link, '' ) - self.assertRaises( ClientFatalError, parse_link, 'u' ) - self.assertRaises( ClientFatalError, parse_link, 'u,' ) - self.assertRaises( ClientFatalError, parse_link, 'u,h,,' ) - self.assertRaises( ClientFatalError, parse_link, 'u,h,a' ) - self.assertRaises( ClientFatalError, parse_link, 'u,h,a=' ) - self.assertRaises( ClientFatalError, parse_link, 'u,h,a=b,=c' ) + self.assertEqual(parse_link('u,h'), {'href': 'h', 'rel': 'u'}) + self.assertEqual(parse_link('u,h,a=b'), { + 'a': 'b', 'href': 'h', 'rel': 'u'}) + self.assertEqual(parse_link('u,h,a=b,c=d'), { + 'a': 'b', 'c': 'd', 'href': 'h', 'rel': 'u'}) + self.assertEqual(parse_link('u,h,a=b,a=d'), { + 'a': 'd', 'href': 'h', 'rel': 'u'}) # desired?? + self.assertRaises(ClientFatalError, parse_link, '') + self.assertRaises(ClientFatalError, parse_link, 'u') + self.assertRaises(ClientFatalError, parse_link, 'u,') + self.assertRaises(ClientFatalError, parse_link, 'u,h,,') + self.assertRaises(ClientFatalError, parse_link, 'u,h,a') + self.assertRaises(ClientFatalError, parse_link, 'u,h,a=') + self.assertRaises(ClientFatalError, parse_link, 'u,h,a=b,=c') def test05_parse_capabilities(self): - # Input string of the form: cap_name=uri,cap_name=uri + # Input string of the form: cap_name=uri,cap_name=uri # good - c = parse_capabilities( 'a=' ) - self.assertEqual( len(c), 1 ) - self.assertEqual( c['a'], '' ) - c = parse_capabilities( 'a=b,c=' ) - self.assertEqual( len(c), 2 ) - self.assertEqual( c['a'], 'b' ) + c = parse_capabilities('a=') + self.assertEqual(len(c), 1) + self.assertEqual(c['a'], '') + c = parse_capabilities('a=b,c=') + self.assertEqual(len(c), 2) + self.assertEqual(c['a'], 'b') # bad - self.assertRaises( ClientFatalError, parse_capabilities, 'a' ) - self.assertRaises( ClientFatalError, parse_capabilities, 'a=b,' ) + self.assertRaises(ClientFatalError, parse_capabilities, 'a') + self.assertRaises(ClientFatalError, parse_capabilities, 'a=b,') def test06_parse_capability_lists(self): # Input string of the form: uri,uri - self.assertEqual( parse_capability_lists('a,b'), ['a','b'] ) + self.assertEqual(parse_capability_lists('a,b'), ['a', 'b']) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestClientUtils) diff --git a/tests/test_dump.py b/tests/test_dump.py index 0a21ac1..d96e839 100644 --- a/tests/test_dump.py +++ b/tests/test_dump.py @@ -12,151 +12,158 @@ from resync.resource import Resource from resync.resource_dump_manifest import ResourceDumpManifest from resync.change_dump_manifest import ChangeDumpManifest + class TestDump(TestCase): def test00_dump_zip_resource_list(self): - rl=ResourceDumpManifest() - rl.add( Resource('http://ex.org/a', length=7, path='tests/testdata/a') ) - rl.add( Resource('http://ex.org/b', length=21, path='tests/testdata/b') ) - d=Dump() - zipf=os.path.join(self.tmpdir,"test00_dump.zip") - d.write_zip(resources=rl,dumpfile=zipf) # named args - self.assertTrue( os.path.exists(zipf) ) - self.assertTrue( zipfile.is_zipfile(zipf) ) - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( len(zo.namelist()), 3 ) + rl = ResourceDumpManifest() + rl.add(Resource('http://ex.org/a', length=7, path='tests/testdata/a')) + rl.add(Resource('http://ex.org/b', length=21, path='tests/testdata/b')) + d = Dump() + zipf = os.path.join(self.tmpdir, "test00_dump.zip") + d.write_zip(resources=rl, dumpfile=zipf) # named args + self.assertTrue(os.path.exists(zipf)) + self.assertTrue(zipfile.is_zipfile(zipf)) + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual(len(zo.namelist()), 3) zo.close() os.unlink(zipf) def test01_dump_zip_change_list(self): - cl=ChangeDumpManifest() - cl.add( Resource('http://ex.org/a', length=7, path='tests/testdata/a', change="updated") ) - cl.add( Resource('http://ex.org/b', length=21, path='tests/testdata/b', change="updated") ) - d=Dump() - zipf=os.path.join(self.tmpdir,"test01_dump.zip") - d.write_zip(cl,zipf) # positional args - self.assertTrue( os.path.exists(zipf) ) - self.assertTrue( zipfile.is_zipfile(zipf) ) - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( len(zo.namelist()), 3 ) + cl = ChangeDumpManifest() + cl.add(Resource('http://ex.org/a', length=7, + path='tests/testdata/a', change="updated")) + cl.add(Resource('http://ex.org/b', length=21, + path='tests/testdata/b', change="updated")) + d = Dump() + zipf = os.path.join(self.tmpdir, "test01_dump.zip") + d.write_zip(cl, zipf) # positional args + self.assertTrue(os.path.exists(zipf)) + self.assertTrue(zipfile.is_zipfile(zipf)) + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual(len(zo.namelist()), 3) zo.close() os.unlink(zipf) def test02_dump_check_files(self): - cl=ChangeList() - cl.add( Resource('http://ex.org/a', length=7, path='tests/testdata/a', change="updated") ) - cl.add( Resource('http://ex.org/b', length=21, path='tests/testdata/b', change="updated") ) - d=Dump(resources=cl) + cl = ChangeList() + cl.add(Resource('http://ex.org/a', length=7, + path='tests/testdata/a', change="updated")) + cl.add(Resource('http://ex.org/b', length=21, + path='tests/testdata/b', change="updated")) + d = Dump(resources=cl) self.assertTrue(d.check_files()) self.assertEqual(d.total_size, 28) - + def test03_dump_multi_file_max_size(self): - rl=ResourceList() - for letter in map(chr,range(ord('a'),ord('l')+1)): - uri='http://ex.org/%s' % (letter) - fname='tests/testdata/a_to_z/%s' % (letter) - rl.add( Resource(uri, path=fname) ) - self.assertEqual( len(rl), 12 ) - #d=Dump(rl) - #tmpdir=tempfile.mkdtemp() - #tmpbase=os.path.join(tmpdir,'base') - #d.max_size=2000 # start new zip after size exceeds 2000 bytes - #n=d.write(tmpbase) - #self.assertEqual( n, 2, 'expect to write 2 dump files' ) - # + rl = ResourceList() + for letter in map(chr, range(ord('a'), ord('l') + 1)): + uri = 'http://ex.org/%s' % (letter) + fname = 'tests/testdata/a_to_z/%s' % (letter) + rl.add(Resource(uri, path=fname)) + self.assertEqual(len(rl), 12) + # d=Dump(rl) + # tmpdir=tempfile.mkdtemp() + # tmpbase=os.path.join(tmpdir,'base') + # d.max_size=2000 # start new zip after size exceeds 2000 bytes + # n=d.write(tmpbase) + # self.assertEqual( n, 2, 'expect to write 2 dump files' ) + # # Now repeat with large size limit but small number of files limit - d2=Dump(rl) - tmpbase=os.path.join(self.tmpdir,'test03_') - d2.max_files=4 - n=d2.write(tmpbase) - self.assertEqual( n, 3, 'expect to write 3 dump files' ) - self.assertTrue( os.path.isfile(tmpbase+'00000.zip') ) - self.assertTrue( os.path.isfile(tmpbase+'00001.zip') ) - self.assertTrue( os.path.isfile(tmpbase+'00002.zip') ) + d2 = Dump(rl) + tmpbase = os.path.join(self.tmpdir, 'test03_') + d2.max_files = 4 + n = d2.write(tmpbase) + self.assertEqual(n, 3, 'expect to write 3 dump files') + self.assertTrue(os.path.isfile(tmpbase + '00000.zip')) + self.assertTrue(os.path.isfile(tmpbase + '00001.zip')) + self.assertTrue(os.path.isfile(tmpbase + '00002.zip')) # Look at the first file in detail - zipf=tmpbase+'00000.zip' - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d'] ) - #self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) - self.assertEqual( zo.getinfo('a').file_size, 9 ) - self.assertEqual( zo.getinfo('b').file_size, 1116 ) - self.assertEqual( zo.getinfo('c').file_size, 32 ) - self.assertEqual( zo.getinfo('d').file_size, 13 ) + zipf = tmpbase + '00000.zip' + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual(zo.namelist(), ['manifest.xml', 'a', 'b', 'c', 'd']) + # self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) + self.assertEqual(zo.getinfo('a').file_size, 9) + self.assertEqual(zo.getinfo('b').file_size, 1116) + self.assertEqual(zo.getinfo('c').file_size, 32) + self.assertEqual(zo.getinfo('d').file_size, 13) zo.close() os.unlink(zipf) # Check second and third files have expected contents - zipf=tmpbase+'00001.zip' - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( zo.namelist(), ['manifest.xml','e','f','g','h'] ) + zipf = tmpbase + '00001.zip' + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual(zo.namelist(), ['manifest.xml', 'e', 'f', 'g', 'h']) zo.close() os.unlink(zipf) - zipf=tmpbase+'00002.zip' - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( zo.namelist(), ['manifest.xml','i','j','k','l'] ) + zipf = tmpbase + '00002.zip' + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual(zo.namelist(), ['manifest.xml', 'i', 'j', 'k', 'l']) zo.close() os.unlink(zipf) def test04_dump_multi_file_max_size(self): - rl=ResourceList() - for letter in map(chr,range(ord('a'),ord('l')+1)): - uri='http://ex.org/%s' % (letter) - fname='tests/testdata/a_to_z/%s' % (letter) - rl.add( Resource(uri, path=fname) ) - self.assertEqual( len(rl), 12 ) - d2=Dump(rl) - tmpbase=os.path.join(self.tmpdir,'test0f_') - d2.max_size=2000 - n=d2.write(tmpbase) - self.assertEqual( n, 2, 'expect to write 2 dump files' ) - self.assertTrue( os.path.isfile(tmpbase+'00000.zip') ) - self.assertTrue( os.path.isfile(tmpbase+'00001.zip') ) + rl = ResourceList() + for letter in map(chr, range(ord('a'), ord('l') + 1)): + uri = 'http://ex.org/%s' % (letter) + fname = 'tests/testdata/a_to_z/%s' % (letter) + rl.add(Resource(uri, path=fname)) + self.assertEqual(len(rl), 12) + d2 = Dump(rl) + tmpbase = os.path.join(self.tmpdir, 'test0f_') + d2.max_size = 2000 + n = d2.write(tmpbase) + self.assertEqual(n, 2, 'expect to write 2 dump files') + self.assertTrue(os.path.isfile(tmpbase + '00000.zip')) + self.assertTrue(os.path.isfile(tmpbase + '00001.zip')) # Look at the first file in detail - zipf=tmpbase+'00000.zip' - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d','e','f'] ) - #self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) - self.assertEqual( zo.getinfo('a').file_size, 9 ) - self.assertEqual( zo.getinfo('b').file_size, 1116 ) - self.assertEqual( zo.getinfo('c').file_size, 32 ) - self.assertEqual( zo.getinfo('d').file_size, 13 ) - self.assertEqual( zo.getinfo('e').file_size, 20 ) - self.assertEqual( zo.getinfo('f').file_size, 1625 ) + zipf = tmpbase + '00000.zip' + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual( + zo.namelist(), ['manifest.xml', 'a', 'b', 'c', 'd', 'e', 'f']) + # self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) + self.assertEqual(zo.getinfo('a').file_size, 9) + self.assertEqual(zo.getinfo('b').file_size, 1116) + self.assertEqual(zo.getinfo('c').file_size, 32) + self.assertEqual(zo.getinfo('d').file_size, 13) + self.assertEqual(zo.getinfo('e').file_size, 20) + self.assertEqual(zo.getinfo('f').file_size, 1625) zo.close() os.unlink(zipf) # Check second and third files have expected contents - zipf=tmpbase+'00001.zip' - zo=zipfile.ZipFile(zipf,'r') - self.assertEqual( zo.namelist(), ['manifest.xml','g','h','i','j','k','l'] ) + zipf = tmpbase + '00001.zip' + zo = zipfile.ZipFile(zipf, 'r') + self.assertEqual( + zo.namelist(), ['manifest.xml', 'g', 'h', 'i', 'j', 'k', 'l']) zo.close() os.unlink(zipf) def test10_no_path(self): - rl=ResourceList() - rl.add( Resource('http://ex.org/a', length=7, path='tests/testdata/a') ) - rl.add( Resource('http://ex.org/b', length=21 ) ) - d=Dump(rl) - self.assertRaises( DumpError, d.check_files ) + rl = ResourceList() + rl.add(Resource('http://ex.org/a', length=7, path='tests/testdata/a')) + rl.add(Resource('http://ex.org/b', length=21)) + d = Dump(rl) + self.assertRaises(DumpError, d.check_files) def test11_bad_size(self): - rl=ResourceList() - rl.add( Resource('http://ex.org/a', length=9999, path='tests/testdata/a') ) - d=Dump(rl) - self.assertTrue( d.check_files(check_length=False) ) - self.assertRaises( DumpError, d.check_files ) + rl = ResourceList() + rl.add(Resource('http://ex.org/a', length=9999, path='tests/testdata/a')) + d = Dump(rl) + self.assertTrue(d.check_files(check_length=False)) + self.assertRaises(DumpError, d.check_files) def test10_no_path(self): - rl=ResourceList() - rl.add( Resource('http://ex.org/a', length=7, path='tests/testdata/a') ) - rl.add( Resource('http://ex.org/b', length=21 ) ) - d=Dump(rl) - self.assertRaises( DumpError, d.check_files ) + rl = ResourceList() + rl.add(Resource('http://ex.org/a', length=7, path='tests/testdata/a')) + rl.add(Resource('http://ex.org/b', length=21)) + d = Dump(rl) + self.assertRaises(DumpError, d.check_files) def test11_bad_size(self): - rl=ResourceList() - rl.add( Resource('http://ex.org/a', length=9999, path='tests/testdata/a') ) - d=Dump(rl) - self.assertTrue( d.check_files(check_length=False) ) - self.assertRaises( DumpError, d.check_files ) + rl = ResourceList() + rl.add(Resource('http://ex.org/a', length=9999, path='tests/testdata/a')) + d = Dump(rl) + self.assertTrue(d.check_files(check_length=False)) + self.assertRaises(DumpError, d.check_files) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDump) diff --git a/tests/test_examples_from_spec.py b/tests/test_examples_from_spec.py index 0d9c0d3..a8e3edd 100644 --- a/tests/test_examples_from_spec.py +++ b/tests/test_examples_from_spec.py @@ -6,249 +6,275 @@ import unittest from resync.resource import Resource from resync.source_description import SourceDescription from resync.capability_list import CapabilityList -from resync.resource_list import ResourceList,ResourceListOrdered +from resync.resource_list import ResourceList, ResourceListOrdered from resync.resource_dump import ResourceDump from resync.resource_dump_manifest import ResourceDumpManifest from resync.change_list import ChangeList from resync.change_dump import ChangeDump from resync.change_dump_manifest import ChangeDumpManifest -from resync.archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive +from resync.archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive from resync.sitemap import Sitemap + class TestExamplesFromSpec(unittest.TestCase): def test_all_simple_read(self): """Just try to read each one""" - for ex in ('archives_ex_2_1','archives_ex_2_2', - 'archives_ex_3_1','archives_ex_3_2', + for ex in ('archives_ex_2_1', 'archives_ex_2_2', + 'archives_ex_3_1', 'archives_ex_3_2', 'archives_ex_4_1', 'archives_ex_5_1', 'archives_ex_6_1', - 'resourcesync_ex_1','resourcesync_ex_2','resourcesync_ex_3', - 'resourcesync_ex_4','resourcesync_ex_5','resourcesync_ex_6', - 'resourcesync_ex_7','resourcesync_ex_8','resourcesync_ex_12', - 'resourcesync_ex_13','resourcesync_ex_14','resourcesync_ex_15', - 'resourcesync_ex_16','resourcesync_ex_17','resourcesync_ex_18', - 'resourcesync_ex_19','resourcesync_ex_20','resourcesync_ex_21', - 'resourcesync_ex_22','resourcesync_ex_23','resourcesync_ex_24', - 'resourcesync_ex_25','resourcesync_ex_26','resourcesync_ex_27', - 'resourcesync_ex_28','resourcesync_ex_29','resourcesync_ex_30', - 'resourcesync_ex_31','resourcesync_ex_32','resourcesync_ex_33'): - s=Sitemap() + 'resourcesync_ex_1', 'resourcesync_ex_2', 'resourcesync_ex_3', + 'resourcesync_ex_4', 'resourcesync_ex_5', 'resourcesync_ex_6', + 'resourcesync_ex_7', 'resourcesync_ex_8', 'resourcesync_ex_12', + 'resourcesync_ex_13', 'resourcesync_ex_14', 'resourcesync_ex_15', + 'resourcesync_ex_16', 'resourcesync_ex_17', 'resourcesync_ex_18', + 'resourcesync_ex_19', 'resourcesync_ex_20', 'resourcesync_ex_21', + 'resourcesync_ex_22', 'resourcesync_ex_23', 'resourcesync_ex_24', + 'resourcesync_ex_25', 'resourcesync_ex_26', 'resourcesync_ex_27', + 'resourcesync_ex_28', 'resourcesync_ex_29', 'resourcesync_ex_30', + 'resourcesync_ex_31', 'resourcesync_ex_32', 'resourcesync_ex_33'): + s = Sitemap() fh = self._open_ex(ex) - si = s.parse_xml( fh=fh ) + si = s.parse_xml(fh=fh) def test_ex_01(self): """resourcesync_ex_1 is a simple resource_list with 2 resources, no metadata""" - rl=ResourceList() + rl = ResourceList() rl.parse(uri='tests/testdata/examples_from_spec/resourcesync_ex_1.xml') - self.assertEqual( rl.capability, 'resourcelist' ) - self.assertEqual( len(rl.resources), 2, '2 resources') + self.assertEqual(rl.capability, 'resourcelist') + self.assertEqual(len(rl.resources), 2, '2 resources') sms = sorted(rl.uris()) - self.assertEqual( sms, ['http://example.com/res1','http://example.com/res2'] ) - self.assertEqual( rl.resources['http://example.com/res1'].lastmod, None ) + self.assertEqual( + sms, ['http://example.com/res1', 'http://example.com/res2']) + self.assertEqual(rl.resources['http://example.com/res1'].lastmod, None) def test_ex_02(self): """resourcesync_ex_2 is a simple resource_list with 2 resources, some metadata""" - rl=ResourceList() + rl = ResourceList() rl.parse(uri='tests/testdata/examples_from_spec/resourcesync_ex_2.xml') - self.assertEqual( len(rl.resources), 2, '2 resources') + self.assertEqual(len(rl.resources), 2, '2 resources') sms = sorted(rl.uris()) - self.assertEqual( sms, ['http://example.com/res1','http://example.com/res2'] ) - self.assertEqual( rl.resources['http://example.com/res1'].lastmod, '2013-01-02T13:00:00Z' ) - self.assertEqual( rl.resources['http://example.com/res2'].lastmod, '2013-01-02T14:00:00Z' ) - self.assertEqual( rl.resources['http://example.com/res1'].md5, '1584abdf8ebdc9802ac0c6a7402c03b6' ) - self.assertEqual( rl.resources['http://example.com/res2'].md5, '1e0d5cb8ef6ba40c99b14c0237be735e' ) + self.assertEqual( + sms, ['http://example.com/res1', 'http://example.com/res2']) + self.assertEqual( + rl.resources['http://example.com/res1'].lastmod, '2013-01-02T13:00:00Z') + self.assertEqual( + rl.resources['http://example.com/res2'].lastmod, '2013-01-02T14:00:00Z') + self.assertEqual( + rl.resources['http://example.com/res1'].md5, '1584abdf8ebdc9802ac0c6a7402c03b6') + self.assertEqual( + rl.resources['http://example.com/res2'].md5, '1e0d5cb8ef6ba40c99b14c0237be735e') def test_ex_03(self): """resourcesync_ex_3 is a simple change_list with 2 resources""" - cl=ChangeList() + cl = ChangeList() cl.parse('tests/testdata/examples_from_spec/resourcesync_ex_3.xml') - self.assertEqual( len(cl.resources), 2, '2 resources') + self.assertEqual(len(cl.resources), 2, '2 resources') sms = sorted(cl.uris()) - self.assertEqual( sms, ['http://example.com/res2.pdf','http://example.com/res3.tiff'] ) - self.assertEqual( cl.resources[0].lastmod, '2013-01-02T13:00:00Z' ) - self.assertEqual( cl.resources[1].lastmod, '2013-01-02T18:00:00Z' ) - self.assertEqual( cl.resources[0].change, 'updated' ) - self.assertEqual( cl.resources[1].change, 'deleted' ) + self.assertEqual( + sms, ['http://example.com/res2.pdf', 'http://example.com/res3.tiff']) + self.assertEqual(cl.resources[0].lastmod, '2013-01-02T13:00:00Z') + self.assertEqual(cl.resources[1].lastmod, '2013-01-02T18:00:00Z') + self.assertEqual(cl.resources[0].change, 'updated') + self.assertEqual(cl.resources[1].change, 'deleted') def test_ex_04(self): """resourcesync_ex_4 is a simple resource dump with one ZIP listed""" - rd=ResourceDump() + rd = ResourceDump() rd.parse('tests/testdata/examples_from_spec/resourcesync_ex_4.xml') - self.assertEqual( len(rd.resources), 1, '1 resources') - self.assertTrue( 'http://example.com/resourcedump.zip' in rd.resources ) - self.assertEqual( rd.resources['http://example.com/resourcedump.zip'].lastmod, '2013-01-03T09:00:00Z' ) + self.assertEqual(len(rd.resources), 1, '1 resources') + self.assertTrue('http://example.com/resourcedump.zip' in rd.resources) + self.assertEqual(rd.resources[ + 'http://example.com/resourcedump.zip'].lastmod, '2013-01-03T09:00:00Z') def test_ex_05(self): """resourcesync_ex_5 is a simple resource dump manifest with two files listed""" - rdm=ResourceDumpManifest() + rdm = ResourceDumpManifest() rdm.parse('tests/testdata/examples_from_spec/resourcesync_ex_5.xml') - self.assertEqual( len(rdm.resources), 2, '2 resources') + self.assertEqual(len(rdm.resources), 2, '2 resources') sms = sorted(rdm.uris()) - self.assertEqual( sms, ['http://example.com/res1','http://example.com/res2'] ) - self.assertEqual( rdm.resources['http://example.com/res1'].lastmod, '2013-01-03T03:00:00Z' ) - self.assertEqual( rdm.resources['http://example.com/res1'].md5, '1584abdf8ebdc9802ac0c6a7402c03b6' ) - self.assertEqual( rdm.resources['http://example.com/res1'].path, '/resources/res1' ) - self.assertEqual( rdm.resources['http://example.com/res2'].lastmod, '2013-01-03T04:00:00Z' ) - self.assertEqual( rdm.resources['http://example.com/res2'].md5, '1e0d5cb8ef6ba40c99b14c0237be735e' ) - self.assertEqual( rdm.resources['http://example.com/res2'].path, '/resources/res2' ) + self.assertEqual( + sms, ['http://example.com/res1', 'http://example.com/res2']) + self.assertEqual( + rdm.resources['http://example.com/res1'].lastmod, '2013-01-03T03:00:00Z') + self.assertEqual(rdm.resources[ + 'http://example.com/res1'].md5, '1584abdf8ebdc9802ac0c6a7402c03b6') + self.assertEqual( + rdm.resources['http://example.com/res1'].path, '/resources/res1') + self.assertEqual( + rdm.resources['http://example.com/res2'].lastmod, '2013-01-03T04:00:00Z') + self.assertEqual(rdm.resources[ + 'http://example.com/res2'].md5, '1e0d5cb8ef6ba40c99b14c0237be735e') + self.assertEqual( + rdm.resources['http://example.com/res2'].path, '/resources/res2') def test_ex_06(self): """resourcesync_ex_6 is a simple capability list with three capabilities""" - capl=CapabilityList() + capl = CapabilityList() capl.parse('tests/testdata/examples_from_spec/resourcesync_ex_6.xml') - self.assertEqual( len(capl.resources), 3, '3 capabilities') + self.assertEqual(len(capl.resources), 3, '3 capabilities') # What capabilities are present? - self.assertTrue( capl.has_capability('resourcelist') ) - self.assertEqual( capl.capability_info('resourcelist').uri, 'http://example.com/dataset1/resourcelist.xml') - self.assertTrue( capl.has_capability('resourcedump') ) - self.assertEqual( capl.capability_info('resourcedump').uri, 'http://example.com/dataset1/resourcedump.xml') - self.assertTrue( capl.has_capability('changelist') ) - self.assertEqual( capl.capability_info('changelist').uri, 'http://example.com/dataset1/changelist.xml') + self.assertTrue(capl.has_capability('resourcelist')) + self.assertEqual(capl.capability_info('resourcelist').uri, + 'http://example.com/dataset1/resourcelist.xml') + self.assertTrue(capl.has_capability('resourcedump')) + self.assertEqual(capl.capability_info('resourcedump').uri, + 'http://example.com/dataset1/resourcedump.xml') + self.assertTrue(capl.has_capability('changelist')) + self.assertEqual(capl.capability_info('changelist').uri, + 'http://example.com/dataset1/changelist.xml') # Check some that aren't - self.assertFalse( capl.has_capability() ) - self.assertFalse( capl.has_capability('bogus') ) - self.assertFalse( capl.has_capability('capabilitylist') ) + self.assertFalse(capl.has_capability()) + self.assertFalse(capl.has_capability('bogus')) + self.assertFalse(capl.has_capability('capabilitylist')) def text_ex_07(self): """resourcesync_ex_7 is a source description that list a single Capability List""" - sd=SourceDescription() + sd = SourceDescription() sd.read(uri='tests/testdata/examples_from_spec/resourcesync_ex_7.xml') - self.assertEqual( len(sd.resources), 1, '1 capability list' ) - cl=sd.resources[0] - self.assertEqual( cl.uri, 'http://example.com/dataset1/capabilitylist.xml' ) - self.assertEqual( cl.capability, 'resourcelist' ) - self.assertEqual( cl.describedby, 'http://example.com/info_about_set1_of_resources.xml' ) + self.assertEqual(len(sd.resources), 1, '1 capability list') + cl = sd.resources[0] + self.assertEqual( + cl.uri, 'http://example.com/dataset1/capabilitylist.xml') + self.assertEqual(cl.capability, 'resourcelist') + self.assertEqual( + cl.describedby, 'http://example.com/info_about_set1_of_resources.xml') def test_ex_08(self): """resourcesync_ex_8 is a simple Resource List Index with 2 Resource Lists""" - rl=ResourceList() - rl.read(uri='tests/testdata/examples_from_spec/resourcesync_ex_8.xml',index_only=True) - self.assertEqual( rl.capability, 'resourcelist' ) - self.assertEqual( rl.md_at, '2013-01-03T09:00:00Z' ) - self.assertEqual( len(rl.resources), 2, '2 resources') + rl = ResourceList() + rl.read( + uri='tests/testdata/examples_from_spec/resourcesync_ex_8.xml', index_only=True) + self.assertEqual(rl.capability, 'resourcelist') + self.assertEqual(rl.md_at, '2013-01-03T09:00:00Z') + self.assertEqual(len(rl.resources), 2, '2 resources') sms = sorted(rl.uris()) - self.assertEqual( sms, ['http://example.com/resourcelist-part1.xml', - 'http://example.com/resourcelist-part2.xml'] ) + self.assertEqual(sms, ['http://example.com/resourcelist-part1.xml', + 'http://example.com/resourcelist-part2.xml']) # Examples 9, 10, 11 in the spec are not XML documents def test_ex_12(self): """resourcesync_ex_12 is a Source Description that talks about 3 sets of resources""" - sd=SourceDescription() + sd = SourceDescription() sd.read(uri='tests/testdata/examples_from_spec/resourcesync_ex_12.xml') - self.assertEqual( len(sd), 3 ) - self.assertEqual( sd.uris(), ['http://example.com/capabilitylist1.xml', - 'http://example.com/capabilitylist2.xml', - 'http://example.com/capabilitylist3.xml'] ) - cl1=sd['http://example.com/capabilitylist1.xml'] - self.assertEqual( cl1.capability, 'capabilitylist' ) - self.assertEqual( cl1.describedby, 'http://example.com/info_about_set1_of_resources.xml') + self.assertEqual(len(sd), 3) + self.assertEqual(sd.uris(), ['http://example.com/capabilitylist1.xml', + 'http://example.com/capabilitylist2.xml', + 'http://example.com/capabilitylist3.xml']) + cl1 = sd['http://example.com/capabilitylist1.xml'] + self.assertEqual(cl1.capability, 'capabilitylist') + self.assertEqual( + cl1.describedby, 'http://example.com/info_about_set1_of_resources.xml') -##### BUILD EXAMPLES ##### +# BUILD EXAMPLES --------------------------- def test_build_ex_01(self): """Simple Resource List document """ rl = ResourceList() rl.md_at = '2013-01-03T09:00:00Z' - rl.add( Resource('http://example.com/res1') ) - rl.add( Resource('http://example.com/res2') ) + rl.add(Resource('http://example.com/res1')) + rl.add(Resource('http://example.com/res2')) ex_xml = self._open_ex('resourcesync_ex_1').read() - self._assert_xml_equal( rl.as_xml(), ex_xml ) + self._assert_xml_equal(rl.as_xml(), ex_xml) def test_build_ex_02(self): """Slightly more complex Resource List document """ rl = ResourceList() rl.md_at = '2013-01-03T09:00:00Z' - rl.add( Resource(uri='http://example.com/res1', - lastmod='2013-01-02T13:00:00Z', - md5='1584abdf8ebdc9802ac0c6a7402c03b6') ) - r2 = Resource(uri='http://example.com/res2', - lastmod='2013-01-02T14:00:00Z', - md5='1e0d5cb8ef6ba40c99b14c0237be735e') - r2.link_set(rel="duplicate",href="http://mirror.example.com/res2") - rl.add( r2 ) + rl.add(Resource(uri='http://example.com/res1', + lastmod='2013-01-02T13:00:00Z', + md5='1584abdf8ebdc9802ac0c6a7402c03b6')) + r2 = Resource(uri='http://example.com/res2', + lastmod='2013-01-02T14:00:00Z', + md5='1e0d5cb8ef6ba40c99b14c0237be735e') + r2.link_set(rel="duplicate", href="http://mirror.example.com/res2") + rl.add(r2) ex_xml = self._open_ex('resourcesync_ex_2').read() - self._assert_xml_equal( rl.as_xml(), ex_xml ) + self._assert_xml_equal(rl.as_xml(), ex_xml) def test_build_ex_03(self): """Simple Change List document """ cl = ChangeList() cl.md_from = '2013-01-02T00:00:00Z' - cl.md_until= '2013-01-03T00:00:00Z' - cl.add( Resource(uri='http://example.com/res2.pdf', - lastmod='2013-01-02T13:00:00Z', - change="updated") ) - cl.add( Resource(uri='http://example.com/res3.tiff', - lastmod='2013-01-02T18:00:00Z', - change='deleted') ) + cl.md_until = '2013-01-03T00:00:00Z' + cl.add(Resource(uri='http://example.com/res2.pdf', + lastmod='2013-01-02T13:00:00Z', + change="updated")) + cl.add(Resource(uri='http://example.com/res3.tiff', + lastmod='2013-01-02T18:00:00Z', + change='deleted')) ex_xml = self._open_ex('resourcesync_ex_3').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_04(self): """Simple Resource Dump document """ rd = ResourceDump() rd.md_at = '2013-01-03T09:00:00Z' - rd.add( Resource(uri='http://example.com/resourcedump.zip', - lastmod='2013-01-03T09:00:00Z') ) + rd.add(Resource(uri='http://example.com/resourcedump.zip', + lastmod='2013-01-03T09:00:00Z')) ex_xml = self._open_ex('resourcesync_ex_4').read() - self._assert_xml_equal( rd.as_xml(), ex_xml ) + self._assert_xml_equal(rd.as_xml(), ex_xml) def test_build_ex_05(self): """Simple Resource Dump Manifest document """ rdm = ResourceDumpManifest() rdm.md_at = '2013-01-03T09:00:00Z' - rdm.add( Resource(uri='http://example.com/res1', - lastmod='2013-01-03T03:00:00Z', - md5='1584abdf8ebdc9802ac0c6a7402c03b6', - path='/resources/res1') ) - rdm.add( Resource(uri='http://example.com/res2', - lastmod='2013-01-03T04:00:00Z', - md5='1e0d5cb8ef6ba40c99b14c0237be735e', - path='/resources/res2') ) + rdm.add(Resource(uri='http://example.com/res1', + lastmod='2013-01-03T03:00:00Z', + md5='1584abdf8ebdc9802ac0c6a7402c03b6', + path='/resources/res1')) + rdm.add(Resource(uri='http://example.com/res2', + lastmod='2013-01-03T04:00:00Z', + md5='1e0d5cb8ef6ba40c99b14c0237be735e', + path='/resources/res2')) ex_xml = self._open_ex('resourcesync_ex_5').read() - self._assert_xml_equal( rdm.as_xml(), ex_xml ) + self._assert_xml_equal(rdm.as_xml(), ex_xml) def test_build_ex_06(self): """Simple Capability List document """ cl = CapabilityList() cl.describedby = 'http://example.com/info_about_set1_of_resources.xml' cl.up = 'http://example.com/resourcesync_description.xml' - cl.add_capability( uri='http://example.com/dataset1/resourcelist.xml', name='resourcelist' ) - cl.add_capability( uri='http://example.com/dataset1/resourcedump.xml', name='resourcedump' ) - cl.add_capability( uri='http://example.com/dataset1/changelist.xml', name='changelist' ) + cl.add_capability( + uri='http://example.com/dataset1/resourcelist.xml', name='resourcelist') + cl.add_capability( + uri='http://example.com/dataset1/resourcedump.xml', name='resourcedump') + cl.add_capability( + uri='http://example.com/dataset1/changelist.xml', name='changelist') ex_xml = self._open_ex('resourcesync_ex_6').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_07(self): """A Source Description document """ sd = SourceDescription() sd.describedby = 'http://example.com/info-about-source.xml' - r = Resource( uri='http://example.com/dataset1/capabilitylist.xml', - capability='capabilitylist' ) - r.link_set( rel='describedby', - href='http://example.com/info_about_set1_of_resources.xml' ) - sd.add( r ) + r = Resource(uri='http://example.com/dataset1/capabilitylist.xml', + capability='capabilitylist') + r.link_set(rel='describedby', + href='http://example.com/info_about_set1_of_resources.xml') + sd.add(r) ex_xml = self._open_ex('resourcesync_ex_7').read() - self._assert_xml_equal( sd.as_xml(), ex_xml ) + self._assert_xml_equal(sd.as_xml(), ex_xml) def test_build_ex_08(self): """Simple Resource List Index document - - This is not something that would usually be created directly - but instead would be created as part of the process of + + This is not something that would usually be created directly + but instead would be created as part of the process of writing a large Resource List in multiple files. However, it is possible to create manually. """ rli = ResourceList() - rli.sitemapindex=True + rli.sitemapindex = True rli.md_at = '2013-01-03T09:00:00Z' - rli.add( Resource(uri='http://example.com/resourcelist-part1.xml') ) - rli.add( Resource(uri='http://example.com/resourcelist-part2.xml') ) + rli.add(Resource(uri='http://example.com/resourcelist-part1.xml')) + rli.add(Resource(uri='http://example.com/resourcelist-part2.xml')) ex_xml = self._open_ex('resourcesync_ex_8').read() - self._assert_xml_equal( rli.as_xml(), ex_xml ) + self._assert_xml_equal(rli.as_xml(), ex_xml) # Examples 9, 10, 11 in the spec are not XML documents @@ -256,262 +282,268 @@ class TestExamplesFromSpec(unittest.TestCase): """Source Description document with describedby links""" sd = SourceDescription() sd.describedby = 'http://example.com/info_about_source.xml' - cl1 = CapabilityList( uri='http://example.com/capabilitylist1.xml' ) + cl1 = CapabilityList(uri='http://example.com/capabilitylist1.xml') cl1.describedby = 'http://example.com/info_about_set1_of_resources.xml' - sd.add_capability_list( cl1 ) - cl2 = CapabilityList( uri='http://example.com/capabilitylist2.xml' ) + sd.add_capability_list(cl1) + cl2 = CapabilityList(uri='http://example.com/capabilitylist2.xml') cl2.describedby = 'http://example.com/info_about_set2_of_resources.xml' - sd.add_capability_list( cl2 ) - cl3 = CapabilityList( uri='http://example.com/capabilitylist3.xml' ) + sd.add_capability_list(cl2) + cl3 = CapabilityList(uri='http://example.com/capabilitylist3.xml') cl3.describedby = 'http://example.com/info_about_set3_of_resources.xml' - sd.add_capability_list( cl3 ) + sd.add_capability_list(cl3) ex_xml = self._open_ex('resourcesync_ex_12').read() - self._assert_xml_equal( sd.as_xml(), ex_xml ) + self._assert_xml_equal(sd.as_xml(), ex_xml) def test_build_ex_13(self): """Capability List document with 4 entries""" cl = CapabilityList() cl.describedby = 'http://example.com/info_about_set1_of_resources.xml' cl.up = 'http://example.com/resourcesync_description.xml' - cl.add_capability( capability=ResourceList( uri='http://example.com/dataset1/resourcelist.xml' ) ) - cl.add_capability( capability=ResourceDump( uri='http://example.com/dataset1/resourcedump.xml' ) ) - cl.add_capability( capability=ChangeList( uri='http://example.com/dataset1/changelist.xml' ) ) - cl.add_capability( capability=ChangeDump( uri='http://example.com/dataset1/changedump.xml' ) ) + cl.add_capability(capability=ResourceList( + uri='http://example.com/dataset1/resourcelist.xml')) + cl.add_capability(capability=ResourceDump( + uri='http://example.com/dataset1/resourcedump.xml')) + cl.add_capability(capability=ChangeList( + uri='http://example.com/dataset1/changelist.xml')) + cl.add_capability(capability=ChangeDump( + uri='http://example.com/dataset1/changedump.xml')) ex_xml = self._open_ex('resourcesync_ex_13').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_14(self): """Resource List with 2 entries and some metadata""" rl = ResourceList() - rl.up='http://example.com/dataset1/capabilitylist.xml' - rl.md_at="2013-01-03T09:00:00Z" - rl.md_completed="2013-01-03T09:01:00Z" - rl.add( Resource( uri='http://example.com/res1', - lastmod='2013-01-02T13:00:00Z', - md5='1584abdf8ebdc9802ac0c6a7402c03b6', - length=8876, - mime_type="text/html" )) - rl.add( Resource( uri='http://example.com/res2', - lastmod='2013-01-02T14:00:00Z', - md5='1e0d5cb8ef6ba40c99b14c0237be735e', - sha256='854f61290e2e197a11bc91063afce22e43f8ccc655237050ace766adc68dc784', - length=14599, - mime_type="application/pdf" )) + rl.up = 'http://example.com/dataset1/capabilitylist.xml' + rl.md_at = "2013-01-03T09:00:00Z" + rl.md_completed = "2013-01-03T09:01:00Z" + rl.add(Resource(uri='http://example.com/res1', + lastmod='2013-01-02T13:00:00Z', + md5='1584abdf8ebdc9802ac0c6a7402c03b6', + length=8876, + mime_type="text/html")) + rl.add(Resource(uri='http://example.com/res2', + lastmod='2013-01-02T14:00:00Z', + md5='1e0d5cb8ef6ba40c99b14c0237be735e', + sha256='854f61290e2e197a11bc91063afce22e43f8ccc655237050ace766adc68dc784', + length=14599, + mime_type="application/pdf")) ex_xml = self._open_ex('resourcesync_ex_14').read() - self._assert_xml_equal( rl.as_xml(), ex_xml ) + self._assert_xml_equal(rl.as_xml(), ex_xml) def test_build_ex_15(self): """Resource List Index with metadata""" - rl = ResourceList(resources_class=ResourceListOrdered) #order in example is non-canonical - rl.sitemapindex=True - rl.up='http://example.com/dataset1/capabilitylist.xml' - rl.md_at="2013-01-03T09:00:00Z" - rl.md_completed="2013-01-03T09:10:00Z" - rl.add( Resource( uri='http://example.com/resourcelist1.xml', - md_at='2013-01-03T09:00:00Z' )) - rl.add( Resource( uri='http://example.com/resourcelist2.xml', - md_at='2013-01-03T09:03:00Z' )) - rl.add( Resource( uri='http://example.com/resourcelist3.xml', - md_at='2013-01-03T09:07:00Z' )) + rl = ResourceList( + resources_class=ResourceListOrdered) # order in example is non-canonical + rl.sitemapindex = True + rl.up = 'http://example.com/dataset1/capabilitylist.xml' + rl.md_at = "2013-01-03T09:00:00Z" + rl.md_completed = "2013-01-03T09:10:00Z" + rl.add(Resource(uri='http://example.com/resourcelist1.xml', + md_at='2013-01-03T09:00:00Z')) + rl.add(Resource(uri='http://example.com/resourcelist2.xml', + md_at='2013-01-03T09:03:00Z')) + rl.add(Resource(uri='http://example.com/resourcelist3.xml', + md_at='2013-01-03T09:07:00Z')) ex_xml = self._open_ex('resourcesync_ex_15').read() - self._assert_xml_equal( rl.as_xml(), ex_xml ) + self._assert_xml_equal(rl.as_xml(), ex_xml) def test_build_ex_16(self): rl = ResourceList() rl.up = 'http://example.com/dataset1/capabilitylist.xml' rl.index = 'http://example.com/dataset1/resourcelist-index.xml' - rl.md_at="2013-01-03T09:00:00Z" - rl.add( Resource( uri='http://example.com/res3', - lastmod='2013-01-02T13:00:00Z', - md5='1584abdf8ebdc9802ac0c6a7402c8753', - length=4385, - mime_type="application/pdf" )) - rl.add( Resource( uri='http://example.com/res4', - lastmod='2013-01-02T14:00:00Z', - md5='4556abdf8ebdc9802ac0c6a7402c9881', - length=883, - mime_type="image/png" )) + rl.md_at = "2013-01-03T09:00:00Z" + rl.add(Resource(uri='http://example.com/res3', + lastmod='2013-01-02T13:00:00Z', + md5='1584abdf8ebdc9802ac0c6a7402c8753', + length=4385, + mime_type="application/pdf")) + rl.add(Resource(uri='http://example.com/res4', + lastmod='2013-01-02T14:00:00Z', + md5='4556abdf8ebdc9802ac0c6a7402c9881', + length=883, + mime_type="image/png")) ex_xml = self._open_ex('resourcesync_ex_16').read() - self._assert_xml_equal( rl.as_xml(), ex_xml ) + self._assert_xml_equal(rl.as_xml(), ex_xml) def test_build_ex_17(self): """Resource Dump with 3 entries and some metadata""" rd = ResourceDump() - rd.up='http://example.com/dataset1/capabilitylist.xml' - rd.md_at="2013-01-03T09:00:00Z" - rd.md_completed="2013-01-03T09:04:00Z" - z1 = Resource( uri='http://example.com/resourcedump-part1.zip', - mime_type="application/zip", - length=4765, - md_at="2013-01-03T09:00:00Z", - md_completed="2013-01-03T09:02:00Z" ) - z1.link_set( rel="contents", - href="http://example.com/resourcedump_manifest-part1.xml", - mime_type="application/xml" ) - rd.add( z1 ) - z2 = Resource( uri='http://example.com/resourcedump-part2.zip', - mime_type="application/zip", - length=9875, - md_at="2013-01-03T09:01:00Z", - md_completed="2013-01-03T09:03:00Z" ) - z2.link_set( rel="contents", - href="http://example.com/resourcedump_manifest-part2.xml", - mime_type="application/xml" ) - rd.add( z2 ) - z3 = Resource( uri='http://example.com/resourcedump-part3.zip', - mime_type="application/zip", - length=2298, - md_at="2013-01-03T09:03:00Z", - md_completed="2013-01-03T09:04:00Z" ) - z3.link_set( rel="contents", - href="http://example.com/resourcedump_manifest-part3.xml", - mime_type="application/xml" ) - rd.add( z3 ) + rd.up = 'http://example.com/dataset1/capabilitylist.xml' + rd.md_at = "2013-01-03T09:00:00Z" + rd.md_completed = "2013-01-03T09:04:00Z" + z1 = Resource(uri='http://example.com/resourcedump-part1.zip', + mime_type="application/zip", + length=4765, + md_at="2013-01-03T09:00:00Z", + md_completed="2013-01-03T09:02:00Z") + z1.link_set(rel="contents", + href="http://example.com/resourcedump_manifest-part1.xml", + mime_type="application/xml") + rd.add(z1) + z2 = Resource(uri='http://example.com/resourcedump-part2.zip', + mime_type="application/zip", + length=9875, + md_at="2013-01-03T09:01:00Z", + md_completed="2013-01-03T09:03:00Z") + z2.link_set(rel="contents", + href="http://example.com/resourcedump_manifest-part2.xml", + mime_type="application/xml") + rd.add(z2) + z3 = Resource(uri='http://example.com/resourcedump-part3.zip', + mime_type="application/zip", + length=2298, + md_at="2013-01-03T09:03:00Z", + md_completed="2013-01-03T09:04:00Z") + z3.link_set(rel="contents", + href="http://example.com/resourcedump_manifest-part3.xml", + mime_type="application/xml") + rd.add(z3) ex_xml = self._open_ex('resourcesync_ex_17').read() - self._assert_xml_equal( rd.as_xml(), ex_xml ) + self._assert_xml_equal(rd.as_xml(), ex_xml) def test_build_ex_18(self): """Resource Dump Manifest with 2 entries and some metadata""" rdm = ResourceDumpManifest() - rdm.up='http://example.com/dataset1/capabilitylist.xml' - rdm.md_at="2013-01-03T09:00:00Z" - rdm.md_completed="2013-01-03T09:02:00Z" - rdm.add( Resource( uri='http://example.com/res1', - lastmod='2013-01-02T13:00:00Z', - md5='1584abdf8ebdc9802ac0c6a7402c03b6', - length=8876, - mime_type='text/html', - path='/resources/res1') ) - rdm.add( Resource( uri='http://example.com/res2', - lastmod='2013-01-02T14:00:00Z', - md5='1e0d5cb8ef6ba40c99b14c0237be735e', - sha256='854f61290e2e197a11bc91063afce22e43f8ccc655237050ace766adc68dc784', - length=14599, - mime_type='application/pdf', - path='/resources/res2') ) + rdm.up = 'http://example.com/dataset1/capabilitylist.xml' + rdm.md_at = "2013-01-03T09:00:00Z" + rdm.md_completed = "2013-01-03T09:02:00Z" + rdm.add(Resource(uri='http://example.com/res1', + lastmod='2013-01-02T13:00:00Z', + md5='1584abdf8ebdc9802ac0c6a7402c03b6', + length=8876, + mime_type='text/html', + path='/resources/res1')) + rdm.add(Resource(uri='http://example.com/res2', + lastmod='2013-01-02T14:00:00Z', + md5='1e0d5cb8ef6ba40c99b14c0237be735e', + sha256='854f61290e2e197a11bc91063afce22e43f8ccc655237050ace766adc68dc784', + length=14599, + mime_type='application/pdf', + path='/resources/res2')) ex_xml = self._open_ex('resourcesync_ex_18').read() - self._assert_xml_equal( rdm.as_xml(), ex_xml ) + self._assert_xml_equal(rdm.as_xml(), ex_xml) def test_build_ex_19(self): """Change List with 4 changes, 'open' as no until""" cl = ChangeList() cl.up = 'http://example.com/dataset1/capabilitylist.xml' - cl.md_from="2013-01-03T00:00:00Z" - cl.add( Resource( uri='http://example.com/res1.html', - lastmod='2013-01-03T11:00:00Z', - change='created' ) ) - cl.add( Resource( uri='http://example.com/res2.pdf', - lastmod='2013-01-03T13:00:00Z', - change='updated' ) ) - cl.add( Resource( uri='http://example.com/res3.tiff', - lastmod='2013-01-03T18:00:00Z', - change='deleted' ) ) - cl.add( Resource( uri='http://example.com/res2.pdf', - lastmod='2013-01-03T21:00:00Z', - change='updated' ) ) + cl.md_from = "2013-01-03T00:00:00Z" + cl.add(Resource(uri='http://example.com/res1.html', + lastmod='2013-01-03T11:00:00Z', + change='created')) + cl.add(Resource(uri='http://example.com/res2.pdf', + lastmod='2013-01-03T13:00:00Z', + change='updated')) + cl.add(Resource(uri='http://example.com/res3.tiff', + lastmod='2013-01-03T18:00:00Z', + change='deleted')) + cl.add(Resource(uri='http://example.com/res2.pdf', + lastmod='2013-01-03T21:00:00Z', + change='updated')) ex_xml = self._open_ex('resourcesync_ex_19').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_20(self): """Change List Index listing 3 Change Lists, the last one 'open'""" - cl = ChangeListArchive(resources_class=ResourceListOrdered) #order in example is non-canonical - cl.sitemapindex=True - cl.capability_name='changelist' + cl = ChangeListArchive( + resources_class=ResourceListOrdered) # order in example is non-canonical + cl.sitemapindex = True + cl.capability_name = 'changelist' cl.up = 'http://example.com/dataset1/capabilitylist.xml' - cl.md_from="2013-01-01T00:00:00Z" - cl.add( Resource( uri='http://example.com/20130101-changelist.xml', - md_from='2013-01-01T00:00:00Z', - md_until='2013-01-02T00:00:00Z') ) - cl.add( Resource( uri='http://example.com/20130102-changelist.xml', - md_from='2013-01-02T00:00:00Z', - md_until='2013-01-03T00:00:00Z') ) - cl.add( Resource( uri='http://example.com/20130103-changelist.xml', - md_from='2013-01-03T00:00:00Z') ) + cl.md_from = "2013-01-01T00:00:00Z" + cl.add(Resource(uri='http://example.com/20130101-changelist.xml', + md_from='2013-01-01T00:00:00Z', + md_until='2013-01-02T00:00:00Z')) + cl.add(Resource(uri='http://example.com/20130102-changelist.xml', + md_from='2013-01-02T00:00:00Z', + md_until='2013-01-03T00:00:00Z')) + cl.add(Resource(uri='http://example.com/20130103-changelist.xml', + md_from='2013-01-03T00:00:00Z')) ex_xml = self._open_ex('resourcesync_ex_20').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_21(self): """Change List which points back to index""" cl = ChangeList() cl.up = 'http://example.com/dataset1/capabilitylist.xml' cl.index = 'http://example.com/dataset1/changelist.xml' - cl.md_from="2013-01-02T00:00:00Z" - cl.md_until="2013-01-03T00:00:00Z" - cl.add( Resource( uri='http://example.com/res7.html', - lastmod='2013-01-02T12:00:00Z', - change='created' ) ) - cl.add( Resource( uri='http://example.com/res9.pdf', - lastmod='2013-01-02T13:00:00Z', - change='updated' ) ) - cl.add( Resource( uri='http://example.com/res5.tiff', - lastmod='2013-01-02T19:00:00Z', - change='deleted' ) ) - cl.add( Resource( uri='http://example.com/res7.html', - lastmod='2013-01-02T20:00:00Z', - change='updated' ) ) + cl.md_from = "2013-01-02T00:00:00Z" + cl.md_until = "2013-01-03T00:00:00Z" + cl.add(Resource(uri='http://example.com/res7.html', + lastmod='2013-01-02T12:00:00Z', + change='created')) + cl.add(Resource(uri='http://example.com/res9.pdf', + lastmod='2013-01-02T13:00:00Z', + change='updated')) + cl.add(Resource(uri='http://example.com/res5.tiff', + lastmod='2013-01-02T19:00:00Z', + change='deleted')) + cl.add(Resource(uri='http://example.com/res7.html', + lastmod='2013-01-02T20:00:00Z', + change='updated')) ex_xml = self._open_ex('resourcesync_ex_21').read() - self._assert_xml_equal( cl.as_xml(), ex_xml ) + self._assert_xml_equal(cl.as_xml(), ex_xml) def test_build_ex_22(self): """Change Dump with three dump files""" cd = ChangeDump() cd.up = 'http://example.com/dataset1/capabilitylist.xml' - cd.md_from="2013-01-01T00:00:00Z" - z1 = Resource( uri='http://example.com/20130101-changedump.zip', - lastmod='2013-01-01T23:59:59Z', - length=3109, - md_from="2013-01-01T00:00:00Z", - md_until="2013-01-02T00:00:00Z", - mime_type="application/zip" ) - z1.contents='http://example.com/20130101-changedump-manifest.xml' - z2 = Resource( uri='http://example.com/20130102-changedump.zip', - lastmod='2013-01-02T23:59:59Z', - length=6629, - md_from="2013-01-02T00:00:00Z", - md_until="2013-01-03T00:00:00Z", - mime_type="application/zip" ) - z2.contents='http://example.com/20130102-changedump-manifest.xml' - z3 = Resource( uri='http://example.com/20130103-changedump.zip', - lastmod='2013-01-03T23:59:59Z', - length=8124, - md_from="2013-01-03T00:00:00Z", - md_until="2013-01-04T00:00:00Z", - mime_type="application/zip" ) - z3.contents='http://example.com/20130103-changedump-manifest.xml' - cd.add( [z1, z2, z3] ) + cd.md_from = "2013-01-01T00:00:00Z" + z1 = Resource(uri='http://example.com/20130101-changedump.zip', + lastmod='2013-01-01T23:59:59Z', + length=3109, + md_from="2013-01-01T00:00:00Z", + md_until="2013-01-02T00:00:00Z", + mime_type="application/zip") + z1.contents = 'http://example.com/20130101-changedump-manifest.xml' + z2 = Resource(uri='http://example.com/20130102-changedump.zip', + lastmod='2013-01-02T23:59:59Z', + length=6629, + md_from="2013-01-02T00:00:00Z", + md_until="2013-01-03T00:00:00Z", + mime_type="application/zip") + z2.contents = 'http://example.com/20130102-changedump-manifest.xml' + z3 = Resource(uri='http://example.com/20130103-changedump.zip', + lastmod='2013-01-03T23:59:59Z', + length=8124, + md_from="2013-01-03T00:00:00Z", + md_until="2013-01-04T00:00:00Z", + mime_type="application/zip") + z3.contents = 'http://example.com/20130103-changedump-manifest.xml' + cd.add([z1, z2, z3]) ex_xml = self._open_ex('resourcesync_ex_22').read() - self._assert_xml_equal( cd.as_xml(), ex_xml ) + self._assert_xml_equal(cd.as_xml(), ex_xml) def test_build_ex_23(self): cdm = ChangeDumpManifest() cdm.up = "http://example.com/dataset1/capabilitylist.xml" cdm.md_from = "2013-01-02T00:00:00Z" cdm.md_until = "2013-01-03T00:00:00Z" - cdm.add( Resource(uri="http://example.com/res7.html", - lastmod="2013-01-02T12:00:00Z", - change="created", - md5="1c1b0e264fa9b7e1e9aa6f9db8d6362b", - length=4339, - mime_type="text/html", - path="/changes/res7.html") ) - cdm.add( Resource(uri="http://example.com/res9.pdf", - lastmod="2013-01-02T13:00:00Z", - change="updated", - md5="f906610c3d4aa745cb2b986f25b37c5a", - length=38297, - mime_type="application/pdf", - path="/changes/res9.pdf") ) - cdm.add( Resource(uri="http://example.com/res5.tiff", - lastmod="2013-01-02T19:00:00Z", - change="deleted") ) - cdm.add( Resource(uri="http://example.com/res7.html", - lastmod="2013-01-02T20:00:00Z", - change="updated", - md5="0988647082c8bc51778894a48ec3b576", - length="5426", #should also take string - mime_type="text/html", - path="/changes/res7-v2.html") ) - self._assert_xml_equal_ex( cdm.as_xml(), 'resourcesync_ex_23' ) + cdm.add(Resource(uri="http://example.com/res7.html", + lastmod="2013-01-02T12:00:00Z", + change="created", + md5="1c1b0e264fa9b7e1e9aa6f9db8d6362b", + length=4339, + mime_type="text/html", + path="/changes/res7.html")) + cdm.add(Resource(uri="http://example.com/res9.pdf", + lastmod="2013-01-02T13:00:00Z", + change="updated", + md5="f906610c3d4aa745cb2b986f25b37c5a", + length=38297, + mime_type="application/pdf", + path="/changes/res9.pdf")) + cdm.add(Resource(uri="http://example.com/res5.tiff", + lastmod="2013-01-02T19:00:00Z", + change="deleted")) + cdm.add(Resource(uri="http://example.com/res7.html", + lastmod="2013-01-02T20:00:00Z", + change="updated", + md5="0988647082c8bc51778894a48ec3b576", + length="5426", # should also take string + mime_type="text/html", + path="/changes/res7-v2.html")) + self._assert_xml_equal_ex(cdm.as_xml(), 'resourcesync_ex_23') def test_build_ex_24(self): cl = ChangeList() @@ -523,8 +555,8 @@ class TestExamplesFromSpec(unittest.TestCase): md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, mime_type="text/html") - # Resource.link_set with add or change link depending on one with - # the particular rel exists unless allow_duplicates=True. + # Resource.link_set with add or change link depending on one with + # the particular rel exists unless allow_duplicates=True. # Resource.link_add will always add. Test both here... c1.link_set(rel="duplicate", href="http://mirror1.example.com/res1", @@ -539,8 +571,8 @@ class TestExamplesFromSpec(unittest.TestCase): href="gsiftp://gridftp.example.com/res1", pri="3", modified="2013-01-03T18:00:00Z") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_24' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_24') def test_build_ex_25(self): cl = ChangeList() @@ -552,13 +584,13 @@ class TestExamplesFromSpec(unittest.TestCase): c1.link_add(rel="alternate", href="http://example.com/res1.html", modified="2013-01-03T18:00:00Z", - type="text/html") #FIXME - inconsistent + type="text/html") # FIXME - inconsistent c1.link_add(rel="alternate", href="http://example.com/res1.pdf", modified="2013-01-03T18:00:00Z", type="application/pdf") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_25' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_25') def test_build_ex_26(self): cl = ChangeList() @@ -568,12 +600,12 @@ class TestExamplesFromSpec(unittest.TestCase): lastmod="2013-01-03T18:00:00Z", change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", - length=8876 ) + length=8876) c1.link_add(rel="canonical", href="http://example.com/res1", modified="2013-01-03T18:00:00Z") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_26' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_26') def test_build_ex_27(self): cl = ChangeList() @@ -584,11 +616,11 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", sha256="f4OxZX_x_DFGFDgghgdfb6rtSx-iosjf6735432nklj", length=56778, - mime_type="application/json" ) + mime_type="application/json") c1.link_set(rel="http://www.openarchives.org/rs/terms/patch", href="http://example.com/res4-json-patch", modified="2013-01-03T17:00:00Z", - hash="sha-256:y66dER_t_HWEIKpesdkeb7rtSc-ippjf9823742opld", #FIXME - inconsistent + hash="sha-256:y66dER_t_HWEIKpesdkeb7rtSc-ippjf9823742opld", # FIXME - inconsistent length=73, type="application/json-patch") c2 = Resource(uri="http://example.com/res5-full.tiff", @@ -602,9 +634,9 @@ class TestExamplesFromSpec(unittest.TestCase): modified="2013-01-03T18:00:00Z", hash="sha-256:h986gT_t_87HTkjHYE76G558hY-jdfgy76t55sadJUYT", length=4533, - type="application/x-tiff-diff" ) - cl.add( [c1,c2] ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_27' ) + type="application/x-tiff-diff") + cl.add([c1, c2]) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_27') def test_build_ex_28(self): cl = ChangeList() @@ -615,7 +647,7 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="application/pdf" ) + mime_type="application/pdf") c1.link_set(rel="describedby", href="http://example.com/res2_dublin-core_metadata.xml", modified="2013-01-01T12:00:00Z", @@ -632,8 +664,8 @@ class TestExamplesFromSpec(unittest.TestCase): type="application/pdf") c2.link_set(rel="profile", href="http://purl.org/dc/elements/1.1/") - cl.add( [c1,c2] ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_28' ) + cl.add([c1, c2]) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_28') def test_build_ex_29(self): cl = ChangeList() @@ -644,7 +676,7 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="text/html" ) + mime_type="text/html") c1.link_add(rel="memento", href="http://example.com/20130103070000/res1", modified="2013-01-02T18:00:00Z", @@ -656,8 +688,8 @@ class TestExamplesFromSpec(unittest.TestCase): c1.link_add(rel="timemap", href="http://example.com/timemap/http://example.com/res1", type="application/link-format") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_29' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_29') def test_build_ex_30(self): cl = ChangeList() @@ -668,11 +700,11 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="text/html" ) + mime_type="text/html") c1.link_add(rel="collection", href="http://example.com/aggregation/0601007") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_30' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_30') def test_build_ex_31(self): cl = ChangeList() @@ -683,9 +715,9 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="text/html" ) - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_31' ) + mime_type="text/html") + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_31') def test_build_ex_32(self): cl = ChangeList() @@ -696,15 +728,15 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="text/html" ) + mime_type="text/html") c1.link_add(rel="via", href="http://original.example.com/res1.html", modified="2013-01-03T07:00:00Z", hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6", length="8876", type="text/html") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_32' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_32') def test_build_ex_33(self): cl = ChangeList() @@ -715,140 +747,142 @@ class TestExamplesFromSpec(unittest.TestCase): change="updated", md5="1584abdf8ebdc9802ac0c6a7402c03b6", length=8876, - mime_type="text/html" ) + mime_type="text/html") c1.link_add(rel="via", href="http://original.example.com/res1.html", modified="2013-01-03T07:00:00Z", hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6", length="8876", type="text/html") - cl.add( c1 ) - self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_33' ) + cl.add(c1) + self._assert_xml_equal_ex(cl.as_xml(), 'resourcesync_ex_33') - ##### Archives tests + # Archives tests def test_build_archives_ex_3_1(self): """Resource List Archive listing 3 Resource Lists""" rla = ResourceListArchive() rla.up = 'http://example.com/dataset1/capabilitylist.xml' - rla.add( Resource( uri='http://example.com/resourcelist1.xml', - md_at='2012-11-03T09:00:00Z') ) - rla.add( Resource( uri='http://example.com/resourcelist2.xml', - md_at='2012-12-03T09:00:00Z') ) - rla.add( Resource( uri='http://example.com/resourcelist3.xml', - md_at='2013-01-03T09:00:00Z') ) + rla.add(Resource(uri='http://example.com/resourcelist1.xml', + md_at='2012-11-03T09:00:00Z')) + rla.add(Resource(uri='http://example.com/resourcelist2.xml', + md_at='2012-12-03T09:00:00Z')) + rla.add(Resource(uri='http://example.com/resourcelist3.xml', + md_at='2013-01-03T09:00:00Z')) ex_xml = self._open_ex('archives_ex_3_1').read() - self._assert_xml_equal( rla.as_xml(), ex_xml ) + self._assert_xml_equal(rla.as_xml(), ex_xml) def test_build_archives_ex_3_2(self): """Resource List Archive Index listing 2 component Resource List Archives""" rlai = ResourceListArchive() rlai.sitemapindex = True rlai.up = 'http://example.com/dataset1/capabilitylist.xml' - rlai.add( Resource( uri='http://example.com/resourcelistarchive00001.xml' )) - rlai.add( Resource( uri='http://example.com/resourcelistarchive00002.xml' )) + rlai.add(Resource(uri='http://example.com/resourcelistarchive00001.xml')) + rlai.add(Resource(uri='http://example.com/resourcelistarchive00002.xml')) ex_xml = self._open_ex('archives_ex_3_2').read() - self._assert_xml_equal( rlai.as_xml(), ex_xml ) + self._assert_xml_equal(rlai.as_xml(), ex_xml) def test_build_archives_ex_4_1(self): """Resource Dump Archive listing 2 Resource Dumps""" rda = ResourceDumpArchive() rda.up = 'http://example.com/dataset1/capabilitylist.xml' - rda.add( Resource( uri='http://example.com/resourcedump1.xml', - lastmod='2012-11-03T09:05:42Z', - md_at="2012-11-03T09:00:00Z", - md_completed="2012-11-03T09:05:01Z" ) ) + rda.add(Resource(uri='http://example.com/resourcedump1.xml', + lastmod='2012-11-03T09:05:42Z', + md_at="2012-11-03T09:00:00Z", + md_completed="2012-11-03T09:05:01Z")) - rda.add( Resource( uri='http://example.com/resourcedump2.xml', - lastmod='2012-12-03T09:06:12Z', - md_at="2012-12-03T09:00:00Z", - md_completed="2012-12-03T09:05:17Z" ) ) + rda.add(Resource(uri='http://example.com/resourcedump2.xml', + lastmod='2012-12-03T09:06:12Z', + md_at="2012-12-03T09:00:00Z", + md_completed="2012-12-03T09:05:17Z")) ex_xml = self._open_ex('archives_ex_4_1').read() - self._assert_xml_equal( rda.as_xml(), ex_xml ) + self._assert_xml_equal(rda.as_xml(), ex_xml) def test_build_archives_ex_5_1(self): """Change List Archive listing 3 Change Lists""" cla = ChangeListArchive() cla.up = 'http://example.com/dataset1/capabilitylist.xml' - cla.add( Resource( uri='http://example.com/changelist1.xml', - md_from='2013-01-01T09:00:00Z', - md_until='2013-01-02T09:00:00Z') ) - cla.add( Resource( uri='http://example.com/changelist2.xml', - md_from='2013-01-02T09:00:00Z', - md_until='2013-01-03T09:00:00Z') ) - cla.add( Resource( uri='http://example.com/changelist3.xml', - md_from='2013-01-03T09:00:00Z', - md_until='2013-01-04T09:00:00Z') ) + cla.add(Resource(uri='http://example.com/changelist1.xml', + md_from='2013-01-01T09:00:00Z', + md_until='2013-01-02T09:00:00Z')) + cla.add(Resource(uri='http://example.com/changelist2.xml', + md_from='2013-01-02T09:00:00Z', + md_until='2013-01-03T09:00:00Z')) + cla.add(Resource(uri='http://example.com/changelist3.xml', + md_from='2013-01-03T09:00:00Z', + md_until='2013-01-04T09:00:00Z')) ex_xml = self._open_ex('archives_ex_5_1').read() - self._assert_xml_equal( cla.as_xml(), ex_xml ) + self._assert_xml_equal(cla.as_xml(), ex_xml) def test_build_archives_ex_6_1(self): """Change Dump Archive listing 2 Change Dumps""" cda = ChangeDumpArchive() cda.up = 'http://example.com/dataset1/capabilitylist.xml' - cda.add( Resource( uri='http://example.com/changedump-w1.xml', - lastmod='2012-12-20T09:02:43Z', - md_from="2012-01-13T09:00:00Z", - md_until="2013-01-20T09:00:00Z" ) ) - cda.add( Resource( uri='http://example.com/changedump-w2.xml', - lastmod='2012-12-27T09:01:57Z', - md_from="2012-01-20T09:00:00Z", - md_until="2013-01-27T09:00:00Z" ) ) + cda.add(Resource(uri='http://example.com/changedump-w1.xml', + lastmod='2012-12-20T09:02:43Z', + md_from="2012-01-13T09:00:00Z", + md_until="2013-01-20T09:00:00Z")) + cda.add(Resource(uri='http://example.com/changedump-w2.xml', + lastmod='2012-12-27T09:01:57Z', + md_from="2012-01-20T09:00:00Z", + md_until="2013-01-27T09:00:00Z")) ex_xml = self._open_ex('archives_ex_6_1').read() - self._assert_xml_equal( cda.as_xml(), ex_xml ) + self._assert_xml_equal(cda.as_xml(), ex_xml) -##### UTILITIES FOR (APPROX) COMPARISON OF XML IN EXAMPLES AND OUTPUT +# UTILITIES FOR (APPROX) COMPARISON OF XML IN EXAMPLES AND OUTPUT - def _assert_xml_equal_ex(self,xml,ex): + def _assert_xml_equal_ex(self, xml, ex): """Compare XML supplied with XML from example file ex""" ex_xml = self._open_ex(ex).read() - self._assert_xml_equal( xml, ex_xml ) + self._assert_xml_equal(xml, ex_xml) - def _assert_xml_equal(self,a,b): - context = "Element mismatch in\n%s\nvs\n%s\n" % (a,b); + def _assert_xml_equal(self, a, b): + context = "Element mismatch in\n%s\nvs\n%s\n" % (a, b) aa = self._xml_massage_split(a) bb = self._xml_massage_split(b) - ia=iter(aa) - ib=iter(bb) + ia = iter(aa) + ib = iter(bb) try: while (1): - self._assert_xml_elements_equal( self._xml_reorder_attributes(next(ia)), - self._xml_reorder_attributes(next(ib)), - context ) + self._assert_xml_elements_equal(self._xml_reorder_attributes(next(ia)), + self._xml_reorder_attributes( + next(ib)), + context) except StopIteration: # all is good provided there were the same number of elements pass - self.assertEqual( len(aa), len(bb), "Same length check\n%s" % (context) ) + self.assertEqual(len(aa), len(bb), "Same length check\n%s" % (context)) - def _assert_xml_elements_equal(self,a,b,context): - context = "Elements %s != %s\n%s" % (a,b,context) - self.assertEqual( a, b, context ) + def _assert_xml_elements_equal(self, a, b, context): + context = "Elements %s != %s\n%s" % (a, b, context) + self.assertEqual(a, b, context) - def _xml_reorder_attributes(self,xml): + def _xml_reorder_attributes(self, xml): """Manipulate string for single element with atts in alpha order - + This is a bit of a fudge because of pattern matching. Should give - correct match for all matches, but might give matches in rare cases + correct match for all matches, but might give matches in rare cases that should not. """ - return(' '.join( sorted( xml.split(' ') ) ) ) + return(' '.join(sorted(xml.split(' ')))) - def _xml_massage_split(self,xml): + def _xml_massage_split(self, xml): """Massage XML for comparison and split by elements (on >)""" - xml = re.sub( r'\s+$', '', xml) - xml = re.sub( r'^\s+', '', xml) - xml = re.sub( r'\s+', ' ', xml) - xml = re.sub( r'\s*/>', ' />', xml) #always one space before end of self-closing element - xml = re.sub( r'>\s+<', '><', xml) #remove space between elements + xml = re.sub(r'\s+$', '', xml) + xml = re.sub(r'^\s+', '', xml) + xml = re.sub(r'\s+', ' ', xml) + # always one space before end of self-closing element + xml = re.sub(r'\s*/>', ' />', xml) + xml = re.sub(r'>\s+<', '><', xml) # remove space between elements # FUDGES, need to check these are OK - xml = re.sub( r"version='1.0'", 'version="1.0"', xml ) - xml = re.sub( r"encoding='UTF-8'", 'encoding="UTF-8"', xml ) + xml = re.sub(r"version='1.0'", 'version="1.0"', xml) + xml = re.sub(r"encoding='UTF-8'", 'encoding="UTF-8"', xml) # return self.assertEqual( x, 'xx' ) - return( xml.split('>') ) + return(xml.split('>')) - def _open_ex(self,ex): - return open('tests/testdata/examples_from_spec/%s.xml'%(ex),'r') + def _open_ex(self, ex): + return open('tests/testdata/examples_from_spec/%s.xml' % (ex), 'r') -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestExamplesFromSpec) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_explorer.py b/tests/test_explorer.py index 052e4f2..17c16d8 100644 --- a/tests/test_explorer.py +++ b/tests/test_explorer.py @@ -21,23 +21,23 @@ class TestExplorer(unittest.TestCase): def test01_create(self): # dumb test that we can create Exporer object e = Explorer() - self.assertTrue( e ) + self.assertTrue(e) def test01_xresource(self): - x = XResource( uri='abc', acceptable_capabilities=[1,2], - checks=[3,4], context='http://example.org/def' ) - self.assertEqual( x.uri, 'http://example.org/abc' ) - self.assertEqual( x.acceptable_capabilities, [1,2] ) - self.assertEqual( x.checks, [3,4] ) + x = XResource(uri='abc', acceptable_capabilities=[1, 2], + checks=[3, 4], context='http://example.org/def') + self.assertEqual(x.uri, 'http://example.org/abc') + self.assertEqual(x.acceptable_capabilities, [1, 2]) + self.assertEqual(x.checks, [3, 4]) def test02_head_response(self): hr = HeadResponse() - self.assertEqual( hr.status_code, None ) - self.assertEqual( len(hr.headers), 0) + self.assertEqual(hr.status_code, None) + self.assertEqual(len(hr.headers), 0) def test03_explorer_quit(self): eq = ExplorerQuit() - self.assertTrue( isinstance(eq, Exception) ) + self.assertTrue(isinstance(eq, Exception)) def test04_explore(self): e = Explorer() @@ -46,81 +46,90 @@ class TestExplorer(unittest.TestCase): # IOError generated from attempt to read stdin e.fake_input = 'q' e.explore() - self.assertTrue( re.search(r'resync-explorer done', capturer.result) ) + self.assertTrue(re.search(r'resync-explorer done', capturer.result)) def test05_explore_uri(self): e = Explorer() with capture_stdout() as capturer: e.fake_input = 'q' - self.assertRaises( ExplorerQuit, e.explore_uri, - XResource('tests/testdata/explore1/caps1.xml') ) - self.assertTrue( re.search(r'Reading tests/testdata/explore1/caps1.xml', - capturer.result) ) - self.assertTrue( re.search(r'Parsed capabilitylist document with 4 entries:', - capturer.result) ) + self.assertRaises(ExplorerQuit, e.explore_uri, + XResource('tests/testdata/explore1/caps1.xml')) + self.assertTrue(re.search(r'Reading tests/testdata/explore1/caps1.xml', + capturer.result)) + self.assertTrue(re.search(r'Parsed capabilitylist document with 4 entries:', + capturer.result)) def test06_explore_show_summary(self): e = Explorer() # file that exists with matching with capture_stdout() as capturer: - e.explore_show_summary( list=CapabilityList() ) - self.assertTrue( re.search(r'Parsed \(unknown capability\) document with 0 entries:', - capturer.result) ) + e.explore_show_summary(list=CapabilityList()) + self.assertTrue(re.search(r'Parsed \(unknown capability\) document with 0 entries:', + capturer.result)) # dummy capabilities object and display cl = CapabilityList() - cl.add( Resource('uri:resourcelist') ) - cl.add( Resource('uri:changelist') ) + cl.add(Resource('uri:resourcelist')) + cl.add(Resource('uri:changelist')) with capture_stdout() as capturer: - e.explore_show_summary(cl,False,[]) - self.assertTrue( re.search(r'Parsed \(unknown capability\) document with 2 entries:',capturer.result) ) - self.assertTrue( re.search(r'\[1\] uri:changelist',capturer.result) ) - self.assertTrue( re.search(r'\[2\] uri:resourcelist',capturer.result) ) - + e.explore_show_summary(cl, False, []) + self.assertTrue(re.search( + r'Parsed \(unknown capability\) document with 2 entries:', capturer.result)) + self.assertTrue(re.search(r'\[1\] uri:changelist', capturer.result)) + self.assertTrue(re.search(r'\[2\] uri:resourcelist', capturer.result)) + def test07_explore_show_head(self): e = Explorer() # file that exists with matching with capture_stdout() as capturer: - e.explore_show_head( uri='tests/testdata/dir1/file_a', - check_headers={ 'content-length': 20, - 'unknown': 'abc' } ) - self.assertTrue( re.search(r'HEAD tests/testdata/dir1/file_a', capturer.result) ) - self.assertTrue( re.search(r'content-length: 20 MATCHES EXPECTED VALUE', capturer.result) ) + e.explore_show_head(uri='tests/testdata/dir1/file_a', + check_headers={'content-length': 20, + 'unknown': 'abc'}) + self.assertTrue( + re.search(r'HEAD tests/testdata/dir1/file_a', capturer.result)) + self.assertTrue( + re.search(r'content-length: 20 MATCHES EXPECTED VALUE', capturer.result)) # same file, bad header check with capture_stdout() as capturer: - e.explore_show_head( 'tests/testdata/dir1/file_a', - check_headers={ 'content-length': 99 } ) - self.assertTrue( re.search(r'HEAD tests/testdata/dir1/file_a', capturer.result) ) - self.assertTrue( re.search(r'content-length: 20 EXPECTED 99', capturer.result) ) + e.explore_show_head('tests/testdata/dir1/file_a', + check_headers={'content-length': 99}) + self.assertTrue( + re.search(r'HEAD tests/testdata/dir1/file_a', capturer.result)) + self.assertTrue( + re.search(r'content-length: 20 EXPECTED 99', capturer.result)) # file that does not exist with capture_stdout() as capturer: - e.explore_show_head( 'tests/testdata/does_not_exist' ) - self.assertTrue( re.search(r'HEAD tests/testdata/does_not_exist', capturer.result) ) - self.assertTrue( re.search(r'status: 404', capturer.result) ) + e.explore_show_head('tests/testdata/does_not_exist') + self.assertTrue( + re.search(r'HEAD tests/testdata/does_not_exist', capturer.result)) + self.assertTrue(re.search(r'status: 404', capturer.result)) def test08_head_on_file(self): e = Explorer() r1 = e.head_on_file('tests/testdata/does_not_exist') - self.assertEqual( r1.status_code, '404' ) + self.assertEqual(r1.status_code, '404') r2 = e.head_on_file('tests/testdata/dir1/file_a') - self.assertEqual( r2.status_code, '200' ) - self.assertTrue( re.match(r'^\d\d\d\d\-\d\d\-\d\d', r2.headers['last-modified']) ) - self.assertEqual( r2.headers['content-length'], 20 ) + self.assertEqual(r2.status_code, '200') + self.assertTrue(re.match(r'^\d\d\d\d\-\d\d\-\d\d', + r2.headers['last-modified'])) + self.assertEqual(r2.headers['content-length'], 20) def test09_allowed_entries(self): e = Explorer() - self.assertEqual( e.allowed_entries('capabilitylistindex'), [] ) - self.assertEqual( e.allowed_entries('resourcelistindex'), ['resourcelist'] ) - self.assertEqual( e.allowed_entries('changelist-archive'), ['changelist'] ) - self.assertEqual( e.allowed_entries('description'),['capabilitylist'] ) - self.assertEqual( len(e.allowed_entries('capabilitylist')), 8 ) - self.assertEqual( e.allowed_entries('unknown'),[] ) + self.assertEqual(e.allowed_entries('capabilitylistindex'), []) + self.assertEqual(e.allowed_entries( + 'resourcelistindex'), ['resourcelist']) + self.assertEqual(e.allowed_entries( + 'changelist-archive'), ['changelist']) + self.assertEqual(e.allowed_entries('description'), ['capabilitylist']) + self.assertEqual(len(e.allowed_entries('capabilitylist')), 8) + self.assertEqual(e.allowed_entries('unknown'), []) def test10_expand_relative_uri(self): e = Explorer() with capture_stdout() as capturer: - uri = e.expand_relative_uri('https://example.org/ctx','abc') - self.assertEqual( uri, 'https://example.org/abc' ) - self.assertTrue( re.search(r'expanded relative URI', capturer.result) ) + uri = e.expand_relative_uri('https://example.org/ctx', 'abc') + self.assertEqual(uri, 'https://example.org/abc') + self.assertTrue(re.search(r'expanded relative URI', capturer.result)) if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestExplorer) diff --git a/tests/test_list_base.py b/tests/test_list_base.py index acc3e8f..7ff9895 100644 --- a/tests/test_list_base.py +++ b/tests/test_list_base.py @@ -3,11 +3,11 @@ from tests.testcase_with_tmpdir import TestCase import sys import os.path import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io from resync.resource import Resource @@ -16,38 +16,39 @@ from resync.sitemap import Sitemap, SitemapIndexError # etree gives ParseError in 2.7, ExpatError in 2.6 etree_error_class = None -if (sys.version_info < (2,7)): +if (sys.version_info < (2, 7)): from xml.parsers.expat import ExpatError etree_error_class = ExpatError else: from xml.etree.ElementTree import ParseError etree_error_class = ParseError + class TestListBase(TestCase): def test01_len_count(self): # count sets explicit number of resources, len(resources) not used lb = ListBase() - self.assertEqual( len(lb), 0 ) + self.assertEqual(len(lb), 0) lb.add(Resource('a')) - self.assertEqual( len(lb), 1 ) - lb = ListBase( count=100 ) - self.assertEqual( len(lb), 100 ) + self.assertEqual(len(lb), 1) + lb = ListBase(count=100) + self.assertEqual(len(lb), 100) def test02_print(self): lb = ListBase() - lb.add( Resource(uri='a',lastmod='2001-01-01',length=1234) ) - lb.add( Resource(uri='b',lastmod='2002-02-02',length=56789) ) - lb.add( Resource(uri='c',lastmod='2003-03-03',length=0) ) - lb.md['from']=None #avoid now being added + lb.add(Resource(uri='a', lastmod='2001-01-01', length=1234)) + lb.add(Resource(uri='b', lastmod='2002-02-02', length=56789)) + lb.add(Resource(uri='c', lastmod='2003-03-03', length=0)) + lb.md['from'] = None # avoid now being added x = lb.as_xml() - self.assertEqual( x, '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z' ) + self.assertEqual(x, '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z') def test03_read(self): # just does parse - lb=ListBase() + lb = ListBase() lb.read(uri='tests/testdata/unknown/unknown1.xml') - self.assertEqual( len(lb.resources), 2 ) + self.assertEqual(len(lb.resources), 2) def test04_parse(self): xml = '\n\ @@ -57,38 +58,38 @@ class TestListBase(TestCase): /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' # parse from string - lb1=ListBase() + lb1 = ListBase() lb1.parse(str_data=xml) - self.assertEqual( len(lb1.resources), 2, 'got 2 resources') + self.assertEqual(len(lb1.resources), 2, 'got 2 resources') # parse from fh - lb2=ListBase() + lb2 = ListBase() lb2.parse(fh=io.StringIO(xml)) - self.assertEqual( len(lb2.resources), 2, 'got 2 resources') + self.assertEqual(len(lb2.resources), 2, 'got 2 resources') # parse from string (LEGACY parameter name, to be removed) - lb3=ListBase() + lb3 = ListBase() lb3.parse(str=xml) - self.assertEqual( len(lb3.resources), 2, 'got 2 resources') + self.assertEqual(len(lb3.resources), 2, 'got 2 resources') # file that doesn't exits lb4 = ListBase() - self.assertRaises( Exception, lb4.parse, - uri='tests/testdata/does_not_exist' ) + self.assertRaises(Exception, lb4.parse, + uri='tests/testdata/does_not_exist') # nothing - self.assertRaises( Exception, lb4.parse ) + self.assertRaises(Exception, lb4.parse) def test04_write(self): - lb = ListBase(capability_name = 'special') - lb.add( Resource(uri='http://example.org/lemon') ) - lb.add( Resource(uri='http://example.org/orange') ) - basename = os.path.join(self.tmpdir,'lb.xml') + lb = ListBase(capability_name='special') + lb.add(Resource(uri='http://example.org/lemon')) + lb.add(Resource(uri='http://example.org/orange')) + basename = os.path.join(self.tmpdir, 'lb.xml') lb.write(basename=basename) - self.assertTrue( os.path.exists(basename) ) + self.assertTrue(os.path.exists(basename)) # and now parse back - fh = open(basename,'r') - lb2 = ListBase(capability_name = 'special') + fh = open(basename, 'r') + lb2 = ListBase(capability_name='special') lb2.parse(fh=fh) - self.assertEqual( lb2.capability, 'special' ) - self.assertEqual( len(lb2), 2 ) + self.assertEqual(lb2.capability, 'special') + self.assertEqual(len(lb2), 2) -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestListBase) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_list_base_with_index.py b/tests/test_list_base_with_index.py index d19c545..43b13fe 100644 --- a/tests/test_list_base_with_index.py +++ b/tests/test_list_base_with_index.py @@ -1,11 +1,11 @@ import re import sys import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io from resync.resource import Resource @@ -14,118 +14,122 @@ from resync.sitemap import Sitemap, SitemapIndexError # etree gives ParseError in 2.7, ExpatError in 2.6 etree_error_class = None -if (sys.version_info < (2,7)): +if (sys.version_info < (2, 7)): from xml.parsers.expat import ExpatError etree_error_class = ExpatError else: from xml.etree.ElementTree import ParseError etree_error_class = ParseError + class TestListBaseWithIndex(unittest.TestCase): def test01_print(self): lb = ListBaseWithIndex() - lb.add( Resource(uri='a',lastmod='2001-01-01',length=1234) ) - lb.add( Resource(uri='b',lastmod='2002-02-02',length=56789) ) - lb.add( Resource(uri='c',lastmod='2003-03-03',length=0) ) - lb.md['from']=None #avoid now being added - self.assertEqual( lb.as_xml(), '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z' ) + lb.add(Resource(uri='a', lastmod='2001-01-01', length=1234)) + lb.add(Resource(uri='b', lastmod='2002-02-02', length=56789)) + lb.add(Resource(uri='c', lastmod='2003-03-03', length=0)) + lb.md['from'] = None # avoid now being added + self.assertEqual(lb.as_xml(), '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z') def test02_print_iter(self): - r = [ Resource(uri='a',lastmod='2001-01-01',length=1234), - Resource(uri='b',lastmod='2002-02-02',length=56789), - Resource(uri='c',lastmod='2003-03-03',length=0) ] + r = [Resource(uri='a', lastmod='2001-01-01', length=1234), + Resource(uri='b', lastmod='2002-02-02', length=56789), + Resource(uri='c', lastmod='2003-03-03', length=0)] # without setting count will barf on len() attempt - lb = ListBaseWithIndex( resources=iter(r) ) - self.assertRaises( TypeError, lb.as_xml ) - # set explicit count larger than max_sitemap_entiries and as_xml will throw exception - lb = ListBaseWithIndex( resources=iter(r), count=3 ) + lb = ListBaseWithIndex(resources=iter(r)) + self.assertRaises(TypeError, lb.as_xml) + # set explicit count larger than max_sitemap_entiries and as_xml will + # throw exception + lb = ListBaseWithIndex(resources=iter(r), count=3) lb.max_sitemap_entries = 2 - self.assertRaises( ListBaseIndexError, lb.as_xml ) + self.assertRaises(ListBaseIndexError, lb.as_xml) # set explicit count and all will be OK - lb = ListBaseWithIndex( resources=iter(r), count=3 ) - lb.md['from']=None #avoid now being added - self.assertEqual( lb.as_xml(), '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z' ) + lb = ListBaseWithIndex(resources=iter(r), count=3) + lb.md['from'] = None # avoid now being added + self.assertEqual(lb.as_xml(), '\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z') def test03_parse_2(self): - xml='\n\ + xml = '\n\ \ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' - lb=ListBaseWithIndex() + lb = ListBaseWithIndex() lb.parse(fh=io.StringIO(xml)) - self.assertEqual( len(lb.resources), 2, 'got 2 resources') - + self.assertEqual(len(lb.resources), 2, 'got 2 resources') + def test16_as_xml(self): - r = [ Resource(uri='a',lastmod='2014-04-14',length=14), - Resource(uri='b',lastmod='2015-05-15',length=15), - Resource(uri='c',lastmod='2016-06-16',length=16) ] - lb=ListBaseWithIndex( resources=r ) + r = [Resource(uri='a', lastmod='2014-04-14', length=14), + Resource(uri='b', lastmod='2015-05-15', length=15), + Resource(uri='c', lastmod='2016-06-16', length=16)] + lb = ListBaseWithIndex(resources=r) lb.max_sitemap_entries = 3 # One file xml = lb.as_xml() - self.assertTrue( re.search(r'a',xml) ) - self.assertTrue( re.search(r'b',xml) ) - self.assertTrue( re.search(r'c',xml) ) + self.assertTrue(re.search(r'a', xml)) + self.assertTrue(re.search(r'b', xml)) + self.assertTrue(re.search(r'c', xml)) # Needs multifile bit not allowed lb.max_sitemap_entries = 1 - self.assertRaises( ListBaseIndexError, lb.as_xml ) + self.assertRaises(ListBaseIndexError, lb.as_xml) # Allow multifile... - xml = lb.as_xml( allow_multifile=True ) - self.assertTrue( re.search(r'/tmp/sitemap00001.xml', xml) ) - self.assertTrue( re.search(r'/tmp/sitemap00002.xml', xml) ) - self.assertFalse( re.search(r'/tmp/sitemap00003.xml', xml) ) + xml = lb.as_xml(allow_multifile=True) + self.assertTrue(re.search(r'/tmp/sitemap00001.xml', xml)) + self.assertTrue(re.search(r'/tmp/sitemap00002.xml', xml)) + self.assertFalse(re.search(r'/tmp/sitemap00003.xml', xml)) def test17_as_xml_index(self): - r = [ Resource(uri='a',lastmod='2006-01-01',length=12), - Resource(uri='b',lastmod='2007-02-02',length=34), - Resource(uri='c',lastmod='2008-03-03',length=56) ] - lb=ListBaseWithIndex( resources=r ) + r = [Resource(uri='a', lastmod='2006-01-01', length=12), + Resource(uri='b', lastmod='2007-02-02', length=34), + Resource(uri='c', lastmod='2008-03-03', length=56)] + lb = ListBaseWithIndex(resources=r) lb.max_sitemap_entries = 2 xml = lb.as_xml_index() - self.assertTrue( re.search(r'/tmp/sitemap00001.xml', xml) ) - self.assertFalse( re.search(r'/tmp/sitemap00002.xml', xml) ) + self.assertTrue(re.search(r'/tmp/sitemap00001.xml', xml)) + self.assertFalse(re.search(r'/tmp/sitemap00002.xml', xml)) # Index not required lb.max_sitemap_entries = 3 - self.assertRaises( ListBaseIndexError, lb.as_xml_index ) + self.assertRaises(ListBaseIndexError, lb.as_xml_index) def test18_as_xml_part(self): - r = [ Resource(uri='a',lastmod='2006-01-01',length=12), - Resource(uri='b',lastmod='2007-02-02',length=34), - Resource(uri='c',lastmod='2008-03-03',length=56) ] - lb=ListBaseWithIndex( resources=r ) + r = [Resource(uri='a', lastmod='2006-01-01', length=12), + Resource(uri='b', lastmod='2007-02-02', length=34), + Resource(uri='c', lastmod='2008-03-03', length=56)] + lb = ListBaseWithIndex(resources=r) # Allow unlimited entries, part makes no sense lb.max_sitemap_entries = None - self.assertRaises( ListBaseIndexError, lb.as_xml_part ) + self.assertRaises(ListBaseIndexError, lb.as_xml_part) # Request after end lb.max_sitemap_entries = 1 - self.assertRaises( ListBaseIndexError, lb.as_xml_part, part_number=9 ) + self.assertRaises(ListBaseIndexError, lb.as_xml_part, part_number=9) # Allow only 1 entry lb.max_sitemap_entries = 1 - xml = lb.as_xml_part( part_number=1 ) - self.assertFalse( re.search(r'a',xml) ) - self.assertTrue( re.search(r'b',xml) ) - self.assertFalse( re.search(r'c',xml) ) + xml = lb.as_xml_part(part_number=1) + self.assertFalse(re.search(r'a', xml)) + self.assertTrue(re.search(r'b', xml)) + self.assertFalse(re.search(r'c', xml)) # Request truncated lb.max_sitemap_entries = 2 - xml = lb.as_xml_part( part_number=1 ) - self.assertFalse( re.search(r'a',xml) ) - self.assertFalse( re.search(r'b',xml) ) - self.assertTrue( re.search(r'c',xml) ) + xml = lb.as_xml_part(part_number=1) + self.assertFalse(re.search(r'a', xml)) + self.assertFalse(re.search(r'b', xml)) + self.assertTrue(re.search(r'c', xml)) def test20_index_as_xml(self): # Check XML for empty case - lb=ListBaseWithIndex() - self.assertEqual( lb.index_as_xml(), '\n' ) + lb = ListBaseWithIndex() + self.assertEqual(lb.index_as_xml( + ), '\n') # Add a resource and make sure we find that - lb.add( Resource(uri='a',lastmod='2001-01-01',length=1234) ) + lb.add(Resource(uri='a', lastmod='2001-01-01', length=1234)) xml = lb.index_as_xml() - self.assertTrue( re.search(r'a2001-01-01T00:00:00Z',xml) ) + self.assertTrue( + re.search(r'a2001-01-01T00:00:00Z', xml)) -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestListBaseWithIndex) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_mapper.py b/tests/test_mapper.py index 15f8128..fa9b111 100644 --- a/tests/test_mapper.py +++ b/tests/test_mapper.py @@ -25,8 +25,8 @@ class TestMapper(unittest.TestCase): self.assertRaises(MapperError, m6.parse, ['a=b=c']) self.assertRaises(MapperError, m6.parse, ['a=b=c=d']) # dupes - self.assertRaises(MapperError, m6.parse, ['a=b','a=c']) - self.assertRaises(MapperError, m6.parse, ['x=z','y=z']) + self.assertRaises(MapperError, m6.parse, ['a=b', 'a=c']) + self.assertRaises(MapperError, m6.parse, ['x=z', 'y=z']) def test01_mapper_src_to_dst(self): m = Mapper(['http://e.org/p/', '/tmp/q/']) diff --git a/tests/test_resource.py b/tests/test_resource.py index 812a8b3..84a37ad 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -2,19 +2,20 @@ import unittest import re from resync.resource import Resource, ChangeTypeError + class TestResource(unittest.TestCase): def test01a_same(self): r1 = Resource('a') r2 = Resource('a') - self.assertEqual( r1, r1 ) - self.assertEqual( r1, r2 ) + self.assertEqual(r1, r1) + self.assertEqual(r1, r2) def test01b_same(self): - r1 = Resource(uri='a',timestamp=1234.0) - r2 = Resource(uri='a',timestamp=1234.0) - self.assertEqual( r1, r1 ) - self.assertEqual( r1, r2 ) + r1 = Resource(uri='a', timestamp=1234.0) + r2 = Resource(uri='a', timestamp=1234.0) + self.assertEqual(r1, r1) + self.assertEqual(r1, r2) def test01c_same(self): """Same with lastmod instead of direct timestamp""" @@ -29,128 +30,132 @@ class TestResource(unittest.TestCase): '2012-01-01T00:00:00Z', '2012-01-01T00:00:00.000000Z', '2012-01-01T00:00:00.000000000000Z', - '2012-01-01T00:00:00.000000000001Z', #below resolution + '2012-01-01T00:00:00.000000000001Z', # below resolution '2012-01-01T00:00:00.00+00:00', '2012-01-01T00:00:00.00-00:00', '2012-01-01T02:00:00.00-02:00', '2011-12-31T23:00:00.00+01:00' ): r2.lastmod = r2lm - self.assertEqual( r1.timestamp, r2.timestamp, ('%s (%f) == %s (%f)' % (r1lm,r1.timestamp,r2lm,r2.timestamp)) ) - self.assertEqual( r1, r2 ) + self.assertEqual(r1.timestamp, r2.timestamp, ('%s (%f) == %s (%f)' % ( + r1lm, r1.timestamp, r2lm, r2.timestamp))) + self.assertEqual(r1, r2) def test01d_same(self): """Same with slight timestamp diff""" r1 = Resource('a') - r1.lastmod='2012-01-02T01:02:03Z' + r1.lastmod = '2012-01-02T01:02:03Z' r2 = Resource('a') - r2.lastmod='2012-01-02T01:02:03.99Z' - self.assertNotEqual( r1.timestamp, r2.timestamp ) - self.assertEqual( r1, r2 ) + r2.lastmod = '2012-01-02T01:02:03.99Z' + self.assertNotEqual(r1.timestamp, r2.timestamp) + self.assertEqual(r1, r2) def test02a_diff(self): r1 = Resource('a') r2 = Resource('b') - self.assertNotEqual(r1,r2) + self.assertNotEqual(r1, r2) def test02b_diff(self): - r1 = Resource('a',lastmod='2012-01-11') - r2 = Resource('a',lastmod='2012-01-22') - #print 'r1 == r2 : '+str(r1==r2) - self.assertNotEqual( r1, r2 ) + r1 = Resource('a', lastmod='2012-01-11') + r2 = Resource('a', lastmod='2012-01-22') + # print 'r1 == r2 : '+str(r1==r2) + self.assertNotEqual(r1, r2) def test04_bad_lastmod(self): - def setlastmod(r,v): - r.lastmod=v + def setlastmod(r, v): + r.lastmod = v r = Resource('4') # Bad formats - self.assertRaises( ValueError, setlastmod, r, "bad_lastmod" ) - self.assertRaises( ValueError, setlastmod, r, "" ) - self.assertRaises( ValueError, setlastmod, r, "2012-13-01" ) - self.assertRaises( ValueError, setlastmod, r, "2012-12-32" ) - self.assertRaises( ValueError, setlastmod, r, "2012-11-01T10:10:60" ) - self.assertRaises( ValueError, setlastmod, r, "2012-11-01T10:10:59.9x" ) + self.assertRaises(ValueError, setlastmod, r, "bad_lastmod") + self.assertRaises(ValueError, setlastmod, r, "") + self.assertRaises(ValueError, setlastmod, r, "2012-13-01") + self.assertRaises(ValueError, setlastmod, r, "2012-12-32") + self.assertRaises(ValueError, setlastmod, r, "2012-11-01T10:10:60") + self.assertRaises(ValueError, setlastmod, r, "2012-11-01T10:10:59.9x") # Valid ISO8601 but not alloed in W3C Datetime - self.assertRaises( ValueError, setlastmod, r, "2012-11-01T01:01:01" ) - self.assertRaises( ValueError, setlastmod, r, "2012-11-01 01:01:01Z" ) - self.assertRaises( ValueError, setlastmod, r, "2012-11-01T01:01:01+0000" ) - self.assertRaises( ValueError, setlastmod, r, "2012-11-01T01:01:01-1000" ) - + self.assertRaises(ValueError, setlastmod, r, "2012-11-01T01:01:01") + self.assertRaises(ValueError, setlastmod, r, "2012-11-01 01:01:01Z") + self.assertRaises(ValueError, setlastmod, r, + "2012-11-01T01:01:01+0000") + self.assertRaises(ValueError, setlastmod, r, + "2012-11-01T01:01:01-1000") + def test05_lastmod_roundtrips(self): r = Resource('a') - r.lastmod='2012-03-14' - self.assertEqual( r.lastmod, '2012-03-14T00:00:00Z' ) - r.lastmod='2012-03-14T00:00:00+00:00' - #print r.timestamp - self.assertEqual( r.lastmod, '2012-03-14T00:00:00Z' ) - r.lastmod='2012-03-14T00:00:00-00:00' - #print r.timestamp - self.assertEqual( r.lastmod, '2012-03-14T00:00:00Z' ) - r.lastmod='2012-03-14T18:37:36Z' - #print r.timestamp - self.assertEqual( r.lastmod, '2012-03-14T18:37:36Z' ) + r.lastmod = '2012-03-14' + self.assertEqual(r.lastmod, '2012-03-14T00:00:00Z') + r.lastmod = '2012-03-14T00:00:00+00:00' + # print r.timestamp + self.assertEqual(r.lastmod, '2012-03-14T00:00:00Z') + r.lastmod = '2012-03-14T00:00:00-00:00' + # print r.timestamp + self.assertEqual(r.lastmod, '2012-03-14T00:00:00Z') + r.lastmod = '2012-03-14T18:37:36Z' + # print r.timestamp + self.assertEqual(r.lastmod, '2012-03-14T18:37:36Z') def test06_str(self): - r1 = Resource('abc',lastmod='2012-01-01') - self.assertTrue( re.match( r"\[ abc \| 2012-01-01T", str(r1) ) ) + r1 = Resource('abc', lastmod='2012-01-01') + self.assertTrue(re.match(r"\[ abc \| 2012-01-01T", str(r1))) def test07_repr(self): - r1 = Resource('def',lastmod='2012-01-01') - self.assertEqual( repr(r1), "{'uri': 'def', 'timestamp': 1325376000}" ) + r1 = Resource('def', lastmod='2012-01-01') + self.assertEqual(repr(r1), "{'uri': 'def', 'timestamp': 1325376000}") def test08_multiple_hashes(self): r1 = Resource('abcd') r1.md5 = "some_md5" r1.sha1 = "some_sha1" r1.sha256 = "some_sha256" - self.assertEqual( r1.md5, "some_md5" ) - self.assertEqual( r1.sha1, "some_sha1" ) - self.assertEqual( r1.sha256, "some_sha256" ) - self.assertEqual( r1.hash, "md5:some_md5 sha-1:some_sha1 sha-256:some_sha256" ) + self.assertEqual(r1.md5, "some_md5") + self.assertEqual(r1.sha1, "some_sha1") + self.assertEqual(r1.sha256, "some_sha256") + self.assertEqual( + r1.hash, "md5:some_md5 sha-1:some_sha1 sha-256:some_sha256") r2 = Resource('def') r2.hash = "md5:ddd" - self.assertEqual( r2.md5, 'ddd' ) - self.assertEqual( r2.sha1, None ) + self.assertEqual(r2.md5, 'ddd') + self.assertEqual(r2.sha1, None) r2.hash = "sha-1:eee" - self.assertEqual( r2.md5, None ) - self.assertEqual( r2.sha1, 'eee' ) + self.assertEqual(r2.md5, None) + self.assertEqual(r2.sha1, 'eee') r2.hash = "md5:fff sha-1:eee sha-256:ggg" - self.assertEqual( r2.md5, 'fff' ) - self.assertEqual( r2.sha1, 'eee' ) - self.assertEqual( r2.sha256, 'ggg' ) + self.assertEqual(r2.md5, 'fff') + self.assertEqual(r2.sha1, 'eee') + self.assertEqual(r2.sha256, 'ggg') def test09_changetypeerror(self): r1 = Resource('a') - self.assertEqual( r1.change, None ) + self.assertEqual(r1.change, None) r1.change = 'deleted' - self.assertEqual( r1.change, 'deleted' ) - self.assertRaises( ChangeTypeError, Resource, 'a', change="bad" ) + self.assertEqual(r1.change, 'deleted') + self.assertRaises(ChangeTypeError, Resource, 'a', change="bad") # disable checking Resource.CHANGE_TYPES = False - r1 = Resource( 'a', change="bad" ) - self.assertEqual( r1.change, 'bad' ) + r1 = Resource('a', change="bad") + self.assertEqual(r1.change, 'bad') def test10_md_at_roundtrips(self): r = Resource('a') - r.md_at='2013-03-14' - self.assertEqual( r.md_at, '2013-03-14T00:00:00Z' ) - r.md_at='2013-03-14T00:00:00+00:00' - self.assertEqual( r.md_at, '2013-03-14T00:00:00Z' ) - r.md_at='2013-03-14T00:00:00-00:00' - self.assertEqual( r.md_at, '2013-03-14T00:00:00Z' ) - r.md_at='2013-03-14T18:37:36Z' - self.assertEqual( r.md_at, '2013-03-14T18:37:36Z' ) + r.md_at = '2013-03-14' + self.assertEqual(r.md_at, '2013-03-14T00:00:00Z') + r.md_at = '2013-03-14T00:00:00+00:00' + self.assertEqual(r.md_at, '2013-03-14T00:00:00Z') + r.md_at = '2013-03-14T00:00:00-00:00' + self.assertEqual(r.md_at, '2013-03-14T00:00:00Z') + r.md_at = '2013-03-14T18:37:36Z' + self.assertEqual(r.md_at, '2013-03-14T18:37:36Z') def test11_md_completed_roundtrips(self): r = Resource('a') - r.md_completed='2013-04-14' - self.assertEqual( r.md_completed, '2013-04-14T00:00:00Z' ) - r.md_completed='2013-04-14T00:00:00+00:00' - self.assertEqual( r.md_completed, '2013-04-14T00:00:00Z' ) - r.md_completed='2013-04-14T00:00:00-00:00' - self.assertEqual( r.md_completed, '2013-04-14T00:00:00Z' ) - r.md_completed='2013-04-14T18:37:36Z' - self.assertEqual( r.md_completed, '2013-04-14T18:37:36Z' ) + r.md_completed = '2013-04-14' + self.assertEqual(r.md_completed, '2013-04-14T00:00:00Z') + r.md_completed = '2013-04-14T00:00:00+00:00' + self.assertEqual(r.md_completed, '2013-04-14T00:00:00Z') + r.md_completed = '2013-04-14T00:00:00-00:00' + self.assertEqual(r.md_completed, '2013-04-14T00:00:00Z') + r.md_completed = '2013-04-14T18:37:36Z' + self.assertEqual(r.md_completed, '2013-04-14T18:37:36Z') def test12_timevalues(self): r = Resource(uri='tv', @@ -159,19 +164,19 @@ class TestResource(unittest.TestCase): md_completed="2000-01-03", md_from="2000-01-04", md_until="2000-01-05") - self.assertEqual( r.lastmod, '2000-01-01T00:00:00Z' ) - self.assertEqual( r.md_at, '2000-01-02T00:00:00Z' ) - self.assertEqual( r.md_completed, '2000-01-03T00:00:00Z' ) - self.assertEqual( r.md_from, '2000-01-04T00:00:00Z' ) - self.assertEqual( r.md_until, '2000-01-05T00:00:00Z' ) + self.assertEqual(r.lastmod, '2000-01-01T00:00:00Z') + self.assertEqual(r.md_at, '2000-01-02T00:00:00Z') + self.assertEqual(r.md_completed, '2000-01-03T00:00:00Z') + self.assertEqual(r.md_from, '2000-01-04T00:00:00Z') + self.assertEqual(r.md_until, '2000-01-05T00:00:00Z') def test13_mime_type(self): r = Resource(uri='tv1', mime_type='text/plain') - self.assertEqual( r.mime_type, 'text/plain' ) + self.assertEqual(r.mime_type, 'text/plain') r.mime_type = None - self.assertEqual( r.mime_type, None ) + self.assertEqual(r.mime_type, None) r = Resource(uri='tv2') - self.assertEqual( r.mime_type, None ) + self.assertEqual(r.mime_type, None) if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestResource) diff --git a/tests/test_resource_container.py b/tests/test_resource_container.py index e402a41..706b050 100644 --- a/tests/test_resource_container.py +++ b/tests/test_resource_container.py @@ -2,118 +2,120 @@ import unittest from resync.resource import Resource from resync.resource_container import ResourceContainer + class TestResourceContainer(unittest.TestCase): def test01_create_and_add(self): - rc = ResourceContainer( resources=[] ) - self.assertEqual( len(rc.resources), 0, "empty" ) - rc.resources.append( Resource('a',timestamp=1) ) - rc.resources.append( Resource('b',timestamp=2) ) - self.assertEqual( len(rc.resources), 2, "two resources" ) + rc = ResourceContainer(resources=[]) + self.assertEqual(len(rc.resources), 0, "empty") + rc.resources.append(Resource('a', timestamp=1)) + rc.resources.append(Resource('b', timestamp=2)) + self.assertEqual(len(rc.resources), 2, "two resources") def test02_iter(self): - rc = ResourceContainer( resources=[] ) - rc.resources.append( Resource('a',timestamp=1) ) - rc.resources.append( Resource('b',timestamp=2) ) - rc.resources.append( Resource('c',timestamp=3) ) - rc.resources.append( Resource('d',timestamp=4) ) - resources=[] + rc = ResourceContainer(resources=[]) + rc.resources.append(Resource('a', timestamp=1)) + rc.resources.append(Resource('b', timestamp=2)) + rc.resources.append(Resource('c', timestamp=3)) + rc.resources.append(Resource('d', timestamp=4)) + resources = [] for r in rc: resources.append(r) self.assertEqual(len(resources), 4) - self.assertEqual( resources[0].uri, 'a') - self.assertEqual( resources[3].uri, 'd') + self.assertEqual(resources[0].uri, 'a') + self.assertEqual(resources[3].uri, 'd') def test03_prune_before(self): rc = ResourceContainer() - rc.resources.append( Resource('a',timestamp=1) ) - rc.resources.append( Resource('b',timestamp=2) ) - rc.resources.append( Resource('c',timestamp=3) ) - rc.resources.append( Resource('d',timestamp=4) ) + rc.resources.append(Resource('a', timestamp=1)) + rc.resources.append(Resource('b', timestamp=2)) + rc.resources.append(Resource('c', timestamp=3)) + rc.resources.append(Resource('d', timestamp=4)) rc.prune_before(3) - self.assertEqual( len(rc.resources), 2 ) + self.assertEqual(len(rc.resources), 2) i = iter(rc) - self.assertEqual( next(i).uri, 'c' ) - self.assertEqual( next(i).uri, 'd' ) + self.assertEqual(next(i).uri, 'c') + self.assertEqual(next(i).uri, 'd') # put some more back out of order - rc.resources.append( Resource('a',timestamp=1) ) - rc.resources.append( Resource('b',timestamp=2) ) - rc.resources.append( Resource('e',timestamp=1000) ) + rc.resources.append(Resource('a', timestamp=1)) + rc.resources.append(Resource('b', timestamp=2)) + rc.resources.append(Resource('e', timestamp=1000)) rc.prune_before(3.5) - self.assertEqual( len(rc.resources), 2 ) + self.assertEqual(len(rc.resources), 2) i = iter(rc) - self.assertEqual( next(i).uri, 'd' ) - self.assertEqual( next(i).uri, 'e' ) + self.assertEqual(next(i).uri, 'd') + self.assertEqual(next(i).uri, 'e') def test04_prune_dupes(self): rc = ResourceContainer() - rc.resources.append( Resource('a',timestamp=1, change='created') ) - rc.resources.append( Resource('b',timestamp=2, change='created') ) - rc.resources.append( Resource('c',timestamp=3, change='updated') ) - rc.resources.append( Resource('d',timestamp=4, change='deleted') ) - rc.resources.append( Resource('a',timestamp=4, change='updated') ) - rc.resources.append( Resource('b',timestamp=4, change='deleted') ) - rc.resources.append( Resource('b',timestamp=5, change='created') ) - rc.resources.append( Resource('b',timestamp=6, change='deleted') ) - rc.resources.append( Resource('c',timestamp=7, change='deleted') ) - rc.resources.append( Resource('d',timestamp=8, change='created') ) + rc.resources.append(Resource('a', timestamp=1, change='created')) + rc.resources.append(Resource('b', timestamp=2, change='created')) + rc.resources.append(Resource('c', timestamp=3, change='updated')) + rc.resources.append(Resource('d', timestamp=4, change='deleted')) + rc.resources.append(Resource('a', timestamp=4, change='updated')) + rc.resources.append(Resource('b', timestamp=4, change='deleted')) + rc.resources.append(Resource('b', timestamp=5, change='created')) + rc.resources.append(Resource('b', timestamp=6, change='deleted')) + rc.resources.append(Resource('c', timestamp=7, change='deleted')) + rc.resources.append(Resource('d', timestamp=8, change='created')) rc.prune_dupes() - self.assertEqual( len(rc.resources), 3 ) + self.assertEqual(len(rc.resources), 3) i = iter(rc) i1 = next(i) - self.assertEqual( i1.uri, 'a' ) - self.assertEqual( i1.change, 'updated' ) + self.assertEqual(i1.uri, 'a') + self.assertEqual(i1.change, 'updated') i2 = next(i) - self.assertEqual( i2.uri, 'c' ) - self.assertEqual( i2.change, 'deleted' ) + self.assertEqual(i2.uri, 'c') + self.assertEqual(i2.change, 'deleted') i3 = next(i) - self.assertEqual( i3.uri, 'd' ) - self.assertEqual( i3.change, 'created' ) + self.assertEqual(i3.uri, 'd') + self.assertEqual(i3.change, 'created') def test05_from_until(self): rc = ResourceContainer() # via convenience methods - self.assertEqual( rc.md_from, None ) - self.assertEqual( rc.md_until, None ) + self.assertEqual(rc.md_from, None) + self.assertEqual(rc.md_until, None) rc.md_from = "ftime" rc.md_until = "utime" - self.assertEqual( rc.md_from, "ftime" ) - self.assertEqual( rc.md_until, "utime" ) + self.assertEqual(rc.md_from, "ftime") + self.assertEqual(rc.md_until, "utime") # via underlying dict rc.md['md_from'] = "ftime2" rc.md['md_until'] = "utime2too" - self.assertEqual( rc.md_from, "ftime2" ) - self.assertEqual( rc.md_until, "utime2too" ) + self.assertEqual(rc.md_from, "ftime2") + self.assertEqual(rc.md_until, "utime2too") def test06_source_description(self): # up links used to point to the source description rc = ResourceContainer() - # via convenience methods - self.assertEqual( rc.up, None ) + # via convenience methods + self.assertEqual(rc.up, None) rc.up = "well-known" - self.assertEqual( rc.up, "well-known" ) + self.assertEqual(rc.up, "well-known") rc.up = "well-known2" - self.assertEqual( rc.up, "well-known2" ) + self.assertEqual(rc.up, "well-known2") # via link link = rc.link("up") - self.assertEqual( link['href'], "well-known2" ) + self.assertEqual(link['href'], "well-known2") link['href'] = "wk3" - self.assertEqual( rc.up, "wk3" ) + self.assertEqual(rc.up, "wk3") def test07_describedby(self): rc = ResourceContainer() - # via convenience methods - self.assertEqual( rc.describedby, None ) + # via convenience methods + self.assertEqual(rc.describedby, None) rc.describedby = "db_uri" - self.assertEqual( rc.describedby, "db_uri" ) + self.assertEqual(rc.describedby, "db_uri") def test08_up(self): rc = ResourceContainer() - # via convenience methods - self.assertEqual( rc.up, None ) + # via convenience methods + self.assertEqual(rc.up, None) rc.up = "up_uri" - self.assertEqual( rc.up, "up_uri" ) + self.assertEqual(rc.up, "up_uri") if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceContainer) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + TestResourceContainer) unittest.TextTestRunner().run(suite) diff --git a/tests/test_resource_dump_manifest.py b/tests/test_resource_dump_manifest.py index e61b57a..ec8b613 100644 --- a/tests/test_resource_dump_manifest.py +++ b/tests/test_resource_dump_manifest.py @@ -1,9 +1,9 @@ import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io import re @@ -11,55 +11,60 @@ from resync.resource import Resource from resync.resource_dump_manifest import ResourceDumpManifest from resync.sitemap import SitemapParseError + class TestResourceDumpManifest(unittest.TestCase): def test01_as_xml(self): rdm = ResourceDumpManifest() - rdm.add( Resource('a.zip',timestamp=1) ) - rdm.add( Resource('b.zip',timestamp=2) ) + rdm.add(Resource('a.zip', timestamp=1)) + rdm.add(Resource('b.zip', timestamp=2)) xml = rdm.as_xml() - self.assertTrue( re.search(r'a.zip1970-01-01T00:00:01Z', xml), 'XML has resource a' ) + self.assertTrue(re.search( + r'a.zip1970-01-01T00:00:01Z', xml), 'XML has resource a') def test10_parse(self): - xml='\n\ + xml = '\n\ \ \ http://example.com/res12012-03-14T18:37:36Z\ http://example.com/res22012-03-14T18:37:36Z\ ' - rdm=ResourceDumpManifest() + rdm = ResourceDumpManifest() rdm.parse(fh=io.StringIO(xml)) - self.assertEqual( len(rdm.resources), 2, 'got 2 resource dumps') - self.assertEqual( rdm.capability, 'resourcedump-manifest', 'capability set' ) - self.assertEqual( rdm.md_at, '2013-08-08' ) - self.assertTrue( 'http://example.com/res1' in rdm.resources ) - self.assertTrue( rdm.resources['http://example.com/res1'].length, 12 ) - self.assertTrue( rdm.resources['http://example.com/res1'].path, '/res1' ) - self.assertTrue( 'http://example.com/res2' in rdm.resources ) - self.assertTrue( rdm.resources['http://example.com/res2'].path, '/res2' ) + self.assertEqual(len(rdm.resources), 2, 'got 2 resource dumps') + self.assertEqual( + rdm.capability, 'resourcedump-manifest', 'capability set') + self.assertEqual(rdm.md_at, '2013-08-08') + self.assertTrue('http://example.com/res1' in rdm.resources) + self.assertTrue(rdm.resources['http://example.com/res1'].length, 12) + self.assertTrue(rdm.resources['http://example.com/res1'].path, '/res1') + self.assertTrue('http://example.com/res2' in rdm.resources) + self.assertTrue(rdm.resources['http://example.com/res2'].path, '/res2') def test11_parse_no_capability(self): # For a resource dump this should be an error - xml='\n\ + xml = '\n\ \ \ http://example.com/res12012-03-14T18:37:36Z\ http://example.com/res22012-03-14T18:37:36Z\ ' - rdm=ResourceDumpManifest() - self.assertRaises( SitemapParseError, rdm.parse, fh=io.StringIO(xml) ) + rdm = ResourceDumpManifest() + self.assertRaises(SitemapParseError, rdm.parse, fh=io.StringIO(xml)) def test12_parse_bad_capability(self): # the \n\ + xml = '\n\ \ \ http://example.com/a.zip2012-03-14T18:37:36Z\ ' - rdm=ResourceDumpManifest() - self.assertRaises( SitemapParseError, rdm.parse, fh=io.StringIO(xml) ) + rdm = ResourceDumpManifest() + self.assertRaises(SitemapParseError, rdm.parse, fh=io.StringIO(xml)) if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceDumpManifest) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + TestResourceDumpManifest) unittest.TextTestRunner().run(suite) diff --git a/tests/test_resource_list.py b/tests/test_resource_list.py index 89ea147..ae954dc 100644 --- a/tests/test_resource_list.py +++ b/tests/test_resource_list.py @@ -1,9 +1,9 @@ import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io import re @@ -11,169 +11,173 @@ from resync.resource import Resource from resync.resource_list import ResourceList, ResourceListDupeError from resync.sitemap import SitemapParseError + class TestResourceList(unittest.TestCase): def test01_same(self): src = ResourceList() - src.add( Resource('a',timestamp=1) ) - src.add( Resource('b',timestamp=2) ) + src.add(Resource('a', timestamp=1)) + src.add(Resource('b', timestamp=2)) dst = ResourceList() - dst.add( Resource('a',timestamp=1) ) - dst.add( Resource('b',timestamp=2) ) - ( same, changed, deleted, added ) = dst.compare(src) - self.assertEqual( len(same), 2, "2 things unchanged" ) + dst.add(Resource('a', timestamp=1)) + dst.add(Resource('b', timestamp=2)) + (same, changed, deleted, added) = dst.compare(src) + self.assertEqual(len(same), 2, "2 things unchanged") i = iter(same) - self.assertEqual( next(i).uri, 'a', "first was a" ) - self.assertEqual( next(i).uri, 'b', "second was b" ) - self.assertEqual( len(changed), 0, "nothing changed" ) - self.assertEqual( len(deleted), 0, "nothing deleted" ) - self.assertEqual( len(added), 0, "nothing added" ) + self.assertEqual(next(i).uri, 'a', "first was a") + self.assertEqual(next(i).uri, 'b', "second was b") + self.assertEqual(len(changed), 0, "nothing changed") + self.assertEqual(len(deleted), 0, "nothing deleted") + self.assertEqual(len(added), 0, "nothing added") def test02_changed(self): src = ResourceList() - src.add( Resource('a',timestamp=1) ) - src.add( Resource('b',timestamp=2) ) + src.add(Resource('a', timestamp=1)) + src.add(Resource('b', timestamp=2)) dst = ResourceList() - dst.add( Resource('a',timestamp=3) ) - dst.add( Resource('b',timestamp=4) ) - ( same, changed, deleted, added ) = dst.compare(src) - self.assertEqual( len(same), 0, "0 things unchanged" ) - self.assertEqual( len(changed), 2, "2 things changed" ) + dst.add(Resource('a', timestamp=3)) + dst.add(Resource('b', timestamp=4)) + (same, changed, deleted, added) = dst.compare(src) + self.assertEqual(len(same), 0, "0 things unchanged") + self.assertEqual(len(changed), 2, "2 things changed") i = iter(changed) - self.assertEqual( next(i).uri, 'a', "first was a" ) - self.assertEqual( next(i).uri, 'b', "second was b" ) - self.assertEqual( len(deleted), 0, "nothing deleted" ) - self.assertEqual( len(added), 0, "nothing added" ) + self.assertEqual(next(i).uri, 'a', "first was a") + self.assertEqual(next(i).uri, 'b', "second was b") + self.assertEqual(len(deleted), 0, "nothing deleted") + self.assertEqual(len(added), 0, "nothing added") def test03_deleted(self): src = ResourceList() - src.add( Resource('a',timestamp=1) ) - src.add( Resource('b',timestamp=2) ) + src.add(Resource('a', timestamp=1)) + src.add(Resource('b', timestamp=2)) dst = ResourceList() - dst.add( Resource('a',timestamp=1) ) - dst.add( Resource('b',timestamp=2) ) - dst.add( Resource('c',timestamp=3) ) - dst.add( Resource('d',timestamp=4) ) - ( same, changed, deleted, added ) = dst.compare(src) - self.assertEqual( len(same), 2, "2 things unchanged" ) - self.assertEqual( len(changed), 0, "nothing changed" ) - self.assertEqual( len(deleted), 2, "c and d deleted" ) + dst.add(Resource('a', timestamp=1)) + dst.add(Resource('b', timestamp=2)) + dst.add(Resource('c', timestamp=3)) + dst.add(Resource('d', timestamp=4)) + (same, changed, deleted, added) = dst.compare(src) + self.assertEqual(len(same), 2, "2 things unchanged") + self.assertEqual(len(changed), 0, "nothing changed") + self.assertEqual(len(deleted), 2, "c and d deleted") i = iter(deleted) - self.assertEqual( next(i).uri, 'c', "first was c" ) - self.assertEqual( next(i).uri, 'd', "second was d" ) - self.assertEqual( len(added), 0, "nothing added" ) + self.assertEqual(next(i).uri, 'c', "first was c") + self.assertEqual(next(i).uri, 'd', "second was d") + self.assertEqual(len(added), 0, "nothing added") def test04_added(self): src = ResourceList() - src.add( Resource('a',timestamp=1) ) - src.add( Resource('b',timestamp=2) ) - src.add( Resource('c',timestamp=3) ) - src.add( Resource('d',timestamp=4) ) + src.add(Resource('a', timestamp=1)) + src.add(Resource('b', timestamp=2)) + src.add(Resource('c', timestamp=3)) + src.add(Resource('d', timestamp=4)) dst = ResourceList() - dst.add( Resource('a',timestamp=1) ) - dst.add( Resource('c',timestamp=3) ) - ( same, changed, deleted, added ) = dst.compare(src) - self.assertEqual( len(same), 2, "2 things unchanged" ) - self.assertEqual( len(changed), 0, "nothing changed" ) - self.assertEqual( len(deleted), 0, "nothing deleted" ) - self.assertEqual( len(added), 2, "b and d added" ) + dst.add(Resource('a', timestamp=1)) + dst.add(Resource('c', timestamp=3)) + (same, changed, deleted, added) = dst.compare(src) + self.assertEqual(len(same), 2, "2 things unchanged") + self.assertEqual(len(changed), 0, "nothing changed") + self.assertEqual(len(deleted), 0, "nothing deleted") + self.assertEqual(len(added), 2, "b and d added") i = iter(added) - self.assertEqual( next(i).uri, 'b', "first was b" ) - self.assertEqual( next(i).uri, 'd', "second was d" ) + self.assertEqual(next(i).uri, 'b', "first was b") + self.assertEqual(next(i).uri, 'd', "second was d") def test05_add(self): - r1 = Resource(uri='a',length=1) - r2 = Resource(uri='b',length=2) + r1 = Resource(uri='a', length=1) + r2 = Resource(uri='b', length=2) i = ResourceList() i.add(r1) - self.assertRaises( ResourceListDupeError, i.add, r1) + self.assertRaises(ResourceListDupeError, i.add, r1) i.add(r2) - self.assertRaises( ResourceListDupeError, i.add, r2) + self.assertRaises(ResourceListDupeError, i.add, r2) # allow dupes - r1d = Resource(uri='a',length=10) - i.add(r1d,replace=True) - self.assertEqual( len(i), 2 ) - self.assertEqual( i.resources['a'].length, 10 ) + r1d = Resource(uri='a', length=10) + i.add(r1d, replace=True) + self.assertEqual(len(i), 2) + self.assertEqual(i.resources['a'].length, 10) def test06_add_iterable(self): - r1 = Resource(uri='a',length=1) - r2 = Resource(uri='b',length=2) + r1 = Resource(uri='a', length=1) + r2 = Resource(uri='b', length=2) i = ResourceList() - i.add( [r1,r2] ) - self.assertRaises( ResourceListDupeError, i.add, r1) - self.assertRaises( ResourceListDupeError, i.add, r2) + i.add([r1, r2]) + self.assertRaises(ResourceListDupeError, i.add, r1) + self.assertRaises(ResourceListDupeError, i.add, r2) # allow dupes - r1d = Resource(uri='a',length=10) - i.add( [r1d] ,replace=True) - self.assertEqual( len(i), 2 ) - self.assertEqual( i.resources['a'].length, 10 ) + r1d = Resource(uri='a', length=10) + i.add([r1d], replace=True) + self.assertEqual(len(i), 2) + self.assertEqual(i.resources['a'].length, 10) def test07_has_md5(self): r1 = Resource(uri='a') r2 = Resource(uri='b') i = ResourceList() - self.assertFalse( i.has_md5() ) + self.assertFalse(i.has_md5()) i.add(r1) i.add(r2) - self.assertFalse( i.has_md5() ) - r1.md5="aabbcc" - self.assertTrue( i.has_md5() ) + self.assertFalse(i.has_md5()) + r1.md5 = "aabbcc" + self.assertTrue(i.has_md5()) def test08_iter(self): i = ResourceList() - i.add( Resource('a',timestamp=1) ) - i.add( Resource('b',timestamp=2) ) - i.add( Resource('c',timestamp=3) ) - i.add( Resource('d',timestamp=4) ) - resources=[] + i.add(Resource('a', timestamp=1)) + i.add(Resource('b', timestamp=2)) + i.add(Resource('c', timestamp=3)) + i.add(Resource('d', timestamp=4)) + resources = [] for r in i: resources.append(r) self.assertEqual(len(resources), 4) - self.assertEqual( resources[0].uri, 'a') - self.assertEqual( resources[3].uri, 'd') + self.assertEqual(resources[0].uri, 'a') + self.assertEqual(resources[3].uri, 'd') def test20_as_xml(self): rl = ResourceList() - rl.add( Resource('a',timestamp=1) ) - rl.add( Resource('b',timestamp=2) ) + rl.add(Resource('a', timestamp=1)) + rl.add(Resource('b', timestamp=2)) xml = rl.as_xml() - self.assertTrue( re.search(r'a1970-01-01T00:00:01Z', xml), 'XML has resource a' ) + self.assertTrue( + re.search(r'a1970-01-01T00:00:01Z', xml), 'XML has resource a') def test30_parse(self): - xml='\n\ + xml = '\n\ \ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' - rl=ResourceList() + rl = ResourceList() rl.parse(fh=io.StringIO(xml)) - self.assertEqual( len(rl.resources), 2, 'got 2 resources') - self.assertEqual( rl.md['capability'], 'resourcelist', 'capability set' ) - self.assertEqual( rl.md_at, '2013-08-07' ) - self.assertEqual( rl.md_completed, '2013-08-08' ) + self.assertEqual(len(rl.resources), 2, 'got 2 resources') + self.assertEqual(rl.md['capability'], 'resourcelist', 'capability set') + self.assertEqual(rl.md_at, '2013-08-07') + self.assertEqual(rl.md_completed, '2013-08-08') def test31_parse_no_capability(self): - xml='\n\ + xml = '\n\ \ http://example.com/res12012-03-14T18:37:36Z\ ' - rl=ResourceList() + rl = ResourceList() rl.parse(fh=io.StringIO(xml)) - self.assertEqual( len(rl.resources), 1, 'got 1 resource') - self.assertEqual( rl.md['capability'], 'resourcelist', 'capability set by reading routine' ) - self.assertFalse( 'from' in rl.md ) + self.assertEqual(len(rl.resources), 1, 'got 1 resource') + self.assertEqual(rl.md['capability'], 'resourcelist', + 'capability set by reading routine') + self.assertFalse('from' in rl.md) def test32_parse_bad_capability(self): # the \n\ + xml = '\n\ \ \ http://example.com/bad_res_12012-03-14T18:37:36Z\ ' - rl=ResourceList() - self.assertRaises( SitemapParseError, rl.parse, fh=io.StringIO(xml) ) + rl = ResourceList() + self.assertRaises(SitemapParseError, rl.parse, fh=io.StringIO(xml)) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceList) diff --git a/tests/test_resource_list_builder.py b/tests/test_resource_list_builder.py index 6ee13ba..7436786 100644 --- a/tests/test_resource_list_builder.py +++ b/tests/test_resource_list_builder.py @@ -5,6 +5,7 @@ import time from resync.resource_list_builder import ResourceListBuilder from resync.mapper import Mapper + class TestResourceListBuilder(unittest.TestCase): def setUp(self): @@ -12,111 +13,113 @@ class TestResourceListBuilder(unittest.TestCase): # in UTC so no conversion issues. # Test case file_a: 1343236426 = 2012-07-25T17:13:46Z # Test case file_b: 1000000000 = 2001-09-09T01:46:40Z - os.utime( "tests/testdata/dir1/file_a", (0, 1343236426 ) ) - os.utime( "tests/testdata/dir1/file_b", (0, 1000000000 ) ) + os.utime("tests/testdata/dir1/file_a", (0, 1343236426)) + os.utime("tests/testdata/dir1/file_b", (0, 1000000000)) def test01_simple_scan(self): rlb = ResourceListBuilder() - rlb.mapper = Mapper(['http://example.org/t','tests/testdata/dir1']) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() - self.assertEqual( len(rl), 2 ) + self.assertEqual(len(rl), 2) rli = iter(rl) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_a' ) - self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' ) - self.assertEqual( r.md5, None ) - self.assertEqual( r.length, 20 ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_a') + self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') + self.assertEqual(r.md5, None) + self.assertEqual(r.length, 20) + self.assertEqual(r.path, None) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_b' ) - self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' ) - self.assertEqual( r.md5, None ) - self.assertEqual( r.length, 45 ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_b') + self.assertEqual(r.lastmod, '2001-09-09T01:46:40Z') + self.assertEqual(r.md5, None) + self.assertEqual(r.length, 45) + self.assertEqual(r.path, None) # Make sure at and completed were set - self.assertTrue( rl.md_at is not None ) - self.assertTrue( rl.md_completed is not None ) + self.assertTrue(rl.md_at is not None) + self.assertTrue(rl.md_completed is not None) def test02_no_length(self): rlb = ResourceListBuilder(set_length=False) - rlb.mapper = Mapper(['http://example.org/t','tests/testdata/dir1']) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() - self.assertEqual( len(rl), 2 ) + self.assertEqual(len(rl), 2) rli = iter(rl) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_a' ) - self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' ) - self.assertEqual( r.md5, None ) - self.assertEqual( r.length, None ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_a') + self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') + self.assertEqual(r.md5, None) + self.assertEqual(r.length, None) + self.assertEqual(r.path, None) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_b' ) - self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' ) - self.assertEqual( r.md5, None ) - self.assertEqual( r.length, None ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_b') + self.assertEqual(r.lastmod, '2001-09-09T01:46:40Z') + self.assertEqual(r.md5, None) + self.assertEqual(r.length, None) + self.assertEqual(r.path, None) def test03_set_md5(self): rlb = ResourceListBuilder(set_md5=True) - rlb.mapper = Mapper(['http://example.org/t','tests/testdata/dir1']) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() - self.assertEqual( len(rl), 2 ) + self.assertEqual(len(rl), 2) rli = iter(rl) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_a' ) - self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' ) - self.assertEqual( r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' ) - self.assertEqual( r.length, 20 ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_a') + self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') + self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==') + self.assertEqual(r.length, 20) + self.assertEqual(r.path, None) r = next(rli) - self.assertEqual( r.uri, 'http://example.org/t/file_b' ) - self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' ) - self.assertEqual( r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==' ) - self.assertEqual( r.length, 45 ) - self.assertEqual( r.path, None ) + self.assertEqual(r.uri, 'http://example.org/t/file_b') + self.assertEqual(r.lastmod, '2001-09-09T01:46:40Z') + self.assertEqual(r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==') + self.assertEqual(r.length, 45) + self.assertEqual(r.path, None) def test04_data(self): - rlb = ResourceListBuilder(set_path=True,set_md5=True) - rlb.mapper = Mapper(['http://example.org/t','tests/testdata/dir1']) + rlb = ResourceListBuilder(set_path=True, set_md5=True) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() - self.assertEqual( len(rl), 2) + self.assertEqual(len(rl), 2) r = rl.resources.get('http://example.org/t/file_a') - self.assertTrue( r is not None ) - self.assertEqual( r.uri, 'http://example.org/t/file_a' ) - self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' ) - self.assertEqual( r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' ) - self.assertEqual( r.path, 'tests/testdata/dir1/file_a' ) + self.assertTrue(r is not None) + self.assertEqual(r.uri, 'http://example.org/t/file_a') + self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') + self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==') + self.assertEqual(r.path, 'tests/testdata/dir1/file_a') def test05_from_disk_paths(self): rlb = ResourceListBuilder() - rlb.mapper = Mapper(['http://example.org/t','tests/testdata/dir1']) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) # no path, should get no resources rl = rlb.from_disk(paths=[]) - self.assertEqual( len(rl), 0) + self.assertEqual(len(rl), 0) # full path, 2 resources rl = rlb.from_disk(paths=['tests/testdata/dir1']) - self.assertEqual( len(rl), 2) + self.assertEqual(len(rl), 2) # new object with mapper covering larger space of disk rlb = ResourceListBuilder(set_path=True) - rlb.mapper = Mapper(['http://example.org/t','tests/testdata']) + rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata']) # same path with 2 resources rl = rlb.from_disk(paths=['tests/testdata/dir1']) - self.assertEqual( len(rl), 2) + self.assertEqual(len(rl), 2) # same path with 2 resources - rl = rlb.from_disk(paths=['tests/testdata/dir1','tests/testdata/dir2']) - self.assertEqual( len(rl), 3) + rl = rlb.from_disk( + paths=['tests/testdata/dir1', 'tests/testdata/dir2']) + self.assertEqual(len(rl), 3) # path that is just a single file rl = rlb.from_disk(paths=['tests/testdata/dir1/file_a']) - self.assertEqual( len(rl), 1) + self.assertEqual(len(rl), 1) rli = iter(rl) r = next(rli) - self.assertTrue( r is not None ) - self.assertEqual( r.uri, 'http://example.org/t/dir1/file_a' ) - self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' ) - self.assertEqual( r.md5, None ) - self.assertEqual( r.length, 20 ) - self.assertEqual( r.path, 'tests/testdata/dir1/file_a' ) + self.assertTrue(r is not None) + self.assertEqual(r.uri, 'http://example.org/t/dir1/file_a') + self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') + self.assertEqual(r.md5, None) + self.assertEqual(r.length, 20) + self.assertEqual(r.path, 'tests/testdata/dir1/file_a') if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceListBuilder) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + TestResourceListBuilder) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_resource_list_multifile.py b/tests/test_resource_list_multifile.py index 73d91b7..533f635 100644 --- a/tests/test_resource_list_multifile.py +++ b/tests/test_resource_list_multifile.py @@ -11,91 +11,95 @@ from resync.mapper import Mapper # etree gives ParseError in 2.7, ExpatError in 2.6 etree_error_class = None -if (sys.version_info < (2,7)): +if (sys.version_info < (2, 7)): from xml.parsers.expat import ExpatError etree_error_class = ExpatError else: from xml.etree.ElementTree import ParseError etree_error_class = ParseError + class TestResourceListMultifile(unittest.TestCase): def test_01_read_local_filenames(self): rl = ResourceList() rl.read('tests/testdata/sitemapindex2/sitemap.xml') - self.assertEqual( len(rl.resources), 17, '17 resources from 3 sitemaps listed') + self.assertEqual(len(rl.resources), 17, + '17 resources from 3 sitemaps listed') sr = sorted(rl.uris()) - self.assertEqual( sr[0], 'http://localhost:8888/resources/1' ) - self.assertEqual( sr[1], 'http://localhost:8888/resources/10' ) - self.assertEqual( sr[2], 'http://localhost:8888/resources/100' ) - self.assertEqual( sr[3], 'http://localhost:8888/resources/1000' ) - self.assertEqual( sr[16], 'http://localhost:8888/resources/826' ) + self.assertEqual(sr[0], 'http://localhost:8888/resources/1') + self.assertEqual(sr[1], 'http://localhost:8888/resources/10') + self.assertEqual(sr[2], 'http://localhost:8888/resources/100') + self.assertEqual(sr[3], 'http://localhost:8888/resources/1000') + self.assertEqual(sr[16], 'http://localhost:8888/resources/826') def test_02_read_with_mapper(self): rl = ResourceList() rl.mapper = Mapper(['http://localhost/=tests/testdata/sitemapindex2/']) rl.read('tests/testdata/sitemapindex2/sitemap_mapper.xml') - self.assertEqual( len(rl.resources), 17, '17 resources from 3 sitemaps listed') + self.assertEqual(len(rl.resources), 17, + '17 resources from 3 sitemaps listed') sr = sorted(rl.uris()) - self.assertEqual( sr[0], 'http://localhost:8888/resources/1' ) - self.assertEqual( sr[1], 'http://localhost:8888/resources/10' ) - self.assertEqual( sr[2], 'http://localhost:8888/resources/100' ) - self.assertEqual( sr[3], 'http://localhost:8888/resources/1000' ) - self.assertEqual( sr[16], 'http://localhost:8888/resources/826' ) + self.assertEqual(sr[0], 'http://localhost:8888/resources/1') + self.assertEqual(sr[1], 'http://localhost:8888/resources/10') + self.assertEqual(sr[2], 'http://localhost:8888/resources/100') + self.assertEqual(sr[3], 'http://localhost:8888/resources/1000') + self.assertEqual(sr[16], 'http://localhost:8888/resources/826') def test_11_write_multifile(self): tempdir = tempfile.mkdtemp(prefix='test_resource_list_multifile') rl = ResourceList() rl.mapper = Mapper(['http://localhost/=%s/' % (tempdir)]) - rl.add( Resource( uri='http://localhost/a' ) ) - rl.add( Resource( uri='http://localhost/b' ) ) - rl.add( Resource( uri='http://localhost/c' ) ) - rl.add( Resource( uri='http://localhost/d' ) ) + rl.add(Resource(uri='http://localhost/a')) + rl.add(Resource(uri='http://localhost/b')) + rl.add(Resource(uri='http://localhost/c')) + rl.add(Resource(uri='http://localhost/d')) rl.max_sitemap_entries = 2 # first try writing without mutlifile allowed rl.allow_multifile = False - self.assertRaises( ListBaseIndexError, rl.write, basename=os.path.join(tempdir,'sitemap.xml') ) + self.assertRaises(ListBaseIndexError, rl.write, + basename=os.path.join(tempdir, 'sitemap.xml')) # second actually do it rl.allow_multifile = True - rl.write( basename = os.path.join(tempdir,'sitemap.xml') ) + rl.write(basename=os.path.join(tempdir, 'sitemap.xml')) # check the two component sitemaps rl1 = ResourceList() - rl1.read( os.path.join(tempdir,'sitemap00000.xml') ) - self.assertEquals( len(rl1), 2 ) - self.assertEquals( rl1.capability, 'resourcelist' ) - self.assertFalse( rl1.sitemapindex ) + rl1.read(os.path.join(tempdir, 'sitemap00000.xml')) + self.assertEquals(len(rl1), 2) + self.assertEquals(rl1.capability, 'resourcelist') + self.assertFalse(rl1.sitemapindex) i = iter(rl1) - self.assertEquals( next(i).uri, 'http://localhost/a' ) - self.assertEquals( next(i).uri, 'http://localhost/b' ) + self.assertEquals(next(i).uri, 'http://localhost/a') + self.assertEquals(next(i).uri, 'http://localhost/b') rl2 = ResourceList() - rl2.read( os.path.join(tempdir,'sitemap00001.xml') ) - self.assertEquals( len(rl2), 2 ) + rl2.read(os.path.join(tempdir, 'sitemap00001.xml')) + self.assertEquals(len(rl2), 2) i = iter(rl2) - self.assertEquals( next(i).uri, 'http://localhost/c' ) - self.assertEquals( next(i).uri, 'http://localhost/d' ) + self.assertEquals(next(i).uri, 'http://localhost/c') + self.assertEquals(next(i).uri, 'http://localhost/d') # check the sitemapindex (read just as index) rli = ResourceList() - rli.read( os.path.join(tempdir,'sitemap.xml'), index_only=True ) - self.assertEquals( len(rli), 2 ) + rli.read(os.path.join(tempdir, 'sitemap.xml'), index_only=True) + self.assertEquals(len(rli), 2) i = iter(rli) - self.assertEquals( rli.capability, 'resourcelist' ) - self.assertTrue( rli.sitemapindex ) - self.assertEquals( next(i).uri, 'http://localhost/sitemap00000.xml' ) - self.assertEquals( next(i).uri, 'http://localhost/sitemap00001.xml' ) + self.assertEquals(rli.capability, 'resourcelist') + self.assertTrue(rli.sitemapindex) + self.assertEquals(next(i).uri, 'http://localhost/sitemap00000.xml') + self.assertEquals(next(i).uri, 'http://localhost/sitemap00001.xml') # check the sitemapindex and components - rli = ResourceList( mapper=rl.mapper ) - rli.read( os.path.join(tempdir,'sitemap.xml') ) - self.assertEquals( len(rli), 4 ) - self.assertEquals( rli.capability, 'resourcelist' ) - self.assertFalse( rli.sitemapindex ) + rli = ResourceList(mapper=rl.mapper) + rli.read(os.path.join(tempdir, 'sitemap.xml')) + self.assertEquals(len(rli), 4) + self.assertEquals(rli.capability, 'resourcelist') + self.assertFalse(rli.sitemapindex) i = iter(rli) - self.assertEquals( next(i).uri, 'http://localhost/a' ) - self.assertEquals( next(i).uri, 'http://localhost/b' ) - self.assertEquals( next(i).uri, 'http://localhost/c' ) - self.assertEquals( next(i).uri, 'http://localhost/d' ) + self.assertEquals(next(i).uri, 'http://localhost/a') + self.assertEquals(next(i).uri, 'http://localhost/b') + self.assertEquals(next(i).uri, 'http://localhost/c') + self.assertEquals(next(i).uri, 'http://localhost/d') # cleanup tempdir shutil.rmtree(tempdir) -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestResourceListMultifile) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_resource_set.py b/tests/test_resource_set.py index 95818ac..33099b8 100644 --- a/tests/test_resource_set.py +++ b/tests/test_resource_set.py @@ -1,37 +1,38 @@ import unittest import re from resync.resource import Resource -from resync.resource_set import ResourceSet,ResourceSetDupeError +from resync.resource_set import ResourceSet, ResourceSetDupeError + class TestResourceSet(unittest.TestCase): def test01_add(self): rs = ResourceSet() - self.assertEqual( len(rs), 0 ) - rs.add( Resource('a') ) - self.assertEqual( len(rs), 1 ) - rs.add( Resource('b') ) - self.assertEqual( len(rs), 2 ) - rs.add( Resource('c') ) - self.assertEqual( len(rs), 3 ) + self.assertEqual(len(rs), 0) + rs.add(Resource('a')) + self.assertEqual(len(rs), 1) + rs.add(Resource('b')) + self.assertEqual(len(rs), 2) + rs.add(Resource('c')) + self.assertEqual(len(rs), 3) def test02_order(self): rs = ResourceSet() - rs.add( Resource('a2') ) - rs.add( Resource('a3') ) - rs.add( Resource('a1') ) + rs.add(Resource('a2')) + rs.add(Resource('a3')) + rs.add(Resource('a1')) i = iter(rs) - self.assertEqual( next(i).uri, 'a1' ) - self.assertEqual( next(i).uri, 'a2' ) - self.assertEqual( next(i).uri, 'a3' ) - self.assertRaises( StopIteration, next, i ) + self.assertEqual(next(i).uri, 'a1') + self.assertEqual(next(i).uri, 'a2') + self.assertEqual(next(i).uri, 'a3') + self.assertRaises(StopIteration, next, i) def test03_dupe(self): rs = ResourceSet() - self.assertEqual( len(rs), 0 ) - rs.add( Resource('a') ) - self.assertEqual( len(rs), 1 ) - self.assertRaises( ResourceSetDupeError, rs.add, Resource('a') ) + self.assertEqual(len(rs), 0) + rs.add(Resource('a')) + self.assertEqual(len(rs), 1) + self.assertRaises(ResourceSetDupeError, rs.add, Resource('a')) if __name__ == '__main__': diff --git a/tests/test_sitemap.py b/tests/test_sitemap.py index 5972d3a..776feec 100644 --- a/tests/test_sitemap.py +++ b/tests/test_sitemap.py @@ -1,10 +1,10 @@ import sys import unittest -try: #python2 +try: # python2 # Must try this first as io also exists in python2 # but in the wrong one! import StringIO as io -except ImportError: #python3 +except ImportError: # python3 import io from resync.resource import Resource @@ -13,159 +13,173 @@ from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError # etree gives ParseError in 2.7,3.x; ExpatError in 2.6 etree_error_class = None -if (sys.version_info < (2,7)): +if (sys.version_info < (2, 7)): from xml.parsers.expat import ExpatError etree_error_class = ExpatError else: - #In python3 this seems only to work with the full class name?? - #from xml.etree.ElementTree import ParseError + # In python3 this seems only to work with the full class name?? + # from xml.etree.ElementTree import ParseError import xml.etree.ElementTree etree_error_class = xml.etree.ElementTree.ParseError + class TestSitemap(unittest.TestCase): def test_01_resource_str(self): r1 = Resource('a3') - r1.lastmod='2012-01-11T01:02:03Z' - self.assertEqual( Sitemap().resource_as_xml(r1), "a32012-01-11T01:02:03Z" ) + r1.lastmod = '2012-01-11T01:02:03Z' + self.assertEqual(Sitemap().resource_as_xml( + r1), "a32012-01-11T01:02:03Z") def test_02_resource_str(self): - r1 = Resource('3b',1234.1,9999,'ab54de') - self.assertEqual( Sitemap().resource_as_xml(r1), "3b1970-01-01T00:20:34.100000Z" ) + r1 = Resource('3b', 1234.1, 9999, 'ab54de') + self.assertEqual(Sitemap().resource_as_xml( + r1), "3b1970-01-01T00:20:34.100000Z") def test_03_resource_str_hashes(self): - r1 = Resource('03hashes',1234.1) + r1 = Resource('03hashes', 1234.1) r1.md5 = 'aaa' r1.sha1 = 'bbb' - self.assertEqual( Sitemap().resource_as_xml(r1), "03hashes1970-01-01T00:20:34.100000Z" ) + self.assertEqual(Sitemap().resource_as_xml( + r1), "03hashes1970-01-01T00:20:34.100000Z") r1.sha256 = 'ccc' - self.assertEqual( Sitemap().resource_as_xml(r1), "03hashes1970-01-01T00:20:34.100000Z" ) + self.assertEqual(Sitemap().resource_as_xml( + r1), "03hashes1970-01-01T00:20:34.100000Z") r1.sha1 = None - self.assertEqual( Sitemap().resource_as_xml(r1), "03hashes1970-01-01T00:20:34.100000Z" ) + self.assertEqual(Sitemap().resource_as_xml( + r1), "03hashes1970-01-01T00:20:34.100000Z") def test_04_resource_str(self): - r1 = Resource(uri='4a',lastmod="2013-01-02",length=9999,md5='ab54de') - r1.ln = [{ 'rel':'duplicate', - 'pri':'1', - 'href':'http://mirror1.example.com/res1', - 'modified':'2013-01-02T18:00:00Z' }] - self.assertEqual( Sitemap().resource_as_xml(r1), "4a2013-01-02T00:00:00Z" ) + r1 = Resource(uri='4a', lastmod="2013-01-02", + length=9999, md5='ab54de') + r1.ln = [{'rel': 'duplicate', + 'pri': '1', + 'href': 'http://mirror1.example.com/res1', + 'modified': '2013-01-02T18:00:00Z'}] + self.assertEqual(Sitemap().resource_as_xml( + r1), "4a2013-01-02T00:00:00Z") # add another two rs:ln's - r1.ln.append( { 'rel':'num2' } ) - r1.ln.append( { 'rel':'num3' } ) - self.assertEqual( Sitemap().resource_as_xml(r1), "4a2013-01-02T00:00:00Z" ) + r1.ln.append({'rel': 'num2'}) + r1.ln.append({'rel': 'num3'}) + self.assertEqual(Sitemap().resource_as_xml( + r1), "4a2013-01-02T00:00:00Z") def test_08_print(self): - r1 = Resource(uri='a',lastmod='2001-01-01',length=1234) - r2 = Resource(uri='b',lastmod='2002-02-02',length=56789) - r3 = Resource(uri='c',lastmod='2003-03-03',length=0) - m = ResourceList(md={'capability':'resourcelist','modified':None}) + r1 = Resource(uri='a', lastmod='2001-01-01', length=1234) + r2 = Resource(uri='b', lastmod='2002-02-02', length=56789) + r3 = Resource(uri='c', lastmod='2003-03-03', length=0) + m = ResourceList(md={'capability': 'resourcelist', 'modified': None}) m.add(r1) m.add(r2) m.add(r3) - #print m - self.assertEqual( Sitemap().resources_as_xml(m), "\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z") + # print m + self.assertEqual(Sitemap().resources_as_xml(m), "\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Zc2003-03-03T00:00:00Z") - def test_09_print_from_iter(self): - r1 = Resource(uri='a',lastmod='2001-01-01',length=1234) - r2 = Resource(uri='b',lastmod='2002-02-02',length=56789) + def test_09_print_from_iter(self): + r1 = Resource(uri='a', lastmod='2001-01-01', length=1234) + r2 = Resource(uri='b', lastmod='2002-02-02', length=56789) m = ResourceList() m.add(r1) m.add(r2) i = iter(m) - self.assertEqual( Sitemap().resources_as_xml(i), "\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Z") + self.assertEqual(Sitemap().resources_as_xml(i), "\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Z") def test_10_sitemap(self): - xml='\n\ + xml = '\n\ \ http://e.com/a2012-03-14T18:37:36Z\ ' - s=Sitemap() - i=s.parse_xml(fh=io.StringIO(xml)) - self.assertFalse( s.parsed_index, 'was a sitemap') - self.assertEqual( s.resources_created, 1, 'got 1 resources') + s = Sitemap() + i = s.parse_xml(fh=io.StringIO(xml)) + self.assertFalse(s.parsed_index, 'was a sitemap') + self.assertEqual(s.resources_created, 1, 'got 1 resources') for r in i.resources: - self.assertTrue( r is not None, 'got the uri expected') - self.assertEqual( r.uri, 'http://e.com/a' ) - self.assertEqual( r.lastmod, '2012-03-14T18:37:36Z' ) - self.assertEqual( r.length, 12 ) - self.assertEqual( r.md5, 'Q2hlY2sgSW50ZWdyaXR5IQ==' ) + self.assertTrue(r is not None, 'got the uri expected') + self.assertEqual(r.uri, 'http://e.com/a') + self.assertEqual(r.lastmod, '2012-03-14T18:37:36Z') + self.assertEqual(r.length, 12) + self.assertEqual(r.md5, 'Q2hlY2sgSW50ZWdyaXR5IQ==') def test_11_parse_2(self): - xml='\n\ + xml = '\n\ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' - s=Sitemap() - i=s.parse_xml(fh=io.StringIO(xml)) - self.assertFalse( s.parsed_index, 'was a sitemap') - self.assertEqual( s.resources_created, 2, 'got 2 resources') + s = Sitemap() + i = s.parse_xml(fh=io.StringIO(xml)) + self.assertFalse(s.parsed_index, 'was a sitemap') + self.assertEqual(s.resources_created, 2, 'got 2 resources') def test_12_parse_multi_loc(self): - xml_start='\n\ + xml_start = '\n\ \ ' - xml_end='2012-03-14T18:37:36Z' - s=Sitemap() - two_loc='/tmp/rs_test/src/file_a/tmp/rs_test/src/file_b' - self.assertRaises( SitemapParseError, s.parse_xml, - io.StringIO(xml_start+two_loc+xml_end)) - mt_loc='' - self.assertRaises( SitemapParseError, s.parse_xml, - io.StringIO(xml_start+mt_loc+xml_end)) - mt_loc_att='' - self.assertRaises( SitemapParseError, s.parse_xml, - io.StringIO(xml_start+mt_loc_att+xml_end)) + xml_end = '2012-03-14T18:37:36Z' + s = Sitemap() + two_loc = '/tmp/rs_test/src/file_a/tmp/rs_test/src/file_b' + self.assertRaises(SitemapParseError, s.parse_xml, + io.StringIO(xml_start + two_loc + xml_end)) + mt_loc = '' + self.assertRaises(SitemapParseError, s.parse_xml, + io.StringIO(xml_start + mt_loc + xml_end)) + mt_loc_att = '' + self.assertRaises(SitemapParseError, s.parse_xml, + io.StringIO(xml_start + mt_loc_att + xml_end)) def test_13_parse_multi_lastmod(self): - xml_start='\n\ + xml_start = '\n\ \ uri:a' - xml_end='' - s=Sitemap() - two_lastmod='2013-01-012013-01-02' - self.assertRaises( SitemapParseError, s.parse_xml, - io.StringIO(xml_start+two_lastmod+xml_end)) + xml_end = '' + s = Sitemap() + two_lastmod = '2013-01-012013-01-02' + self.assertRaises(SitemapParseError, s.parse_xml, + io.StringIO(xml_start + two_lastmod + xml_end)) # While it not ideal to omit, is not required and # thus either empty lastmod or lastmod with just an attribute # and no content are not ambiguous and thus should be accepted # with resulting None for resource.lastmod - mt_lastmod='' - i=s.parse_xml(fh=io.StringIO(xml_start+mt_lastmod+xml_end)) - self.assertEqual( s.resources_created, 1 ) - self.assertEqual( i.resources[0].lastmod, None ) - mt_lastmod_att='' - i=s.parse_xml(fh=io.StringIO(xml_start+mt_lastmod_att+xml_end)) - self.assertEqual( s.resources_created, 1 ) - self.assertEqual( i.resources[0].lastmod, None ) + mt_lastmod = '' + i = s.parse_xml(fh=io.StringIO(xml_start + mt_lastmod + xml_end)) + self.assertEqual(s.resources_created, 1) + self.assertEqual(i.resources[0].lastmod, None) + mt_lastmod_att = '' + i = s.parse_xml(fh=io.StringIO(xml_start + mt_lastmod_att + xml_end)) + self.assertEqual(s.resources_created, 1) + self.assertEqual(i.resources[0].lastmod, None) def test_14_parse_multi(self): - xml_start='\n\ + xml_start = '\n\ ' - xml_end='uri:a' - s=Sitemap() - two_md='' - self.assertRaises( SitemapParseError, s.parse_xml, - io.StringIO(xml_start+two_md+xml_end)) + xml_end = 'uri:a' + s = Sitemap() + two_md = '' + self.assertRaises(SitemapParseError, s.parse_xml, + io.StringIO(xml_start + two_md + xml_end)) def test_15_parse_illformed(self): - s=Sitemap() + s = Sitemap() # ExpatError in python2.6, ParserError in 2.7,3.x - self.assertRaises( etree_error_class, s.parse_xml, io.StringIO('not xml') ) - self.assertRaises( etree_error_class, s.parse_xml, io.StringIO('something') ) + self.assertRaises(etree_error_class, s.parse_xml, + io.StringIO('not xml')) + self.assertRaises(etree_error_class, s.parse_xml, + io.StringIO('something')) def test_16_parse_valid_xml_but_other(self): - s=Sitemap() - self.assertRaises( SitemapParseError, s.parse_xml, io.StringIO(' ') ) - self.assertRaises( SitemapParseError, s.parse_xml, io.StringIO(' ') ) + s = Sitemap() + self.assertRaises(SitemapParseError, s.parse_xml, io.StringIO( + ' ')) + self.assertRaises(SitemapParseError, s.parse_xml, io.StringIO( + ' ')) def test_17_parse_sitemapindex_as_sitemap(self): - s=Sitemap() - self.assertRaises( SitemapIndexError, s.parse_xml, io.StringIO(' '), sitemapindex=False ) + s = Sitemap() + self.assertRaises(SitemapIndexError, s.parse_xml, io.StringIO( + ' '), sitemapindex=False) def test_18_parse_with_rs_ln_on_resource(self): - xml='\n\ + xml = '\n\ \ \ \ @@ -180,103 +194,108 @@ class TestSitemap(unittest.TestCase): \ \ ' - s=Sitemap() - rc=s.parse_xml(fh=io.StringIO(xml)) - self.assertFalse( s.parsed_index, 'was a sitemap') - self.assertEqual( s.resources_created, 2, 'got 2 resources') + s = Sitemap() + rc = s.parse_xml(fh=io.StringIO(xml)) + self.assertFalse(s.parsed_index, 'was a sitemap') + self.assertEqual(s.resources_created, 2, 'got 2 resources') i = iter(rc) r1 = next(i) r2 = next(i) - self.assertEqual( r1.uri, 'http://example.com/file_a' ) - self.assertEqual( r1.ln[0]['rel'], 'duplicate' ) - self.assertEqual( r1.ln[0]['href'], 'http://mirror1.example.com/res1' ) - self.assertEqual( r1.ln[0]['modified'], '2013-01-02' ) - self.assertEqual( r1.ln[0]['pri'], 1 ) - self.assertEqual( r2.uri, 'http://example.com/file_b' ) + self.assertEqual(r1.uri, 'http://example.com/file_a') + self.assertEqual(r1.ln[0]['rel'], 'duplicate') + self.assertEqual(r1.ln[0]['href'], 'http://mirror1.example.com/res1') + self.assertEqual(r1.ln[0]['modified'], '2013-01-02') + self.assertEqual(r1.ln[0]['pri'], 1) + self.assertEqual(r2.uri, 'http://example.com/file_b') def test_19_parse_with_bad_rs_ln(self): - xmlstart='\n\ + xmlstart = '\n\ \ \ http://example.com/file_a' - xmlend='' - s=Sitemap() + xmlend = '' + s = Sitemap() # # missing href - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) # missing rel - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) # bad length - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) # bad pri - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) - xml=xmlstart+''+xmlend - self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + xml = xmlstart + '' + xmlend + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) # and finally OK with errors fixes - xml=xmlstart+''+xmlend + xml = xmlstart + '' + xmlend rc = s.parse_xml(fh=io.StringIO(xml)) - self.assertEqual( len(rc.resources), 1, 'good at last, extra attribute ignored' ) + self.assertEqual(len(rc.resources), 1, + 'good at last, extra attribute ignored') def test_20_parse_sitemapindex_empty(self): - s=Sitemap() - si = s.parse_xml( fh=io.StringIO(' '), sitemapindex=True ) - self.assertTrue( s.parsed_index, 'was a sitemapindex') - self.assertEqual( len(si.resources), 0, '0 sitemaps') + s = Sitemap() + si = s.parse_xml(fh=io.StringIO( + ' '), sitemapindex=True) + self.assertTrue(s.parsed_index, 'was a sitemapindex') + self.assertEqual(len(si.resources), 0, '0 sitemaps') def test_21_parse_sitemapindex(self): - s=Sitemap() - si = s.parse_xml( fh=io.StringIO('aaabbb'), sitemapindex=True ) - self.assertEqual( len(si.resources), 2, '2 sitemaps') + s = Sitemap() + si = s.parse_xml(fh=io.StringIO( + 'aaabbb'), sitemapindex=True) + self.assertEqual(len(si.resources), 2, '2 sitemaps') sms = sorted(si.uris()) - self.assertEqual( sms, ['aaa','bbb'] ) + self.assertEqual(sms, ['aaa', 'bbb']) # add a couple more - s.parse_xml( fh=io.StringIO('ccdd'), resources=si ) - self.assertTrue( s.parsed_index, 'was a sitemapindex') - self.assertEqual( len(si.resources), 4, '4 sitemaps total') + s.parse_xml(fh=io.StringIO( + 'ccdd'), resources=si) + self.assertTrue(s.parsed_index, 'was a sitemapindex') + self.assertEqual(len(si.resources), 4, '4 sitemaps total') sms = sorted(si.uris()) - self.assertEqual( sms, ['aaa','bbb', 'cc', 'dd'] ) + self.assertEqual(sms, ['aaa', 'bbb', 'cc', 'dd']) def test_22_parse_sitemapindex_file(self): - s=Sitemap() - fh=open('tests/testdata/sitemapindex1/sitemap.xml','r') - si = s.parse_xml( fh=fh, sitemapindex=True ) - self.assertTrue( s.parsed_index, 'was a sitemapindex') - self.assertEqual( len(si.resources), 3, '3 sitemaps') + s = Sitemap() + fh = open('tests/testdata/sitemapindex1/sitemap.xml', 'r') + si = s.parse_xml(fh=fh, sitemapindex=True) + self.assertTrue(s.parsed_index, 'was a sitemapindex') + self.assertEqual(len(si.resources), 3, '3 sitemaps') sms = sorted(si.uris()) - self.assertEqual( sms, ['http://localhost:8888/sitemap00000.xml','http://localhost:8888/sitemap00001.xml','http://localhost:8888/sitemap00002.xml'] ) - #self.assertEqual( si.resources['http://localhost:8888/sitemap00000.xml'].lastmod, '2012-06-13T18:09:13Z' ) + self.assertEqual(sms, ['http://localhost:8888/sitemap00000.xml', + 'http://localhost:8888/sitemap00001.xml', 'http://localhost:8888/sitemap00002.xml']) + # self.assertEqual( si.resources['http://localhost:8888/sitemap00000.xml'].lastmod, '2012-06-13T18:09:13Z' ) def test_21_parse_multi_sitemapindex(self): s = Sitemap() - fh=open('tests/testdata/sitemapindex2/sitemap.xml','r') - si = s.parse_xml( fh=fh, sitemapindex=True ) - self.assertTrue( s.parsed_index, 'was a sitemapindex') - self.assertEqual( len(si.resources), 3, '3 sitemaps listed') + fh = open('tests/testdata/sitemapindex2/sitemap.xml', 'r') + si = s.parse_xml(fh=fh, sitemapindex=True) + self.assertTrue(s.parsed_index, 'was a sitemapindex') + self.assertEqual(len(si.resources), 3, '3 sitemaps listed') def test_30_parse_change_list(self): - xml='\n\ + xml = '\n\ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ /tmp/rs_test/src/file_b2012-03-14T18:37:36Z\ ' - s=Sitemap() - s.resource_class=Resource - c=s.parse_xml(fh=io.StringIO(xml)) - self.assertEqual( s.resources_created, 2, 'got 2 resources') + s = Sitemap() + s.resource_class = Resource + c = s.parse_xml(fh=io.StringIO(xml)) + self.assertEqual(s.resources_created, 2, 'got 2 resources') i = iter(c) r1 = next(i) - self.assertEqual( r1.uri, '/tmp/rs_test/src/file_a' ) - self.assertEqual( r1.change, 'updated' ) + self.assertEqual(r1.uri, '/tmp/rs_test/src/file_a') + self.assertEqual(r1.change, 'updated') r2 = next(i) - self.assertEqual( r2.uri, '/tmp/rs_test/src/file_b' ) - self.assertEqual( r2.change, None ) + self.assertEqual(r2.uri, '/tmp/rs_test/src/file_b') + self.assertEqual(r2.change, None) -if __name__ == '__main__': +if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(TestSitemap) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tests/test_source_description.py b/tests/test_source_description.py index bb97bf8..5ef213c 100644 --- a/tests/test_source_description.py +++ b/tests/test_source_description.py @@ -3,46 +3,48 @@ import re from resync.resource import Resource from resync.source_description import SourceDescription + class TestSourceDescription(unittest.TestCase): def test01_empty(self): rsd = SourceDescription() rsd.describedby = "http://example.org/about" - self.assertEqual( len(rsd), 0 ) + self.assertEqual(len(rsd), 0) rsd.md_at = None - self.assertEqual( rsd.as_xml(), '\n' ) + self.assertEqual(rsd.as_xml(), '\n') def test02_one_caplist(self): rsd = SourceDescription() rsd.describedby = "http://example.org/about" - self.assertEqual( len(rsd), 0 ) + self.assertEqual(len(rsd), 0) rsd.md_at = None rsd.add_capability_list("http://example.org/ds1/cl.xml") - self.assertEqual( len(rsd), 1 ) - self.assertEqual( rsd.as_xml(), '\nhttp://example.org/ds1/cl.xml' ) + self.assertEqual(len(rsd), 1) + self.assertEqual(rsd.as_xml(), '\nhttp://example.org/ds1/cl.xml') def test03_a_bunch(self): rsd = SourceDescription() rsd.describedby = "http://example.org/about" - self.assertEqual( len(rsd), 0 ) + self.assertEqual(len(rsd), 0) rsd.add_capability_list("http://example.org/ds1/cl.xml") rsd.add_capability_list("http://example.org/ds2/cl.xml") rsd.add_capability_list("http://example.org/ds3/cl.xml") - self.assertEqual( len(rsd), 3 ) - self.assertEqual( rsd.as_xml(), '\nhttp://example.org/ds1/cl.xmlhttp://example.org/ds2/cl.xmlhttp://example.org/ds3/cl.xml' ) + self.assertEqual(len(rsd), 3) + self.assertEqual(rsd.as_xml(), '\nhttp://example.org/ds1/cl.xmlhttp://example.org/ds2/cl.xmlhttp://example.org/ds3/cl.xml') def test04_parse(self): - xml='\nhttp://example.org/ds1/cl.xmlhttp://example.org/ds2/cl.xmlhttp://example.org/ds3/cl.xml' - sd=SourceDescription() + xml = '\nhttp://example.org/ds1/cl.xmlhttp://example.org/ds2/cl.xmlhttp://example.org/ds3/cl.xml' + sd = SourceDescription() sd.parse(str_data=xml) - self.assertEqual( sd.link_href('describedby'), 'http://example.org/about', - 'describedby link' ) - self.assertEqual( sd.capability, 'description' ) - self.assertEqual( len(sd.resources), 3, 'got 3 capacility lists' ) - [r1,r2,r3]=sd.resources - self.assertEqual( r1.uri, 'http://example.org/ds1/cl.xml' ) - self.assertEqual( r1.capability, 'capabilitylist' ) + self.assertEqual(sd.link_href('describedby'), 'http://example.org/about', + 'describedby link') + self.assertEqual(sd.capability, 'description') + self.assertEqual(len(sd.resources), 3, 'got 3 capacility lists') + [r1, r2, r3] = sd.resources + self.assertEqual(r1.uri, 'http://example.org/ds1/cl.xml') + self.assertEqual(r1.capability, 'capabilitylist') if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestSourceDescription) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( + TestSourceDescription) unittest.TextTestRunner().run(suite) diff --git a/tests/test_url_authority.py b/tests/test_url_authority.py index b534484..0362a47 100644 --- a/tests/test_url_authority.py +++ b/tests/test_url_authority.py @@ -3,6 +3,7 @@ import unittest from resync.url_authority import UrlAuthority + class TestUrlAuthority(unittest.TestCase): def test01_strict_authority(self): diff --git a/tests/test_utils.py b/tests/test_utils.py index eb08325..de775c9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,16 +1,17 @@ import unittest import resync.utils + class TestUtill(unittest.TestCase): def test1_string(self): - self.assertEqual( resync.utils.compute_md5_for_string('A file\n'), - 'j912liHgA/48DCHpkptJHg==') + self.assertEqual(resync.utils.compute_md5_for_string('A file\n'), + 'j912liHgA/48DCHpkptJHg==') def test2_file(self): # Should be same as the string above - self.assertEqual( resync.utils.compute_md5_for_file('tests/testdata/a'), - 'j912liHgA/48DCHpkptJHg==') + self.assertEqual(resync.utils.compute_md5_for_file('tests/testdata/a'), + 'j912liHgA/48DCHpkptJHg==') if __name__ == '__main__': unittest.main() diff --git a/tests/test_w3c_datetime.py b/tests/test_w3c_datetime.py index d93acb8..18f82a4 100644 --- a/tests/test_w3c_datetime.py +++ b/tests/test_w3c_datetime.py @@ -12,20 +12,27 @@ class TestW3cDatetime(unittest.TestCase): def test01_datetime_to_str(self): """Writing.""" - self.assertEqual(datetime_to_str(0), "1970-01-01T00:00:00Z") + self.assertEqual(datetime_to_str(0), + "1970-01-01T00:00:00Z") self.assertEqual(datetime_to_str(0.000001), "1970-01-01T00:00:00.000001Z") - self.assertEqual(datetime_to_str(0.1), "1970-01-01T00:00:00.100000Z") - self.assertEqual(datetime_to_str(1), "1970-01-01T00:00:01Z") - self.assertEqual(datetime_to_str(60), "1970-01-01T00:01:00Z") - self.assertEqual(datetime_to_str(60 * 60), "1970-01-01T01:00:00Z") - self.assertEqual(datetime_to_str(60 * 60 * 24), "1970-01-02T00:00:00Z") + self.assertEqual(datetime_to_str(0.1), + "1970-01-01T00:00:00.100000Z") + self.assertEqual(datetime_to_str(1), + "1970-01-01T00:00:01Z") + self.assertEqual(datetime_to_str(60), + "1970-01-01T00:01:00Z") + self.assertEqual(datetime_to_str(60 * 60), + "1970-01-01T01:00:00Z") + self.assertEqual(datetime_to_str(60 * 60 * 24), + "1970-01-02T00:00:00Z") self.assertEqual(datetime_to_str(60 * 60 * 24 * 31), "1970-02-01T00:00:00Z") self.assertEqual(datetime_to_str(60 * 60 * 24 * 365), "1971-01-01T00:00:00Z") - # - self.assertEqual(datetime_to_str(1234567890), "2009-02-13T23:31:30Z") + # Random other datetime + self.assertEqual(datetime_to_str(1234567890), + "2009-02-13T23:31:30Z") # Rounding issues self.assertEqual(datetime_to_str(0.199999), "1970-01-01T00:00:00.199999Z") @@ -42,11 +49,12 @@ class TestW3cDatetime(unittest.TestCase): self.assertEqual(datetime_to_str(0.200001), "1970-01-01T00:00:00.200001Z") # No fractions - self.assertEqual(datetime_to_str(100, True), "1970-01-01T00:01:40Z") + self.assertEqual(datetime_to_str(100, True), + "1970-01-01T00:01:40Z") self.assertEqual(datetime_to_str(0.2000009, True), "1970-01-01T00:00:00Z") - self.assertEqual(datetime_to_str( - 0.200001, no_fractions=True), "1970-01-01T00:00:00Z") + self.assertEqual(datetime_to_str(0.200001, no_fractions=True), + "1970-01-01T00:00:00Z") def test02_str_to_datetime(self): """Reading.""" @@ -54,11 +62,10 @@ class TestW3cDatetime(unittest.TestCase): self.assertEqual(str_to_datetime("1970-01-01T00:00:00.000Z"), 0) self.assertEqual(str_to_datetime("1970-01-01T00:00:00+00:00"), 0) self.assertEqual(str_to_datetime("1970-01-01T00:00:00-00:00"), 0) - self.assertEqual(str_to_datetime( - "1970-01-01T00:00:00.000001Z"), 0.000001) + self.assertEqual(str_to_datetime("1970-01-01T00:00:00.000001Z"), 0.000001) self.assertEqual(str_to_datetime("1970-01-01T00:00:00.1Z"), 0.1) self.assertEqual(str_to_datetime("1970-01-01T00:00:00.100000Z"), 0.1) - # + # Random other datetime self.assertEqual(str_to_datetime("2009-02-13T23:31:30Z"), 1234567890) def test03_same(self): @@ -88,15 +95,12 @@ class TestW3cDatetime(unittest.TestCase): self.assertRaises(ValueError, str_to_datetime, "2012-13-01") self.assertRaises(ValueError, str_to_datetime, "2012-12-32") self.assertRaises(ValueError, str_to_datetime, "2012-11-01T10:10:60") - self.assertRaises(ValueError, str_to_datetime, - "2012-11-01T10:10:59.9x") + self.assertRaises(ValueError, str_to_datetime, "2012-11-01T10:10:59.9x") # Valid ISO8601 but not allowed in W3C Datetime self.assertRaises(ValueError, str_to_datetime, "2012-11-01T01:01:01") self.assertRaises(ValueError, str_to_datetime, "2012-11-01 01:01:01Z") - self.assertRaises(ValueError, str_to_datetime, - "2012-11-01T01:01:01+0000") - self.assertRaises(ValueError, str_to_datetime, - "2012-11-01T01:01:01-1000") + self.assertRaises(ValueError, str_to_datetime, "2012-11-01T01:01:01+0000") + self.assertRaises(ValueError, str_to_datetime, "2012-11-01T01:01:01-1000") # Bad values self.assertRaises(ValueError, str_to_datetime, "2012-00-01T01:01:01Z") self.assertRaises(ValueError, str_to_datetime, "2012-13-01T01:01:01Z") @@ -105,10 +109,8 @@ class TestW3cDatetime(unittest.TestCase): self.assertRaises(ValueError, str_to_datetime, "2012-13-01T24:01:01Z") self.assertRaises(ValueError, str_to_datetime, "2012-13-01T00:60:01Z") self.assertRaises(ValueError, str_to_datetime, "2012-13-01T00:00:60Z") - self.assertRaises(ValueError, str_to_datetime, - "2012-11-01T01:01:01+99:00") - self.assertRaises(ValueError, str_to_datetime, - "2012-11-01T01:01:01-25:00") + self.assertRaises(ValueError, str_to_datetime, "2012-11-01T01:01:01+99:00") + self.assertRaises(ValueError, str_to_datetime, "2012-11-01T01:01:01-25:00") def test05_roundtrips(self): """Round trips.""" diff --git a/tests/testcase_with_tmpdir.py b/tests/testcase_with_tmpdir.py index a6bcf1e..8ae7125 100644 --- a/tests/testcase_with_tmpdir.py +++ b/tests/testcase_with_tmpdir.py @@ -5,15 +5,16 @@ import unittest import tempfile import shutil + class TestCase(unittest.TestCase): """Adds setUpClass an tearDownClass that create and destroy tmpdir.""" - _tmpdir=None + _tmpdir = None @classmethod def setUpClass(cls): # Create tmp dir to write to and check - cls._tmpdir=tempfile.mkdtemp() + cls._tmpdir = tempfile.mkdtemp() if (not os.path.isdir(cls._tmpdir)): raise Exception("Failed to create tempdir to use for dump tests") try: @@ -39,6 +40,6 @@ class TestCase(unittest.TestCase): # FIXME - Hack to work on python2.6 where setUpClass is not called, will # FIXME - not have proper tidy as tearDownClass will not be called. # FIXME - Remove when 2.6 no longer supported - if (not self._tmpdir and sys.version_info < (2,7)): + if (not self._tmpdir and sys.version_info < (2, 7)): self.setUpClass() return(self._tmpdir)