Merge pull request #48 from resync/develop

Release 2.0.1

.gitignore

@@ -11,10 +11,8 @@ htmlcov
 # Files generated by client / local config
 .resync-client-status.cfg
 resync-client.log
-resync.library.cornell.edu_sim100
-resync.library.cornell.edu_sim1000
-resync.library.cornell.edu_sim10000
 localhost_8888
+tmp
 # Related to packaging
 resync.egg-info
 # Other...
@@ -22,5 +20,4 @@ resync.egg-info
 *.pyc
 *~
 .cache
-rc
-re
+

CHANGES.md

@@ -1,5 +1,12 @@
 # resync change log
 
+v2.0.1 2021-03-23
+  * Route all URI and file requests through `resync/url_or_file_open.py` so that settings such as authentication headers can be consistently applied
+  * Do not exclude any directories from sync by default, specify with --exclude
+  * Fix comparison of using possibly multiple checksums
+  * Improve test coverage
+  * Remove Python 2 cruft
+
 v2.0.0 2020-12-16
   * Supports ResourceSync v1.1 ANSI/NISO Z39.99-2017 <http://www.openarchives.org/rs/1.1/toc> as default
   * Support for the prior v1.0 ANSI/NISO Z39.99-2014 <http://www.openarchives.org/rs/1.0/toc> is retained with `spec-version='1.0'` option in scripts and `spec_version='1.0'` in various classes

README.md

@@ -1,7 +1,7 @@
 # resync
 
-[![Build Status](https://travis-ci.org/resync/resync.svg?branch=master)](https://travis-ci.org/resync/resync)
-[![Test Coverage](https://coveralls.io/repos/github/resync/resync/badge.svg?branch=master)](https://coveralls.io/github/resync/resync)
+[![Build Status](https://travis-ci.org/resync/resync.svg?branch=main)](https://travis-ci.org/resync/resync)
+[![Test Coverage](https://coveralls.io/repos/github/resync/resync/badge.svg?branch=main)](https://coveralls.io/github/resync/resync)
 
 **resync** is a ResourceSync library with supporting client scipts,
 written in python.
@@ -102,7 +102,7 @@ Thanks to: [Bernhard Haslhofer](https://github.com/behas),
 [Robert Sanderson](<https://github.com/azaroth42)
 and [other contributors](https://github.com/resync/resync/graphs/contributors).
 
-See [CONTRIBUTING.md](https://github.com/resync/resync/blob/master/CONTRIBUTING.md)
+See [CONTRIBUTING.md](https://github.com/resync/resync/blob/main/CONTRIBUTING.md)
 for guidelines for contributing.
 
 ## Copyright and License

docs/pypi_upload.md

@@ -9,11 +9,11 @@ resync is at <https://pypi.python.org/pypi/resync> on pypi
 0. In `develop` branch: bump version number in `resync/__init__.py` and check `CHANGES.md` is up to date
 1. Check all tests good with appropriate Python 3.x (`python setup.py test` and CI)
 2. Check code is up-to-date with github version
-3. Check out `master` and merge in `develop`
+3. Check out `main` and merge in `develop`
 4. Check all tests still good (`python setup.py test` and CI)
-5. Make sure master `README.md` has correct travis-ci icon link
+5. Make sure main `README.md` has correct travis-ci icon link
 6. Check branches as expected (`git branch -a`)
-7. Check local build and version reported OK (`python setup.py build; python setup.py install; resync-sync --version`)
+7. Check local build and version reported OK (`python setup.py build; python setup.py install; resync-sync -h`)
 8. Check client works with simulator:
 
    ```
@@ -58,14 +58,22 @@ resync is at <https://pypi.python.org/pypi/resync> on pypi
 
 9. If all checks out OK, tag and push the new version to github:
 
-    ```
-    git tag -n1
-    #...current tags
-    git tag -a -m "ResourceSync v1.1 specification, add --delay" v2.0.0
-    git push --tags
+```
+git tag -n1
+#...current tags
+git tag -a -m "ResourceSync library and client" v2.0.0
+git push --tags
+```
 
-    python setup.py sdist upload
-    ```
+10. Upload to PyPI
 
-10. Then check on PyPI at <https://pypi.python.org/pypi/resync>
+
+```
+rm -r dist
+python setup.py sdist bdist_wheel; ls dist
+# Should be source and wheel files for just this version
+twine upload dist/*
+```
+
+10. Check on PyPI at <https://pypi.python.org/pypi/resync>
 11. Finally, back on `develop` branch start new version number by editing `resync/__init__.py` and `CHANGES.md`

resync-explorer

@@ -70,7 +70,7 @@ def main():
                  verbose=args.verbose)
 
     print("----- ResourceSync Explorer -----")
-    process_shared_misc_options(args)
+    process_shared_misc_options(args, include_remote=True)
 
     c = Explorer(hashes=args.hash,
                  verbose=args.verbose)

resync/__init__.py

@@ -2,7 +2,7 @@
 
 This is the one place the version number for resync is stored.
 """
-__version__ = '2.0.0'
+__version__ = '2.0.1'
 
 # Enable easy import for core classes, e.g.
 # from resync import Resource

resync/client.py

@@ -11,7 +11,8 @@ import distutils.dir_util
 import re
 import time
 import logging
-import requests
+import shutil
+import socket
 
 from .resource_list_builder import ResourceListBuilder
 from .resource_list import ResourceList
@@ -224,7 +225,7 @@ class Client(object):
         rlb = ResourceListBuilder(set_hashes=self.hashes, mapper=self.mapper)
         rlb.set_path = set_path
         try:
-            rlb.add_exclude_files(self.exclude_patterns)
+            rlb.add_exclude_patterns(self.exclude_patterns)
             rl = rlb.from_disk(paths=paths)
         except ValueError as e:
             raise ClientFatalError(str(e))
@@ -504,15 +505,12 @@ class Client(object):
             # 1. GET
             for try_i in range(1, self.tries + 1):
                 try:
-                    r = requests.get(resource.uri, timeout=self.timeout, stream=True)
-                    # Fail on 4xx or 5xx
-                    r.raise_for_status()
-                    with open(filename, 'wb') as fd:
-                        for chunk in r.iter_content(chunk_size=1024):
-                            fd.write(chunk)
+                    with url_or_file_open(resource.uri, timeout=self.timeout) as fh_in:
+                        with open(filename, 'wb') as fh_out:
+                            shutil.copyfileobj(fh_in, fh_out)
                     num_updated += 1
                     break
-                except requests.Timeout as e:
+                except socket.timeout as e:
                     if try_i < self.tries:
                         msg = 'Download timed out, retrying...'
                         self.logger.info(msg)
@@ -525,7 +523,7 @@ class Client(object):
                             return(num_updated)
                         else:
                             raise ClientFatalError(msg)
-                except (requests.RequestException, IOError) as e:
+                except IOError as e:
                     msg = "Failed to GET %s -- %s" % (resource.uri, str(e))
                     if (self.ignore_failures):
                         self.logger.warning(msg)

resync/client_utils.py

@@ -7,8 +7,6 @@ import argparse
 from datetime import datetime
 import logging
 import logging.config
-import re
-from urllib.request import urlopen
 
 from .url_or_file_open import set_url_or_file_open_config
 
@@ -181,7 +179,7 @@ def add_shared_misc_options(opt, default_logfile, include_remote=False):
                           "--incremental sync")
     opt.add_argument('--exclude', type=str, action='append',
                      help="exclude resources with URI or filename matching the python regex "
-                          "supplied (see: <https://docs.python.org/2/howto/regex.html> for regex "
+                          "supplied (see: <https://docs.python.org/3/howto/regex.html> for regex "
                           "information, repeat option for multiple excludes)")
     opt.add_argument('--multifile', '-m', action='store_true',
                      help="disable reading and output of sitemapindex for multifile sitemap")

resync/explorer.py

@@ -17,7 +17,6 @@ import distutils.dir_util
 import re
 import time
 import logging
-import requests
 
 from .mapper import Mapper
 from .sitemap import Sitemap
@@ -164,7 +163,7 @@ class Explorer(Client):
                 caps = 'resource'
             else:
                 caps = self.allowed_entries(capability)
-        elif (r.capability is 'resource'):
+        elif (r.capability == 'resource'):
             caps = r.capability
         else:
             caps = [r.capability]
@@ -278,38 +277,36 @@ class Explorer(Client):
         print("HEAD %s" % (uri))
         if (re.match(r'^\w+:', uri)):
             # Looks like a URI
-            response = requests.head(uri)
+            with url_or_file_open(uri, method='HEAD') as response:
+                status_code = response.code()
+                headers = response.headers()
         else:
             # Mock up response if we have a local file
-            response = self.head_on_file(uri)
-        print("  status: %s" % (response.status_code))
-        if (response.status_code == '200'):
+            (status_code, headers) = self.head_on_file(uri)
+        print("  status: %s" % (status_code))
+        if (status_code == '200'):
             # print some of the headers
             for header in ['content-length', 'last-modified',
                            'lastmod', 'content-type', 'etag']:
-                if header in response.headers:
+                if header in headers:
                     check_str = ''
                     if (check_headers is not None and header in check_headers):
-                        if (response.headers[header] == check_headers[header]):
+                        if (headers[header] == check_headers[header]):
                             check_str = ' MATCHES EXPECTED VALUE'
                         else:
                             check_str = ' EXPECTED %s' % (
                                 check_headers[header])
-                    print(
-                        "  %s: %s%s" %
-                        (header, response.headers[header], check_str))
+                    print("  %s: %s%s" % (header, headers[header], check_str))
 
     def head_on_file(self, file):
-        """Mock up requests.head(..) response on local file."""
-        response = HeadResponse()
-        if (not os.path.isfile(file)):
-            response.status_code = '404'
-        else:
-            response.status_code = '200'
-            response.headers[
-                'last-modified'] = datetime_to_str(os.path.getmtime(file))
-            response.headers['content-length'] = os.path.getsize(file)
-        return(response)
+        """Get fake status code and headers from local file."""
+        status_code = '404'
+        headers = {}
+        if os.path.isfile(file):
+            status_code = '200'
+            headers['last-modified'] = datetime_to_str(os.path.getmtime(file))
+            headers['content-length'] = os.path.getsize(file)
+        return(status_code, headers)
 
     def allowed_entries(self, capability):
         """Return list of allowed entries for given capability document.
@@ -367,15 +364,6 @@ class XResource(object):
         self.checks = checks
 
 
-class HeadResponse(object):
-    """Object to mock up requests.head(...) response."""
-
-    def __init__(self):
-        """Initialize with no status_code and no headers."""
-        self.status_code = None
-        self.headers = {}
-
-
 class ExplorerQuit(Exception):
     """Exception raised when user quits normally, no error."""
 

resync/hashes.py

@@ -34,14 +34,16 @@ class Hashes(object):
     up [http://docs.python.org/library/base64.html]
     """
 
+    NAME_TO_ATTRIBUTE = {'md5': 'md5', 'sha-1': 'sha1', 'sha-256': 'sha256'}
+
     def __init__(self, hashes=None, file=None):
-        """Initialize Hasher object with types of hash to caluclate.
+        """Initialize Hashes object with types of hash to caluclate.
 
         If file is supplied then compute for that file.
         """
         self.hashes = set()
         for hash in hashes:
-            if (hash not in ['md5', 'sha-1', 'sha-256']):
+            if (hash not in self.NAME_TO_ATTRIBUTE.keys()):
                 raise Exception("Hash type %s not supported" % (hash))
             self.hashes.add(hash)
         #
@@ -74,14 +76,24 @@ class Hashes(object):
             data = f.read(block_size)
             if not data:
                 break
-            if (self.md5_calc is not None):
+            if self.md5_calc is not None:
                 self.md5_calc.update(data)
-            if (self.sha1_calc is not None):
+            if self.sha1_calc is not None:
                 self.sha1_calc.update(data)
-            if (self.sha256_calc is not None):
+            if self.sha256_calc is not None:
                 self.sha256_calc.update(data)
         f.close()
 
+    def set(self, resource):
+        """Set hash values for resource from current file.
+
+        Assumes that resource has appropriate attributes or setters
+        with names md5, sha1, etc. and that hashes have been calculated.
+        """
+        for hash in self.hashes:
+            att = self.NAME_TO_ATTRIBUTE[hash]
+            setattr(resource, att, getattr(self, att))
+
     @property
     def md5(self):
         """Return MD5 hash calculated."""

resync/list_base_with_index.py

@@ -101,7 +101,7 @@ class ListBaseWithIndex(ListBase):
             self.logger.debug(
                 "Read %d bytes from %s" %
                 (self.content_length, uri))
-        except KeyError:
+        except (KeyError, TypeError):
             # If we don't get a length then c'est la vie
             self.logger.debug("Read ????? bytes from %s" % (uri))
             pass
@@ -170,7 +170,7 @@ class ListBaseWithIndex(ListBase):
         try:
             self.content_length = int(fh.info()['Content-Length'])
             self.bytes_read += self.content_length
-        except KeyError:
+        except (KeyError, TypeError):
             # If we don't get a length then c'est la vie
             pass
         self.logger.info(

resync/mapper.py

@@ -173,23 +173,35 @@ class Map:
         This does not rely on the destination filepath actually
         existing on the local filesystem, just on pattern matching.
         Return source URI on success, None on failure.
+
+        Relies upon self.dst_path and self.src_path not including trailing
+        slashes. However, a match of just self.dst_path withouth a trailing
+        slash will return self.src_path with a trailing slash.
         """
-        m = re.match(self.dst_path + "/(.*)$", dst_file)
+        m = re.match(self.dst_path + "(/.*)?$", dst_file)
         if (m is None):
             return(None)
         rel_path = m.group(1)
-        return(self.src_uri + '/' + rel_path)
+        if rel_path is None:
+            rel_path = '/'
+        return self.src_uri + rel_path
 
     def src_to_dst(self, src_uri):
         """Return the dst filepath from the src URI.
 
-        Returns None on failure, destination path on success.
+        Returns None on failure, local destination path on success.
+
+        Relies upon self.dst_path and self.src_path not including trailing
+        slashes. However, a match of just self.src_path withouth a trailing
+        slash will return self.dst_path with a trailing slash.
         """
-        m = re.match(self.src_uri + "/(.*)$", src_uri)
+        m = re.match(self.src_uri + "(/.*)?$", src_uri)
         if (m is None):
             return(None)
         rel_path = m.group(1)
-        return(self.dst_path + '/' + rel_path)
+        if rel_path is None:
+            rel_path = '/'
+        return self.dst_path + rel_path
 
     def unsafe(self):
         """True if the mapping is unsafe for an update.

resync/resource.py

@@ -1,11 +1,8 @@
 """ResourceSync Resources - information about a web resource and changes."""
 
 import re
-try:  # python3
-    from urllib.parse import urlparse
-except ImportError:  # python2
-    from urlparse import urlparse
 from posixpath import basename
+from urllib.parse import urlparse
 
 from .w3c_datetime import str_to_datetime, datetime_to_str
 
@@ -58,19 +55,21 @@ class Resource(object):
 
     __slots__ = ('uri', 'timestamp', 'length', 'mime_type',
                  'md5', 'sha1', 'sha256', 'change', 'ts_datetime',
-                 'path', '_extra', 'ln')
+                 'path', 'ln', '_extra')
 
     CHANGE_TYPES = ['created', 'updated', 'deleted']
 
     def __init__(self, uri=None, timestamp=None, length=None,
-                 md5=None, sha1=None, sha256=None, mime_type=None,
-                 change=None, ts_datetime=None, datetime=None, path=None,
-                 lastmod=None, capability=None,
-                 ts_at=None, md_at=None,
-                 ts_completed=None, md_completed=None,
-                 ts_from=None, md_from=None,
-                 ts_until=None, md_until=None,
-                 resource=None, ln=None):
+                 mime_type=None, md5=None, sha1=None, sha256=None,
+                 change=None, ts_datetime=None, path=None, ln=None,
+                 # the following in _extra
+                 capability=None,
+                 ts_at=None, ts_completed=None, ts_from=None, ts_until=None,
+                 # and the following via setters
+                 lastmod=None, datetime=None,
+                 md_at=None, md_completed=None, md_from=None, md_until=None,
+                 # and finally another Resource to copy from
+                 resource=None):
         """Initialize Resource object.
 
         Initialize either from parameters specified or from an existing
@@ -88,14 +87,15 @@ class Resource(object):
         self.change = None
         self.ts_datetime = None  # Added in ResourceSync v1.1
         self.path = None
-        self._extra = None
         self.ln = None
-        # Create from a Resource-like object? Copy any relevant attributes
+        self._extra = None
+        # Create from a Resource-like object? Copy any attributes, both ones
+        # that have slots and ones that live in _extra
         if (resource is not None):
-            for att in ['uri', 'timestamp', 'length', 'md5', 'sha1', 'sha256',
-                        'change', 'ts_datetime', 'path', 'capability',
-                        'ts_at', 'md_at', 'ts_completed', 'md_completed',
-                        'ts_from', 'md_from', 'ts_until', 'md_until', 'ln']:
+            for att in ['uri', 'timestamp', 'length', 'mime_type',
+                        'md5', 'sha1', 'sha256', 'change', 'ts_datetime', 'path', 'ln',
+                        # the following in _extra
+                        'capability', 'ts_at', 'ts_completed', 'ts_from', 'ts_until']:
                 if hasattr(resource, att):
                     setattr(self, att, getattr(resource, att))
         # Any arguments will then override
@@ -146,7 +146,7 @@ class Resource(object):
             self.md_until = md_until
         # Sanity check
         if (self.uri is None):
-            raise ValueError("Cannot create resource without a URI")
+            raise ValueError("Cannot create Resource without a URI")
 
     def __setattr__(self, prop, value):
         """Attribute setter with check and support for extra attributes.
@@ -199,6 +199,11 @@ class Resource(object):
         """Set timestamp from a W3C Datetime Last-Modified value."""
         self.ts_datetime = str_to_datetime(datetime, context='ts_datetime')
 
+    @property
+    def ts_at(self):
+        """ts_at is the timestamp for md_at."""
+        return self._get_extra('ts_at')
+
     @property
     def md_at(self):
         """md_at values in W3C Datetime syntax, Z notation."""
@@ -211,6 +216,11 @@ class Resource(object):
             'ts_at',
             str_to_datetime(md_at, context='md_at datetime'))
 
+    @property
+    def ts_completed(self):
+        """ts_completed is the timestamp for md_completed."""
+        return self._get_extra('ts_completed')
+
     @property
     def md_completed(self):
         """md_completed value in W3C Datetime syntax, Z notation."""
@@ -223,6 +233,11 @@ class Resource(object):
             'ts_completed',
             str_to_datetime(md_completed, context='md_completed datetime'))
 
+    @property
+    def ts_from(self):
+        """ts_from is the timestamp for md_from."""
+        return self._get_extra('ts_from')
+
     @property
     def md_from(self):
         """md_from value in W3C Datetime syntax, Z notation."""
@@ -235,6 +250,11 @@ class Resource(object):
             'ts_from',
             str_to_datetime(md_from, context='md_from datetime'))
 
+    @property
+    def ts_until(self):
+        """ts_until is the timestamp for md_until."""
+        return self._get_extra('ts_until')
+
     @property
     def md_until(self):
         """md_until value in W3C Datetime syntax, Z notation."""
@@ -277,8 +297,10 @@ class Resource(object):
         return(None)
 
     @hash.setter
-    def hash(self, hash):
-        """Parse space separated set of values.
+    def hash(self, hashes_str):
+        """Parse space separated set of values to set md5, sha1 and sha256.
+
+        Any existing values for md5, sha1 and sha256 will be removed first.
 
         See specification at:
         http://tools.ietf.org/html/draft-snell-atompub-link-extensions-09
@@ -287,27 +309,24 @@ class Resource(object):
         self.md5 = None
         self.sha1 = None
         self.sha256 = None
-        if (hash is None):
-            return
         hash_seen = set()
         errors = []
-        for entry in hash.split():
+        for entry in hashes_str.split():
             (hash_type, value) = entry.split(':', 1)
-            if (hash_type in hash_seen):
-                errors.append("Ignored duplicate hash type %s" % (hash_type))
+            if hash_type in hash_seen:
+                errors.append("duplicate hash type %s" % (hash_type))
             else:
                 hash_seen.add(hash_type)
-                if (hash_type == 'md5'):
+                if hash_type == 'md5':
                     self.md5 = value
-                elif (hash_type == 'sha-1'):
+                elif hash_type == 'sha-1':
                     self.sha1 = value
-                elif (hash_type == 'sha-256'):
+                elif hash_type == 'sha-256':
                     self.sha256 = value
                 else:
-                    errors.append("Ignored unsupported hash type (%s)" %
-                                  (hash_type))
-        if (len(errors) > 0):
-            raise ValueError(". ".join(errors))
+                    errors.append("unsupported hash type %s" % (hash_type))
+        if len(errors) > 0:
+            raise ValueError("Ignored " + ", ".join(errors))
 
     def link(self, rel):
         """Look for link with specified rel, return else None.
@@ -316,19 +335,16 @@ class Resource(object):
         specified rel value. If there are multiple links with the
         same rel then just the first will be returned
         """
-        if (self.ln is None):
-            return(None)
-        for link in self.ln:
-            if ('rel' in link and link['rel'] == rel):
-                return(link)
-        return(None)
+        if self.ln is not None:
+            for link in self.ln:
+                if 'rel' in link and link['rel'] == rel:
+                    return link
+        return None
 
     def link_href(self, rel):
         """Look for link with specified rel, return href from it or None."""
         link = self.link(rel)
-        if (link is not None):
-            link = link['href']
-        return(link)
+        return None if link is None else link['href']
 
     def link_set(self, rel, href, allow_duplicates=False, **atts):
         """Set/create link with specified rel, set href and any other attributes.
@@ -342,13 +358,13 @@ class Resource(object):
         Be aware that adding links to a Resource object will
         significantly increase the size of the object.
         """
-        if (self.ln is None):
+        if self.ln is None:
             # automagically create a self.ln list
             self.ln = []
             link = None
         else:
             link = self.link(rel)
-        if (link is not None and not allow_duplicates):
+        if link is not None and not allow_duplicates:
             # overwrite current value
             link['href'] = href
         else:
@@ -398,7 +414,7 @@ class Resource(object):
     @property
     def contents(self):
         """Get the URI of and ResourceSync rel="contents" link."""
-        return(self.link_href('index'))
+        return(self.link_href('contents'))
 
     @contents.setter
     def contents(self, uri, type='application/xml'):
@@ -420,36 +436,43 @@ class Resource(object):
     def __eq__(self, other):
         """Equality test for resources allowing <1s difference in timestamp.
 
-        See equal(...) for more details of equality test
+        See equal(...) for more details of equality test.
         """
         return(self.equal(other, delta=1.0))
 
     def equal(self, other, delta=0.0):
         """Equality or near equality test for resources.
 
-        Equality means:
+        Equality means that any parameters that exist for both resources match,
+        essentially "no proof of inequality":
         1. same uri, AND
         2. same timestamp WITHIN delta if specified for either, AND
-        3. same md5 if specified for both, AND
+        3. same md5|sha1|sha256 if specified for both, AND
         4. same length if specified for both
         """
-        if (other is None):
+        if other is None:
+            return False
+        if self.uri != other.uri:
             return False
-        if (self.uri != other.uri):
-            return(False)
         if (self.timestamp is not None or other.timestamp is not None):
             # not equal if only one timestamp specified
             if (self.timestamp is None
                     or other.timestamp is None
                     or abs(self.timestamp - other.timestamp) >= delta):
-                return(False)
+                return False
         if ((self.md5 is not None and other.md5 is not None)
                 and self.md5 != other.md5):
-            return(False)
+            return False
+        if ((self.sha1 is not None and other.sha1 is not None)
+                and self.sha1 != other.sha1):
+            return False
+        if ((self.sha256 is not None and other.sha256 is not None)
+                and self.sha256 != other.sha256):
+            return False
         if ((self.length is not None and other.length is not None)
                 and self.length != other.length):
-            return(False)
-        return(True)
+            return False
+        return True
 
     def __str__(self):
         """Return a human readable string for this resource.
@@ -458,11 +481,12 @@ class Resource(object):
         designed to support logging.
         """
         s = [str(self.uri), str(self.lastmod), str(self.length),
-             str(self.md5 if self.md5 else self.sha1)]
+             str(self.md5 if self.md5 else (self.sha1 if self.sha1 else self.sha256))]
         if (self.change is not None):
-            s.append(str(self.change))
+            ch = str(self.change)
             if self.datetime is not None:
-                s.append(" @ " + str(self.datetime))
+                ch += " @ " + str(self.datetime)
+            s.append(ch)
         if (self.path is not None):
             s.append(str(self.path))
         return "[ " + " | ".join(s) + " ]"

resync/resource_container.py

@@ -14,6 +14,22 @@ import collections.abc
 from .w3c_datetime import datetime_to_str
 
 
+def _str_datetime_now(x='now'):
+    """Return datetime string for use with time attributes.
+
+    Handling depends on input:
+      'now'   - returns datetime for now
+      other string - no change, return same value
+      otherwise - assume datetime value, generate string
+    """
+    if (x == 'now' or not isinstance(x, str)):
+        # Now, this is wht datetime_to_str() with no arg gives
+        return datetime_to_str(x)
+    else:
+        # Assume valid datetime string
+        return x
+
+
 class ResourceContainer(object):
     """Class containing resource-like objects.
 
@@ -26,6 +42,11 @@ class ResourceContainer(object):
     - uri is optional identifier of this container object
     - capability_name - name of this capability
 
+    The properties md_from, md_at, md_until and md_completed are datetime
+    strings. The setters acceopt either a string value (not checked), the value
+    'now' to generate the current datetime, or otherwise an integer timestamp
+    value that is converted to a string.
+
     Derived classes may add extra functionality such as len() etc..
     However, any code designed to work with any ResourceContainer
     should use only the core functionality.
@@ -45,19 +66,16 @@ class ResourceContainer(object):
 
         Baseline implementation use iterator given by resources property.
         """
-        return(iter(self.resources))
+        return iter(self.resources)
 
     def __getitem__(self, index):
         """Feed through for __getitem__ of resources property."""
-        return(self.resources[index])
+        return self.resources[index]
 
     @property
     def capability(self):
-        """Get/set the <rs:md capability="" .../> attribute."""
-        if ('capability' in self.md):
-            return(self.md['capability'])
-        else:
-            return(None)
+        """Get/set the <rs:md capability="" .../> attribute, None if not set."""
+        return self.md.get('capability')
 
     @capability.setter
     def capability(self, capability):
@@ -65,65 +83,53 @@ class ResourceContainer(object):
 
     @property
     def md_from(self):
-        """Get/set the <rs:md from="" .../> attribute."""
-        if ('md_from' in self.md):
-            return(self.md['md_from'])
-        else:
-            return(None)
+        """Get/set the <rs:md from="" .../> attribute, None if not set."""
+        return self.md.get('md_from')
 
     @md_from.setter
     def md_from(self, md_from):
-        self.md['md_from'] = self._str_datetime_now(md_from)
+        self.md['md_from'] = _str_datetime_now(md_from)
 
     @property
     def md_until(self):
-        """Get/set the <rs:md until="" .../> attribute."""
-        if ('md_until' in self.md):
-            return(self.md['md_until'])
-        else:
-            return(None)
+        """Get/set the <rs:md until="" .../> attribute, None if not set."""
+        return self.md.get('md_until')
 
     @md_until.setter
     def md_until(self, md_until):
-        self.md['md_until'] = self._str_datetime_now(md_until)
+        self.md['md_until'] = _str_datetime_now(md_until)
 
     @property
     def md_at(self):
         """Get/set the <rs:md at="" attribute."""
-        if ('md_at' in self.md):
-            return(self.md['md_at'])
-        else:
-            return(None)
+        return self.md.get('md_at')
 
     @md_at.setter
     def md_at(self, md_at):
-        self.md['md_at'] = self._str_datetime_now(md_at)
+        self.md['md_at'] = _str_datetime_now(md_at)
 
     @property
     def md_completed(self):
-        """Get/set the <rs:md completed="" .../> attribute."""
-        if ('md_completed' in self.md):
-            return(self.md['md_completed'])
-        else:
-            return(None)
+        """Get/set the <rs:md completed="" .../> attribute, None if not set."""
+        return self.md.get('md_completed')
 
     @md_completed.setter
     def md_completed(self, md_completed):
-        self.md['md_completed'] = self._str_datetime_now(md_completed)
+        self.md['md_completed'] = _str_datetime_now(md_completed)
 
     def link(self, rel):
         """Look for link with specified rel, return else None."""
         for link in self.ln:
             if ('rel' in link and link['rel'] == rel):
-                return(link)
-        return(None)
+                return link
+        return None
 
     def link_href(self, rel):
         """Look for link with specified rel, return href from it or None."""
         link = self.link(rel)
         if (link is not None):
             link = link['href']
-        return(link)
+        return link
 
     def link_set(self, rel, href, **atts):
         """Set/create link with specified rel, set href and any other attributes.
@@ -132,16 +138,16 @@ class ResourceContainer(object):
         also defines the type attributes and others are permitted also. See
         description of allowed formats in
 
-        http://www.openarchives.org/rs/resourcesync.html#DocumentFormats
+        http://www.openarchives.org/rs/resourcesync#DocumentFormats
         """
         link = self.link(rel)
-        if (link is not None):
-            # overwrite current value
-            link['href'] = href
-        else:
+        if link is None:
             # create new link
             link = {'rel': rel, 'href': href}
             self.ln.append(link)
+        else:
+            # overwrite current value
+            link['href'] = href
         for k in atts:
             link[k] = atts[k]
 
@@ -232,29 +238,3 @@ class ResourceContainer(object):
                 pruned2.append(r)
         self.resources = pruned2
         return(n)
-
-    def __str__(self):
-        """Return string of all resources in order given by interator."""
-        s = ''
-        for resource in self:
-            s += str(resource) + "\n"
-        return(s)
-
-    def _str_datetime_now(self, x=None):
-        """Return datetime string for use with time attributes.
-
-        Handling depends on input:
-          'now'   - returns datetime for now
-          number  - assume datetime values, generate string
-          other   - no change, return same value
-        """
-        if (x == 'now'):
-            # Now, this is wht datetime_to_str() with no arg gives
-            return(datetime_to_str())
-        try:
-            # Test for number
-            junk = x + 0.0
-            return datetime_to_str(x)
-        except TypeError:
-            # Didn't look like a number, treat as string
-            return x

resync/resource_list.py

@@ -1,16 +1,16 @@
 """ResourceSync Resource List object.
 
 A Resource List is a set of resources with some metadata for
-each resource. Comparison of resource lists from a source and
-a destination allows understanding of whether the two are in
-sync or whether some resources need to be updated at the
-destination.
+each resource. Every resource must have a uri attribute.
+Comparison of resource lists from a source and a destination
+allows understanding of whether the two are in sync or whether
+some resources need to be updated at the destination.
 
 There may also be metadata about the Resource List, and links
 to other ResourceSync documents. Metadata include the
 timestamp of the Resource List (md_at) and, optionally, the
 timestamp when creation of the Resource List was completed
-(md_completed).at the top level. These include a creation timestamp
+(md_completed) at the top level. These include a creation timestamp
 (from) and links to the Capability List.
 
 Described in specification at:
@@ -18,6 +18,7 @@ http://www.openarchives.org/rs/resourcesync#DescResources
 """
 
 import collections.abc
+from collections import OrderedDict
 import os
 from datetime import datetime
 import re
@@ -29,29 +30,62 @@ from .mapper import Mapper, MapperError
 from .url_authority import UrlAuthority
 
 
-class ResourceListDict(dict):
+class SortedIterMixin(object):
+    """Mixin to provide sorted_iter() method."""
+
+    def sorted_iter(self):
+        """Iterator over all the resources in this dict by sorted key order."""
+        self._sorted_iter_next_list = sorted(self.keys(), reverse=True)
+        return iter(self._sorted_iter_next, None)
+
+    def _sorted_iter_next(self):
+        if (len(self._sorted_iter_next_list) > 0):
+            return self[self._sorted_iter_next_list.pop()]
+        else:
+            return None
+
+
+class ResourceListDict(dict, SortedIterMixin):
     """Default implementation of class to store resources in ResourceList.
 
     Key properties of this class are:
     - has add(resource) method
-    - is iterable and results given in alphanumeric order by resource.uri
+    - is iterable and gives resources (not keys) in alphanumeric order by
+      resource.uri
     """
 
     def __iter__(self):
         """Iterator over all the resources in this ResourceListDict."""
-        self._iter_next_list = sorted(self.keys())
-        self._iter_next_list.reverse()
-        return(iter(self._iter_next, None))
-
-    def _iter_next(self):
-        if (len(self._iter_next_list) > 0):
-            return(self[self._iter_next_list.pop()])
-        else:
-            return(None)
+        return self.sorted_iter()
 
     def uris(self):
         """Extract sorted list of URIs for resources in this ResourceListDict."""
-        return(sorted(self.keys()))
+        return sorted(self.keys())
+
+    def add(self, resource, replace=False):
+        """Add just a single resource."""
+        uri = resource.uri
+        if (uri in self.keys() and not replace):
+            raise ResourceListDupeError(
+                "Attempt to add resource already in resource_list")
+        self[uri] = resource
+
+
+class ResourceListOrdered(OrderedDict, SortedIterMixin):
+    """Alternative implementation of class to store resources in ResourceList.
+
+    Key properties of this class are:
+    - has add(resource) method
+    - is iterable and gives resources (not keys) in the order added
+    """
+
+    def __iter__(self):
+        """Iterator over the resources in the list in the order added."""
+        return iter(self.values())
+
+    def uris(self):
+        """Extract list of all resource URIs in the order added."""
+        return self.keys()
 
     def add(self, resource, replace=False):
         """Add just a single resource."""
@@ -62,42 +96,6 @@ class ResourceListDict(dict):
         self[uri] = resource
 
 
-class ResourceListOrdered(list):
-    """Alternative implementation of class to store resources in ResourceList.
-
-    FIXME - This is a rather inefficient implementation which involves
-    scanning all resources to check for duplicates. Designed just to enable
-    re-creation of examples in the spec. Something dictionary based would
-    likely be better. Might be best to use OrderedDict but that is available
-    natively only in python >= 2.7 and this library is designed for 2.6,2.7.
-
-    Key properties of this class are:
-    - has add(resource) method
-    - is iterable and results given in order added (not the usual one!)
-    """
-
-    def uris(self):
-        """Extract list of all resource URIs (in the order added)."""
-        uris = []
-        for r in self:
-            uris.append(r.uri)
-        return(uris)
-
-    def add(self, resource, replace=False):
-        """Add a single resource, check for dupes."""
-        uri = resource.uri
-        for r in self:
-            if (uri == r.uri):
-                if (replace):
-                    r = resource
-                    return
-                else:
-                    raise ResourceListDupeError(
-                        "Attempt to add resource already in resource_list")
-        # didn't find it in list, add to end
-        self.append(resource)
-
-
 class ResourceListDupeError(Exception):
     """Exception in case of duplicate resource."""
 
@@ -123,11 +121,13 @@ class ResourceList(ListBaseWithIndex):
 
     The default storage is unordered but the iterator imposes a canonical
     order which is alphabetical by URI. If it is desired to have
-    resources listed in the order they are added then the ResourceDictOrdered
+    resources listed in the order they are added then the ResourceListOrdered
     class may be specified on creation:
 
         rl = ResourceList( resources_class=ResourceDictOrdered )
 
+    Use of ResourceListOrdered will be faster than the default.
+
     In normal use it is expected that any Resource List Index will be
     created automatically when writing out a large Resource List in
     multiple sitemap files. However, should it be necessary to
@@ -176,11 +176,11 @@ class ResourceList(ListBaseWithIndex):
         written to work for any objects in self and sc, provided that the
         == operator can be used to compare them.
 
-        The functioning of this method depends on the iterators for self and
-        src providing access to the resource objects in URI order.
+        The functioning of this method depends on the sorted_iter() iterators
+        for self and src providing access to the resource objects in URI order.
         """
-        dst_iter = iter(self.resources)
-        src_iter = iter(src.resources)
+        dst_iter = self.resources.sorted_iter()
+        src_iter = src.resources.sorted_iter()
         same = ResourceList()
         updated = ResourceList()
         deleted = ResourceList()
@@ -189,21 +189,19 @@ class ResourceList(ListBaseWithIndex):
         src_cur = next(src_iter, None)
         while ((dst_cur is not None) and (src_cur is not None)):
             # print 'dst='+dst_cur+'  src='+src_cur
-            if (dst_cur.uri == src_cur.uri):
+            if dst_cur.uri == src_cur.uri:
                 if (dst_cur == src_cur):
                     same.add(dst_cur)
                 else:
                     updated.add(src_cur)
                 dst_cur = next(dst_iter, None)
                 src_cur = next(src_iter, None)
-            elif (not src_cur or dst_cur.uri < src_cur.uri):
+            elif dst_cur.uri < src_cur.uri:
                 deleted.add(dst_cur)
                 dst_cur = next(dst_iter, None)
-            elif (not dst_cur or dst_cur.uri > src_cur.uri):
+            else:  # dst_cur.uri > src_cur.uri:
                 created.add(src_cur)
                 src_cur = next(src_iter, None)
-            else:
-                raise Exception("this should not be possible")
         # what do we have leftover in src or dst lists?
         while (dst_cur is not None):
             deleted.add(dst_cur)

resync/resource_list_builder.py

@@ -3,7 +3,6 @@
 import os
 import os.path
 import re
-import sys
 import time
 import logging
 from defusedxml.ElementTree import parse
@@ -24,8 +23,10 @@ class ResourceListBuilder():
     - set_hashes to indicate which hashes should be calculated for each resource
     - set_path set true to add path attribute for each resource
     - set_length set true to include file length in resource_list (defaults true)
-    - exclude_dirs is a list of directory names to exclude
-      (defaults to ['CVS','.git'))
+    - exclude_patterns is a list of files and directory names patterns to exclude
+      using re.match(..). These patterns are left anchored so thus need to be
+      preceded with .* if there may be arbitrary leading characters (defaults to
+      empty)
     """
 
     def __init__(self, mapper=None, set_hashes=None,
@@ -37,9 +38,10 @@ class ResourceListBuilder():
 
         The following attributes may be set to determine information added to
         each Resource object based on the disk scan:
-        - set_hashes - Set of digests to computer for each resource. This may add
+        - set_hashes - Set of digests to compute for each resource. This may add
         significant time to the scan process as each file has to be read to
-        compute the hash. Empty set or None means no hashes calculated
+        compute the hash. Empty set or None means no hashes calculated, names
+        defined in Hashes object
         - set_length - False to not add length for each resources
         - set_path - True to add local path information for each file/resource
         """
@@ -47,34 +49,38 @@ class ResourceListBuilder():
         self.set_path = set_path
         self.set_hashes = set_hashes if (set_hashes and len(set_hashes) > 0) else None
         self.set_length = set_length
-        self.exclude_files = [r'sitemap\d{0,5}.xml']
-        self.exclude_dirs = ['CVS', '.git']
+        self.exclude_patterns = []
         self.include_symlinks = False
+        self.log_count_increment = 50000  # Write log message after 50000 files
         # Used internally only:
         self.logger = logging.getLogger('resync.resource_list_builder')
-        self.compiled_exclude_files = []
+        self.compiled_exclude_patterns = []
 
-    def add_exclude_files(self, exclude_patterns):
-        """Add more patterns of files to exclude while building resource_list."""
+    def add_exclude_patterns(self, exclude_patterns):
+        """Add more patterns of files or directories to exclude while building resource_list.
+
+        After patterns are added the set of compiled patterns is updated.
+        """
         for pattern in exclude_patterns:
-            self.exclude_files.append(pattern)
+            self.exclude_patterns.append(pattern)
+        self._compile_excludes()
 
-    def compile_excludes(self):
-        """Compile a set of regexps for files to be exlcuded from scans."""
-        self.compiled_exclude_files = []
-        for pattern in self.exclude_files:
+    def _compile_excludes(self):
+        # Compile a set of regexps for files and directories to be exlcuded from scans
+        self.compiled_exclude_patterns = []
+        for pattern in self.exclude_patterns:
             try:
-                self.compiled_exclude_files.append(re.compile(pattern))
+                self.compiled_exclude_patterns.append(re.compile(pattern))
             except re.error as e:
                 raise ValueError(
-                    "Bad python regex in exclude '%s': %s" % (pattern, str(e)))
+                    "Bad python regex in exclude pattern '%s': %s" % (pattern, str(e)))
 
-    def exclude_file(self, file):
-        """True if file should be exclude based on name pattern."""
-        for pattern in self.compiled_exclude_files:
-            if (pattern.match(file)):
-                return(True)
-        return(False)
+    def _exclude(self, file):
+        # True if file should be exclude based on name pattern.
+        for pattern in self.compiled_exclude_patterns:
+            if pattern.match(file):
+                return True
+        return False
 
     def from_disk(self, resource_list=None, paths=None):
         """Create or extend resource_list with resources from disk scan.
@@ -105,8 +111,6 @@ class ResourceListBuilder():
         # Either use resource_list passed in or make a new one
         if (resource_list is None):
             resource_list = ResourceList()
-        # Compile exclude pattern matches
-        self.compile_excludes()
         # Work out start paths from map if not explicitly specified
         if (paths is None):
             paths = []
@@ -131,23 +135,24 @@ class ResourceListBuilder():
             raise ValueError("Must specify path, resource_list and mapper")
         # is path a directory or a file? for each file: create Resource object,
         # add, increment counter
-        if (sys.version_info < (3, 0)):
-            path = path.decode('utf-8')
         if os.path.isdir(path):
             num_files = 0
             for dirpath, dirs, files in os.walk(path, topdown=True):
                 for file_in_dirpath in files:
                     num_files += 1
-                    if (num_files % 50000 == 0):
+                    if (num_files % self.log_count_increment == 0):
                         self.logger.info(
                             "ResourceListBuilder.from_disk_add_path: %d files..." % (num_files))
                     self.add_file(resource_list=resource_list,
                                   dir=dirpath, file=file_in_dirpath)
-                    # prune list of dirs based on self.exclude_dirs
-                    for exclude in self.exclude_dirs:
-                        if exclude in dirs:
-                            self.logger.debug("Excluding dir %s" % (exclude))
-                            dirs.remove(exclude)
+                # prune list of dirs based on self.exclude_dirs
+                prune = []
+                for dir in dirs:
+                    if self._exclude(dir):
+                        self.logger.debug("Excluding dir '%s'" % (dir))
+                        prune.append(dir)
+                for dir in prune:
+                    dirs.remove(dir)
         else:
             # single file
             self.add_file(resource_list=resource_list, file=path)
@@ -157,37 +162,27 @@ class ResourceListBuilder():
 
         Follows object settings of set_path, set_hashes and set_length.
         """
+        if self._exclude(file):
+            self.logger.debug("Excluding file '%s'" % (file))
+            return
+        # get abs filename and also URL
+        if (dir is not None):
+            file = os.path.join(dir, file)
+        if os.path.islink(file) and not self.include_symlinks:
+            self.logger.warning("Ignoring symlink '%s'" % (file))
+            return
         try:
-            if self.exclude_file(file):
-                self.logger.debug("Excluding file %s" % (file))
-                return
-            # get abs filename and also URL
-            if (dir is not None):
-                file = os.path.join(dir, file)
-            if (not os.path.isfile(file) or not (
-                    self.include_symlinks or not os.path.islink(file))):
-                return
-            uri = self.mapper.dst_to_src(file)
-            if (uri is None):
-                raise Exception("Internal error, mapping failed")
+            uri = self.mapper.dst_to_src(file)  # might throw MapperError
             file_stat = os.stat(file)
         except OSError as e:
-            sys.stderr.write("Ignoring file %s (error: %s)" % (file, str(e)))
+            self.logger.warning("Ignoring file '%s' (error: %s)" % (file, str(e)))
             return
         timestamp = file_stat.st_mtime  # UTC
         r = Resource(uri=uri, timestamp=timestamp)
-        if (self.set_path):
-            # add full local path
+        if self.set_path:  # add full local path
             r.path = file
-        if (self.set_hashes):
-            hasher = Hashes(self.set_hashes, file)
-            if ('md5' in self.set_hashes):
-                r.md5 = hasher.md5
-            if ('sha-1' in self.set_hashes):
-                r.sha1 = hasher.sha1
-            if ('sha-256' in self.set_hashes):
-                r.sha256 = hasher.sha256
-        if (self.set_length):
-            # add length
+        if self.set_hashes:  # add any hashes requested
+            Hashes(self.set_hashes, file).set(r)
+        if self.set_length:  # add length
             r.length = file_stat.st_size
         resource_list.add(r)

resync/sitemap.py

@@ -3,7 +3,6 @@
 import io
 import logging
 import os
-import re
 import sys
 from defusedxml.ElementTree import parse
 from xml.etree.ElementTree import ElementTree, Element, tostring
@@ -35,7 +34,7 @@ class SitemapIndexError(Exception):
         self.message = message
         self.etree = etree
 
-    def __repr__(self):
+    def __str__(self):
         """Return just the message attribute."""
         return(self.message)
 
@@ -93,6 +92,13 @@ class Sitemap(object):
         self.md_att_keys = ['md_at', 'capability', 'change', 'datetime',
                             'md_completed', 'md_from', 'hash', 'length',
                             'path', 'mime_type', 'md_until']
+        # capabilities
+        self.capabilities = ['resourcelist', 'changelist', 'resourcedump',
+                             'changedump', 'resourcedump-manifest',
+                             'changedump-manifest', 'capabilitylist',
+                             'description',
+                             'resourcelist-archive', 'resourcedump-archive',
+                             'changelist-archive', 'changedump-archive']
         if self.spec_1_0:
             self.md_att_keys.remove('datetime')
 
@@ -198,17 +204,13 @@ class Sitemap(object):
         self.resources_created = 0
         seen_top_level_md = False
         for e in list(etree.getroot()):
-            # look for <rs:md> and <rs:ln>, first <url> ends
-            # then look for resources in <url> blocks
+            # look for <rs:md> and <rs:ln>, first <url>/<sitemap> ends
+            # then look for resources in <url>/<sitemap> blocks.
+            # ignore any elements we don't recognize
             if (e.tag == resource_tag):
                 in_preamble = False  # any later rs:md or rs:ln is error
                 r = self.resource_from_etree(e, self.resource_class)
-                try:
-                    resources.add(r)
-                except SitemapDupeError:
-                    self.logger.warning(
-                        "dupe of: %s (lastmod=%s)" %
-                        (r.uri, r.lastmod))
+                resources.add(r)
                 self.resources_created += 1
             elif (e.tag == "{" + RS_NS + "}md"):
                 if (in_preamble):
@@ -227,9 +229,6 @@ class Sitemap(object):
                 else:
                     raise SitemapParseError(
                         "Found <rs:ln> after first <url> in sitemap")
-            else:
-                # element we don't recognize, ignore
-                pass
         # check that we read to right capability document
         if (capability is not None):
             if ('capability' not in resources.md):
@@ -288,19 +287,11 @@ class Sitemap(object):
         """Return string for the resource as part of an XML sitemap.
 
         Returns a string with the XML snippet representing the resource,
-        without any XML declaration.
+        without any XML declaration. (So much simpler now only Python 3.x
+        supported, see earlier versions for 2.6, 2.7 etc.)
         """
         e = self.resource_etree_element(resource)
-        if (sys.version_info >= (3, 0)):
-            # python3.x
-            return(tostring(e, encoding='unicode', method='xml'))
-        elif (sys.version_info >= (2, 7)):
-            s = tostring(e, encoding='UTF-8', method='xml')
-        else:
-            # must not specify method='xml' in python2.6
-            s = tostring(e, encoding='UTF-8')
-        # Chop off XML declaration that is added in 2.x... sigh
-        return(s.replace("<?xml version='1.0' encoding='UTF-8'?>\n", ''))
+        return(tostring(e, encoding='unicode', method='xml'))
 
     def resource_from_etree(self, etree, resource_class):
         """Construct a Resource from an etree.
@@ -382,27 +373,20 @@ class Sitemap(object):
             val = md_element.attrib.get(xml_att, None)
             if (val is not None):
                 md[att] = val
-        # capability. Allow this to be missing but do a very simple syntax
-        # check on plausible values if present
-        if ('capability' in md):
-            if (re.match(r"^[\w\-]+$", md['capability']) is None):
-                raise SitemapParseError(
-                    "Bad capability name '%s' in %s" %
-                    (capability, context))
-        # change should be one of defined values
-        if ('change' in md):
-            if (md['change'] not in ['created', 'updated', 'deleted']):
-                self.logger.warning(
-                    "Bad change attribute in <rs:md> for %s" %
-                    (context))
-        # length should be an integer
+        # capability. Allow this to be missing or a new value but warn if it
+        # isn't recognized
+        if ('capability' in md and md['capability'] not in self.capabilities):
+            self.logger.warning("Unknown capability name '%s' in %s" % (md['capability'], context))
+        # change must be one of defined values
+        if ('change' in md and md['change'] not in ['created', 'updated', 'deleted']):
+            raise SitemapParseError("Bad change attribute in <rs:md> for %s" % (context))
+        # length must be an integer
         if ('length' in md):
             try:
                 md['length'] = int(md['length'])
             except ValueError as e:
-                raise SitemapParseError(
-                    "Invalid length element in <rs:md> for %s" %
-                    (context))
+                raise SitemapParseError("Invalid length element in <rs:md> for %s" %
+                                        (context))
         return(md)
 
     def ln_from_etree(self, ln_element, context=''):
@@ -423,9 +407,8 @@ class Sitemap(object):
         # now do some checks and conversions...
         # href (MANDATORY)
         if ('href' not in ln):
-            raise SitemapParseError(
-                "Missing href in <rs:ln> in %s" %
-                (context))
+            raise SitemapParseError("Missing href in <rs:ln> in %s" %
+                                    (context))
         # rel (MANDATORY)
         if ('rel' not in ln):
             raise SitemapParseError("Missing rel in <rs:ln> in %s" % (context))
@@ -434,9 +417,8 @@ class Sitemap(object):
             try:
                 ln['length'] = int(ln['length'])
             except ValueError as e:
-                raise SitemapParseError(
-                    "Invalid length attribute value in <rs:ln> for %s" %
-                    (context))
+                raise SitemapParseError("Invalid length attribute value in <rs:ln> for %s" %
+                                        (context))
         # pri - priority, must be a number between 1 and 999999
         if ('pri' in ln):
             try:

resync/url_authority.py

@@ -1,9 +1,6 @@
 """Determine URI authority based on DNS and paths."""
 
-try:  # python3
-    from urllib.parse import urlparse
-except ImportError:  # pragma: no cover  python2
-    from urlparse import urlparse  # pragma: no cover
+from urllib.parse import urlparse
 import os.path
 
 

resync/url_or_file_open.py

@@ -8,7 +8,7 @@ from . import __version__
 
 
 # Global configuration settings
-FIRST_REQUEST = False
+NUM_REQUESTS = 0
 CONFIG = {
     'bearer_token': None,
     'delay': None
@@ -21,21 +21,29 @@ def set_url_or_file_open_config(key, value):
     CONFIG[key] = value
 
 
-def url_or_file_open(uri):
-    """Wrapper around urlopen() to prepend file: if no scheme provided."""
+def url_or_file_open(uri, method=None, timeout=None):
+    """Wrapper around urlopen() to prepend file: if no scheme provided.
+
+    Can be used as a context manager because the return value from urlopen(...)
+    supports both that and straightforwrd use as simple file handle object.
+
+    If timeout is exceeded then urlopen(..) will raise a socket.timeout exception. If
+    no timeout is specified then the global default will be used.
+    """
     if (not re.match(r'''\w+:''', uri)):
         uri = 'file:' + uri
+    headers = {'User-Agent': 'resync/' + __version__}
     # Do we need to send an Authorization header?
     # FIXME - This token will be added blindy to all requests. This is insecure
     # if the --noauth setting is used allowing requests across different domains.
     # It would be better to have some scheme where a token is tied to a particular
     # domain, or domain pattern.
-    headers = {'User-Agent': 'resync/' + __version__}
     if CONFIG['bearer_token'] is not None:
         headers['Authorization'] = 'Bearer ' + CONFIG['bearer_token']
     # Have we got a delay set? Apply only to web requests after first
-    global FIRST_REQUEST
-    if not FIRST_REQUEST and CONFIG['delay'] is not None and not uri.startswith('file:'):
-        FIRST_REQUEST = False
+    global NUM_REQUESTS
+    if NUM_REQUESTS != 0 and CONFIG['delay'] is not None and not uri.startswith('file:'):
         time.sleep(CONFIG['delay'])
-    return(urlopen(Request(url=uri, headers=headers)))
+    NUM_REQUESTS += 1
+    maybe_timeout = {} if timeout is None else {'timeout': timeout}
+    return urlopen(Request(url=uri, headers=headers, method=method), **maybe_timeout)

setup.py

@@ -61,7 +61,6 @@ setup(
     long_description=open('README.md').read(),
     long_description_content_type='text/markdown',
     install_requires=[
-        "requests",
         "python-dateutil>=1.5",
         "defusedxml>=0.4.1"
     ],

tests/test_explorer.py

@@ -8,7 +8,7 @@ import sys
 from resync.client import Client
 from resync.client_utils import ClientFatalError
 from resync.capability_list import CapabilityList
-from resync.explorer import Explorer, XResource, HeadResponse, ExplorerQuit
+from resync.explorer import Explorer, XResource, ExplorerQuit
 from resync.resource import Resource
 
 
@@ -30,11 +30,6 @@ class TestExplorer(unittest.TestCase):
         self.assertEqual(x.acceptable_capabilities, [1, 2])
         self.assertEqual(x.checks, [3, 4])
 
-    def test02_head_response(self):
-        hr = HeadResponse()
-        self.assertEqual(hr.status_code, None)
-        self.assertEqual(len(hr.headers), 0)
-
     def test03_explorer_quit(self):
         eq = ExplorerQuit()
         self.assertTrue(isinstance(eq, Exception))
@@ -105,13 +100,14 @@ class TestExplorer(unittest.TestCase):
 
     def test08_head_on_file(self):
         e = Explorer()
-        r1 = e.head_on_file('tests/testdata/does_not_exist')
-        self.assertEqual(r1.status_code, '404')
-        r2 = e.head_on_file('tests/testdata/dir1/file_a')
-        self.assertEqual(r2.status_code, '200')
+        (status_code, headers) = e.head_on_file('tests/testdata/does_not_exist')
+        self.assertEqual(status_code, '404')
+        self.assertEqual(headers, {})
+        (status_code, headers) = e.head_on_file('tests/testdata/dir1/file_a')
+        self.assertEqual(status_code, '200')
         self.assertTrue(re.match(r'^\d\d\d\d\-\d\d\-\d\d',
-                                 r2.headers['last-modified']))
-        self.assertEqual(r2.headers['content-length'], 20)
+                        headers['last-modified']))
+        self.assertEqual(headers['content-length'], 20)
 
     def test09_allowed_entries(self):
         e = Explorer()

tests/test_mapper.py

@@ -33,9 +33,8 @@ class TestMapper(unittest.TestCase):
         self.assertEqual(m.src_to_dst('http://e.org/p/'), '/tmp/q/')
         self.assertEqual(m.src_to_dst('http://e.org/p/aa'), '/tmp/q/aa')
         self.assertEqual(m.src_to_dst('http://e.org/p/aa/bb'), '/tmp/q/aa/bb')
-        self.assertEqual(m.src_to_dst(
-            'http://e.org/p/aa/bb/'), '/tmp/q/aa/bb/')
-        self.assertRaises(MapperError, m.src_to_dst, 'http://e.org/p')
+        self.assertEqual(m.src_to_dst('http://e.org/p/aa/bb/'), '/tmp/q/aa/bb/')
+        self.assertEqual(m.src_to_dst('http://e.org/p'), '/tmp/q/')
         self.assertRaises(MapperError, m.src_to_dst, 'http://e.org/pa')
         self.assertRaises(MapperError, m.src_to_dst, 'nomatch')
 
@@ -44,7 +43,7 @@ class TestMapper(unittest.TestCase):
         self.assertEqual(m.dst_to_src('/tmp/q/'), 'http://e.org/p/')
         self.assertEqual(m.dst_to_src('/tmp/q/bb'), 'http://e.org/p/bb')
         self.assertEqual(m.dst_to_src('/tmp/q/bb/cc'), 'http://e.org/p/bb/cc')
-        self.assertRaises(MapperError, m.dst_to_src, '/tmp/q')
+        self.assertEqual(m.dst_to_src('/tmp/q'), 'http://e.org/p/')
         self.assertRaises(MapperError, m.dst_to_src, '/tmp/qa')
         self.assertRaises(MapperError, m.dst_to_src, 'nomatch')
 
@@ -96,17 +95,44 @@ class TestMapper(unittest.TestCase):
         self.assertEqual(Mapper(['a=b', 'b=c']).default_src_uri(), 'a')
         self.assertRaises(MapperError, Mapper().default_src_uri)
 
-    # Tests for Map class
 
-    def test10_map_unsafe(self):
+class TestMap(unittest.TestCase):
+
+    def test01_map_unsafe(self):
+        """Test unsafe method."""
         self.assertFalse(Map('http://example.com/', 'path').unsafe())
         # Note the first is a URI and the second is a (silly) path in the
         # following
-        self.assertFalse(
-            Map('http://example.com/', 'http://example.com/').unsafe())
+        self.assertFalse(Map('http://example.com/', 'http://example.com/').unsafe())
         self.assertFalse(Map('a', 'b').unsafe())
         self.assertFalse(Map('path/a', 'path/b').unsafe())
         # The following are unsafe
         self.assertTrue(Map('path', 'path').unsafe())
         self.assertTrue(Map('path/a', 'path').unsafe())
         self.assertTrue(Map('path', 'path/b').unsafe())
+
+    def test02_dst_to_src(self):
+        """Test dst_to_src method."""
+        m = Map('uri:a/b', 'path/c')
+        self.assertEqual(m.dst_to_src('path/c/file'), 'uri:a/b/file')
+        self.assertEqual(m.dst_to_src('path/c/file/'), 'uri:a/b/file/')
+        self.assertEqual(m.dst_to_src('path/c'), 'uri:a/b/')
+        self.assertEqual(m.dst_to_src('path/c/'), 'uri:a/b/')
+        self.assertEqual(m.dst_to_src('path/cliff'), None)
+        # Note that trailing slashes on the map elements have no effect
+        m = Map('uri:a/b/', 'path/c/')
+        self.assertEqual(m.dst_to_src('path/c'), 'uri:a/b/')
+        self.assertEqual(m.dst_to_src('path/c/'), 'uri:a/b/')
+        self.assertEqual(m.dst_to_src('path/cliff'), None)
+        # Failed maps return None
+        self.assertEqual(m.dst_to_src('c/file'), None)
+        self.assertEqual(m.dst_to_src(''), None)
+
+    def test03_src_to_dst(self):
+        """Test src_to_dst method."""
+        m = Map('uri:x/', '/tmp/gg/')
+        self.assertEqual(m.src_to_dst('uri:other'), None)
+        self.assertEqual(m.src_to_dst('uri:x'), '/tmp/gg/')
+        self.assertEqual(m.src_to_dst('uri:x/'), '/tmp/gg/')
+        self.assertEqual(m.src_to_dst('uri:x/lizard'), '/tmp/gg/lizard')
+        self.assertEqual(m.src_to_dst('uri:x/lizard/'), '/tmp/gg/lizard/')

tests/test_resource.py

@@ -1,3 +1,4 @@
+"""Tests for resync.resource."""
 import unittest
 import re
 from resync.resource import Resource, ChangeTypeError
@@ -5,20 +6,60 @@ from resync.resource import Resource, ChangeTypeError
 
 class TestResource(unittest.TestCase):
 
-    def test01a_same(self):
+    def test01_init(self):
+        """Test __init__ method."""
+        # No uri = error
+        self.assertRaises(ValueError, Resource)
+        self.assertRaises(ValueError, Resource, timestamp=12)
+        # Create with many params
+        r1 = Resource(uri='a', timestamp=0, length=123,
+                      md5='aaa', sha1='bbb', sha256='ccc',
+                      mime_type='text/plain', change='updated',
+                      ts_datetime=1000, path='/a/b/c', ln={'x': 'y'},
+                      ts_at=2000, ts_completed=3000, ts_from=4000,
+                      ts_until=5000)
+        self.assertEqual(r1.uri, 'a')
+        self.assertEqual(r1.timestamp, 0)
+        self.assertEqual(r1.length, 123)
+        self.assertEqual(r1.md5, 'aaa')
+        self.assertEqual(r1.sha1, 'bbb')
+        self.assertEqual(r1.sha256, 'ccc')
+        self.assertEqual(r1.mime_type, 'text/plain')
+        self.assertEqual(r1.change, 'updated')
+        self.assertEqual(r1.ts_datetime, 1000)
+        self.assertEqual(r1.path, '/a/b/c')
+        self.assertEqual(r1.ln, {'x': 'y'})
+        self.assertEqual(r1.ts_at, 2000)
+        self.assertEqual(r1.ts_completed, 3000)
+        self.assertEqual(r1.ts_from, 4000)
+        self.assertEqual(r1.ts_until, 5000)
+        # Create r2 like r1
+        r2 = Resource(resource=r1)
+        self.assertEqual(r2.uri, 'a')
+        self.assertEqual(r2.timestamp, 0)
+        self.assertEqual(r2.length, 123)
+        self.assertEqual(r2.md5, 'aaa')
+        self.assertEqual(r2.sha1, 'bbb')
+        self.assertEqual(r2.sha256, 'ccc')
+        self.assertEqual(r2.mime_type, 'text/plain')
+        self.assertEqual(r2.change, 'updated')
+        self.assertEqual(r2.ts_datetime, 1000)
+        self.assertEqual(r2.path, '/a/b/c')
+        self.assertEqual(r2.ln, {'x': 'y'})
+
+    def test02_equal(self):
+        """Test equal method via = operator."""
+        # just uri
         r1 = Resource('a')
         r2 = Resource('a')
         self.assertEqual(r1, r1)
         self.assertEqual(r1, r2)
-
-    def test01b_same(self):
+        # with timestamps
         r1 = Resource(uri='a', timestamp=1234.0)
         r2 = Resource(uri='a', timestamp=1234.0)
         self.assertEqual(r1, r1)
         self.assertEqual(r1, r2)
-
-    def test01c_same(self):
-        """Same with lastmod instead of direct timestamp"""
+        # with lastmod instead of direct timestamp
         r1 = Resource('a')
         r1lm = '2012-01-01T00:00:00Z'
         r1.lastmod = r1lm
@@ -40,26 +81,45 @@ class TestResource(unittest.TestCase):
             self.assertEqual(r1.timestamp, r2.timestamp, ('%s (%f) == %s (%f)' % (
                 r1lm, r1.timestamp, r2lm, r2.timestamp)))
             self.assertEqual(r1, r2)
-
-    def test01d_same(self):
-        """Same with slight timestamp diff"""
+        # with slight timestamp diff
         r1 = Resource('a')
         r1.lastmod = '2012-01-02T01:02:03Z'
         r2 = Resource('a')
         r2.lastmod = '2012-01-02T01:02:03.99Z'
         self.assertNotEqual(r1.timestamp, r2.timestamp)
         self.assertEqual(r1, r2)
-
-    def test02a_diff(self):
+        # now with too much time diff
+        r1 = Resource('a', lastmod='2012-01-11')
+        r2 = Resource('a', lastmod='2012-01-22')
+        self.assertNotEqual(r1, r2)
+        # different uris
         r1 = Resource('a')
         r2 = Resource('b')
         self.assertNotEqual(r1, r2)
-
-    def test02b_diff(self):
-        r1 = Resource('a', lastmod='2012-01-11')
-        r2 = Resource('a', lastmod='2012-01-22')
-        # print 'r1 == r2 : '+str(r1==r2)
+        # same and different lengths
+        r1 = Resource('a', length=1234)
+        r2 = Resource('a', length=4321)
         self.assertNotEqual(r1, r2)
+        r2.length = r1.md5
+        self.assertEqual(r1, r2)
+        # same and different md5
+        r1.md5 = "3006f84272f2653a6cf5ec3af8f0d773"
+        r2.md5 = "3006f84272f2653a6cf5ec3af8f00000"
+        self.assertNotEqual(r1, r2)
+        r2.md5 = r1.md5
+        self.assertEqual(r1, r2)
+        # same and different sha1
+        r1.sha1 = "3be0f3af2aa4656ce38e0cef305c6eb2af4385d4"
+        r2.sha1 = "555"
+        self.assertNotEqual(r1, r2)
+        r2.sha1 = r1.sha1
+        self.assertEqual(r1, r2)
+        # same and different sha256
+        r1.sha256 = "f41094ad47ef3e93ec1021bfa40f4bf0185f1bf897533638ae5358b61713f84a"
+        r2.sha256 = "fab"
+        self.assertNotEqual(r1, r2)
+        r2.sha256 = r1.sha256
+        self.assertEqual(r1, r2)
 
     def test04_bad_lastmod(self):
         def setlastmod(r, v):
@@ -100,7 +160,8 @@ class TestResource(unittest.TestCase):
         r1 = Resource('def', lastmod='2012-01-01')
         self.assertEqual(repr(r1), "{'uri': 'def', 'timestamp': 1325376000}")
 
-    def test08_multiple_hashes(self):
+    def test08_hash(self):
+        """Test hash getter and setters."""
         r1 = Resource('abcd')
         r1.md5 = "some_md5"
         r1.sha1 = "some_sha1"
@@ -108,8 +169,7 @@ class TestResource(unittest.TestCase):
         self.assertEqual(r1.md5, "some_md5")
         self.assertEqual(r1.sha1, "some_sha1")
         self.assertEqual(r1.sha256, "some_sha256")
-        self.assertEqual(
-            r1.hash, "md5:some_md5 sha-1:some_sha1 sha-256:some_sha256")
+        self.assertEqual(r1.hash, "md5:some_md5 sha-1:some_sha1 sha-256:some_sha256")
         r2 = Resource('def')
         r2.hash = "md5:ddd"
         self.assertEqual(r2.md5, 'ddd')
@@ -121,6 +181,20 @@ class TestResource(unittest.TestCase):
         self.assertEqual(r2.md5, 'fff')
         self.assertEqual(r2.sha1, 'eee')
         self.assertEqual(r2.sha256, 'ggg')
+        # bogus value will reset
+        r2.hash = 11
+        self.assertEqual(r2.md5, None)
+        self.assertEqual(r2.sha1, None)
+        self.assertEqual(r2.sha256, None)
+        # string withough : will raise error
+        with self.assertRaises(ValueError):
+            r2.hash = "no-colon"
+        # dupe
+        with self.assertRaises(ValueError):
+            r2.hash = "md5:aaa md5:bbb"
+        # unknown
+        with self.assertRaises(ValueError):
+            r2.hash = "sha999:aaa"
 
     def test09_changetypeerror(self):
         r1 = Resource('a')
@@ -129,9 +203,11 @@ class TestResource(unittest.TestCase):
         self.assertEqual(r1.change, 'deleted')
         self.assertRaises(ChangeTypeError, Resource, 'a', change="bad")
         # disable checking
+        ct = Resource.CHANGE_TYPES
         Resource.CHANGE_TYPES = False
         r1 = Resource('a', change="bad")
         self.assertEqual(r1.change, 'bad')
+        Resource.CHANGE_TYPES = ct
 
     def test10_md_at_roundtrips(self):
         r = Resource('a')
@@ -182,3 +258,61 @@ class TestResource(unittest.TestCase):
         self.assertEqual(r.datetime, None)
         r = Resource(uri='dt2', datetime='2000-01-04T00:00:00Z')
         self.assertEqual(r.datetime, '2000-01-04T00:00:00Z')
+
+    def test15_link(self):
+        """Test link link_href and link_set methods."""
+        r = Resource(uri='ln1')
+        self.assertEqual(r.link('up'), None)
+        self.assertEqual(r.link_href('up'), None)
+        r.link_set('up', 'uri:up')
+        self.assertEqual(r.link('up'), {'rel': 'up', 'href': 'uri:up'})
+        self.assertEqual(r.link_href('up'), 'uri:up')
+        r.link_set('down', 'uri:down')
+        self.assertEqual(r.link('down'), {'rel': 'down', 'href': 'uri:down'})
+        self.assertEqual(r.link_href('down'), 'uri:down')
+        r.link_set('up', 'uri:up2')
+        self.assertEqual(r.link('up'), {'rel': 'up', 'href': 'uri:up2'})
+        r.link_add('up', 'uri:up3')
+        self.assertEqual(r.link('up'), {'rel': 'up', 'href': 'uri:up2'})  # still get first
+        self.assertEqual(r.ln, [{'rel': 'up', 'href': 'uri:up2'},
+                                {'href': 'uri:down', 'rel': 'down'},
+                                {'rel': 'up', 'href': 'uri:up3'}])
+
+    def test16_specific_links(self):
+        """Test setters/getters for specific link types."""
+        r = Resource(uri='laughing')
+        r.describedby = 'uri:db'
+        self.assertEqual(r.describedby, 'uri:db')
+        r.up = 'uri:up'
+        self.assertEqual(r.up, 'uri:up')
+        r.index = 'uri:index'
+        self.assertEqual(r.index, 'uri:index')
+        r.contents = 'uri:ct'
+        self.assertEqual(r.contents, 'uri:ct')
+
+    def test17_basename(self):
+        """Test basename property derived from uri."""
+        r = Resource(uri='http://example.org/any/complex/path/file')
+        self.assertEqual(r.basename, 'file')
+        r.uri = 'http://example.org/any/complex/path/'
+        self.assertEqual(r.basename, '')
+        r.uri = 'http://example.org'
+        self.assertEqual(r.basename, '')
+
+    def test18_str(self):
+        """Test str method."""
+        self.assertEqual(str(Resource('uri:a')),
+                         '[ uri:a | None | None | None ]')
+        self.assertEqual(str(Resource('uri:a', timestamp=0, length=999)),
+                         '[ uri:a | 1970-01-01T00:00:00Z | 999 | None ]')
+        self.assertEqual(str(Resource('uri:a', timestamp=0, length=999, sha256='abcdef123')),
+                         '[ uri:a | 1970-01-01T00:00:00Z | 999 | abcdef123 ]')
+        self.assertEqual(str(Resource('uri:a', change='updated', ts_datetime=3661)),
+                         '[ uri:a | None | None | None | updated @ 1970-01-01T01:01:01Z ]')
+        self.assertEqual(str(Resource('uri:a', path='/a/b/c')),
+                         '[ uri:a | None | None | None | /a/b/c ]')
+
+    def test19_change_type_error(self):
+        """Test error from bad change type."""
+        cte = ChangeTypeError('unk')
+        self.assertIn('ChangeTypeError: got unk, expected one of ', str(cte))

tests/test_resource_container.py

@@ -1,6 +1,8 @@
+"""Tests for resync.resource_container."""
+
 import unittest
 from resync.resource import Resource
-from resync.resource_container import ResourceContainer
+from resync.resource_container import ResourceContainer, _str_datetime_now
 
 
 class TestResourceContainer(unittest.TestCase):
@@ -8,9 +10,12 @@ class TestResourceContainer(unittest.TestCase):
     def test01_create_and_add(self):
         rc = ResourceContainer(resources=[])
         self.assertEqual(len(rc.resources), 0, "empty")
-        rc.resources.append(Resource('a', timestamp=1))
-        rc.resources.append(Resource('b', timestamp=2))
+        rc.add(Resource('a', timestamp=1))
+        rc.add(Resource('b', timestamp=2))
         self.assertEqual(len(rc.resources), 2, "two resources")
+        # Add two more
+        rc.add([Resource('c'), Resource('d')])
+        self.assertEqual(len(rc.resources), 4, "four resources")
 
     def test02_iter(self):
         rc = ResourceContainer(resources=[])
@@ -93,3 +98,37 @@ class TestResourceContainer(unittest.TestCase):
         self.assertEqual(rc.up, None)
         rc.up = "up_uri"
         self.assertEqual(rc.up, "up_uri")
+
+    def test09_capability(self):
+        """Test capability property."""
+        rc = ResourceContainer()
+        self.assertEqual(rc.capability, None)
+        rc.capability = "fixit"
+        self.assertEqual(rc.capability, 'fixit')
+
+    def test10_link_set(self):
+        """Test link_set method."""
+        rc = ResourceContainer()
+        self.assertEqual(rc.link('alink'), None)  # non-spec rel should be supported
+        rc.link_set('alink', 'uri:l', extra1='one', extra2='two')
+        ln = rc.link('alink')
+        self.assertEqual(rc.link('alink'), {'rel': 'alink', 'href': 'uri:l',
+                                            'extra1': 'one', 'extra2': 'two'})
+        # Can update existing link
+        rc.link_set('alink', 'uri:new', extra1='1')
+        self.assertEqual(rc.link('alink'), {'rel': 'alink', 'href': 'uri:new',
+                                            'extra1': '1', 'extra2': 'two'})
+
+    def test11_index(self):
+        """Test index property."""
+        rc = ResourceContainer()
+        self.assertEqual(rc.index, None)
+        rc.index = 'uri:i'
+        self.assertEqual(rc.index, 'uri:i')
+
+    def test12_str_datetime_now(self):
+        """Test _str_datetime_now function."""
+        self.assertTrue(isinstance(_str_datetime_now(), str))
+        self.assertTrue(isinstance(_str_datetime_now('now'), str))
+        self.assertEqual(_str_datetime_now(0), '1970-01-01T00:00:00Z')
+        self.assertEqual(_str_datetime_now('2020-12-24T16:01:00Z'), '2020-12-24T16:01:00Z')

tests/test_resource_list.py

@@ -1,15 +1,13 @@
+"""Tests for resync.resource_list."""
+
 from .testlib import TestCase
+
 import os.path
-try:  # python2
-    # Must try this first as io also exists in python2
-    # but is the wrong one!
-    import StringIO as io
-except ImportError:  # python3
-    import io
+import io
 import re
 
 from resync.resource import Resource
-from resync.resource_list import ResourceList, ResourceListDupeError
+from resync.resource_list import ResourceList, ResourceListOrdered, ResourceListDupeError
 from resync.sitemap import Sitemap, SitemapParseError
 
 
@@ -121,7 +119,8 @@ class TestResourceList(TestCase):
         r1.md5 = "aabbcc"
         self.assertEqual(i.hashes(), set(['md5']))
         r2.sha1 = "ddeeff"
-        self.assertEqual(i.hashes(), set(['md5', 'sha-1']))
+        r2.sha256 = "hhiijj"
+        self.assertEqual(i.hashes(), set(['md5', 'sha-1', 'sha-256']))
 
     def test08_iter(self):
         i = ResourceList()
@@ -129,13 +128,26 @@ class TestResourceList(TestCase):
         i.add(Resource('b', timestamp=2))
         i.add(Resource('c', timestamp=3))
         i.add(Resource('d', timestamp=4))
-        resources = []
-        for r in i:
-            resources.append(r)
+        resources = list(i.resources)
         self.assertEqual(len(resources), 4)
         self.assertEqual(resources[0].uri, 'a')
         self.assertEqual(resources[3].uri, 'd')
 
+    def test09_resource_list_ordered(self):
+        """Tests for ResourceList with ResourceListOrdered."""
+        i = ResourceList(resources_class=ResourceListOrdered)
+        i.add(Resource('a', timestamp=1))
+        i.add(Resource('d', timestamp=4))
+        i.add(Resource('c', timestamp=3))
+        self.assertEqual(list(i.resources.uris()), ['a', 'd', 'c'])
+        self.assertRaises(ResourceListDupeError, i.add, Resource('a', timestamp=11))
+        self.assertEqual(i.resources['a'].uri, 'a')
+        self.assertEqual(i.resources['a'].timestamp, 1)
+        # With replacement
+        i.add(Resource('a', timestamp=11), replace=True)
+        self.assertEqual(i.resources['a'].uri, 'a')
+        self.assertEqual(i.resources['a'].timestamp, 11)
+
     def test20_as_xml(self):
         rl = ResourceList()
         rl.add(Resource('a', timestamp=1))

tests/test_resource_list_builder.py

@@ -1,10 +1,13 @@
 import unittest
 import re
 import os
+from testfixtures import LogCapture
 import time
+
 from resync.resource_list_builder import ResourceListBuilder
+from resync.resource_list import ResourceList
 from resync.resource import Resource
-from resync.mapper import Mapper
+from resync.mapper import Mapper, MapperError
 
 
 class TestResourceListBuilder(unittest.TestCase):
@@ -137,3 +140,84 @@ class TestResourceListBuilder(unittest.TestCase):
         self.assertFalse(u'x:/A_\u00c3_tilde.txt' in uris)
         # Snowman is single char
         self.assertFalse(u'x:snowman_\u2603.txt' in uris)
+
+    def test10_add_exclude_patterns(self):
+        """Test add_exclude_patterns method."""
+        rlb = ResourceListBuilder()
+        self.assertEqual(len(rlb.exclude_patterns), 0)
+        rlb.add_exclude_patterns(['aaa', 'bbb'])
+        self.assertIn('aaa', rlb.exclude_patterns)
+        self.assertIn('bbb', rlb.exclude_patterns)
+
+    def test11_compile_excludes(self):
+        """Test _compile_excludes method."""
+        rlb = ResourceListBuilder()
+        self.assertEqual(len(rlb.compiled_exclude_patterns), 0)
+        rlb.exclude_patterns = [r'aaa\d+', r'bbb']
+        rlb._compile_excludes()
+        self.assertEqual(len(rlb.compiled_exclude_patterns), 2)
+        # Error case
+        rlb.exclude_patterns.append('bad regex \\')
+        self.assertRaises(ValueError, rlb._compile_excludes)
+
+    def test12_exclude(self):
+        """Test _exclude method."""
+        rlb = ResourceListBuilder()
+        rlb.add_exclude_patterns(['.*frog.*'])
+        rlb._compile_excludes()
+        self.assertTrue(rlb._exclude('a frog'))
+        self.assertFalse(rlb._exclude('toad'))
+
+    def test13_from_disk_add_path(self):
+        """Test from_disk_add_path method."""
+        # Check sanity check - must have path, resource_list and mapper
+        rlb = ResourceListBuilder(mapper=Mapper())
+        self.assertRaises(ValueError, rlb.from_disk_add_path, path='aaa')
+        self.assertRaises(ValueError, rlb.from_disk_add_path, resource_list=ResourceList())
+        rlb = ResourceListBuilder()
+        self.assertRaises(ValueError, rlb.from_disk_add_path, path='aaa', resource_list=ResourceList())
+        # Check log message
+        rlb = ResourceListBuilder(mapper=Mapper(['http://example.org/', 'tests']))
+        rlb.log_count_increment = 2
+        rl = ResourceList()
+        with LogCapture() as lc:
+            rlb.from_disk_add_path(path='tests/testdata/dir1', resource_list=rl)
+            self.assertIn('from_disk_add_path: 2 files...', lc.records[-1].msg)
+        # text excluding dirs -- just one file under find2 not excluced
+        rlb = ResourceListBuilder(mapper=Mapper(['http://example.org/', 'tests']))
+        rl = ResourceList()
+        rlb.add_exclude_patterns(['find1', 'find3'])
+        rlb._compile_excludes()
+        rlb.from_disk_add_path(path='tests/testdata/find', resource_list=rl)
+        self.assertEqual(len(rl), 1)
+
+    def test14_add_file(self):
+        """Test add_file method."""
+        rlb = ResourceListBuilder(mapper=Mapper(['http://example.org/', 'tests']))
+        rl = ResourceList()
+        rlb.add_exclude_patterns(['.*ro'])
+        with LogCapture() as lc:
+            # escluded
+            rlb.add_file(resource_list=rl, file='frog')
+            self.assertIn("Excluding file 'frog'", lc.records[-1].msg)
+            # mapper error
+            self.assertRaises(MapperError, rlb.add_file, resource_list=rl, file='i-dont-exist')
+            # map OK but doesn't exist
+            rlb.add_file(resource_list=rl, file='tests/i-dont-exist')
+            self.assertIn("Ignoring file 'tests/i-dont-exist'", lc.records[-1].msg)
+            # ignore symlink by default
+            rlb.add_file(resource_list=rl, file='tests/testdata/symlink/dir2/a_file.txt')
+            self.assertIn("Ignoring symlink 'tests/testdata/symlink/dir2/a_file.txt'", lc.records[-1].msg)
+            # ...or not
+            rl = ResourceList()
+            rlb.include_symlinks = True
+            rlb.add_file(resource_list=rl, file='tests/testdata/symlink/dir2/a_file.txt')
+            self.assertEqual(len(rl), 1)
+        # Check hashing
+        rlb = ResourceListBuilder(mapper=Mapper(['http://example.org/', 'tests/testdata/dir1/']),
+                                  set_hashes=['md5', 'sha-1', 'sha-256'])
+        rl = ResourceList()
+        rlb.add_file(resource_list=rl, file='tests/testdata/dir1/file_a')
+        self.assertEqual(rl['http://example.org/file_a'].md5, '6bf26fd66601b528d2e0b47eaa87edfd')
+        self.assertEqual(rl['http://example.org/file_a'].sha1, 'c60a598a5d9e489cf50533eeead6d70f15eafcf8')
+        self.assertEqual(rl['http://example.org/file_a'].sha256, '1c6291bfac0322752c4632ebd69bf6d81d53985fbf5ee54de5cc1fefba6566b6')

tests/test_sitemap.py

@@ -1,27 +1,24 @@
+"""Test for resync.sitemap."""
+
+import io
+from testfixtures import LogCapture
 import re
 import sys
 import unittest
-try:  # python2
-    # Must try this first as io also exists in python2
-    # but in the wrong one!
-    import StringIO as io
-except ImportError:  # python3
-    import io
+import xml.etree.ElementTree  # for xml.etree.ElementTree.ParseError
+from defusedxml.ElementTree import parse
 
 from resync.resource import Resource
 from resync.resource_list import ResourceList
 from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError
 
-# etree gives ParseError in 2.7,3.x; ExpatError in 2.6
-etree_error_class = None
-if (sys.version_info < (2, 7)):
-    from xml.parsers.expat import ExpatError
-    etree_error_class = ExpatError
-else:
-    # In python3 this seems only to work with the full class name??
-    # from xml.etree.ElementTree import ParseError
-    import xml.etree.ElementTree
-    etree_error_class = xml.etree.ElementTree.ParseError
+
+class TestSitemapIndexError(unittest.TestCase):
+
+    def test_str(self):
+        """Test str(...) gives just message part."""
+        err = SitemapIndexError("howdy", "this should be the etree")
+        self.assertEqual(str(err), "howdy")
 
 
 class TestSitemap(unittest.TestCase):
@@ -50,7 +47,7 @@ class TestSitemap(unittest.TestCase):
                          '<url><loc>aardvark</loc><lastmod>2012-01-11T04:05:06Z</lastmod><rs:md datetime="2012-01-11T04:05:06Z" /></url>')
 
     def test_02_resource_str(self):
-        r1 = Resource('3b', 1234.1, 9999, 'ab54de')
+        r1 = Resource('3b', 1234.1, length=9999, md5='ab54de')
         self.assertEqual(Sitemap().resource_as_xml(r1),
                          "<url><loc>3b</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /></url>")
         r1 = Resource('3c', datetime='2013-01-02T13:00:00Z')
@@ -115,7 +112,8 @@ class TestSitemap(unittest.TestCase):
         i = iter(m)
         self.assertEqual(Sitemap().resources_as_xml(i), "<?xml version='1.0' encoding='UTF-8'?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:rs=\"http://www.openarchives.org/rs/terms/\"><url><loc>a</loc><lastmod>2001-01-01T00:00:00Z</lastmod><rs:md length=\"1234\" /></url><url><loc>b</loc><lastmod>2002-02-02T00:00:00Z</lastmod><rs:md length=\"56789\" /></url></urlset>")
 
-    def test_10_sitemap(self):
+    def test_10_parse_xml(self):
+        """Test parse_xml method with string XML."""
         xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
 <url><loc>http://e.com/a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md hash="md5:Q2hlY2sgSW50ZWdyaXR5IQ==" length=\"12\" /></url>\
@@ -130,8 +128,7 @@ class TestSitemap(unittest.TestCase):
             self.assertEqual(r.lastmod, '2012-03-14T18:37:36Z')
             self.assertEqual(r.length, 12)
             self.assertEqual(r.md5, 'Q2hlY2sgSW50ZWdyaXR5IQ==')
-
-    def test_11_parse_2(self):
+        # ..another
         xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
 <url><loc>/tmp/rs_test/src/file_a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length=\"12\" /></url>\
@@ -142,6 +139,54 @@ class TestSitemap(unittest.TestCase):
         self.assertFalse(s.parsed_index, 'was a sitemap')
         self.assertEqual(s.resources_created, 2, 'got 2 resources')
 
+    def test_11_parse_xml_error(self):
+        """Test exceptiona from parse_xml method."""
+        # bad params
+        s = Sitemap()
+        self.assertRaises(ValueError, s.parse_xml)
+        # got a sitemap when told to expect and indexp
+        xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+</urlset>'
+        self.assertRaises(SitemapIndexError, s.parse_xml, fh=io.StringIO(xml), sitemapindex=True)
+        # dupe entries DO NOT create an error
+        xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<url><loc>/mouse</loc><lastmod>2020-12-21T00:00:00Z</lastmod><rs:md length=\"12\" /></url>\
+<url><loc>/mouse</loc><lastmod>2020-12-21T00:00:00Z</lastmod><rs:md length=\"12\" /></url>\
+</urlset>'
+        s = Sitemap()
+        i = s.parse_xml(fh=io.StringIO(xml))
+        self.assertEqual(len(i.resources), 2)
+        # preamble rs:md after <url> is error
+        xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<url><loc>/frog</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
+<rs:md capability=\"resourcelist\"/>\
+<url><loc>/toad</loc><lastmod>2020-12-21T00:02:00Z</lastmod><rs:md length=\"8\" /></url>\
+</urlset>'
+        self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
+        # preamble rs:ln after <url> is also error
+        xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<url><loc>/wills</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
+<rs:ln rel="up" href="http://example.com/resourcesync_description.xml"/>\
+</urlset>'
+        s = Sitemap()
+        self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
+        # but random unknown junk should be ignored...
+        xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<junk1>beetle</junk1>\
+<rs:md capability=\"resourcelist\"/>\
+<junk2>fly</junk2>\
+<url><loc>/whale</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
+<junk3>ant</junk3>\
+</urlset>'
+        s = Sitemap()
+        i = s.parse_xml(fh=io.StringIO(xml))
+        self.assertEqual(len(i.resources), 1)
+
     def test_12_parse_multi_loc(self):
         xml_start = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
@@ -192,9 +237,9 @@ class TestSitemap(unittest.TestCase):
     def test_15_parse_illformed(self):
         s = Sitemap()
         # ExpatError in python2.6, ParserError in 2.7,3.x
-        self.assertRaises(etree_error_class, s.parse_xml,
+        self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml,
                           io.StringIO('not xml'))
-        self.assertRaises(etree_error_class, s.parse_xml,
+        self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml,
                           io.StringIO('<urlset><url>something</urlset>'))
 
     def test_16_parse_valid_xml_but_other(self):
@@ -326,3 +371,65 @@ class TestSitemap(unittest.TestCase):
         r2 = next(i)
         self.assertEqual(r2.uri, '/tmp/rs_test/src/file_b')
         self.assertEqual(r2.change, None)
+
+    def test_31_resource_from_etree(self):
+        """Test resource_from_etree method."""
+        # multiple <loc>
+        xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<loc>a_name</loc>
+<loc>another_name_oops</loc>
+</url>'''
+        et = parse(io.StringIO(xml))
+        self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
+        # no <loc>
+        xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<no_loc_element />
+</url>'''
+        et = parse(io.StringIO(xml))
+        self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
+        # muktiple <rs:md> not allowed
+        xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<loc>a_name</loc>
+<rs:md type="text/plain" />
+<rs:md something="123" />
+</url>'''
+        et = parse(io.StringIO(xml))
+        self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
+        # warn is hash invalid
+        with LogCapture() as lc:
+            xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
+<loc>a_name</loc>
+<rs:md hash="UNKNOWN-TYPE:blah" />
+</url>'''
+            et = parse(io.StringIO(xml))
+            Sitemap().resource_from_etree(et, Resource)
+            self.assertIn('Ignored unsupported hash type UNKNOWN-TYPE', lc.records[-1].msg)
+
+    def test_32_md_from_etree(self):
+        """Test md_from_etree method."""
+        # Warning for unknwon capability
+        with LogCapture() as lc:
+            xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" capability="WHY" />
+'''
+            et = parse(io.StringIO(xml)).getroot()
+            Sitemap().md_from_etree(et)
+            self.assertIn("Unknown capability name 'WHY'", lc.records[-1].msg)
+        # Bad value for change is an error
+        with LogCapture() as lc:
+            xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" change="BAD" />
+'''
+            et = parse(io.StringIO(xml)).getroot()
+            self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et)
+        # length must be an integer
+        with LogCapture() as lc:
+            xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
+<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" length="short" />
+'''
+            et = parse(io.StringIO(xml)).getroot()
+            self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et)

tests/test_url_or_file_open.py

@@ -0,0 +1,37 @@
+"""Tests for resync.url_or_file_open."""
+from .testlib import TestCase, webserver
+import time
+
+from resync.url_or_file_open import NUM_REQUESTS, CONFIG, set_url_or_file_open_config, url_or_file_open
+
+
+class TestUrlOrFileOpen(TestCase):
+
+    def test_set_url_or_file_open_config(self):
+        """Test set_url_or_file_open_config function."""
+        self.assertEqual(CONFIG['bearer_token'], None)
+        self.assertEqual(CONFIG['delay'], None)
+        self.assertNotIn('my_thing', CONFIG)
+        set_url_or_file_open_config('bearer_token', 'open seasame')
+        self.assertEqual(CONFIG['bearer_token'], 'open seasame')
+        set_url_or_file_open_config('my_thing', 'special')
+        self.assertEqual(CONFIG['my_thing'], 'special')
+
+    def test_url_or_file_open(self):
+        """Test basic operation of url_or_file_open function."""
+        # Open file
+        fh = url_or_file_open('tests/testdata/dir1/file_a')
+        self.assertIn(b'I am file a', fh.read())
+        fh.close()
+        with url_or_file_open('file:tests/testdata/dir1/file_b') as fh:
+            self.assertIn(b'I am file b', fh.read())
+        # Open URL
+        with webserver('tests/testdata', 'localhost', 9999):
+            with url_or_file_open('http://localhost:9999/dir2/file_x') as fh:
+                self.assertIn(b'I am the mysterious file_x', fh.read())
+            # test delay of 0.1s
+            set_url_or_file_open_config('delay', 0.1)
+            before = time.time()
+            with url_or_file_open('http://localhost:9999/dir1/file_a') as fh:
+                self.assertIn(b'I am file a', fh.read())
+            self.assertGreater(time.time() - before, 0.099)

tests/testdata/symlink/dir1/a_file.txt

@@ -0,0 +1 @@
+I am a real file!

tests/testdata/symlink/dir2/a_file.txt

@@ -0,0 +1 @@
+../dir1/a_file.txt

tests/testlib/testcase_with_tmpdir.py

@@ -36,10 +36,4 @@ class TestCase(unittest.TestCase):
     @property
     def tmpdir(self):
         """Read-only access to _tmpdir, just in case... The rmtree scares me."""
-        #
-        # FIXME - Hack to work on python2.6 where setUpClass is not called, will
-        # FIXME - not have proper tidy as tearDownClass will not be called.
-        # FIXME - Remove when 2.6 no longer supported
-        if (not self._tmpdir and sys.version_info < (2, 7)):
-            self.setUpClass()
         return(self._tmpdir)

tests/testlib/webserver_context.py

@@ -78,6 +78,7 @@ def webserver(dir='/tmp/htdocs', host='localhost', port=9999):
     finally:
         # Close the server
         p.terminate()
+        time.sleep(0.1)
 
 
 if __name__ == '__main__':