Fix errors with 3.6, tidy

.gitignore

@@ -2,6 +2,7 @@
 build
 dist
 MANIFEST
+.tox
 .eggs
 *.egg
 # Testing

.travis.yml

@@ -4,15 +4,14 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
-  - "3.8"
 install:
   - pip install requests
   - pip install python-dateutil
-  - pip install coveralls pep8 pep257 restructuredtext_lint testfixtures
+  - pip install coveralls pycodestyle pep257 restructuredtext_lint testfixtures
   - python setup.py install
 script:
   - python setup.py test
-  - pep8 --ignore=E501 resync tests bin/resync bin/resync-explorer
+  - pycodestyle --ignore=E501,W503 resync tests bin/resync bin/resync-explorer
   - pep257 resync tests bin/resync bin/resync-explorer
   - rst-lint README
   - coverage run --source=resync setup.py test

CHANGES.md

@@ -6,9 +6,11 @@ core specification version. Versions 1.0.x implement the v1.0
 ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014
 <http://www.openarchives.org/rs/1.0/toc>.
 
-=======
 v1.0.10 ???
-  * Drop Pyhton 3.3 & 3.4 from tests, add 3.7 & 3.8
+  * Move and rename command line clients: bin/resync -> resync.pl and bin/resync-explorer -> resync-explorer.py
+  * Drop Python 3.3 & 3.4 from tests, add 3.7. Fix various new warnings from 3.7 (more work is required to support 3.8)
+  * Switch from pep8 to pycodestyle in tests
+  * Move libraries to support tests into test/testlib
 
 v1.0.9 2018-10-23
   * Add -t/--tries and -T/--timeout options (https://github.com/resync/resync/issues/34)

README

@@ -23,16 +23,16 @@ Client usage
 Typical client usage to synchronize from a source at 
 ``http://source.example.com/`` to a set of local files would be::
 
-    resync http://source.example.com/
+    resync.py http://source.example.com/
 
 which will create or update a local directory ``./source.example.com``. 
 Alternatively, the destination directory may be specified explicitly::
 
-    resync http://source.example.com/ /tmp/my_copy
+    resync.py http://source.example.com/ /tmp/my_copy
 
 Option details and a number of different modes are described with::
 
-    resync -h
+    resync.py -h
 
 Library usage
 -------------
@@ -59,7 +59,7 @@ Typical library use in a destination (get and examine a Capability List)::
 Installation
 ------------
 
-The client and library are designed to work with Python 2.7, 3.5, 3.6, 3.7 and 3.8.
+The client and library are designed to work with Python 2.7, 3.5, 3.6 and 3.7.
 
 **Automatic installation**::
 
@@ -78,7 +78,7 @@ rsync is listed in `PyPI
     sudo python setup.py install
 
 This will install the library code in the appropriate place within 
-your python setup, and the client ``resync`` in an appropriate system 
+your python setup, and the client ``resync.py`` in an appropriate system
 path (perhaps ``/usr/local/bin`` or ``/usr/bin`` depending on your system).
 
 The source code is maintained on `Github

resync-explorer.py

@@ -125,5 +125,6 @@ def main():
     except ClientFatalError as e:
         sys.stderr.write("\nFatalError: " + str(e) + "\n")
 
+
 if __name__ == '__main__':
     main()

resync.py

@@ -184,10 +184,10 @@ def main():
 
     # Implement exclusive arguments and default --baseline (support for exclusive
     # groups in argparse is incomplete is python2.6)
-    if (not args.baseline and not args.incremental and not args.audit and
+    if (not args.baseline and not args.incremental and not args.audit
-            not args.parse and not args.write_resourcelist and not args.write_changelist and
+            and not args.parse and not args.write_resourcelist and not args.write_changelist
-            not args.write_capabilitylist and not args.write_sourcedescription and
+            and not args.write_capabilitylist and not args.write_sourcedescription
-            not args.write_resourcedump and not args.write_changedump):
+            and not args.write_resourcedump and not args.write_changedump):
         if (len(map) == 0):
             p.error("No arguments specified (use -h for help)")
             return
@@ -241,9 +241,9 @@ def main():
         # Links apply to anything that writes sitemaps
         links = parse_links(args.link)
         # Add specific links is appropriate cases
-        if (args.capabilitylist_link and
+        if (args.capabilitylist_link
-                not args.write_capabilitylist and
+                and not args.write_capabilitylist
-                not args.write_sourcedescription):
+                and not args.write_sourcedescription):
             # rel="up" to Capability List in all but Capability List
             # and Source Description
             links.insert(0, {'rel': 'up', 'href': args.capabilitylist_link})
@@ -299,5 +299,6 @@ def main():
     except ClientFatalError as e:
         sys.stderr.write("\nFatalError: " + str(e) + "\n")
 
+
 if __name__ == '__main__':
     main()

resync/__init__.py

@@ -1,15 +1,15 @@
 """Module config for resync."""
 
-from resync._version import __version__
+from ._version import __version__
 
 # Enable easy import for core classes, e.g.
 # from resync import Resource
-from resync.source_description import SourceDescription
+from .source_description import SourceDescription
-from resync.capability_list import CapabilityList
+from .capability_list import CapabilityList
-from resync.resource_list import ResourceList
+from .resource_list import ResourceList
-from resync.change_list import ChangeList
+from .change_list import ChangeList
-from resync.resource_dump import ResourceDump
+from .resource_dump import ResourceDump
-from resync.resource_dump_manifest import ResourceDumpManifest
+from .resource_dump_manifest import ResourceDumpManifest
-from resync.change_dump import ChangeDump
+from .change_dump import ChangeDump
-from resync.archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive
+from .archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive
-from resync.resource import Resource
+from .resource import Resource

resync/list_base.py

@@ -107,7 +107,7 @@ class ListBase(ResourceContainer):
             # Legacy support for str argument, see
             # https://github.com/resync/resync/pull/21
             # One test for this in tests/test_list_base.py
-            self.logger.warn(
+            self.logger.warning(
                 "Legacy parse(str=...), use parse(str_data=...) instead")
             fh = io.StringIO(kwargs['str'])
         if (fh is None):

resync/mapper.py

@@ -43,25 +43,25 @@ class Mapper():
         source base URI. In the case that the source base URI is a local
         path already then an indentity mapping is used.
         """
-        if (use_default_path and
+        if (use_default_path
-                len(mappings) == 1 and
+                and len(mappings) == 1
-                re.search(r"=", mappings[0]) is None):
+                and re.search(r"=", mappings[0]) is None):
             path = self.path_from_uri(mappings[0])
             self.logger.warning("Using URI mapping: %s -> %s" %
                                 (mappings[0], path))
             self.mappings.append(Map(mappings[0], path))
-        elif (len(mappings) == 2 and
+        elif (len(mappings) == 2
-              re.search(r"=", mappings[0]) is None and
+                and re.search(r"=", mappings[0]) is None
-              re.search(r"=", mappings[1]) is None):
+                and re.search(r"=", mappings[1]) is None):
             self.mappings.append(Map(mappings[0], mappings[1]))
         else:
             for mapping in mappings:
-                l = mapping.split('=')
+                entry = mapping.split('=')
-                if (len(l) != 2):
+                if (len(entry) != 2):
                     raise MapperError(
                         "Bad mapping argument (%s), got %s" %
-                        (mapping, str(l)))
+                        (mapping, str(entry)))
-                (src_uri, dst_path) = l
+                (src_uri, dst_path) = entry
                 # Check for dupes
                 for map in self.mappings:
                     if (src_uri == map.src_uri):
@@ -129,10 +129,10 @@ class Mapper():
         if (netloc == ''):
             return(uri)
         path = '/'.join([netloc, path])
-        path = re.sub('[^\w\-\.]', '_', path)
+        path = re.sub(r'[^\w\-\.]', '_', path)
-        path = re.sub('__+', '_', path)
+        path = re.sub(r'__+', '_', path)
-        path = re.sub('[_\.]+$', '', path)
+        path = re.sub(r'[_\.]+$', '', path)
-        path = re.sub('^[_\.]+', '', path)
+        path = re.sub(r'^[_\.]+', '', path)
         return(path)
 
     def __repr__(self):

resync/resource.py

@@ -3,7 +3,7 @@
 import re
 try:  # python3
     from urllib.parse import urlparse
-except:  # python2
+except ImportError:  # python2
     from urlparse import urlparse
 from posixpath import basename
 from .w3c_datetime import str_to_datetime, datetime_to_str
@@ -149,8 +149,8 @@ class Resource(object):
         the idea of extra properties.
         """
         # Add validity check for self.change
-        if (prop == 'change' and Resource.CHANGE_TYPES and
+        if (prop == 'change' and Resource.CHANGE_TYPES
-                value is not None and value not in Resource.CHANGE_TYPES):
+                and value is not None and value not in Resource.CHANGE_TYPES):
             raise ChangeTypeError(value)
         else:
             try:
@@ -422,15 +422,15 @@ class Resource(object):
             return(False)
         if (self.timestamp is not None or other.timestamp is not None):
             # not equal if only one timestamp specified
-            if (self.timestamp is None or
+            if (self.timestamp is None
-                    other.timestamp is None or
+                    or other.timestamp is None
-                    abs(self.timestamp - other.timestamp) >= delta):
+                    or abs(self.timestamp - other.timestamp) >= delta):
                 return(False)
-        if ((self.md5 is not None and other.md5 is not None) and
+        if ((self.md5 is not None and other.md5 is not None)
-                self.md5 != other.md5):
+                and self.md5 != other.md5):
             return(False)
-        if ((self.length is not None and other.length is not None) and
+        if ((self.length is not None and other.length is not None)
-                self.length != other.length):
+                and self.length != other.length):
             return(False)
         return(True)
 

resync/resource_container.py

@@ -114,8 +114,7 @@ class ResourceContainer(object):
     def link(self, rel):
         """Look for link with specified rel, return else None."""
         for link in self.ln:
-            if ('rel' in link and
+            if ('rel' in link and link['rel'] == rel):
-                    link['rel'] == rel):
                 return(link)
         return(None)
 

resync/resource_list_builder.py

@@ -50,7 +50,7 @@ class ResourceListBuilder():
         self.set_path = set_path
         self.set_hashes = set_hashes if (set_hashes and len(set_hashes) > 0) else None
         self.set_length = set_length
-        self.exclude_files = ['sitemap\d{0,5}.xml']
+        self.exclude_files = [r'sitemap\d{0,5}.xml']
         self.exclude_dirs = ['CVS', '.git']
         self.include_symlinks = False
         # Used internally only:

resync/sitemap.py

@@ -270,8 +270,7 @@ class Sitemap(object):
             md = Element('rs:md', md_atts)
             e.append(md)
         # add any <rs:ln>
-        if (hasattr(resource, 'ln') and
+        if (hasattr(resource, 'ln') and resource.ln is not None):
-                resource.ln is not None):
             for ln in resource.ln:
                 self.add_element_with_atts_to_etree(e, 'rs:ln', ln)
         if (self.pretty_xml):

resync/url_authority.py

@@ -70,8 +70,7 @@ class UrlAuthority(object):
             # Maybe should allow parallel for 3+ components, eg. a.example.org,
             # b.example.org
         path = os.path.dirname(s.path)
-        if (self.strict and
+        if (self.strict and path != self.master_path
-                path != self.master_path and
+                and not path.startswith(self.master_path)):
-                not path.startswith(self.master_path)):
             return(False)
         return(True)

resync/w3c_datetime.py

@@ -102,7 +102,7 @@ def str_to_datetime(s, context='datetime'):
     # with dt.tzinfo module but this has variation in behavior
     # between python 2.6 and 2.7... so do here for now
     m = re.match(r"(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d(:\d\d)?)(Z|([+-])"
-                 "(\d\d):(\d\d))$", s)
+                 r"(\d\d):(\d\d))$", s)
     if (m is None):
         raise ValueError("Bad datetime format (%s)" % s)
     str = m.group(1) + 'Z'

setup.py

@@ -40,17 +40,16 @@ setup(
     name='resync',
     version=version,
     packages=['resync'],
-    scripts=['bin/resync','bin/resync-explorer'],
+    scripts=['resync.py','resync-explorer.py'],
     classifiers=["Development Status :: 4 - Beta",
                  "Intended Audience :: Developers",
                  "License :: OSI Approved :: Apache Software License",
                  "Operating System :: OS Independent", #is this true? know Linux & OS X ok
                  "Programming Language :: Python",
                  "Programming Language :: Python :: 2.7",
-                 "Programming Language :: Python :: 3.3",
-                 "Programming Language :: Python :: 3.4",
                  "Programming Language :: Python :: 3.5",
                  "Programming Language :: Python :: 3.6",
+                 "Programming Language :: Python :: 3.7",
                  "Topic :: Internet :: WWW/HTTP",
                  "Topic :: Software Development :: Libraries :: Python Modules",
                  "Environment :: Web Environment"],

tests/test_client.py

@@ -1,6 +1,4 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase, capture_stdout, webserver
-from tests.capture_stdout import capture_stdout
-from tests.webserver_context import webserver
 
 import unittest
 import re

tests/test_client_link_options.py

@@ -16,7 +16,7 @@ import subprocess
 
 
 def run_resync(args):
-    args.insert(0, 'bin/resync')
+    args.insert(0, './resync.py')
     proc = subprocess.Popen(args, stdout=subprocess.PIPE)
     (out, err) = proc.communicate()
     return(out)

tests/test_client_state.py

@@ -1,6 +1,6 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
+
 import os.path
-import unittest
 from resync.client_state import ClientState
 
 

tests/test_client_utils.py

@@ -1,4 +1,4 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
 
 import logging
 import os.path

tests/test_dump.py

@@ -1,8 +1,6 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
 
 import os.path
-import unittest
-import sys
 import zipfile
 
 from resync.dump import Dump, DumpError

tests/test_examples_from_spec.py

@@ -169,11 +169,8 @@ class TestExamplesFromSpec(unittest.TestCase):
         self.assertEqual(
             cl1.describedby, 'http://example.com/info_about_set1_of_resources.xml')
 
-
-# BUILD EXAMPLES ---------------------------
-
     def test_build_ex_01(self):
-        """Simple Resource List document """
+        """Simple Resource List document"""
         rl = ResourceList()
         rl.md_at = '2013-01-03T09:00:00Z'
         rl.add(Resource('http://example.com/res1'))
@@ -182,7 +179,7 @@ class TestExamplesFromSpec(unittest.TestCase):
         self._assert_xml_equal(rl.as_xml(), ex_xml)
 
     def test_build_ex_02(self):
-        """Slightly more complex Resource List document """
+        """Slightly more complex Resource List document"""
         rl = ResourceList()
         rl.md_at = '2013-01-03T09:00:00Z'
         rl.add(Resource(uri='http://example.com/res1',

tests/test_explorer.py

@@ -1,4 +1,4 @@
-from tests.capture_stdout import capture_stdout
+from .testlib import capture_stdout
 
 import unittest
 import re

tests/test_explorer_client.py

@@ -10,7 +10,7 @@ except ImportError:  # python3
 
 
 def run_resync_explorer(args):
-    args.insert(0, 'bin/resync-explorer')
+    args.insert(0, './resync-explorer.py')
     proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (out, err) = proc.communicate()
     return(out, err)
@@ -21,7 +21,7 @@ class TestClientLinkOptions(unittest.TestCase):
     def test01_help(self):
         """Check that it runs with -h."""
         txt = run_resync_explorer(['-h'])[0]
-        self.assertTrue(txt.startswith(b'Usage: resync-explorer [options] uri'))
+        self.assertTrue(txt.startswith(b'Usage: resync-explorer.py [options] uri'))
 
     def test02_error(self):
         """Bad parameter."""

tests/test_list_base.py

@@ -1,4 +1,4 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
 
 import sys
 import os.path

tests/test_resource_dump.py

@@ -1,5 +1,6 @@
 """Tests for resync.resource_dump."""
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
+
 import os.path
 try:  # python2
     # Must try this first as io also exists in python2

tests/test_resource_list.py

@@ -1,8 +1,8 @@
-from tests.testcase_with_tmpdir import TestCase
+from .testlib import TestCase
 import os.path
 try:  # python2
     # Must try this first as io also exists in python2
-    # but in the wrong one!
+    # but is the wrong one!
     import StringIO as io
 except ImportError:  # python3
     import io

tests/test_resource_list_multifile.py

@@ -65,37 +65,37 @@ class TestResourceListMultifile(unittest.TestCase):
         # check the two component sitemaps
         rl1 = ResourceList()
         rl1.read(os.path.join(tempdir, 'sitemap00000.xml'))
-        self.assertEquals(len(rl1), 2)
+        self.assertEqual(len(rl1), 2)
-        self.assertEquals(rl1.capability, 'resourcelist')
+        self.assertEqual(rl1.capability, 'resourcelist')
         self.assertFalse(rl1.sitemapindex)
         i = iter(rl1)
-        self.assertEquals(next(i).uri, 'http://localhost/a')
+        self.assertEqual(next(i).uri, 'http://localhost/a')
-        self.assertEquals(next(i).uri, 'http://localhost/b')
+        self.assertEqual(next(i).uri, 'http://localhost/b')
         rl2 = ResourceList()
         rl2.read(os.path.join(tempdir, 'sitemap00001.xml'))
-        self.assertEquals(len(rl2), 2)
+        self.assertEqual(len(rl2), 2)
         i = iter(rl2)
-        self.assertEquals(next(i).uri, 'http://localhost/c')
+        self.assertEqual(next(i).uri, 'http://localhost/c')
-        self.assertEquals(next(i).uri, 'http://localhost/d')
+        self.assertEqual(next(i).uri, 'http://localhost/d')
         # check the sitemapindex (read just as index)
         rli = ResourceList()
         rli.read(os.path.join(tempdir, 'sitemap.xml'), index_only=True)
-        self.assertEquals(len(rli), 2)
+        self.assertEqual(len(rli), 2)
         i = iter(rli)
-        self.assertEquals(rli.capability, 'resourcelist')
+        self.assertEqual(rli.capability, 'resourcelist')
         self.assertTrue(rli.sitemapindex)
-        self.assertEquals(next(i).uri, 'http://localhost/sitemap00000.xml')
+        self.assertEqual(next(i).uri, 'http://localhost/sitemap00000.xml')
-        self.assertEquals(next(i).uri, 'http://localhost/sitemap00001.xml')
+        self.assertEqual(next(i).uri, 'http://localhost/sitemap00001.xml')
         # check the sitemapindex and components
         rli = ResourceList(mapper=rl.mapper)
         rli.read(os.path.join(tempdir, 'sitemap.xml'))
-        self.assertEquals(len(rli), 4)
+        self.assertEqual(len(rli), 4)
-        self.assertEquals(rli.capability, 'resourcelist')
+        self.assertEqual(rli.capability, 'resourcelist')
         self.assertFalse(rli.sitemapindex)
         i = iter(rli)
-        self.assertEquals(next(i).uri, 'http://localhost/a')
+        self.assertEqual(next(i).uri, 'http://localhost/a')
-        self.assertEquals(next(i).uri, 'http://localhost/b')
+        self.assertEqual(next(i).uri, 'http://localhost/b')
-        self.assertEquals(next(i).uri, 'http://localhost/c')
+        self.assertEqual(next(i).uri, 'http://localhost/c')
-        self.assertEquals(next(i).uri, 'http://localhost/d')
+        self.assertEqual(next(i).uri, 'http://localhost/d')
         # cleanup tempdir
         shutil.rmtree(tempdir)

tests/testlib/__init__.py

@@ -0,0 +1,4 @@
+"""Test support libraries for resync tests."""
+from .testcase_with_tmpdir import TestCase
+from .capture_stdout_context import capture_stdout
+from .webserver_context import webserver

tests/testlib/capture_stdout_context.py

@@ -1,4 +1,4 @@
-"""Provide capture_stdout as contect manager."""
+"""Provide capture_stdout as context manager."""
 
 import sys
 import contextlib

tests/testlib/testcase_with_tmpdir.py

@@ -19,7 +19,7 @@ class TestCase(unittest.TestCase):
             raise Exception("Failed to create tempdir to use for dump tests")
         try:
             cls.extraSetUpClass()
-        except:
+        except Exception:
             pass
 
     @classmethod
@@ -30,7 +30,7 @@ class TestCase(unittest.TestCase):
         shutil.rmtree(cls._tmpdir)
         try:
             cls.extraTearUpClass()
-        except:
+        except Exception:
             pass
 
     @property

tests/testlib/webserver_context.py

@@ -1,4 +1,5 @@
-"""Provides content manager that runs local webserver."""
+"""Provides context manager that runs local webserver."""
+
 import contextlib
 import os
 import posixpath
@@ -8,7 +9,7 @@ import time
 from multiprocessing import Process
 try:  # python3
     from http.server import HTTPServer, SimpleHTTPRequestHandler
-except:
+except ImportError:  # python2
     from BaseHTTPServer import HTTPServer
     from SimpleHTTPServer import SimpleHTTPRequestHandler
 try:  # python3

tox.ini

@@ -0,0 +1,13 @@
+# tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py36
+
+[testenv]
+deps =
+
+commands =
+    python setup.py test