gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix errors with 3.6, tidy

Browse Source
This commit is contained in:
Simeon Warner 2020-12-10 07:24:00 -05:00
parent 7d2181a31a
commit a15b4c50ce
33 changed files with 120 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
build
dist
MANIFEST
.tox
.eggs
*.egg
# Testing

5
.travis.yml
View File

@ -4,15 +4,14 @@ python:
- "3.5"
- "3.6"
- "3.7"
- "3.8"
install:
- pip install requests
- pip install python-dateutil
- pip install coveralls pep8 pep257 restructuredtext_lint testfixtures
- pip install coveralls pycodestyle pep257 restructuredtext_lint testfixtures
- python setup.py install
script:
- python setup.py test
- pep8 --ignore=E501 resync tests bin/resync bin/resync-explorer
- pycodestyle --ignore=E501,W503 resync tests bin/resync bin/resync-explorer
- pep257 resync tests bin/resync bin/resync-explorer
- rst-lint README
- coverage run --source=resync setup.py test

6
CHANGES.md
View File

@ -6,9 +6,11 @@ core specification version. Versions 1.0.x implement the v1.0
ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014
<http://www.openarchives.org/rs/1.0/toc>.
=======
v1.0.10 ???
* Drop Pyhton 3.3 & 3.4 from tests, add 3.7 & 3.8
* Move and rename command line clients: bin/resync -> resync.pl and bin/resync-explorer -> resync-explorer.py
* Drop Python 3.3 & 3.4 from tests, add 3.7. Fix various new warnings from 3.7 (more work is required to support 3.8)
* Switch from pep8 to pycodestyle in tests
* Move libraries to support tests into test/testlib
v1.0.9 2018-10-23
* Add -t/--tries and -T/--timeout options (https://github.com/resync/resync/issues/34)

10
README
View File

@ -23,16 +23,16 @@ Client usage
Typical client usage to synchronize from a source at
``http://source.example.com/`` to a set of local files would be::
resync http://source.example.com/
resync.py http://source.example.com/
which will create or update a local directory ``./source.example.com``.
Alternatively, the destination directory may be specified explicitly::
resync http://source.example.com/ /tmp/my_copy
resync.py http://source.example.com/ /tmp/my_copy
Option details and a number of different modes are described with::
resync -h
resync.py -h
Library usage
-------------
@ -59,7 +59,7 @@ Typical library use in a destination (get and examine a Capability List)::
Installation
------------
The client and library are designed to work with Python 2.7, 3.5, 3.6, 3.7 and 3.8.
The client and library are designed to work with Python 2.7, 3.5, 3.6 and 3.7.
**Automatic installation**::
@ -78,7 +78,7 @@ rsync is listed in `PyPI
sudo python setup.py install
This will install the library code in the appropriate place within
your python setup, and the client ``resync`` in an appropriate system
your python setup, and the client ``resync.py`` in an appropriate system
path (perhaps ``/usr/local/bin`` or ``/usr/bin`` depending on your system).
The source code is maintained on `Github

1
bin/resync-explorer → resync-explorer.py
View File

@ -125,5 +125,6 @@ def main():
except ClientFatalError as e:
sys.stderr.write("\nFatalError: " + str(e) + "\n")
if __name__ == '__main__':
main()

15
bin/resync → resync.py
View File

@ -184,10 +184,10 @@ def main():
# Implement exclusive arguments and default --baseline (support for exclusive
# groups in argparse is incomplete is python2.6)
if (not args.baseline and not args.incremental and not args.audit and
not args.parse and not args.write_resourcelist and not args.write_changelist and
not args.write_capabilitylist and not args.write_sourcedescription and
not args.write_resourcedump and not args.write_changedump):
if (not args.baseline and not args.incremental and not args.audit
and not args.parse and not args.write_resourcelist and not args.write_changelist
and not args.write_capabilitylist and not args.write_sourcedescription
and not args.write_resourcedump and not args.write_changedump):
if (len(map) == 0):
p.error("No arguments specified (use -h for help)")
return
@ -241,9 +241,9 @@ def main():
# Links apply to anything that writes sitemaps
links = parse_links(args.link)
# Add specific links is appropriate cases
if (args.capabilitylist_link and
not args.write_capabilitylist and
not args.write_sourcedescription):
if (args.capabilitylist_link
and not args.write_capabilitylist
and not args.write_sourcedescription):
# rel="up" to Capability List in all but Capability List
# and Source Description
links.insert(0, {'rel': 'up', 'href': args.capabilitylist_link})
@ -299,5 +299,6 @@ def main():
except ClientFatalError as e:
sys.stderr.write("\nFatalError: " + str(e) + "\n")
if __name__ == '__main__':
main()

20
resync/__init__.py
View File

@ -1,15 +1,15 @@
"""Module config for resync."""
from resync._version import __version__
from ._version import __version__
# Enable easy import for core classes, e.g.
# from resync import Resource
from resync.source_description import SourceDescription
from resync.capability_list import CapabilityList
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
from resync.resource_dump import ResourceDump
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.change_dump import ChangeDump
from resync.archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive
from resync.resource import Resource
from .source_description import SourceDescription
from .capability_list import CapabilityList
from .resource_list import ResourceList
from .change_list import ChangeList
from .resource_dump import ResourceDump
from .resource_dump_manifest import ResourceDumpManifest
from .change_dump import ChangeDump
from .archives import ResourceListArchive, ResourceDumpArchive, ChangeListArchive, ChangeDumpArchive
from .resource import Resource

2
resync/list_base.py
View File

@ -107,7 +107,7 @@ class ListBase(ResourceContainer):
# Legacy support for str argument, see
# https://github.com/resync/resync/pull/21
# One test for this in tests/test_list_base.py
self.logger.warn(
self.logger.warning(
"Legacy parse(str=...), use parse(str_data=...) instead")
fh = io.StringIO(kwargs['str'])
if (fh is None):

28
resync/mapper.py
View File

@ -43,25 +43,25 @@ class Mapper():
source base URI. In the case that the source base URI is a local
path already then an indentity mapping is used.
"""
if (use_default_path and
len(mappings) == 1 and
re.search(r"=", mappings[0]) is None):
if (use_default_path
and len(mappings) == 1
and re.search(r"=", mappings[0]) is None):
path = self.path_from_uri(mappings[0])
self.logger.warning("Using URI mapping: %s -> %s" %
(mappings[0], path))
self.mappings.append(Map(mappings[0], path))
elif (len(mappings) == 2 and
re.search(r"=", mappings[0]) is None and
re.search(r"=", mappings[1]) is None):
elif (len(mappings) == 2
and re.search(r"=", mappings[0]) is None
and re.search(r"=", mappings[1]) is None):
self.mappings.append(Map(mappings[0], mappings[1]))
else:
for mapping in mappings:
l = mapping.split('=')
if (len(l) != 2):
entry = mapping.split('=')
if (len(entry) != 2):
raise MapperError(
"Bad mapping argument (%s), got %s" %
(mapping, str(l)))
(src_uri, dst_path) = l
(mapping, str(entry)))
(src_uri, dst_path) = entry
# Check for dupes
for map in self.mappings:
if (src_uri == map.src_uri):
@ -129,10 +129,10 @@ class Mapper():
if (netloc == ''):
return(uri)
path = '/'.join([netloc, path])
path = re.sub('[^\w\-\.]', '_', path)
path = re.sub('__+', '_', path)
path = re.sub('[_\.]+$', '', path)
path = re.sub('^[_\.]+', '', path)
path = re.sub(r'[^\w\-\.]', '_', path)
path = re.sub(r'__+', '_', path)
path = re.sub(r'[_\.]+$', '', path)
path = re.sub(r'^[_\.]+', '', path)
return(path)
def __repr__(self):

20
resync/resource.py
View File

@ -3,7 +3,7 @@
import re
try: # python3
from urllib.parse import urlparse
except: # python2
except ImportError: # python2
from urlparse import urlparse
from posixpath import basename
from .w3c_datetime import str_to_datetime, datetime_to_str
@ -149,8 +149,8 @@ class Resource(object):
the idea of extra properties.
"""
# Add validity check for self.change
if (prop == 'change' and Resource.CHANGE_TYPES and
value is not None and value not in Resource.CHANGE_TYPES):
if (prop == 'change' and Resource.CHANGE_TYPES
and value is not None and value not in Resource.CHANGE_TYPES):
raise ChangeTypeError(value)
else:
try:
@ -422,15 +422,15 @@ class Resource(object):
return(False)
if (self.timestamp is not None or other.timestamp is not None):
# not equal if only one timestamp specified
if (self.timestamp is None or
other.timestamp is None or
abs(self.timestamp - other.timestamp) >= delta):
if (self.timestamp is None
or other.timestamp is None
or abs(self.timestamp - other.timestamp) >= delta):
return(False)
if ((self.md5 is not None and other.md5 is not None) and
self.md5 != other.md5):
if ((self.md5 is not None and other.md5 is not None)
and self.md5 != other.md5):
return(False)
if ((self.length is not None and other.length is not None) and
self.length != other.length):
if ((self.length is not None and other.length is not None)
and self.length != other.length):
return(False)
return(True)

3
resync/resource_container.py
View File

@ -114,8 +114,7 @@ class ResourceContainer(object):
def link(self, rel):
"""Look for link with specified rel, return else None."""
for link in self.ln:
if ('rel' in link and
link['rel'] == rel):
if ('rel' in link and link['rel'] == rel):
return(link)
return(None)

2
resync/resource_list_builder.py
View File

@ -50,7 +50,7 @@ class ResourceListBuilder():
self.set_path = set_path
self.set_hashes = set_hashes if (set_hashes and len(set_hashes) > 0) else None
self.set_length = set_length
self.exclude_files = ['sitemap\d{0,5}.xml']
self.exclude_files = [r'sitemap\d{0,5}.xml']
self.exclude_dirs = ['CVS', '.git']
self.include_symlinks = False
# Used internally only:

3
resync/sitemap.py
View File

@ -270,8 +270,7 @@ class Sitemap(object):
md = Element('rs:md', md_atts)
e.append(md)
# add any <rs:ln>
if (hasattr(resource, 'ln') and
resource.ln is not None):
if (hasattr(resource, 'ln') and resource.ln is not None):
for ln in resource.ln:
self.add_element_with_atts_to_etree(e, 'rs:ln', ln)
if (self.pretty_xml):

5
resync/url_authority.py
View File

@ -70,8 +70,7 @@ class UrlAuthority(object):
# Maybe should allow parallel for 3+ components, eg. a.example.org,
# b.example.org
path = os.path.dirname(s.path)
if (self.strict and
path != self.master_path and
not path.startswith(self.master_path)):
if (self.strict and path != self.master_path
and not path.startswith(self.master_path)):
return(False)
return(True)

2
resync/w3c_datetime.py
View File

@ -102,7 +102,7 @@ def str_to_datetime(s, context='datetime'):
# with dt.tzinfo module but this has variation in behavior
# between python 2.6 and 2.7... so do here for now
m = re.match(r"(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d(:\d\d)?)(Z|([+-])"
"(\d\d):(\d\d))$", s)
r"(\d\d):(\d\d))$", s)
if (m is None):
raise ValueError("Bad datetime format (%s)" % s)
str = m.group(1) + 'Z'

5
setup.py
View File

@ -40,17 +40,16 @@ setup(
name='resync',
version=version,
packages=['resync'],
scripts=['bin/resync','bin/resync-explorer'],
scripts=['resync.py','resync-explorer.py'],
classifiers=["Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent", #is this true? know Linux & OS X ok
"Programming Language :: Python",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Software Development :: Libraries :: Python Modules",
"Environment :: Web Environment"],

4
tests/test_client.py
View File

@ -1,6 +1,4 @@
from tests.testcase_with_tmpdir import TestCase
from tests.capture_stdout import capture_stdout
from tests.webserver_context import webserver
from .testlib import TestCase, capture_stdout, webserver
import unittest
import re

2
tests/test_client_link_options.py
View File

@ -16,7 +16,7 @@ import subprocess
def run_resync(args):
args.insert(0, 'bin/resync')
args.insert(0, './resync.py')
proc = subprocess.Popen(args, stdout=subprocess.PIPE)
(out, err) = proc.communicate()
return(out)

4
tests/test_client_state.py
View File

@ -1,6 +1,6 @@
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import os.path
import unittest
from resync.client_state import ClientState

2
tests/test_client_utils.py
View File

@ -1,4 +1,4 @@
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import logging
import os.path

4
tests/test_dump.py
View File

@ -1,8 +1,6 @@
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import os.path
import unittest
import sys
import zipfile
from resync.dump import Dump, DumpError

3
tests/test_examples_from_spec.py
View File

@ -169,9 +169,6 @@ class TestExamplesFromSpec(unittest.TestCase):
self.assertEqual(
cl1.describedby, 'http://example.com/info_about_set1_of_resources.xml')
# BUILD EXAMPLES ---------------------------
def test_build_ex_01(self):
"""Simple Resource List document"""
rl = ResourceList()

2
tests/test_explorer.py
View File

@ -1,4 +1,4 @@
from tests.capture_stdout import capture_stdout
from .testlib import capture_stdout
import unittest
import re

4
tests/test_explorer_client.py
View File

@ -10,7 +10,7 @@ except ImportError: # python3
def run_resync_explorer(args):
args.insert(0, 'bin/resync-explorer')
args.insert(0, './resync-explorer.py')
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = proc.communicate()
return(out, err)
@ -21,7 +21,7 @@ class TestClientLinkOptions(unittest.TestCase):
def test01_help(self):
"""Check that it runs with -h."""
txt = run_resync_explorer(['-h'])[0]
self.assertTrue(txt.startswith(b'Usage: resync-explorer [options] uri'))
self.assertTrue(txt.startswith(b'Usage: resync-explorer.py [options] uri'))
def test02_error(self):
"""Bad parameter."""

2
tests/test_list_base.py
View File

@ -1,4 +1,4 @@
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import sys
import os.path

3
tests/test_resource_dump.py
View File

@ -1,5 +1,6 @@
"""Tests for resync.resource_dump."""
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import os.path
try: # python2
# Must try this first as io also exists in python2

4
tests/test_resource_list.py
View File

@ -1,8 +1,8 @@
from tests.testcase_with_tmpdir import TestCase
from .testlib import TestCase
import os.path
try: # python2
# Must try this first as io also exists in python2
# but in the wrong one!
# but is the wrong one!
import StringIO as io
except ImportError: # python3
import io

34
tests/test_resource_list_multifile.py
View File

@ -65,37 +65,37 @@ class TestResourceListMultifile(unittest.TestCase):
# check the two component sitemaps
rl1 = ResourceList()
rl1.read(os.path.join(tempdir, 'sitemap00000.xml'))
self.assertEquals(len(rl1), 2)
self.assertEquals(rl1.capability, 'resourcelist')
self.assertEqual(len(rl1), 2)
self.assertEqual(rl1.capability, 'resourcelist')
self.assertFalse(rl1.sitemapindex)
i = iter(rl1)
self.assertEquals(next(i).uri, 'http://localhost/a')
self.assertEquals(next(i).uri, 'http://localhost/b')
self.assertEqual(next(i).uri, 'http://localhost/a')
self.assertEqual(next(i).uri, 'http://localhost/b')
rl2 = ResourceList()
rl2.read(os.path.join(tempdir, 'sitemap00001.xml'))
self.assertEquals(len(rl2), 2)
self.assertEqual(len(rl2), 2)
i = iter(rl2)
self.assertEquals(next(i).uri, 'http://localhost/c')
self.assertEquals(next(i).uri, 'http://localhost/d')
self.assertEqual(next(i).uri, 'http://localhost/c')
self.assertEqual(next(i).uri, 'http://localhost/d')
# check the sitemapindex (read just as index)
rli = ResourceList()
rli.read(os.path.join(tempdir, 'sitemap.xml'), index_only=True)
self.assertEquals(len(rli), 2)
self.assertEqual(len(rli), 2)
i = iter(rli)
self.assertEquals(rli.capability, 'resourcelist')
self.assertEqual(rli.capability, 'resourcelist')
self.assertTrue(rli.sitemapindex)
self.assertEquals(next(i).uri, 'http://localhost/sitemap00000.xml')
self.assertEquals(next(i).uri, 'http://localhost/sitemap00001.xml')
self.assertEqual(next(i).uri, 'http://localhost/sitemap00000.xml')
self.assertEqual(next(i).uri, 'http://localhost/sitemap00001.xml')
# check the sitemapindex and components
rli = ResourceList(mapper=rl.mapper)
rli.read(os.path.join(tempdir, 'sitemap.xml'))
self.assertEquals(len(rli), 4)
self.assertEquals(rli.capability, 'resourcelist')
self.assertEqual(len(rli), 4)
self.assertEqual(rli.capability, 'resourcelist')
self.assertFalse(rli.sitemapindex)
i = iter(rli)
self.assertEquals(next(i).uri, 'http://localhost/a')
self.assertEquals(next(i).uri, 'http://localhost/b')
self.assertEquals(next(i).uri, 'http://localhost/c')
self.assertEquals(next(i).uri, 'http://localhost/d')
self.assertEqual(next(i).uri, 'http://localhost/a')
self.assertEqual(next(i).uri, 'http://localhost/b')
self.assertEqual(next(i).uri, 'http://localhost/c')
self.assertEqual(next(i).uri, 'http://localhost/d')
# cleanup tempdir
shutil.rmtree(tempdir)

4
tests/testlib/__init__.py Normal file
View File

@ -0,0 +1,4 @@
"""Test support libraries for resync tests."""
from .testcase_with_tmpdir import TestCase
from .capture_stdout_context import capture_stdout
from .webserver_context import webserver

2
tests/capture_stdout.py → tests/testlib/capture_stdout_context.py
View File

@ -1,4 +1,4 @@
"""Provide capture_stdout as contect manager."""
"""Provide capture_stdout as context manager."""
import sys
import contextlib

4
tests/testcase_with_tmpdir.py → tests/testlib/testcase_with_tmpdir.py
View File

@ -19,7 +19,7 @@ class TestCase(unittest.TestCase):
raise Exception("Failed to create tempdir to use for dump tests")
try:
cls.extraSetUpClass()
except:
except Exception:
pass
@classmethod
@ -30,7 +30,7 @@ class TestCase(unittest.TestCase):
shutil.rmtree(cls._tmpdir)
try:
cls.extraTearUpClass()
except:
except Exception:
pass
@property

5
tests/webserver_context.py → tests/testlib/webserver_context.py
View File

@ -1,4 +1,5 @@
"""Provides content manager that runs local webserver."""
"""Provides context manager that runs local webserver."""
import contextlib
import os
import posixpath
@ -8,7 +9,7 @@ import time
from multiprocessing import Process
try: # python3
from http.server import HTTPServer, SimpleHTTPRequestHandler
except:
except ImportError: # python2
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler
try: # python3

13
tox.ini Normal file
View File

@ -0,0 +1,13 @@
# tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.
[tox]
envlist = py36
[testenv]
deps =
commands =
python setup.py test