gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #26 from resync/develop

Browse Source 
Merge updates for 1.0.6
This commit is contained in:
Simeon Warner 2017-03-20 17:50:17 -04:00 committed by GitHub
commit ac765b27ab
17 changed files with 245 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGES.md
View File

@ -6,8 +6,12 @@ core specification version. Versions 1.0.x implement the v1.0
ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014
<http://www.openarchives.org/rs/1.0/toc>.
v1.0.6 2017-03-20
* Fixed md5 hash format (https://github.com/resync/resync/issues/25)
* Added support for sha-1 and sha-256 hashes
v1.0.5 2017-01-30
* Fix to support non-ASCII URIs/filenames (https://github.com/resync/resync/issues/22)
* Fixed to support non-ASCII URIs/filenames (https://github.com/resync/resync/issues/22)
* Added tests for Python 3.6
* Temporarily disabled tests for Python 2.6 due to problems with coverage (https://github.com/resync/resync/issues/23)

4
README
View File

@ -59,7 +59,7 @@ Typical library use in a destination (get and examine a Capability List)::
Installation
------------
The client and library are designed to work with Python 2.6 or 2.7.
The client and library are designed to work with Python 2.6, 2.7, 3.4, 3.5 and 3.6.
**Automatic installation**::
@ -106,7 +106,7 @@ See also CONTRIBUTING.md
Copyright and License
---------------------
Copyright 2012--2016 Simeon Warner
Copyright 2012--2017 Simeon Warner
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

5
TODO.md
View File

@ -12,8 +12,3 @@ Planned changes and deprecation
-------------------------------
* Remove support for ListBase.parse(str=...) which was deprecated in favor of ListBase.parse(str_data=...) 2016-03-22 in 1.0.1
Issues with specification
=========================
* v1.0 spec Figure 4 shows Capability List indexes but these are not allowed per section 9

13
bin/resync
View File

@ -120,8 +120,12 @@ def main():
# Options that apply to multiple modes
opt = p.add_option_group('MISCELANEOUS OPTIONS')
opt.add_option('--hash', type=str, action='append',
help="use specified hash types in addition to last modification time "
"and size (repeatable, may include `md5`, `sha-1` and `sha-256`)")
opt.add_option('--checksum', action='store_true',
help="use checksum (md5) in addition to last modification time and size")
help="use md5 checksum in addition to last modification time and size "
"(same as --hash=md5)")
opt.add_option('--delete', action='store_true',
help="allow files on destination to be deleted")
opt.add_option('--from', type=str, action='store', dest='from_datetime', metavar="DATETIME",
@ -176,8 +180,7 @@ def main():
not args.capabilitylist and not args.sourcedescription and
not args.resourcedump and not args.changedump):
if (len(map) == 0):
# No args at all, show help
p.print_help()
p.error("No arguments specified (use -h for help)")
return
else:
args.baseline = True
@ -190,7 +193,9 @@ def main():
init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE,
verbose=args.verbose, eval_mode=args.eval)
c = Client(checksum=args.checksum,
if (args.checksum):
args.hash.append('md5')
c = Client(hashes=args.hash,
verbose=args.verbose,
dryrun=args.dryrun)

10
bin/resync-explorer
View File

@ -61,8 +61,12 @@ def main():
# Options that apply to multiple modes
opt = p.add_option_group('MISCELANEOUS OPTIONS')
opt.add_option('--hash', type=str, action='append',
help="use specified hash types in addition to last modification time "
"and size (repeatable, may include `md5`, `sha-1` and `sha-256`)")
opt.add_option('--checksum', action='store_true',
help="use checksum (md5) in addition to last modification time and size")
help="use md5 checksum in addition to last modification time and size "
"(same as --hash=md5)")
opt.add_option('--from', type=str, action='store', dest='from_datetime', metavar="DATETIME",
help="explicit datetime value used to filter updates in change list for "
"--incremental sync")
@ -94,7 +98,9 @@ def main():
verbose=args.verbose)
print("----- ResourceSync Explorer -----")
c = Explorer(checksum=args.checksum,
if (args.checksum):
args.hash.append('md5')
c = Explorer(hashes=args.hashes,
verbose=args.verbose)
try:

2
resync/_version.py
View File

@ -3,4 +3,4 @@
# Format: x.y.z where
# x.y is spec version, see http://www.openarchives.org/rs/x.y/
# z is incremented for revisions within that version, 1...
__version__ = '1.0.5'
__version__ = '1.0.6'

70
resync/client.py
View File

@ -25,7 +25,7 @@ from .sitemap import Sitemap
from .dump import Dump
from .resource import Resource
from .url_authority import UrlAuthority
from .utils import compute_md5_for_file
from .hashes import Hashes
from .client_state import ClientState
from .client_utils import ClientFatalError, url_or_file_open
from .list_base_with_index import ListBaseIndexError
@ -42,9 +42,9 @@ class Client(object):
debug - very verbose for automated analysis
"""
def __init__(self, checksum=False, verbose=False, dryrun=False):
def __init__(self, hashes=None, verbose=False, dryrun=False):
"""Initialize Client object with default parameters."""
self.checksum = checksum
self.hashes = set(hashes) if hashes else set()
self.verbose = verbose
self.dryrun = dryrun
self.logger = logging.getLogger('resync.client')
@ -111,7 +111,7 @@ class Client(object):
# Expect comma separated list of paths
paths = paths.split(',')
# 1. Build from disk
rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper)
rlb = ResourceListBuilder(set_hashes=self.hashes, mapper=self.mapper)
rlb.set_path = set_path
try:
rlb.add_exclude_files(self.exclude_patterns)
@ -165,12 +165,10 @@ class Client(object):
if (len(src_resource_list) == 0):
raise ClientFatalError(
"Aborting as there are no resources to sync")
if (self.checksum and not src_resource_list.has_md5()):
self.checksum = False
self.logger.info(
"Not calculating checksums on destination as not present in source resource list")
if (len(self.hashes) > 0):
self.prune_hashes(src_resource_list.hashes(), 'resource')
# 1.b destination resource list mapped back to source URIs
rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper)
rlb = ResourceListBuilder(set_hashes=self.hashes, mapper=self.mapper)
dst_resource_list = rlb.from_disk()
# 2. Compare these resource lists respecting any comparison options
(same, updated, deleted, created) = dst_resource_list.compare(src_resource_list)
@ -279,10 +277,8 @@ class Client(object):
(len(src_change_list)))
# if (len(src_change_list)==0):
# raise ClientFatalError("Aborting as there are no resources to sync")
if (self.checksum and not src_change_list.has_md5()):
self.checksum = False
self.logger.info(
"Not calculating checksums on destination as not present in source change list")
if (len(self.hashes) > 0):
self.prune_hashes(src_change_list.hashes(), 'change')
# Check all changes have timestamp and record last
self.last_timestamp = 0
for resource in src_change_list:
@ -426,14 +422,40 @@ class Client(object):
self.logger.info(
"Downloaded size for %s of %d bytes does not match expected %d bytes" %
(resource.uri, length, resource.length))
if (self.checksum and resource.md5 is not None):
file_md5 = compute_md5_for_file(filename)
if (resource.md5 != file_md5):
self.logger.info(
"MD5 mismatch for %s, got %s but expected %s bytes" %
(resource.uri, file_md5, resource.md5))
if (len(self.hashes) > 0):
self.check_hashes(filename, resource)
return(num_updated)
def check_hashes(self, filename, resource):
"""Check all hashes present in self.hashes _and_ resource object.
Simply shows warning for mismatch, does not raise exception or
otherwise stop process.
"""
# which hashes to calculate?
hashes = []
if ('md5' in self.hashes and resource.md5 is not None):
hashes.append('md5')
if ('sha-1' in self.hashes and resource.sha1 is not None):
hashes.append('sha-1')
if ('sha-256' in self.hashes and resource.sha256 is not None):
hashes.append('sha-256')
# calculate
hasher = Hashes(hashes, filename)
# check and report
if ('md5' in hashes and resource.md5 != hasher.md5):
self.logger.info(
"MD5 mismatch for %s, got %s but expected %s" %
(resource.uri, hasher.md5, resource.md5))
if ('sha-1' in hashes and resource.sha1 != hasher.sha1):
self.logger.info(
"SHA-1 mismatch for %s, got %s but expected %s" %
(resource.uri, hasher.sha1, resource.sha1))
if ('sha-256' in hashes and resource.sha256 != hasher.sha256):
self.logger.info(
"SHA-256 mismatch for %s, got %s but expected %s" %
(resource.uri, hasher.sha256, resource.sha256))
def delete_resource(self, resource, filename, allow_deletion=False):
"""Delete copy of resource in filename on local system.
@ -645,6 +667,16 @@ class Client(object):
break
return(rl)
def prune_hashes(self, hashes, list_type):
"""Prune any hashes not in source resource or change list."""
discarded = []
for hash in hashes:
if (hash in self.hashes):
self.hashes.discard(hash)
discarded.append(hash)
self.logger.info("Not calculating %s hash(es) on destination as not present "
"in source %s list" % (', '.join(sorted(discarded)), list_type))
def log_status(self, in_sync=True, incremental=False, audit=False,
same=None, created=0, updated=0, deleted=0, to_delete=0):
"""Write log message regarding status in standard form.

104
resync/hashes.py Normal file
View File

@ -0,0 +1,104 @@
"""util.py: A collection of utility functions for source and/or client."""
import base64
import hashlib
class Hashes(object):
"""Compute hash digests for ResourceSync.
These are all base64 encoded according to the rules of
http://www.ietf.org/rfc/rfc4648.txt
MD5
ResourceSync defined to be the same as for Content-MD5 in HTTP,
http://www.ietf.org/rfc/rfc2616.txt which, in turn, defined the
digest string as the "base64 of 128 bit MD5 digest as per RFC 1864"
http://www.ietf.org/rfc/rfc1864.txt
Unfortunately, RFC1864 is rather vague and contains only and example
which doesn't use encoding characters for 62 or 63. It points to
RFC1521 to describe base64 which is explicit that the encoding alphabet
is [A-Za-z0-9+/] with = to pad.
The above corresponds with the alphabet of "3. Base 64 Encoding" in RFC3548
http://www.ietf.org/rfc/rfc3548.txt
and not the url safe version, "Base 64 Encoding with URL and Filename Safe
Alphabet" which replaces + and / with - and _ respectively.
This is the same as the alphabet of "4. Base 64 Encoding" in RFC4648
http://www.ietf.org/rfc/rfc4648.txt.
This algorithm is implemented by base64.standard_b64encode() or
base64.b64encode() with no altchars specified. Available in python2.4 and
up [http://docs.python.org/library/base64.html]
"""
def __init__(self, hashes=None, file=None):
"""Initialize Hasher object with types of hash to caluclate.
If file is supplied then compute for that file.
"""
self.hashes = set()
for hash in hashes:
if (hash not in ['md5', 'sha-1', 'sha-256']):
raise Exception("Hash type %s not supported" % (hash))
self.hashes.add(hash)
#
self.md5_calc = None
self.sha1_calc = None
self.sha256_calc = None
#
if (file is not None):
self.compute_for_file(file)
def initialize_hashes(self):
"""Create new hashlib objects for each hash we are going to calculate."""
if ('md5' in self.hashes):
self.md5_calc = hashlib.md5()
if ('sha-1' in self.hashes):
self.sha1_calc = hashlib.sha1()
if ('sha-256' in self.hashes):
self.sha256_calc = hashlib.sha256()
def compute_for_file(self, file, block_size=2**14):
"""Compute hash digests for a file.
Calculate the hashes based on one read through the file.
Optional block_size parameter controls memory used to do
calculations. This should be a multiple of 128 bytes.
"""
self.initialize_hashes()
f = open(file, 'rb')
while True:
data = f.read(block_size)
if not data:
break
if (self.md5_calc is not None):
self.md5_calc.update(data)
if (self.sha1_calc is not None):
self.sha1_calc.update(data)
if (self.sha256_calc is not None):
self.sha256_calc.update(data)
f.close()
@property
def md5(self):
"""Return MD5 hash calculated."""
if (self.md5_calc is None):
return None
return self.md5_calc.hexdigest()
@property
def sha1(self):
"""Return SHA-1 hash calculated."""
if (self.sha1_calc is None):
return None
return self.sha1_calc.hexdigest()
@property
def sha256(self):
"""Return SHA-256 hash calculated."""
if (self.sha256_calc is None):
return None
return self.sha256_calc.hexdigest()

4
resync/list_base_with_index.py
View File

@ -21,7 +21,7 @@ from .resource import Resource
from .sitemap import Sitemap
from .mapper import Mapper, MapperError
from .url_authority import UrlAuthority
from .utils import compute_md5_for_file
from .hashes import Hashes
class ListBaseWithIndex(ListBase):
@ -326,7 +326,7 @@ class ListBaseWithIndex(ListBase):
# Record information about this sitemap for index
r = Resource(uri=uri,
timestamp=os.stat(file).st_mtime,
md5=compute_md5_for_file(file))
md5=Hashes(['md5'], file).md5)
index.add(r)
# Get next chunk
(chunk, nxt) = self.get_resources_chunk(resources_iter, nxt)

20
resync/resource_list.py
View File

@ -209,11 +209,15 @@ class ResourceList(ListBaseWithIndex):
# have now gone through both lists
return(same, updated, deleted, created)
def has_md5(self):
"""Return true if at least one contained resource-like object has md5 data."""
if (self.resources is None):
return(False)
for resource in self:
if (resource.md5 is not None):
return(True)
return(False)
def hashes(self):
"""Return set of hashes uses in this resource_list."""
hashes = set()
if (self.resources is not None):
for resource in self:
if (resource.md5 is not None):
hashes.add('md5')
if (resource.sha1 is not None):
hashes.add('sha-1')
if (resource.sha256 is not None):
hashes.add('sha-256')
return(hashes)

25
resync/resource_list_builder.py
View File

@ -12,10 +12,10 @@ except ImportError: # python2
from urllib import URLopener
from defusedxml.ElementTree import parse
from .hashes import Hashes
from .resource import Resource
from .resource_list import ResourceList
from .sitemap import Sitemap
from .utils import compute_md5_for_file
from .w3c_datetime import datetime_to_str
@ -25,8 +25,8 @@ class ResourceListBuilder():
Currently implements build from files on disk only.
Attributes:
- set_hashes to indicate which hashes should be calculated for each resource
- set_path set true to add path attribute for each resource
- set_md5 set true to calculate MD5 sums for all files
- set_length set true to include file length in resource_list (defaults true)
- exclude_dirs is a list of directory names to exclude
(defaults to ['CVS','.git'))
@ -35,7 +35,7 @@ class ResourceListBuilder():
alternatives to md5.
"""
def __init__(self, mapper=None, set_md5=False,
def __init__(self, mapper=None, set_hashes=None,
set_length=True, set_path=False):
"""Create ResourceListBuilder object, optionally set options.
@ -44,15 +44,15 @@ class ResourceListBuilder():
The following attributes may be set to determine information added to
each Resource object based on the disk scan:
- set_md5 - True to add md5 digests for each resource. This may add
- set_hashes - Set of digests to computer for each resource. This may add
significant time to the scan process as each file has to be read to
compute the sum
compute the hash. Empty set or None means no hashes calculated
- set_length - False to not add length for each resources
- set_path - True to add local path information for each file/resource
"""
self.mapper = mapper
self.set_path = set_path
self.set_md5 = set_md5
self.set_hashes = set_hashes if (set_hashes and len(set_hashes) > 0) else None
self.set_length = set_length
self.exclude_files = ['sitemap\d{0,5}.xml']
self.exclude_dirs = ['CVS', '.git']
@ -162,7 +162,7 @@ class ResourceListBuilder():
def add_file(self, resource_list=None, dir=None, file=None):
"""Add a single file to resource_list.
Follows object settings of set_path, set_md5 and set_length.
Follows object settings of set_path, set_hashes and set_length.
"""
try:
if self.exclude_file(file):
@ -186,9 +186,14 @@ class ResourceListBuilder():
if (self.set_path):
# add full local path
r.path = file
if (self.set_md5):
# add md5
r.md5 = compute_md5_for_file(file)
if (self.set_hashes):
hasher = Hashes(self.set_hashes, file)
if ('md5' in self.set_hashes):
r.md5 = hasher.md5
if ('sha-1' in self.set_hashes):
r.sha1 = hasher.sha1
if ('sha-256' in self.set_hashes):
r.sha256 = hasher.sha256
if (self.set_length):
# add length
r.length = file_stat.st_size

61
resync/utils.py
View File

@ -1,61 +0,0 @@
"""util.py: A collection of utility functions for source and/or client."""
import base64
import hashlib
"""Compute digests for ResourceSync.
These are all base64 encoded according to the rules of
http://www.ietf.org/rfc/rfc4648.txt
MD5
ResourceSync defined to be the same as for Content-MD5 in HTTP,
http://www.ietf.org/rfc/rfc2616.txt which, in turn, defined the
digest string as the "base64 of 128 bit MD5 digest as per RFC 1864"
http://www.ietf.org/rfc/rfc1864.txt
Unfortunately, RFC1864 is rather vague and contains only and example
which doesn't use encoding characters for 62 or 63. It points to
RFC1521 to describe base64 which is explicit that the encoding alphabet
is [A-Za-z0-9+/] with = to pad.
The above corresponds with the alphabet of "3. Base 64 Encoding" in RFC3548
http://www.ietf.org/rfc/rfc3548.txt
and not the url safe version, "Base 64 Encoding with URL and Filename Safe
Alphabet" which replaces + and / with - and _ respectively.
This is the same as the alphabet of "4. Base 64 Encoding" in RFC4648
http://www.ietf.org/rfc/rfc4648.txt.
This algorithm is implemented by base64.standard_b64encode() or
base64.b64encode() with no altchars specified. Available in python2.4 and
up [http://docs.python.org/library/base64.html]
"""
def compute_md5_for_string(str):
"""Compute MD5 digest over some string or byte payload.
Returns a string containing the digest.
"""
if (not isinstance(str, bytes)):
str = str.encode('utf-8') # make bytes
return base64.b64encode(hashlib.md5(str).digest()).decode('utf-8')
def compute_md5_for_file(file, block_size=2**14):
"""Compute MD5 digest for a file.
Returns a string containing the digest. Optional block_size parameter
controls memory used to do MD5 calculation. This should be a multiple
of 128 bytes.
"""
f = open(file, 'rb')
md5 = hashlib.md5()
while True:
data = f.read(block_size)
if not data:
break
md5.update(data)
f.close()
return base64.b64encode(md5.digest()).decode('utf-8')

2
tests/test_client.py
View File

@ -99,7 +99,7 @@ class TestClient(TestCase):
c.last_timestamp = 0
with LogCapture() as lc:
c.logger = logging.getLogger('resync.client')
c.checksum = True
c.hashes = set(['md5'])
n = c.update_resource(resource, filename)
self.assertEqual(n, 1)
self.assertTrue(lc.records[-1].msg.startswith('MD5 mismatch '))

23
tests/test_hashes.py Normal file
View File

@ -0,0 +1,23 @@
import unittest
import resync.hashes
class TestUtill(unittest.TestCase):
def test01_md5_sha1_file(self):
h = resync.hashes.Hashes(['md5', 'sha-1'])
h.compute_for_file('tests/testdata/a')
self.assertEqual(h.md5, '8fdd769621e003fe3c0c21e9929b491e')
self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33')
self.assertEqual(h.sha256, None)
h = resync.hashes.Hashes(['sha-256', 'sha-1'])
h.compute_for_file('tests/testdata/a')
self.assertEqual(h.md5, None)
self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33')
self.assertEqual(h.sha256, '69fe6314a94800456af959d380f5d6932052478ea03d5ccac7ba0a14bd5e67c6')
def test02_bad_type(self):
self.assertRaises(Exception, resync.hashes.Hashes, ['md5', 'xyz'])
if __name__ == '__main__':
unittest.main()

10
tests/test_resource_list.py
View File

@ -109,16 +109,18 @@ class TestResourceList(unittest.TestCase):
self.assertEqual(len(i), 2)
self.assertEqual(i.resources['a'].length, 10)
def test07_has_md5(self):
def test07_hashes(self):
r1 = Resource(uri='a')
r2 = Resource(uri='b')
i = ResourceList()
self.assertFalse(i.has_md5())
self.assertEqual(i.hashes(), set())
i.add(r1)
i.add(r2)
self.assertFalse(i.has_md5())
self.assertEqual(i.hashes(), set())
r1.md5 = "aabbcc"
self.assertTrue(i.has_md5())
self.assertEqual(i.hashes(), set(['md5']))
r2.sha1 = "ddeeff"
self.assertEqual(i.hashes(), set(['md5', 'sha-1']))
def test08_iter(self):
i = ResourceList()

12
tests/test_resource_list_builder.py
View File

@ -58,8 +58,8 @@ class TestResourceListBuilder(unittest.TestCase):
self.assertEqual(r.length, None)
self.assertEqual(r.path, None)
def test03_set_md5(self):
rlb = ResourceListBuilder(set_md5=True)
def test03_set_hashes(self):
rlb = ResourceListBuilder(set_hashes=['md5'])
rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1'])
rl = rlb.from_disk()
self.assertEqual(len(rl), 2)
@ -67,18 +67,18 @@ class TestResourceListBuilder(unittest.TestCase):
r = next(rli)
self.assertEqual(r.uri, 'http://example.org/t/file_a')
self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z')
self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==')
self.assertEqual(r.md5, '6bf26fd66601b528d2e0b47eaa87edfd')
self.assertEqual(r.length, 20)
self.assertEqual(r.path, None)
r = next(rli)
self.assertEqual(r.uri, 'http://example.org/t/file_b')
self.assertEqual(r.lastmod, '2001-09-09T01:46:40Z')
self.assertEqual(r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==')
self.assertEqual(r.md5, '452e54bdae1626ac5d6e7be81b39de21')
self.assertEqual(r.length, 45)
self.assertEqual(r.path, None)
def test04_data(self):
rlb = ResourceListBuilder(set_path=True, set_md5=True)
rlb = ResourceListBuilder(set_path=True, set_hashes=['md5'])
rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1'])
rl = rlb.from_disk()
self.assertEqual(len(rl), 2)
@ -86,7 +86,7 @@ class TestResourceListBuilder(unittest.TestCase):
self.assertTrue(r is not None)
self.assertEqual(r.uri, 'http://example.org/t/file_a')
self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z')
self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==')
self.assertEqual(r.md5, '6bf26fd66601b528d2e0b47eaa87edfd')
self.assertEqual(r.path, 'tests/testdata/dir1/file_a')
def test05_from_disk_paths(self):

17
tests/test_utils.py
View File

@ -1,17 +0,0 @@
import unittest
import resync.utils
class TestUtill(unittest.TestCase):
def test1_string(self):
self.assertEqual(resync.utils.compute_md5_for_string('A file\n'),
'j912liHgA/48DCHpkptJHg==')
def test2_file(self):
# Should be same as the string above
self.assertEqual(resync.utils.compute_md5_for_file('tests/testdata/a'),
'j912liHgA/48DCHpkptJHg==')
if __name__ == '__main__':
unittest.main()