From 7caf1bb396013e96ef6ba316ef18ee777bf08d8d Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Mon, 30 Jan 2017 13:11:19 -0500 Subject: [PATCH 1/5] Start path to 1.0.6 --- CHANGES.md | 3 +++ README | 4 ++-- resync/_version.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d6300a5..d6ac45e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ core specification version. Versions 1.0.x implement the v1.0 ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014 . +v1.0.6 2017-??-?? + * ... + v1.0.5 2017-01-30 * Fix to support non-ASCII URIs/filenames (https://github.com/resync/resync/issues/22) * Added tests for Python 3.6 diff --git a/README b/README index 2db49d2..fed5179 100644 --- a/README +++ b/README @@ -59,7 +59,7 @@ Typical library use in a destination (get and examine a Capability List):: Installation ------------ -The client and library are designed to work with Python 2.6 or 2.7. +The client and library are designed to work with Python 2.6, 2.7, 3.4, 3.5 and 3.6. **Automatic installation**:: @@ -106,7 +106,7 @@ See also CONTRIBUTING.md Copyright and License --------------------- -Copyright 2012--2016 Simeon Warner +Copyright 2012--2017 Simeon Warner Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/resync/_version.py b/resync/_version.py index 39e6eb2..b739a7b 100644 --- a/resync/_version.py +++ b/resync/_version.py @@ -3,4 +3,4 @@ # Format: x.y.z where # x.y is spec version, see http://www.openarchives.org/rs/x.y/ # z is incremented for revisions within that version, 1... -__version__ = '1.0.5' +__version__ = '1.0.6' From 39334dd9c244b19a9d48441a393f96a091d83f67 Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Wed, 22 Feb 2017 10:54:40 -0500 Subject: [PATCH 2/5] Figure 4 was fixed in 1.1: http://www.openarchives.org/rs/1.1/resourcesync#fig_framework_structure --- TODO.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/TODO.md b/TODO.md index 4ce7ec4..558ad81 100644 --- a/TODO.md +++ b/TODO.md @@ -12,8 +12,3 @@ Planned changes and deprecation ------------------------------- * Remove support for ListBase.parse(str=...) which was deprecated in favor of ListBase.parse(str_data=...) 2016-03-22 in 1.0.1 - -Issues with specification -========================= - - * v1.0 spec Figure 4 shows Capability List indexes but these are not allowed per section 9 From 6ddcd168146b5d5a601abc3e44f8c5e19d932e5e Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Mon, 20 Mar 2017 15:19:54 -0400 Subject: [PATCH 3/5] Support multiple hash types --- bin/resync | 13 ++-- bin/resync-explorer | 10 ++- resync/client.py | 70 ++++++++++++++----- resync/hashes.py | 103 ++++++++++++++++++++++++++++ resync/list_base_with_index.py | 4 +- resync/resource_list.py | 20 +++--- resync/resource_list_builder.py | 25 ++++--- resync/utils.py | 61 ---------------- tests/test_client.py | 2 +- tests/test_hashes.py | 23 +++++++ tests/test_resource_list.py | 10 +-- tests/test_resource_list_builder.py | 6 +- tests/test_utils.py | 17 ----- 13 files changed, 233 insertions(+), 131 deletions(-) create mode 100644 resync/hashes.py delete mode 100644 resync/utils.py create mode 100644 tests/test_hashes.py delete mode 100644 tests/test_utils.py diff --git a/bin/resync b/bin/resync index a65fa82..9be3d2d 100755 --- a/bin/resync +++ b/bin/resync @@ -120,8 +120,12 @@ def main(): # Options that apply to multiple modes opt = p.add_option_group('MISCELANEOUS OPTIONS') + opt.add_option('--hash', type=str, action='append', + help="use specified hash types in addition to last modification time " + "and size (repeatable, may include `md5`, `sha-1` and `sha-256`)") opt.add_option('--checksum', action='store_true', - help="use checksum (md5) in addition to last modification time and size") + help="use md5 checksum in addition to last modification time and size " + "(same as --hash=md5)") opt.add_option('--delete', action='store_true', help="allow files on destination to be deleted") opt.add_option('--from', type=str, action='store', dest='from_datetime', metavar="DATETIME", @@ -176,8 +180,7 @@ def main(): not args.capabilitylist and not args.sourcedescription and not args.resourcedump and not args.changedump): if (len(map) == 0): - # No args at all, show help - p.print_help() + p.error("No arguments specified (use -h for help)") return else: args.baseline = True @@ -190,7 +193,9 @@ def main(): init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE, verbose=args.verbose, eval_mode=args.eval) - c = Client(checksum=args.checksum, + if (args.checksum): + args.hash.append('md5') + c = Client(hashes=args.hash, verbose=args.verbose, dryrun=args.dryrun) diff --git a/bin/resync-explorer b/bin/resync-explorer index d604ef1..50a7247 100755 --- a/bin/resync-explorer +++ b/bin/resync-explorer @@ -61,8 +61,12 @@ def main(): # Options that apply to multiple modes opt = p.add_option_group('MISCELANEOUS OPTIONS') + opt.add_option('--hash', type=str, action='append', + help="use specified hash types in addition to last modification time " + "and size (repeatable, may include `md5`, `sha-1` and `sha-256`)") opt.add_option('--checksum', action='store_true', - help="use checksum (md5) in addition to last modification time and size") + help="use md5 checksum in addition to last modification time and size " + "(same as --hash=md5)") opt.add_option('--from', type=str, action='store', dest='from_datetime', metavar="DATETIME", help="explicit datetime value used to filter updates in change list for " "--incremental sync") @@ -94,7 +98,9 @@ def main(): verbose=args.verbose) print("----- ResourceSync Explorer -----") - c = Explorer(checksum=args.checksum, + if (args.checksum): + args.hash.append('md5') + c = Explorer(hashes=args.hashes, verbose=args.verbose) try: diff --git a/resync/client.py b/resync/client.py index efe05fa..cc779a8 100644 --- a/resync/client.py +++ b/resync/client.py @@ -25,7 +25,7 @@ from .sitemap import Sitemap from .dump import Dump from .resource import Resource from .url_authority import UrlAuthority -from .utils import compute_md5_for_file +from .hashes import Hashes from .client_state import ClientState from .client_utils import ClientFatalError, url_or_file_open from .list_base_with_index import ListBaseIndexError @@ -42,9 +42,9 @@ class Client(object): debug - very verbose for automated analysis """ - def __init__(self, checksum=False, verbose=False, dryrun=False): + def __init__(self, hashes=None, verbose=False, dryrun=False): """Initialize Client object with default parameters.""" - self.checksum = checksum + self.hashes = set(hashes) if hashes else set() self.verbose = verbose self.dryrun = dryrun self.logger = logging.getLogger('resync.client') @@ -111,7 +111,7 @@ class Client(object): # Expect comma separated list of paths paths = paths.split(',') # 1. Build from disk - rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper) + rlb = ResourceListBuilder(set_hashes=self.hashes, mapper=self.mapper) rlb.set_path = set_path try: rlb.add_exclude_files(self.exclude_patterns) @@ -165,12 +165,10 @@ class Client(object): if (len(src_resource_list) == 0): raise ClientFatalError( "Aborting as there are no resources to sync") - if (self.checksum and not src_resource_list.has_md5()): - self.checksum = False - self.logger.info( - "Not calculating checksums on destination as not present in source resource list") + if (len(self.hashes) > 0): + self.prune_hashes(src_resource_list.hashes(), 'resource') # 1.b destination resource list mapped back to source URIs - rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper) + rlb = ResourceListBuilder(set_hashes=self.hashes, mapper=self.mapper) dst_resource_list = rlb.from_disk() # 2. Compare these resource lists respecting any comparison options (same, updated, deleted, created) = dst_resource_list.compare(src_resource_list) @@ -279,10 +277,8 @@ class Client(object): (len(src_change_list))) # if (len(src_change_list)==0): # raise ClientFatalError("Aborting as there are no resources to sync") - if (self.checksum and not src_change_list.has_md5()): - self.checksum = False - self.logger.info( - "Not calculating checksums on destination as not present in source change list") + if (len(self.hashes) > 0): + self.prune_hashes(src_change_list.hashes(), 'change') # Check all changes have timestamp and record last self.last_timestamp = 0 for resource in src_change_list: @@ -426,14 +422,40 @@ class Client(object): self.logger.info( "Downloaded size for %s of %d bytes does not match expected %d bytes" % (resource.uri, length, resource.length)) - if (self.checksum and resource.md5 is not None): - file_md5 = compute_md5_for_file(filename) - if (resource.md5 != file_md5): - self.logger.info( - "MD5 mismatch for %s, got %s but expected %s bytes" % - (resource.uri, file_md5, resource.md5)) + if (len(self.hashes) > 0): + self.check_hashes(filename, resource) return(num_updated) + def check_hashes(self, filename, resource): + """Check all hashes present in self.hashes _and_ resource object. + + Simply shows warning for mismatch, does not raise exception or + otherwise stop process. + """ + # which hashes to calculate? + hashes = [] + if ('md5' in self.hashes and resource.md5 is not None): + hashes.append('md5') + if ('sha-1' in self.hashes and resource.sha1 is not None): + hashes.append('sha-1') + if ('sha-256' in self.hashes and resource.sha256 is not None): + hashes.append('sha-256') + # calculate + hasher = Hashes(hashes, filename) + # check and report + if ('md5' in hashes and resource.md5 != hasher.md5): + self.logger.info( + "MD5 mismatch for %s, got %s but expected %s" % + (resource.uri, hasher.md5, resource.md5)) + if ('sha-1' in hashes and resource.sha1 != hasher.sha1): + self.logger.info( + "SHA-1 mismatch for %s, got %s but expected %s" % + (resource.uri, hasher.sha1, resource.sha1)) + if ('sha-256' in hashes and resource.sha256 != hasher.sha256): + self.logger.info( + "SHA-256 mismatch for %s, got %s but expected %s" % + (resource.uri, hasher.sha256, resource.sha256)) + def delete_resource(self, resource, filename, allow_deletion=False): """Delete copy of resource in filename on local system. @@ -645,6 +667,16 @@ class Client(object): break return(rl) + def prune_hashes(self, hashes, list_type): + """Prune any hashes not in source resource or change list.""" + discarded = [] + for hash in hashes: + if (hash in self.hashes): + self.hashes.discard(hash) + discarded.append(hash) + self.logger.info("Not calculating %s hash(es) on destination as not present " + "in source %s list" % (', '.join(sorted(discarded)), list_type)) + def log_status(self, in_sync=True, incremental=False, audit=False, same=None, created=0, updated=0, deleted=0, to_delete=0): """Write log message regarding status in standard form. diff --git a/resync/hashes.py b/resync/hashes.py new file mode 100644 index 0000000..28615b8 --- /dev/null +++ b/resync/hashes.py @@ -0,0 +1,103 @@ +"""util.py: A collection of utility functions for source and/or client.""" +import base64 +import hashlib + +class Hashes(object): + """Compute hash digests for ResourceSync. + + These are all base64 encoded according to the rules of + http://www.ietf.org/rfc/rfc4648.txt + + MD5 + + ResourceSync defined to be the same as for Content-MD5 in HTTP, + http://www.ietf.org/rfc/rfc2616.txt which, in turn, defined the + digest string as the "base64 of 128 bit MD5 digest as per RFC 1864" + http://www.ietf.org/rfc/rfc1864.txt + + Unfortunately, RFC1864 is rather vague and contains only and example + which doesn't use encoding characters for 62 or 63. It points to + RFC1521 to describe base64 which is explicit that the encoding alphabet + is [A-Za-z0-9+/] with = to pad. + + The above corresponds with the alphabet of "3. Base 64 Encoding" in RFC3548 + http://www.ietf.org/rfc/rfc3548.txt + and not the url safe version, "Base 64 Encoding with URL and Filename Safe + Alphabet" which replaces + and / with - and _ respectively. + + This is the same as the alphabet of "4. Base 64 Encoding" in RFC4648 + http://www.ietf.org/rfc/rfc4648.txt. + + This algorithm is implemented by base64.standard_b64encode() or + base64.b64encode() with no altchars specified. Available in python2.4 and + up [http://docs.python.org/library/base64.html] + """ + + def __init__(self, hashes=None, file=None): + """Initialize Hasher object with types of hash to caluclate. + + If file is supplied then compute for that file. + """ + self.hashes = set() + for hash in hashes: + if (hash not in ['md5', 'sha-1', 'sha-256']): + raise Exception("Hash type %s not supported" % (hash)) + self.hashes.add(hash) + # + self.md5_calc = None + self.sha1_calc = None + self.sha256_calc = None + # + if (file is not None): + self.compute_for_file(file) + + def initialize_hashes(self): + """Create new hashlib objects for each hash we are going to calculate.""" + if ('md5' in self.hashes): + self.md5_calc = hashlib.md5() + if ('sha-1' in self.hashes): + self.sha1_calc = hashlib.sha1() + if ('sha-256' in self.hashes): + self.sha256_calc = hashlib.sha256() + + def compute_for_file(self, file, block_size=2**14): + """Compute hash digests for a file. + + Calculate the hashes based on one read through the file. + Optional block_size parameter controls memory used to do + calculations. This should be a multiple of 128 bytes. + """ + self.initialize_hashes() + f = open(file, 'rb') + while True: + data = f.read(block_size) + if not data: + break + if (self.md5_calc is not None): + self.md5_calc.update(data) + if (self.sha1_calc is not None): + self.sha1_calc.update(data) + if (self.sha256_calc is not None): + self.sha256_calc.update(data) + f.close() + + @property + def md5(self): + """Return MD5 hash calculated.""" + if (self.md5_calc is None): + return None + return base64.b64encode(self.md5_calc.digest()).decode('utf-8') + + @property + def sha1(self): + """Return SHA-1 hash calculated.""" + if (self.sha1_calc is None): + return None + return self.sha1_calc.hexdigest() + + @property + def sha256(self): + """Return SHA-256 hash calculated.""" + if (self.sha256_calc is None): + return None + return self.sha256_calc.hexdigest() diff --git a/resync/list_base_with_index.py b/resync/list_base_with_index.py index c88e29a..dd05c97 100644 --- a/resync/list_base_with_index.py +++ b/resync/list_base_with_index.py @@ -21,7 +21,7 @@ from .resource import Resource from .sitemap import Sitemap from .mapper import Mapper, MapperError from .url_authority import UrlAuthority -from .utils import compute_md5_for_file +from .hashes import Hashes class ListBaseWithIndex(ListBase): @@ -326,7 +326,7 @@ class ListBaseWithIndex(ListBase): # Record information about this sitemap for index r = Resource(uri=uri, timestamp=os.stat(file).st_mtime, - md5=compute_md5_for_file(file)) + md5=Hashes(['md5'], file).md5) index.add(r) # Get next chunk (chunk, nxt) = self.get_resources_chunk(resources_iter, nxt) diff --git a/resync/resource_list.py b/resync/resource_list.py index 4f9599c..a35818c 100644 --- a/resync/resource_list.py +++ b/resync/resource_list.py @@ -209,11 +209,15 @@ class ResourceList(ListBaseWithIndex): # have now gone through both lists return(same, updated, deleted, created) - def has_md5(self): - """Return true if at least one contained resource-like object has md5 data.""" - if (self.resources is None): - return(False) - for resource in self: - if (resource.md5 is not None): - return(True) - return(False) + def hashes(self): + """Return set of hashes uses in this resource_list.""" + hashes = set() + if (self.resources is not None): + for resource in self: + if (resource.md5 is not None): + hashes.add('md5') + if (resource.sha1 is not None): + hashes.add('sha-1') + if (resource.sha256 is not None): + hashes.add('sha-256') + return(hashes) diff --git a/resync/resource_list_builder.py b/resync/resource_list_builder.py index f19fa6b..aa81bbc 100644 --- a/resync/resource_list_builder.py +++ b/resync/resource_list_builder.py @@ -12,10 +12,10 @@ except ImportError: # python2 from urllib import URLopener from defusedxml.ElementTree import parse +from .hashes import Hashes from .resource import Resource from .resource_list import ResourceList from .sitemap import Sitemap -from .utils import compute_md5_for_file from .w3c_datetime import datetime_to_str @@ -25,8 +25,8 @@ class ResourceListBuilder(): Currently implements build from files on disk only. Attributes: + - set_hashes to indicate which hashes should be calculated for each resource - set_path set true to add path attribute for each resource - - set_md5 set true to calculate MD5 sums for all files - set_length set true to include file length in resource_list (defaults true) - exclude_dirs is a list of directory names to exclude (defaults to ['CVS','.git')) @@ -35,7 +35,7 @@ class ResourceListBuilder(): alternatives to md5. """ - def __init__(self, mapper=None, set_md5=False, + def __init__(self, mapper=None, set_hashes=None, set_length=True, set_path=False): """Create ResourceListBuilder object, optionally set options. @@ -44,15 +44,15 @@ class ResourceListBuilder(): The following attributes may be set to determine information added to each Resource object based on the disk scan: - - set_md5 - True to add md5 digests for each resource. This may add + - set_hashes - Set of digests to computer for each resource. This may add significant time to the scan process as each file has to be read to - compute the sum + compute the hash. Empty set or None means no hashes calculated - set_length - False to not add length for each resources - set_path - True to add local path information for each file/resource """ self.mapper = mapper self.set_path = set_path - self.set_md5 = set_md5 + self.set_hashes = set_hashes if (set_hashes and len(set_hashes) > 0) else None self.set_length = set_length self.exclude_files = ['sitemap\d{0,5}.xml'] self.exclude_dirs = ['CVS', '.git'] @@ -162,7 +162,7 @@ class ResourceListBuilder(): def add_file(self, resource_list=None, dir=None, file=None): """Add a single file to resource_list. - Follows object settings of set_path, set_md5 and set_length. + Follows object settings of set_path, set_hashes and set_length. """ try: if self.exclude_file(file): @@ -186,9 +186,14 @@ class ResourceListBuilder(): if (self.set_path): # add full local path r.path = file - if (self.set_md5): - # add md5 - r.md5 = compute_md5_for_file(file) + if (self.set_hashes): + hasher = Hashes(self.set_hashes, file) + if ('md5' in self.set_hashes): + r.md5 = hasher.md5 + if ('sha-1' in self.set_hashes): + r.sha1 = hasher.sha1 + if ('sha-256' in self.set_hashes): + r.sha256 = hasher.sha256 if (self.set_length): # add length r.length = file_stat.st_size diff --git a/resync/utils.py b/resync/utils.py deleted file mode 100644 index 17d737a..0000000 --- a/resync/utils.py +++ /dev/null @@ -1,61 +0,0 @@ -"""util.py: A collection of utility functions for source and/or client.""" -import base64 -import hashlib - -"""Compute digests for ResourceSync. - -These are all base64 encoded according to the rules of -http://www.ietf.org/rfc/rfc4648.txt - -MD5 - -ResourceSync defined to be the same as for Content-MD5 in HTTP, -http://www.ietf.org/rfc/rfc2616.txt which, in turn, defined the -digest string as the "base64 of 128 bit MD5 digest as per RFC 1864" -http://www.ietf.org/rfc/rfc1864.txt - -Unfortunately, RFC1864 is rather vague and contains only and example -which doesn't use encoding characters for 62 or 63. It points to -RFC1521 to describe base64 which is explicit that the encoding alphabet -is [A-Za-z0-9+/] with = to pad. - -The above corresponds with the alphabet of "3. Base 64 Encoding" in RFC3548 -http://www.ietf.org/rfc/rfc3548.txt -and not the url safe version, "Base 64 Encoding with URL and Filename Safe -Alphabet" which replaces + and / with - and _ respectively. - -This is the same as the alphabet of "4. Base 64 Encoding" in RFC4648 -http://www.ietf.org/rfc/rfc4648.txt. - -This algorithm is implemented by base64.standard_b64encode() or -base64.b64encode() with no altchars specified. Available in python2.4 and -up [http://docs.python.org/library/base64.html] -""" - - -def compute_md5_for_string(str): - """Compute MD5 digest over some string or byte payload. - - Returns a string containing the digest. - """ - if (not isinstance(str, bytes)): - str = str.encode('utf-8') # make bytes - return base64.b64encode(hashlib.md5(str).digest()).decode('utf-8') - - -def compute_md5_for_file(file, block_size=2**14): - """Compute MD5 digest for a file. - - Returns a string containing the digest. Optional block_size parameter - controls memory used to do MD5 calculation. This should be a multiple - of 128 bytes. - """ - f = open(file, 'rb') - md5 = hashlib.md5() - while True: - data = f.read(block_size) - if not data: - break - md5.update(data) - f.close() - return base64.b64encode(md5.digest()).decode('utf-8') diff --git a/tests/test_client.py b/tests/test_client.py index 04f40d4..473640d 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -99,7 +99,7 @@ class TestClient(TestCase): c.last_timestamp = 0 with LogCapture() as lc: c.logger = logging.getLogger('resync.client') - c.checksum = True + c.hashes = set(['md5']) n = c.update_resource(resource, filename) self.assertEqual(n, 1) self.assertTrue(lc.records[-1].msg.startswith('MD5 mismatch ')) diff --git a/tests/test_hashes.py b/tests/test_hashes.py new file mode 100644 index 0000000..75830a5 --- /dev/null +++ b/tests/test_hashes.py @@ -0,0 +1,23 @@ +import unittest +import resync.hashes + + +class TestUtill(unittest.TestCase): + + def test01_md5_sha1_file(self): + h = resync.hashes.Hashes(['md5','sha-1']) + h.compute_for_file('tests/testdata/a') + self.assertEqual(h.md5, 'j912liHgA/48DCHpkptJHg==') + self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33') + self.assertEqual(h.sha256, None) + h = resync.hashes.Hashes(['sha-256','sha-1']) + h.compute_for_file('tests/testdata/a') + self.assertEqual(h.md5, None) + self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33') + self.assertEqual(h.sha256, '69fe6314a94800456af959d380f5d6932052478ea03d5ccac7ba0a14bd5e67c6') + + def test02_bad_type(self): + self.assertRaises(Exception, resync.hashes.Hashes, ['md5','xyz']) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_resource_list.py b/tests/test_resource_list.py index ae954dc..5bae589 100644 --- a/tests/test_resource_list.py +++ b/tests/test_resource_list.py @@ -109,16 +109,18 @@ class TestResourceList(unittest.TestCase): self.assertEqual(len(i), 2) self.assertEqual(i.resources['a'].length, 10) - def test07_has_md5(self): + def test07_hashes(self): r1 = Resource(uri='a') r2 = Resource(uri='b') i = ResourceList() - self.assertFalse(i.has_md5()) + self.assertEqual(i.hashes(), set()) i.add(r1) i.add(r2) - self.assertFalse(i.has_md5()) + self.assertEqual(i.hashes(), set()) r1.md5 = "aabbcc" - self.assertTrue(i.has_md5()) + self.assertEqual(i.hashes(), set(['md5'])) + r2.sha1 = "ddeeff" + self.assertEqual(i.hashes(), set(['md5','sha-1'])) def test08_iter(self): i = ResourceList() diff --git a/tests/test_resource_list_builder.py b/tests/test_resource_list_builder.py index 72a9a95..5664cba 100644 --- a/tests/test_resource_list_builder.py +++ b/tests/test_resource_list_builder.py @@ -58,8 +58,8 @@ class TestResourceListBuilder(unittest.TestCase): self.assertEqual(r.length, None) self.assertEqual(r.path, None) - def test03_set_md5(self): - rlb = ResourceListBuilder(set_md5=True) + def test03_set_hashes(self): + rlb = ResourceListBuilder(set_hashes=['md5']) rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() self.assertEqual(len(rl), 2) @@ -78,7 +78,7 @@ class TestResourceListBuilder(unittest.TestCase): self.assertEqual(r.path, None) def test04_data(self): - rlb = ResourceListBuilder(set_path=True, set_md5=True) + rlb = ResourceListBuilder(set_path=True, set_hashes=['md5']) rlb.mapper = Mapper(['http://example.org/t', 'tests/testdata/dir1']) rl = rlb.from_disk() self.assertEqual(len(rl), 2) diff --git a/tests/test_utils.py b/tests/test_utils.py deleted file mode 100644 index de775c9..0000000 --- a/tests/test_utils.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest -import resync.utils - - -class TestUtill(unittest.TestCase): - - def test1_string(self): - self.assertEqual(resync.utils.compute_md5_for_string('A file\n'), - 'j912liHgA/48DCHpkptJHg==') - - def test2_file(self): - # Should be same as the string above - self.assertEqual(resync.utils.compute_md5_for_file('tests/testdata/a'), - 'j912liHgA/48DCHpkptJHg==') - -if __name__ == '__main__': - unittest.main() From e0b9e24b6bba1264ef01ee24cfc49a883d0e78be Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Mon, 20 Mar 2017 15:25:20 -0400 Subject: [PATCH 4/5] pep8 fixes --- resync/hashes.py | 3 ++- tests/test_hashes.py | 6 +++--- tests/test_resource_list.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/resync/hashes.py b/resync/hashes.py index 28615b8..47aa8e8 100644 --- a/resync/hashes.py +++ b/resync/hashes.py @@ -2,6 +2,7 @@ import base64 import hashlib + class Hashes(object): """Compute hash digests for ResourceSync. @@ -64,7 +65,7 @@ class Hashes(object): """Compute hash digests for a file. Calculate the hashes based on one read through the file. - Optional block_size parameter controls memory used to do + Optional block_size parameter controls memory used to do calculations. This should be a multiple of 128 bytes. """ self.initialize_hashes() diff --git a/tests/test_hashes.py b/tests/test_hashes.py index 75830a5..94ef9db 100644 --- a/tests/test_hashes.py +++ b/tests/test_hashes.py @@ -5,19 +5,19 @@ import resync.hashes class TestUtill(unittest.TestCase): def test01_md5_sha1_file(self): - h = resync.hashes.Hashes(['md5','sha-1']) + h = resync.hashes.Hashes(['md5', 'sha-1']) h.compute_for_file('tests/testdata/a') self.assertEqual(h.md5, 'j912liHgA/48DCHpkptJHg==') self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33') self.assertEqual(h.sha256, None) - h = resync.hashes.Hashes(['sha-256','sha-1']) + h = resync.hashes.Hashes(['sha-256', 'sha-1']) h.compute_for_file('tests/testdata/a') self.assertEqual(h.md5, None) self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33') self.assertEqual(h.sha256, '69fe6314a94800456af959d380f5d6932052478ea03d5ccac7ba0a14bd5e67c6') def test02_bad_type(self): - self.assertRaises(Exception, resync.hashes.Hashes, ['md5','xyz']) + self.assertRaises(Exception, resync.hashes.Hashes, ['md5', 'xyz']) if __name__ == '__main__': unittest.main() diff --git a/tests/test_resource_list.py b/tests/test_resource_list.py index 5bae589..6e353c2 100644 --- a/tests/test_resource_list.py +++ b/tests/test_resource_list.py @@ -120,7 +120,7 @@ class TestResourceList(unittest.TestCase): r1.md5 = "aabbcc" self.assertEqual(i.hashes(), set(['md5'])) r2.sha1 = "ddeeff" - self.assertEqual(i.hashes(), set(['md5','sha-1'])) + self.assertEqual(i.hashes(), set(['md5', 'sha-1'])) def test08_iter(self): i = ResourceList() From 941cba41b189f8e6a5ec56ab7b729bbb23168405 Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Mon, 20 Mar 2017 17:08:44 -0400 Subject: [PATCH 5/5] Fix MD5 hash formatting --- CHANGES.md | 7 ++++--- resync/hashes.py | 2 +- tests/test_hashes.py | 2 +- tests/test_resource_list_builder.py | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d6ac45e..d24402f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,11 +6,12 @@ core specification version. Versions 1.0.x implement the v1.0 ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014 . -v1.0.6 2017-??-?? - * ... +v1.0.6 2017-03-20 + * Fixed md5 hash format (https://github.com/resync/resync/issues/25) + * Added support for sha-1 and sha-256 hashes v1.0.5 2017-01-30 - * Fix to support non-ASCII URIs/filenames (https://github.com/resync/resync/issues/22) + * Fixed to support non-ASCII URIs/filenames (https://github.com/resync/resync/issues/22) * Added tests for Python 3.6 * Temporarily disabled tests for Python 2.6 due to problems with coverage (https://github.com/resync/resync/issues/23) diff --git a/resync/hashes.py b/resync/hashes.py index 47aa8e8..dac3d85 100644 --- a/resync/hashes.py +++ b/resync/hashes.py @@ -87,7 +87,7 @@ class Hashes(object): """Return MD5 hash calculated.""" if (self.md5_calc is None): return None - return base64.b64encode(self.md5_calc.digest()).decode('utf-8') + return self.md5_calc.hexdigest() @property def sha1(self): diff --git a/tests/test_hashes.py b/tests/test_hashes.py index 94ef9db..51a00cf 100644 --- a/tests/test_hashes.py +++ b/tests/test_hashes.py @@ -7,7 +7,7 @@ class TestUtill(unittest.TestCase): def test01_md5_sha1_file(self): h = resync.hashes.Hashes(['md5', 'sha-1']) h.compute_for_file('tests/testdata/a') - self.assertEqual(h.md5, 'j912liHgA/48DCHpkptJHg==') + self.assertEqual(h.md5, '8fdd769621e003fe3c0c21e9929b491e') self.assertEqual(h.sha1, '49844dd211aa33071a252d7cdc250a52cf39af33') self.assertEqual(h.sha256, None) h = resync.hashes.Hashes(['sha-256', 'sha-1']) diff --git a/tests/test_resource_list_builder.py b/tests/test_resource_list_builder.py index 5664cba..e3f1db8 100644 --- a/tests/test_resource_list_builder.py +++ b/tests/test_resource_list_builder.py @@ -67,13 +67,13 @@ class TestResourceListBuilder(unittest.TestCase): r = next(rli) self.assertEqual(r.uri, 'http://example.org/t/file_a') self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') - self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==') + self.assertEqual(r.md5, '6bf26fd66601b528d2e0b47eaa87edfd') self.assertEqual(r.length, 20) self.assertEqual(r.path, None) r = next(rli) self.assertEqual(r.uri, 'http://example.org/t/file_b') self.assertEqual(r.lastmod, '2001-09-09T01:46:40Z') - self.assertEqual(r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==') + self.assertEqual(r.md5, '452e54bdae1626ac5d6e7be81b39de21') self.assertEqual(r.length, 45) self.assertEqual(r.path, None) @@ -86,7 +86,7 @@ class TestResourceListBuilder(unittest.TestCase): self.assertTrue(r is not None) self.assertEqual(r.uri, 'http://example.org/t/file_a') self.assertEqual(r.lastmod, '2012-07-25T17:13:46Z') - self.assertEqual(r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==') + self.assertEqual(r.md5, '6bf26fd66601b528d2e0b47eaa87edfd') self.assertEqual(r.path, 'tests/testdata/dir1/file_a') def test05_from_disk_paths(self):