From 0ff75050f23c2a04073eb9b1d0149281b6ba3e72 Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Thu, 19 Jun 2014 09:23:48 -0400 Subject: [PATCH 1/3] Towards dump implementation. Add logging to dump.py. Use and clean up temp dir for dump tests --- resync/dump.py | 140 +++++++++++++++++++++-------- resync/resource_dump.py | 10 +++ resync/test/test_dump.py | 161 ++++++++++++++++++++++++++++++++-- resync/test/testdata/a_to_z/a | 1 + resync/test/testdata/a_to_z/b | 18 ++++ resync/test/testdata/a_to_z/c | 1 + resync/test/testdata/a_to_z/d | 1 + resync/test/testdata/a_to_z/e | 1 + resync/test/testdata/a_to_z/f | 25 ++++++ resync/test/testdata/a_to_z/g | 1 + resync/test/testdata/a_to_z/h | 1 + resync/test/testdata/a_to_z/i | 1 + resync/test/testdata/a_to_z/j | 1 + resync/test/testdata/a_to_z/k | 24 +++++ resync/test/testdata/a_to_z/l | 1 + resync/test/testdata/a_to_z/m | 1 + 16 files changed, 345 insertions(+), 43 deletions(-) create mode 100644 resync/test/testdata/a_to_z/a create mode 100644 resync/test/testdata/a_to_z/b create mode 100644 resync/test/testdata/a_to_z/c create mode 100644 resync/test/testdata/a_to_z/d create mode 100644 resync/test/testdata/a_to_z/e create mode 100644 resync/test/testdata/a_to_z/f create mode 100644 resync/test/testdata/a_to_z/g create mode 100644 resync/test/testdata/a_to_z/h create mode 100644 resync/test/testdata/a_to_z/i create mode 100644 resync/test/testdata/a_to_z/j create mode 100644 resync/test/testdata/a_to_z/k create mode 100644 resync/test/testdata/a_to_z/l create mode 100644 resync/test/testdata/a_to_z/m diff --git a/resync/dump.py b/resync/dump.py index ddff6af..ca98c1c 100644 --- a/resync/dump.py +++ b/resync/dump.py @@ -1,8 +1,9 @@ """Dump handler for ResourceSync""" +import logging import os.path from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED -from sitemap import Sitemap +from resync.resource_dump_manifest import ResourceDumpManifest class DumpError(Exception): pass @@ -10,49 +11,79 @@ class DumpError(Exception): class Dump(object): """Dump of content for a Resource Dump or Change Dump - The resource_list must be comprised of Resource objects + The resources itearable must be comprised of Resource objects which have the path attributes set to indicate the local location of the copies of the resources. + + rl = ResourceList() + # ... add items by whatever means, may have >50k items and/or + # >100MB total size of files ... + d = Dump(rl) + d.write(dumpfile="/tmp/rd") """ - def __init__(self, format=None, compress=True): + def __init__(self, resources=None, format=None, compress=True): + self.resources = resources self.format = ('zip' if (format is None) else format) self.compress = compress + self.manifest_class = ResourceDumpManifest #FIXME self.max_size = 100*1024*1024 #100MB self.max_files = 50000 self.path_prefix = None + self.logger = logging.getLogger('dump') - def write(self, resource_list=None, dumpfile=None): - """Wrapper to Write a dump file""" - self.check_files(resource_list) - if (self.format == 'zip'): - self.write_zip(resource_list,dumpfile) - elif (self.format == 'warc'): - self.write_warc(resource_list,dumpfile) - else: - raise DumpError("Unknown dump format '%s'" % (self.format)) + def write(self, basename=None, write_separate_manifests=True): + """Write one or more dump files to complete this dump - def write_zip(self, resource_list=None, dumpfile=None): - """Write a ZIP format dump file""" + Returns the number of dump/archive files written. + """ + self.check_files() + n=0 + for manifest in self.partition_dumps(): + dumpbase="%s%05d" % (basename,n) + dumpfile="%s.%s" % (dumpbase,self.format) + if (write_separate_manifests): + manifest.write(basename=dumpbase+'.xml') + if (self.format == 'zip'): + self.write_zip(manifest.resources,dumpfile) + elif (self.format == 'warc'): + self.write_warc(manifest.resources,dumpfile) + else: + raise DumpError("Unknown dump format requested (%s)" % (self.format)) + n+=1 + self.logger.info("Wrote %d dump files" % (n)) + return(n) + + def write_zip(self, resources=None, dumpfile=None): + """Write a ZIP format dump file + + Writes a ZIP file containing the resources in the iterable resources along with + a manifest file manifest.xml (written first). No checks on the size of files + or total size are performed, this is expected to have been done beforehand. + """ compression = ( ZIP_DEFLATED if self.compress else ZIP_STORED ) zf = ZipFile(dumpfile, mode="w", compression=compression, allowZip64=True) - # Write resource_list first - s = Sitemap() + # Write resources first + rdm = ResourceDumpManifest(resources=resources) real_path = {} - for resource in resource_list: + for resource in resources: archive_path = self.archive_path(resource.path) real_path[archive_path] = resource.path resource.path = archive_path - zf.writestr('manifest.xml',s.resources_as_xml(resource_list)) - # Add all files in the resource_list - for resource in resource_list: + zf.writestr('manifest.xml',rdm.as_xml()) + # Add all files in the resources + for resource in resources: zf.write(real_path[resource.path], arcname=resource.path) zf.close() zipsize = os.path.getsize(dumpfile) - print "Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize) + self.logger.info("Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize)) - def write_warc(self, resource_list=None, dumpfile=None): - """Write a WARC dump file""" + def write_warc(self, resources=None, dumpfile=None): + """Write a WARC dump file + + WARC support is not part of ResourceSync v1.0 (Z39.99 2014) but is left + in this library for experimentation. + """ # Load library late as we want to be able to run rest of code # without this installed try: @@ -60,8 +91,8 @@ class Dump(object): except: raise DumpError("Failed to load WARC library") wf = WARCFile(dumpfile, mode="w", compress=self.compress) - # Add all files in the resource_list - for resource in resource_list: + # Add all files in the resources + for resource in resources: wh = WARCHeader({}) wh.url = resource.uri wh.ip_address = None @@ -73,19 +104,21 @@ class Dump(object): wf.write_record( WARCRecord( header=wh, payload=resource.path ) ) wf.close() warcsize = os.path.getsize(dumpfile) - print "Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize) + self.logging.info("Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize)) - def check_files(self,resource_list): - """Go though and check all files in resource_list, add up size, find common path + def check_files(self,set_length=True,check_length=True): + """Go though and check all files in self.resources, add up size, and find + longest common path that can be used when writing the dump file. Saved in + self.path_prefix. - Will find the longest common path prefix that can be used when writing the - dump file. Saved in self.path_prefix. + Parameters set_length and check_length control control whether then set_length + attribute should be set from the file size if not specified, and whether any + length specified should be checked. By default both are True. In any event, the + total size calculated is the size of files on disk. """ - if (len(resource_list) > self.max_files): - raise DumpError("Number of files to dump (%d) exceeds maximum (%d)" % (len(resource_list),self.max_files)) total_size = 0 #total size of all files in bytes path_prefix = None - for resource in resource_list: + for resource in self.resources: if (resource.path is None): #explicit test because exception raised by getsize otherwise confusing raise DumpError("No file path defined for resource %s" % resource.uri) @@ -93,12 +126,45 @@ class Dump(object): path_prefix = os.path.dirname(resource.path) else: path_prefix = os.path.commonprefix( [ path_prefix, os.path.dirname(resource.path) ]) - total_size += os.path.getsize(resource.path) + size = os.path.getsize(resource.path) + if (resource.length is not None): + if (check_length and resource.length!=size): + raise DumpError("Size of resource %s is %d on disk, not %d as specified" % + (resource.uri, size, resource.length) ) + elif (set_length): + resource.length = size + if (size > self.max_size): + raise DumpError("Size of file (%s, %d) exceeds maximum (%d) dump size" % (resource.path,size,self.max_size)) + total_size += size self.path_prefix = path_prefix self.total_size = total_size - print "Total size of files to include in dump %d bytes" % (total_size) - if (total_size > self.max_size): - raise DumpError("Size of files to dump (%d) exceeds maximum (%d)" % (total_size,self.max_size)) + self.logger.info("Total size of files to include in dump %d bytes" % (total_size)) + return True + + def partition_dumps(self): + """Yeild a set of manifest object that parition the dumps + + Simply adds resources/files to a manifest until their are either the + the correct number of files or the size limit is exceeded, then yields + that manifest. + """ + manifest=self.manifest_class() + manifest_size=0 + manifest_files=0 + for resource in self.resources: + manifest.add(resource) + manifest_size+=resource.length + manifest_files+=1 + if (manifest_size>=self.max_size or + manifest_files>=self.max_files): + yield(manifest) + # Need to start a new manifest + manifest=self.manifest_class() + manifest_size=0 + manifest_files=0 + if (manifest_files>0): + yield(manifest) + def archive_path(self,real_path): """Return the archive path for file with real_path diff --git a/resync/resource_dump.py b/resync/resource_dump.py index e8b1a29..e0cc112 100644 --- a/resync/resource_dump.py +++ b/resync/resource_dump.py @@ -17,9 +17,19 @@ class ResourceDump(ResourceList): A Resource Dump comprises a set of content packages that are the resources listed. Properties similar to a ResourceList and implemented as a sub-class of ResourceList. + + See the Dump class for details of how to create a Resource + Dump from a ResourceList. """ def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None): super(ResourceDump, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper) self.capability_name='resourcedump' + + def write(self, basename="/tmp/resource_dump.xml", dumpfile=None): + """Write out a Resource Dump document and optionally also the actual dump files + + Dump files will be written if the dumpfile argument is set to a base filename. + """ + super(ResourceDump,self).write(basename) diff --git a/resync/test/test_dump.py b/resync/test/test_dump.py index 2c3c56b..e538d3d 100644 --- a/resync/test/test_dump.py +++ b/resync/test/test_dump.py @@ -1,19 +1,168 @@ +import os.path import unittest +import tempfile +import shutil +import sys +import zipfile from resync.dump import Dump, DumpError from resync.resource_list import ResourceList +from resync.change_list import ChangeList from resync.resource import Resource +from resync.resource_dump_manifest import ResourceDumpManifest +from resync.change_dump_manifest import ChangeDumpManifest class TestDump(unittest.TestCase): - def test00_dump_creation(self): - i=ResourceList() - i.add( Resource('http://ex.org/a', length=1, path='resync/test/testdata/a') ) - i.add( Resource('http://ex.org/b', length=2, path='resync/test/testdata/b') ) + _tmpdir=None + + @classmethod + def setUpClass(cls): + # Create tmp dir to write to and check + cls._tmpdir=tempfile.mkdtemp() + if (not os.path.isdir(cls._tmpdir)): + raise Exception("Failed to create tempdir to use for dump tests") + + @classmethod + def tearDownClass(cls): + # Cleanup + if (not os.path.isdir(cls._tmpdir)): + raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir)) + shutil.rmtree(cls._tmpdir) + + @property + def tmpdir(self): + # read-only access to _tmpdir, just in case... The rmtree scares me + return(self._tmpdir) + + def test00_dump_zip_resource_list(self): + rl=ResourceDumpManifest() + rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') ) + rl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b') ) d=Dump() - d.check_files(resource_list=i) + zipf=os.path.join(self.tmpdir,"test00_dump.zip") + d.write_zip(resources=rl,dumpfile=zipf) # named args + self.assertTrue( os.path.exists(zipf) ) + self.assertTrue( zipfile.is_zipfile(zipf) ) + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( len(zo.namelist()), 3 ) + zo.close() + os.unlink(zipf) + + def test01_dump_zip_change_list(self): + cl=ChangeDumpManifest() + cl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a', change="updated") ) + cl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b', change="updated") ) + d=Dump() + zipf=os.path.join(self.tmpdir,"test01_dump.zip") + d.write_zip(cl,zipf) # positional args + self.assertTrue( os.path.exists(zipf) ) + self.assertTrue( zipfile.is_zipfile(zipf) ) + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( len(zo.namelist()), 3 ) + zo.close() + os.unlink(zipf) + + def test02_dump_check_files(self): + cl=ChangeList() + cl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a', change="updated") ) + cl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b', change="updated") ) + d=Dump(resources=cl) + self.assertTrue(d.check_files()) self.assertEqual(d.total_size, 28) - #FIXME -- need some code to actually write and read dump + def test03_dump_multi_file_max_size(self): + rl=ResourceList() + for letter in map(chr,xrange(ord('a'),ord('l')+1)): + uri='http://ex.org/%s' % (letter) + fname='resync/test/testdata/a_to_z/%s' % (letter) + rl.add( Resource(uri, path=fname) ) + self.assertEqual( len(rl), 12 ) + #d=Dump(rl) + #tmpdir=tempfile.mkdtemp() + #tmpbase=os.path.join(tmpdir,'base') + #d.max_size=2000 # start new zip after size exceeds 2000 bytes + #n=d.write(tmpbase) + #self.assertEqual( n, 2, 'expect to write 2 dump files' ) + # + # Now repeat with large size limit but small number of files limit + d2=Dump(rl) + tmpbase=os.path.join(self.tmpdir,'test03_') + d2.max_files=4 + n=d2.write(tmpbase) + self.assertEqual( n, 3, 'expect to write 3 dump files' ) + self.assertTrue( os.path.isfile(tmpbase+'00000.zip') ) + self.assertTrue( os.path.isfile(tmpbase+'00001.zip') ) + self.assertTrue( os.path.isfile(tmpbase+'00002.zip') ) + # Look at the first file in detail + zipf=tmpbase+'00000.zip' + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d'] ) + #self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) + self.assertEqual( zo.getinfo('a').file_size, 9 ) + self.assertEqual( zo.getinfo('b').file_size, 1116 ) + self.assertEqual( zo.getinfo('c').file_size, 32 ) + self.assertEqual( zo.getinfo('d').file_size, 13 ) + zo.close() + os.unlink(zipf) + # Check second and third files have expected contents + zipf=tmpbase+'00001.zip' + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( zo.namelist(), ['manifest.xml','e','f','g','h'] ) + zo.close() + os.unlink(zipf) + zipf=tmpbase+'00002.zip' + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( zo.namelist(), ['manifest.xml','i','j','k','l'] ) + zo.close() + os.unlink(zipf) + + def test04_dump_multi_file_max_size(self): + rl=ResourceList() + for letter in map(chr,xrange(ord('a'),ord('l')+1)): + uri='http://ex.org/%s' % (letter) + fname='resync/test/testdata/a_to_z/%s' % (letter) + rl.add( Resource(uri, path=fname) ) + self.assertEqual( len(rl), 12 ) + d2=Dump(rl) + tmpbase=os.path.join(self.tmpdir,'test0f_') + d2.max_size=2000 + n=d2.write(tmpbase) + self.assertEqual( n, 2, 'expect to write 2 dump files' ) + self.assertTrue( os.path.isfile(tmpbase+'00000.zip') ) + self.assertTrue( os.path.isfile(tmpbase+'00001.zip') ) + # Look at the first file in detail + zipf=tmpbase+'00000.zip' + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d','e','f'] ) + #self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 ) + self.assertEqual( zo.getinfo('a').file_size, 9 ) + self.assertEqual( zo.getinfo('b').file_size, 1116 ) + self.assertEqual( zo.getinfo('c').file_size, 32 ) + self.assertEqual( zo.getinfo('d').file_size, 13 ) + self.assertEqual( zo.getinfo('e').file_size, 20 ) + self.assertEqual( zo.getinfo('f').file_size, 1625 ) + zo.close() + os.unlink(zipf) + # Check second and third files have expected contents + zipf=tmpbase+'00001.zip' + zo=zipfile.ZipFile(zipf,'r') + self.assertEqual( zo.namelist(), ['manifest.xml','g','h','i','j','k','l'] ) + zo.close() + os.unlink(zipf) + + def test10_no_path(self): + rl=ResourceList() + rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') ) + rl.add( Resource('http://ex.org/b', length=21 ) ) + d=Dump(rl) + self.assertRaises( DumpError, d.check_files ) + + def test11_bad_size(self): + rl=ResourceList() + rl.add( Resource('http://ex.org/a', length=9999, path='resync/test/testdata/a') ) + d=Dump(rl) + self.assertTrue( d.check_files(check_length=False) ) + self.assertRaises( DumpError, d.check_files ) if __name__ == '__main__': suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDump) diff --git a/resync/test/testdata/a_to_z/a b/resync/test/testdata/a_to_z/a new file mode 100644 index 0000000..9606d7d --- /dev/null +++ b/resync/test/testdata/a_to_z/a @@ -0,0 +1 @@ +aaaaaaaaa \ No newline at end of file diff --git a/resync/test/testdata/a_to_z/b b/resync/test/testdata/a_to_z/b new file mode 100644 index 0000000..742ded7 --- /dev/null +++ b/resync/test/testdata/a_to_z/b @@ -0,0 +1,18 @@ +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb diff --git a/resync/test/testdata/a_to_z/c b/resync/test/testdata/a_to_z/c new file mode 100644 index 0000000..935af58 --- /dev/null +++ b/resync/test/testdata/a_to_z/c @@ -0,0 +1 @@ +ccccccccccccccccccccccccccccccc diff --git a/resync/test/testdata/a_to_z/d b/resync/test/testdata/a_to_z/d new file mode 100644 index 0000000..b7a5ef5 --- /dev/null +++ b/resync/test/testdata/a_to_z/d @@ -0,0 +1 @@ +dddddddddddd diff --git a/resync/test/testdata/a_to_z/e b/resync/test/testdata/a_to_z/e new file mode 100644 index 0000000..a41d7e9 --- /dev/null +++ b/resync/test/testdata/a_to_z/e @@ -0,0 +1 @@ +eeeeeeeeeeeeeeeeeee diff --git a/resync/test/testdata/a_to_z/f b/resync/test/testdata/a_to_z/f new file mode 100644 index 0000000..086e67e --- /dev/null +++ b/resync/test/testdata/a_to_z/f @@ -0,0 +1,25 @@ +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff diff --git a/resync/test/testdata/a_to_z/g b/resync/test/testdata/a_to_z/g new file mode 100644 index 0000000..101e504 --- /dev/null +++ b/resync/test/testdata/a_to_z/g @@ -0,0 +1 @@ +ggggggg diff --git a/resync/test/testdata/a_to_z/h b/resync/test/testdata/a_to_z/h new file mode 100644 index 0000000..0fd2680 --- /dev/null +++ b/resync/test/testdata/a_to_z/h @@ -0,0 +1 @@ +hhhhhhhhhhhhhhhhhh diff --git a/resync/test/testdata/a_to_z/i b/resync/test/testdata/a_to_z/i new file mode 100644 index 0000000..a9534b1 --- /dev/null +++ b/resync/test/testdata/a_to_z/i @@ -0,0 +1 @@ +iiii diff --git a/resync/test/testdata/a_to_z/j b/resync/test/testdata/a_to_z/j new file mode 100644 index 0000000..51a877a --- /dev/null +++ b/resync/test/testdata/a_to_z/j @@ -0,0 +1 @@ +jjjjjjjj diff --git a/resync/test/testdata/a_to_z/k b/resync/test/testdata/a_to_z/k new file mode 100644 index 0000000..f33060d --- /dev/null +++ b/resync/test/testdata/a_to_z/k @@ -0,0 +1,24 @@ +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk diff --git a/resync/test/testdata/a_to_z/l b/resync/test/testdata/a_to_z/l new file mode 100644 index 0000000..a4a8dab --- /dev/null +++ b/resync/test/testdata/a_to_z/l @@ -0,0 +1 @@ +lllll diff --git a/resync/test/testdata/a_to_z/m b/resync/test/testdata/a_to_z/m new file mode 100644 index 0000000..28ce6a8 --- /dev/null +++ b/resync/test/testdata/a_to_z/m @@ -0,0 +1 @@ +m From 5a716c47ceb86aad29652a7ecf8817f88e6ec42f Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Wed, 2 Jul 2014 19:07:22 -0400 Subject: [PATCH 2/3] Fix version number --- CHANGES.md | 59 +++++++++++++++++++++++----------------------- resync/_version.py | 2 +- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 33e9c03..c589679 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,56 +4,57 @@ resync change log The first two components of the version tags are tied to the ResourceSync core specification version. Versions 1.0.x implement the v1.0 ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014 -(http://www.openarchives.org/rs/1.0/). +. v1.0.0 2014-06-XX -- Update for v1.0, Z39.99-2014 specification (http://www.openarchives.org/rs/1.0/) + * Update for v1.0, Z39.99-2014 specification (http://www.openarchives.org/rs/1.0/) + * Partial implementation of dump output (no read yet) v0.9.5 2013-11-06 -- Still working toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/) -- Set up for Travis-CI working from github -- Use /usr/bin/env to find python in executables -- Fix timezone handling in w3c_datetime.py -- Fix a number of documentation inconsistencies + * Still working toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/) + * Set up for Travis-CI working from github + * Use /usr/bin/env to find python in executables + * Fix timezone handling in w3c_datetime.py + * Fix a number of documentation inconsistencies v0.9.4 2013-09-06 -- Work toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/) -- ResourceSync Description becomes Source Description + * Work toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/) + * ResourceSync Description becomes Source Description v0.9.3 2013-07-31 -- Add dependencies and tests to setup, no code changes + * Add dependencies and tests to setup, no code changes v0.9.2 2013-07-04 -- Cosmetic only + * Cosmetic only v0.9.1 2013-07-04 -- First release working toward v0.9 specification -- Use test examples from v0.9 specification, add code to build them too -- Added ResourceSync Description document -- Improved resync-explore binary replaces --explore mode -- Relax default notion of URL authority, add --strictauth + * First release working toward v0.9 specification + * Use test examples from v0.9 specification, add code to build them too + * Added ResourceSync Description document + * Improved resync-explore binary replaces --explore mode + * Relax default notion of URL authority, add --strictauth v0.6.3 2013-05... -- Improved --explore mode + * Improved --explore mode v0.6.2 2013-05-14 -- Fixed example code in README! -- Added --version flag -- Added --paths option to specify local paths to search rather than mappings + * Fixed example code in README! + * Added --version flag + * Added --paths option to specify local paths to search rather than mappings v0.6.1 2013-05-09 -- First release working toward v0.6 specification -- Changed test examples to be those from v0.6 specification -- Added first stab at ResourceDump and ResourceDumpManifest objects + * First release working toward v0.6 specification + * Changed test examples to be those from v0.6 specification + * Added first stab at ResourceDump and ResourceDumpManifest objects v0.5.3 2013-05-08 -- Final release working with v0.5 specification -- Improved handling of assumed mappings for testing client on a local filesystem -- Fixed sitemapindex support for large resource lists, add rel="up" for component sitemaps + * Final release working with v0.5 specification + * Improved handling of assumed mappings for testing client on a local filesystem + * Fixed sitemapindex support for large resource lists, add rel="up" for component sitemaps v0.5.2 2013-03-26 -- Fixed setup.py + * Fixed setup.py v0.5.1 2013-03-22 -- Code reworked for 0.5 specification (http://www.openarchives.org/rs/0.5/) -- Client code handles --baseline, --audit and --incremental sync against the simulator + * Code reworked for 0.5 specification (http://www.openarchives.org/rs/0.5/) + * Client code handles --baseline, --audit and --incremental sync against the simulator diff --git a/resync/_version.py b/resync/_version.py index 7e07f9d..1bb8e2b 100644 --- a/resync/_version.py +++ b/resync/_version.py @@ -3,4 +3,4 @@ # Format: x.y.z where # x.y is spec version, see http://www.openarchives.org/rs/x.y/ # z is incremented for revisions within that version, 1... -__version__ = '0.9.5' +__version__ = '1.0.0' From c62416fd119b5be57fd03541aac3f392f552e420 Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Wed, 2 Jul 2014 19:45:26 -0400 Subject: [PATCH 3/3] Use hierarchical logger names --- bin/resync | 2 +- resync/client.py | 2 +- resync/client_utils.py | 2 +- resync/dump.py | 7 ++++--- resync/list_base.py | 2 +- resync/mapper.py | 4 ++-- resync/resource_list_builder.py | 2 +- resync/sitemap.py | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/bin/resync b/bin/resync index a98ed3e..9bfe899 100755 --- a/bin/resync +++ b/bin/resync @@ -156,7 +156,7 @@ def main(): help="override default size limits") # Want these to show at the end opt.add_option('--verbose', '-v', action='store_true', - help="verbose") + help="verbose, show additional informational messages") opt.add_option('--logger', '-l', action='store_true', help="create detailed log of client actions (will write " "to %s unless specified with --logfile" % diff --git a/resync/client.py b/resync/client.py index 63286c6..296d267 100644 --- a/resync/client.py +++ b/resync/client.py @@ -45,7 +45,7 @@ class Client(object): self.checksum = checksum self.verbose = verbose self.dryrun = dryrun - self.logger = logging.getLogger('client') + self.logger = logging.getLogger('resync.client') self.mapper = Mapper() self.resource_list_name = 'resourcelist.xml' self.change_list_name = 'changelist.xml' diff --git a/resync/client_utils.py b/resync/client_utils.py index 499c292..b5d14f2 100644 --- a/resync/client_utils.py +++ b/resync/client_utils.py @@ -57,7 +57,7 @@ def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log', fh.setFormatter(formatter) fh.setLevel( logging.DEBUG if (eval_mode) else logging.INFO ) - loggers = [default_logger,'resourcelist_builder','sitemap'] + loggers = [default_logger,'resync'] if (extra_loggers is not None): for logger in extra_loggers: loggers.append(logger) diff --git a/resync/dump.py b/resync/dump.py index ca98c1c..b9bdb68 100644 --- a/resync/dump.py +++ b/resync/dump.py @@ -18,8 +18,9 @@ class Dump(object): rl = ResourceList() # ... add items by whatever means, may have >50k items and/or # >100MB total size of files ... - d = Dump(rl) - d.write(dumpfile="/tmp/rd") + d = Dump() + d.write(resources=rl,basename="/tmp/rd_") + # will create dump files /tmp/rd_00001.zip etc. """ def __init__(self, resources=None, format=None, compress=True): @@ -30,7 +31,7 @@ class Dump(object): self.max_size = 100*1024*1024 #100MB self.max_files = 50000 self.path_prefix = None - self.logger = logging.getLogger('dump') + self.logger = logging.getLogger('resync.dump') def write(self, basename=None, write_separate_manifests=True): """Write one or more dump files to complete this dump diff --git a/resync/list_base.py b/resync/list_base.py index f81bf24..20d321f 100644 --- a/resync/list_base.py +++ b/resync/list_base.py @@ -41,7 +41,7 @@ class ListBase(ResourceContainer): self.sitemapindex = False self.pretty_xml = False # - self.logger = logging.getLogger('list_base') + self.logger = logging.getLogger('resync.list_base') self.bytes_read = 0 self.parsed_index = None diff --git a/resync/mapper.py b/resync/mapper.py index 6aa1516..303e698 100644 --- a/resync/mapper.py +++ b/resync/mapper.py @@ -11,7 +11,7 @@ class MapperError(Exception): class Mapper(): def __init__(self, mappings=None, use_default_path=False): - self.logger = logging.getLogger('mapper') + self.logger = logging.getLogger('resync.mapper') self.mappings=[] if (mappings): self.parse(mappings, use_default_path) @@ -39,7 +39,7 @@ class Mapper(): len(mappings)==1 and re.search(r"=",mappings[0])==None): path = self.path_from_uri(mappings[0]) - self.logger.info("Assuming mapping: %s -> %s" % (mappings[0],path)) + self.logger.warning("Using URI mapping: %s -> %s" % (mappings[0],path)) self.mappings.append(Map(mappings[0],path)) elif (len(mappings)==2 and re.search(r"=",mappings[0])==None and diff --git a/resync/resource_list_builder.py b/resync/resource_list_builder.py index 10a22d1..b424022 100644 --- a/resync/resource_list_builder.py +++ b/resync/resource_list_builder.py @@ -51,7 +51,7 @@ class ResourceListBuilder(): self.exclude_dirs = ['CVS','.git'] self.include_symlinks = False # Used internally only: - self.logger = logging.getLogger('resource_list_builder') + self.logger = logging.getLogger('resync.resource_list_builder') self.compiled_exclude_files = [] def add_exclude_files(self, exclude_patterns): diff --git a/resync/sitemap.py b/resync/sitemap.py index 6531191..1f2b183 100644 --- a/resync/sitemap.py +++ b/resync/sitemap.py @@ -56,7 +56,7 @@ class Sitemap(object): """ def __init__(self, pretty_xml=False): - self.logger = logging.getLogger('sitemap') + self.logger = logging.getLogger('resync.sitemap') self.pretty_xml=pretty_xml # Classes used when parsing self.resource_class=Resource