gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged changes form laptop

Browse Source
This commit is contained in:
Simeon Warner 2014-08-09 15:48:31 -04:00
commit cbe4a1691e
11 changed files with 174 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
CHANGES.md
View File

@ -4,56 +4,57 @@ resync change log
The first two components of the version tags are tied to the ResourceSync
core specification version. Versions 1.0.x implement the v1.0
ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014
(http://www.openarchives.org/rs/1.0/).
<http://www.openarchives.org/rs/1.0/toc>.
v1.0.0 2014-06-XX
- Update for v1.0, Z39.99-2014 specification (http://www.openarchives.org/rs/1.0/)
* Update for v1.0, Z39.99-2014 specification (http://www.openarchives.org/rs/1.0/)
* Partial implementation of dump output (no read yet)
v0.9.5 2013-11-06
- Still working toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/)
- Set up for Travis-CI working from github
- Use /usr/bin/env to find python in executables
- Fix timezone handling in w3c_datetime.py
- Fix a number of documentation inconsistencies
* Still working toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/)
* Set up for Travis-CI working from github
* Use /usr/bin/env to find python in executables
* Fix timezone handling in w3c_datetime.py
* Fix a number of documentation inconsistencies
v0.9.4 2013-09-06
- Work toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/)
- ResourceSync Description becomes Source Description
* Work toward v0.9.1 specification (http://www.openarchives.org/rs/0.9.1/)
* ResourceSync Description becomes Source Description
v0.9.3 2013-07-31
- Add dependencies and tests to setup, no code changes
* Add dependencies and tests to setup, no code changes
v0.9.2 2013-07-04
- Cosmetic only
* Cosmetic only
v0.9.1 2013-07-04
- First release working toward v0.9 specification
- Use test examples from v0.9 specification, add code to build them too
- Added ResourceSync Description document
- Improved resync-explore binary replaces --explore mode
- Relax default notion of URL authority, add --strictauth
* First release working toward v0.9 specification
* Use test examples from v0.9 specification, add code to build them too
* Added ResourceSync Description document
* Improved resync-explore binary replaces --explore mode
* Relax default notion of URL authority, add --strictauth
v0.6.3 2013-05...
- Improved --explore mode
* Improved --explore mode
v0.6.2 2013-05-14
- Fixed example code in README!
- Added --version flag
- Added --paths option to specify local paths to search rather than mappings
* Fixed example code in README!
* Added --version flag
* Added --paths option to specify local paths to search rather than mappings
v0.6.1 2013-05-09
- First release working toward v0.6 specification
- Changed test examples to be those from v0.6 specification
- Added first stab at ResourceDump and ResourceDumpManifest objects
* First release working toward v0.6 specification
* Changed test examples to be those from v0.6 specification
* Added first stab at ResourceDump and ResourceDumpManifest objects
v0.5.3 2013-05-08
- Final release working with v0.5 specification
- Improved handling of assumed mappings for testing client on a local filesystem
- Fixed sitemapindex support for large resource lists, add rel="up" for component sitemaps
* Final release working with v0.5 specification
* Improved handling of assumed mappings for testing client on a local filesystem
* Fixed sitemapindex support for large resource lists, add rel="up" for component sitemaps
v0.5.2 2013-03-26
- Fixed setup.py
* Fixed setup.py
v0.5.1 2013-03-22
- Code reworked for 0.5 specification (http://www.openarchives.org/rs/0.5/)
- Client code handles --baseline, --audit and --incremental sync against the simulator
* Code reworked for 0.5 specification (http://www.openarchives.org/rs/0.5/)
* Client code handles --baseline, --audit and --incremental sync against the simulator

2
bin/resync
View File

@ -156,7 +156,7 @@ def main():
help="override default size limits")
# Want these to show at the end
opt.add_option('--verbose', '-v', action='store_true',
help="verbose")
help="verbose, show additional informational messages")
opt.add_option('--logger', '-l', action='store_true',
help="create detailed log of client actions (will write "
"to %s unless specified with --logfile" %

2
resync/_version.py
View File

@ -3,4 +3,4 @@
# Format: x.y.z where
# x.y is spec version, see http://www.openarchives.org/rs/x.y/
# z is incremented for revisions within that version, 1...
__version__ = '0.9.5'
__version__ = '1.0.0'

2
resync/client.py
View File

@ -45,7 +45,7 @@ class Client(object):
self.checksum = checksum
self.verbose = verbose
self.dryrun = dryrun
self.logger = logging.getLogger('client')
self.logger = logging.getLogger('resync.client')
self.mapper = Mapper()
self.resource_list_name = 'resourcelist.xml'
self.change_list_name = 'changelist.xml'

2
resync/client_utils.py
View File

@ -57,7 +57,7 @@ def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log',
fh.setFormatter(formatter)
fh.setLevel( logging.DEBUG if (eval_mode) else logging.INFO )
loggers = [default_logger,'resourcelist_builder','sitemap']
loggers = [default_logger,'resync']
if (extra_loggers is not None):
for logger in extra_loggers:
loggers.append(logger)

27
resync/dump.py
View File

@ -1,5 +1,6 @@
"""Dump handler for ResourceSync"""
import logging
import os.path
from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
from resync.resource_dump_manifest import ResourceDumpManifest
@ -17,8 +18,8 @@ class Dump(object):
rl = ResourceList()
# ... add items by whatever means, may have >50k items and/or
# >100MB total size of files ...
d = Dump(rl)
d.write(dumpfile="/tmp/rd")
d.write(resources=rl,basename="/tmp/rd_")
# will create dump files /tmp/rd_00001.zip etc.
"""
def __init__(self, resources=None, format=None, compress=True):
@ -29,6 +30,7 @@ class Dump(object):
self.max_size = 100*1024*1024 #100MB
self.max_files = 50000
self.path_prefix = None
self.logger = logging.getLogger('resync.dump')
def write(self, basename=None, write_separate_manifests=True):
"""Write one or more dump files to complete this dump
@ -49,11 +51,16 @@ class Dump(object):
else:
raise DumpError("Unknown dump format requested (%s)" % (self.format))
n+=1
print "Wrote %d dump files" % (n)
self.logger.info("Wrote %d dump files" % (n))
return(n)
def write_zip(self, resources=None, dumpfile=None):
"""Write a ZIP format dump file"""
"""Write a ZIP format dump file
Writes a ZIP file containing the resources in the iterable resources along with
a manifest file manifest.xml (written first). No checks on the size of files
or total size are performed, this is expected to have been done beforehand.
"""
compression = ( ZIP_DEFLATED if self.compress else ZIP_STORED )
zf = ZipFile(dumpfile, mode="w", compression=compression, allowZip64=True)
# Write resources first
@ -69,10 +76,14 @@ class Dump(object):
zf.write(real_path[resource.path], arcname=resource.path)
zf.close()
zipsize = os.path.getsize(dumpfile)
print "Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize)
self.logger.info("Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize))
def write_warc(self, resources=None, dumpfile=None):
"""Write a WARC dump file"""
"""Write a WARC dump file
WARC support is not part of ResourceSync v1.0 (Z39.99 2014) but is left
in this library for experimentation.
"""
# Load library late as we want to be able to run rest of code
# without this installed
try:
@ -93,7 +104,7 @@ class Dump(object):
wf.write_record( WARCRecord( header=wh, payload=resource.path ) )
wf.close()
warcsize = os.path.getsize(dumpfile)
print "Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize)
self.logging.info("Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize))
def check_files(self,set_length=True,check_length=True):
"""Go though and check all files in self.resources, add up size, and find
@ -127,7 +138,7 @@ class Dump(object):
total_size += size
self.path_prefix = path_prefix
self.total_size = total_size
print "Total size of files to include in dump %d bytes" % (total_size)
self.logger.info("Total size of files to include in dump %d bytes" % (total_size))
return True
def partition_dumps(self):

2
resync/list_base.py
View File

@ -41,7 +41,7 @@ class ListBase(ResourceContainer):
self.sitemapindex = False
self.pretty_xml = False
#
self.logger = logging.getLogger('list_base')
self.logger = logging.getLogger('resync.list_base')
self.bytes_read = 0
self.parsed_index = None

4
resync/mapper.py
View File

@ -11,7 +11,7 @@ class MapperError(Exception):
class Mapper():
def __init__(self, mappings=None, use_default_path=False):
self.logger = logging.getLogger('mapper')
self.logger = logging.getLogger('resync.mapper')
self.mappings=[]
if (mappings):
self.parse(mappings, use_default_path)
@ -39,7 +39,7 @@ class Mapper():
len(mappings)==1 and
re.search(r"=",mappings[0])==None):
path = self.path_from_uri(mappings[0])
self.logger.info("Assuming mapping: %s -> %s" % (mappings[0],path))
self.logger.warning("Using URI mapping: %s -> %s" % (mappings[0],path))
self.mappings.append(Map(mappings[0],path))
elif (len(mappings)==2 and
re.search(r"=",mappings[0])==None and

2
resync/resource_list_builder.py
View File

@ -51,7 +51,7 @@ class ResourceListBuilder():
self.exclude_dirs = ['CVS','.git']
self.include_symlinks = False
# Used internally only:
self.logger = logging.getLogger('resource_list_builder')
self.logger = logging.getLogger('resync.resource_list_builder')
self.compiled_exclude_files = []
def add_exclude_files(self, exclude_patterns):

2
resync/sitemap.py
View File

@ -56,7 +56,7 @@ class Sitemap(object):
"""
def __init__(self, pretty_xml=False):
self.logger = logging.getLogger('sitemap')
self.logger = logging.getLogger('resync.sitemap')
self.pretty_xml=pretty_xml
# Classes used when parsing
self.resource_class=Resource

128
resync/test/test_dump.py
View File

@ -1,7 +1,9 @@
import os.path
import unittest
import tempfile
import subprocess
import shutil
import sys
import zipfile
from resync.dump import Dump, DumpError
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
@ -11,21 +13,54 @@ from resync.change_dump_manifest import ChangeDumpManifest
class TestDump(unittest.TestCase):
_tmpdir=None
@classmethod
def setUpClass(cls):
# Create tmp dir to write to and check
cls._tmpdir=tempfile.mkdtemp()
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Failed to create tempdir to use for dump tests")
@classmethod
def tearDownClass(cls):
# Cleanup
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir))
shutil.rmtree(cls._tmpdir)
@property
def tmpdir(self):
# read-only access to _tmpdir, just in case... The rmtree scares me
return(self._tmpdir)
def test00_dump_zip_resource_list(self):
rl=ResourceDumpManifest()
rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') )
rl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b') )
d=Dump(rl)
zipfile="test00_dump.zip"
d.write_zip("test00_dump.zip")
self.assertTrue( os.path.exists(zipfile) )
d=Dump()
zipf=os.path.join(self.tmpdir,"test00_dump.zip")
d.write_zip(resources=rl,dumpfile=zipf) # named args
self.assertTrue( os.path.exists(zipf) )
self.assertTrue( zipfile.is_zipfile(zipf) )
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( len(zo.namelist()), 3 )
zo.close()
os.unlink(zipf)
def test01_dump_zip_change_list(self):
cl=ChangeDumpManifest()
cl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a', change="updated") )
cl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b', change="updated") )
d=Dump(cl)
d.write_zip("test00_dump.zip")
d=Dump()
zipf=os.path.join(self.tmpdir,"test01_dump.zip")
d.write_zip(cl,zipf) # positional args
self.assertTrue( os.path.exists(zipf) )
self.assertTrue( zipfile.is_zipfile(zipf) )
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( len(zo.namelist()), 3 )
zo.close()
os.unlink(zipf)
def test02_dump_check_files(self):
cl=ChangeList()
@ -35,7 +70,7 @@ class TestDump(unittest.TestCase):
self.assertTrue(d.check_files())
self.assertEqual(d.total_size, 28)
def test03_dump_multi_file(self):
def test03_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
@ -51,14 +86,83 @@ class TestDump(unittest.TestCase):
#
# Now repeat with large size limit but small number of files limit
d2=Dump(rl)
tmpdir=tempfile.mkdtemp()
tmpbase=os.path.join(tmpdir,'basez')
tmpbase=os.path.join(self.tmpdir,'test03_')
d2.max_files=4
n=d2.write(tmpbase)
self.assertEqual( n, 3, 'expect to write 3 dump files' )
self.assertEqual( 'a', subprocess.check_output(['/bin/ls','-l',tmpdir]) )
#FIXME -- need some code to actually write and read dump
self.assertTrue( os.path.isfile(tmpbase+'00000.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00001.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00002.zip') )
# Look at the first file in detail
zipf=tmpbase+'00000.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d'] )
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual( zo.getinfo('a').file_size, 9 )
self.assertEqual( zo.getinfo('b').file_size, 1116 )
self.assertEqual( zo.getinfo('c').file_size, 32 )
self.assertEqual( zo.getinfo('d').file_size, 13 )
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf=tmpbase+'00001.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','e','f','g','h'] )
zo.close()
os.unlink(zipf)
zipf=tmpbase+'00002.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','i','j','k','l'] )
zo.close()
os.unlink(zipf)
def test04_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
fname='resync/test/testdata/a_to_z/%s' % (letter)
rl.add( Resource(uri, path=fname) )
self.assertEqual( len(rl), 12 )
d2=Dump(rl)
tmpbase=os.path.join(self.tmpdir,'test0f_')
d2.max_size=2000
n=d2.write(tmpbase)
self.assertEqual( n, 2, 'expect to write 2 dump files' )
self.assertTrue( os.path.isfile(tmpbase+'00000.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00001.zip') )
# Look at the first file in detail
zipf=tmpbase+'00000.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d','e','f'] )
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual( zo.getinfo('a').file_size, 9 )
self.assertEqual( zo.getinfo('b').file_size, 1116 )
self.assertEqual( zo.getinfo('c').file_size, 32 )
self.assertEqual( zo.getinfo('d').file_size, 13 )
self.assertEqual( zo.getinfo('e').file_size, 20 )
self.assertEqual( zo.getinfo('f').file_size, 1625 )
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf=tmpbase+'00001.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','g','h','i','j','k','l'] )
zo.close()
os.unlink(zipf)
def test10_no_path(self):
rl=ResourceList()
rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') )
rl.add( Resource('http://ex.org/b', length=21 ) )
d=Dump(rl)
self.assertRaises( DumpError, d.check_files )
def test11_bad_size(self):
rl=ResourceList()
rl.add( Resource('http://ex.org/a', length=9999, path='resync/test/testdata/a') )
d=Dump(rl)
self.assertTrue( d.check_files(check_length=False) )
self.assertRaises( DumpError, d.check_files )
def test10_no_path(self):
rl=ResourceList()