gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add --paths option in client and handle in building resource list

Browse Source
This commit is contained in:
Simeon Warner 2013-05-14 12:27:30 -04:00
parent 7a238d82c7
commit dc95fc3dea
6 changed files with 274 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGES.md
View File

@ -6,9 +6,10 @@ The first two components of the version tags are tied to the ResourceSync specif
version. Versions 0.6.x are intended to implement the v0.6 ResourceSync specification
(http://www.openarchives.org/rs/0.6/),
v0.6.2 2013-05..
- Fix example code in README!
v0.6.2 2013-05-14
- Fixed example code in README!
- Added --version flag
- Added --paths option to specify local paths to search rather than mappings
v0.6.1 2013-05-09
- First release working toward v0.6 specification
@ -17,11 +18,11 @@ v0.6.1 2013-05-09
v0.5.3 2013-05-08
- Final release working with v0.5 specification
- Improve handling of assumed mappings for testing client on a local filesystem
- Fix sitemapindex support for large resource lists, add rel="up" for component sitemaps
- Improved handling of assumed mappings for testing client on a local filesystem
- Fixed sitemapindex support for large resource lists, add rel="up" for component sitemaps
v0.5.2 2013-03-26
- Fix setup.py
- Fixed setup.py
v0.5.1 2013-03-22
- Code reworked for 0.5 specification (http://www.openarchives.org/rs/0.5/)

50
bin/resync
View File

@ -2,7 +2,19 @@
"""
resync: The ResourceSync command line client
Created by Simeon Warner on 2012-04...
Copyright 2012,2013 Simeon Warner
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License
"""
import logging
@ -122,8 +134,7 @@ def main():
# Options and arguments
p = optparse.OptionParser(description='ResourceSync command line client',
usage='usage: %prog [options] uri_path local_path (-h for help)',
version='%prog '+__version__,
add_help_option=False)
version='%prog '+__version__ )
# Modes
# a. modes using remote sitemap/resources
@ -146,8 +157,10 @@ def main():
'These modes act on files on the local disk')
loc.add_option('--resourcelist', '--resource-list', '-r', action='store_true',
help="write a resource list based on files on disk using uri=path mappings "
"in reverse to calculate URIs from the local paths. To STDOUT by "
"default, override with --outfile")
"in reverse to calculate URIs from the local paths. Scans local disk "
"based either on explicit --paths setting, else starting from all local "
"paths specified in the mappings. Writes to STDOUT by default, override "
"with --outfile")
loc.add_option('--changelist', '--change-list', '-c', action='store_true',
help="write a change list based on comparison of a reference sitemap "
"(specify file with --reference) and either files on disk (using "
@ -167,19 +180,22 @@ def main():
# Specification of map between remote URI and local file paths, and remote sitemap
nam = p.add_option_group('FILE/URI NAMING OPTIONS')
nam.add_option('--outfile', type=str, action='store',
help='write sitemap to specified file rather than STDOUT')
help="write sitemap to specified file rather than STDOUT")
nam.add_option('--paths', type=str, action='store',
help="explicit set of paths for disk scan --resourceslist or --changelist "
"generation")
nam.add_option('--sitemap', type=str, action='store',
help='explicitly set sitemap name, overriding default sitemap.xml '
'appended to first source URI specified in the mappings')
help="explicitly set sitemap name, overriding default sitemap.xml "
"appended to first source URI specified in the mappings")
nam.add_option('--reference', type=str, action='store',
help='reference sitemap name for --changelist calculation')
help="reference sitemap name for --changelist calculation")
nam.add_option('--newreference', type=str, action='store',
help='updated reference sitemap name for --changelist calculation')
nam.add_option('--dump', metavar="DUMPFILE", type=str, action='store',
help='write dump to specified file for --resourcelist or --changelist')
help="updated reference sitemap name for --changelist calculation")
nam.add_option('--dump', metavar='DUMPFILE', type=str, action='store',
help="write dump to specified file for --resourcelist or --changelist")
nam.add_option('--changelist-uri','--change-list-uri', type=str, action='store',
help='explicitly set the changelist URI that will be use in --inc mode, '
'overrides process of getting this from the sitemap')
help="explicitly set the changelist URI that will be use in --inc mode, "
"overrides process of getting this from the sitemap")
# Options that apply to multiple modes
opt = p.add_option_group('MISCELANEOUS OPTIONS')
@ -228,8 +244,6 @@ def main():
opt.add_option('--eval', '-e', action='store_true',
help="output evaluation of source/client synchronization performance... "
"be warned, this is very verbose")
opt.add_option('--help', '-h', action='help',
help="this help")
(args, map) = p.parse_args()
@ -296,7 +310,8 @@ def main():
elif (args.explore):
c.explore()
elif (args.resourcelist):
c.write_resource_list(outfile=args.outfile,
c.write_resource_list(paths.args.paths,
outfile=args.outfile,
links=links,
dump=args.dump)
elif (args.changelist):
@ -305,6 +320,7 @@ def main():
c.write_change_list(ref_sitemap=args.reference,
newref_sitemap=( args.newreference if (args.newreference) else None ),
empty=args.empty,
paths=args.paths,
outfile=args.outfile,
links=links,
dump=args.dump)

42
resync/client.py
View File

@ -81,19 +81,24 @@ class Client(object):
return(self.sitemap_name)
return(self.sitemap_uri(self.resource_list_name))
@property
def resource_list(self):
"""Return a resource list for files on disk based on current mappings
def build_resource_list(self, paths=None):
"""Return a resource list for files on local disk
The set of files is taken by disk scan from the paths specified or
else defaults to the paths specified in the current mappings
Return resource_list. Uses existing self.mapper settings.
"""
# 0. Sanity checks
# 0. Sanity checks, parse paths is specified
if (len(self.mapper)<1):
raise ClientFatalError("No source to destination mapping specified")
if (paths is not None):
# Expect comma separated list of paths
paths=paths.split(',')
# 1. Build from disk
rlb = ResourceListBuilder(do_md5=self.checksum,mapper=self.mapper)
rlb = ResourceListBuilder(set_md5=self.checksum,mapper=self.mapper)
rlb.add_exclude_files(self.exclude_patterns)
rl = rlb.from_disk()
rl = rlb.from_disk(paths=paths)
# 2. Set defaults and overrides
rl.allow_multifile = self.allow_multifile
rl.pretty_xml = self.pretty_xml
@ -136,8 +141,7 @@ class Client(object):
self.checksum=False
self.logger.info("Not calculating checksums on destination as not present in source resource_list")
# 1.b destination resource_list mapped back to source URIs
rlb = ResourceListBuilder(mapper=self.mapper)
rlb.do_md5=self.checksum
rlb = ResourceListBuilder(set_md5=self.checksum, mapper=self.mapper)
dst_resource_list = rlb.from_disk()
### 2. Compare these resource_lists respecting any comparison options
(same,updated,deleted,created)=dst_resource_list.compare(src_resource_list)
@ -490,23 +494,31 @@ class Client(object):
caps = ['capabilitylist']
return( options[inp].uri, caps, inp )
def write_resource_list(self,outfile=None,links=None,dump=None):
def write_resource_list(self,paths=None,outfile=None,links=None,dump=None):
"""Write a resource list sitemap for files on local disk
Set of resources included is based on the mappings. Optionally
links can be added. Output will be to stdout unless outfile
Set of resources included is based on paths setting or else the mappings.
Optionally links can be added. Output will be to stdout unless outfile
is specified.
"""
rl = self.resource_list
rl.ln = links
rl = self.build_resource_list(paths=paths)
if (links is not None):
rl.ln = links
if (outfile is None):
print rl.as_xml()
else:
rl.write(basename=outfile)
self.write_dump_if_requested(rl,dump)
def write_change_list(self,outfile=None,ref_sitemap=None,newref_sitemap=None,
def write_change_list(self,paths=None,outfile=None,ref_sitemap=None,newref_sitemap=None,
empty=None,links=None,dump=None):
"""Write a change list
Unless the both ref_sitemap and newref_sitemap are specified then the Change
List is calculated between the reference an the current state of files on
disk. The files on disk are scanned based either on the paths setting or
else on the mappings.
"""
cl = ChangeList(ln=links)
if (not empty):
# 1. Get and parse reference sitemap
@ -515,7 +527,7 @@ class Client(object):
# or build resource_list from files on disk
if (newref_sitemap is None):
# Get resource list from disk
new_rl = self.resource_list
new_rl = self.build_resource_list(paths=paths)
else:
new_rl = self.read_reference_resource_list(newref_sitemap,name='new reference')
# 3. Calculate change list

167
resync/resource_list_builder.py
View File

@ -1,12 +1,16 @@
"""ResourceListBuilder to create ResourceList objects
Currently implements build from files on disk only.
Currently implements build from files on disk only.
Attributes:
- do_md5 set true to calculate MD5 sums for all files
- do_length set true to include file length in resource_list
- set_path set true to add path attribute for each resource
- set_md5 set true to calculate MD5 sums for all files
- set_length set true to include file length in resource_list (defaults true)
- exclude_dirs is a list of directory names to exclude
(defaults to ['CVS','.git'))
FIXME - should add options to set sha1 and sha256 in addition or as
alternatives to md5.
"""
import os
@ -24,16 +28,24 @@ from utils import compute_md5_for_file
class ResourceListBuilder():
def __init__(self, do_md5=False, do_length=True, mapper=None):
def __init__(self, mapper=None, set_md5=False, set_length=True, set_path=False):
"""Create ResourceListBuilder object, optionally set options
Optionaly sets the following attributes:
- do_md5 - True to add md5 digests for each resource
- do_length - False to not add length for each resources
The mapper attribute must be set before a call to from_disk() in order to
map between local filenames and URIs.
The following attributes may be set to determine information added to
each Resource object based on the disk scan:
- set_md5 - True to add md5 digests for each resource. This may add
significant time to the scan process as each file has to be read to
compute the sum
- set_length - False to not add length for each resources
- set_path - True to add local path information for each file/resource
"""
self.do_md5 = do_md5
self.do_length = do_length
self.mapper = mapper
self.set_path = set_path
self.set_md5 = set_md5
self.set_length = set_length
self.exclude_files = ['sitemap\d{0,5}.xml']
self.exclude_dirs = ['CVS','.git']
self.include_symlinks = False
@ -47,6 +59,7 @@ class ResourceListBuilder():
self.exclude_files.append(pattern)
def compile_excludes(self):
"""Compile a set of regexps for files to be exlcuded from scans"""
self.compiled_exclude_files = []
for pattern in self.exclude_files:
self.compiled_exclude_files.append(re.compile(pattern))
@ -58,22 +71,30 @@ class ResourceListBuilder():
return(True)
return(False)
def from_disk(self, resource_list=None, set_path=False):
def from_disk(self, resource_list=None, paths=None):
"""Create or extend resource_list with resources from disk scan
Assumes very simple disk path to URL mapping: chop path and
replace with url_path. Returns the new or extended ResourceList
Assumes very simple disk path to URL mapping (in self.mapping): chop
path and replace with url_path. Returns the new or extended ResourceList
object.
If a resource_list is specified then items are added to that rather
than creating a new one.
If set_path is True then the path attribue will be set with the
local path for each Resource.
If paths is specified then these are used instead of the set
of local paths in self.mapping.
Example usage with mapping start paths:
mapper=Mapper('http://example.org/path','/path/to/files')
rlb = ResourceListBuilder(mapper=mapper)
m = rlb.from_disk()
Example usage with explicit paths:
mapper=Mapper('http://example.org/path','/path/to/files')
mb = ResourceListBuilder(mapper=mapper)
m = resource_list_from_disk()
rlb = ResourceListBuilder(mapper=mapper)
m = rlb.from_disk(paths=['/path/to/files/a','/path/to/files/b'])
"""
num=0
# Either use resource_list passed in or make a new one
@ -81,58 +102,72 @@ class ResourceListBuilder():
resource_list = ResourceList()
# Compile exclude pattern matches
self.compile_excludes()
# Work out start paths from map if not explicitly specified
if (paths is None):
paths=[]
for map in self.mapper.mappings:
paths.append(map.dst_path)
# Run for each map in the mappings
for map in self.mapper.mappings:
self.logger.info("Scanning disk for %s" % (str(map)))
self.from_disk_add_map(resource_list=resource_list, map=map, set_path=set_path)
for path in paths:
self.logger.info("Scanning disk from %s" % (path))
self.from_disk_add_path(path=path, resource_list=resource_list)
return(resource_list)
def from_disk_add_map(self, resource_list=None, map=None, set_path=False):
"""Add to resource_list with resources from disk scan based one map
If set_path is True then the path attribue will be set with the
local path for each Resource.
def from_disk_add_path(self, path=None, resource_list=None):
"""Add to resource_list with resources from disk scan starting at path
"""
# sanity
if (resource_list is None or map is None):
raise ValueError("Must specify resource_list and map")
path=map.dst_path
#print "walking: %s" % (path)
# for each file: create Resource object, add, increment counter
num_files=0
for dirpath, dirs, files in os.walk(path,topdown=True):
for file_in_dirpath in files:
num_files+=1
if (num_files%50000 == 0):
self.logger.info("ResourceListBuilder.from_disk_add_map: %d files..." % (num_files))
try:
if self.exclude_file(file_in_dirpath):
self.logger.debug("Excluding file %s" % (file_in_dirpath))
continue
# get abs filename and also URL
file = os.path.join(dirpath,file_in_dirpath)
if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))):
continue
uri = map.dst_to_src(file)
if (uri is None):
raise Exception("Internal error, mapping failed")
file_stat=os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e)))
continue
timestamp = file_stat.st_mtime #UTC
r = Resource(uri=uri,timestamp=timestamp)
if (set_path):
r.path=file
if (self.do_md5):
# add md5
r.md5=compute_md5_for_file(file)
if (self.do_length):
# add length
r.length=file_stat.st_size
resource_list.add(r)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
self.logger.debug("Excluding dir %s" % (exclude))
dirs.remove(exclude)
if (path is None or resource_list is None or self.mapper is None):
raise ValueError("Must specify path, resource_list and mapper")
# is path a directory or a file? for each file: create Resource object,
# add, increment counter
if os.path.isdir(path):
num_files=0
for dirpath, dirs, files in os.walk(path,topdown=True):
for file_in_dirpath in files:
num_files+=1
if (num_files%50000 == 0):
self.logger.info("ResourceListBuilder.from_disk_add_path: %d files..." % (num_files))
self.add_file(resource_list=resource_list,dir=dirpath,file=file_in_dirpath)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
self.logger.debug("Excluding dir %s" % (exclude))
dirs.remove(exclude)
else:
# single file
self.add_file(resource_list=resource_list,file=path)
def add_file(self, resource_list=None, dir=None, file=None):
"""Add a single file to resource_list
Follows object settings of set_path, set_md5 and set_length.
"""
try:
if self.exclude_file(file):
self.logger.debug("Excluding file %s" % (file))
return
# get abs filename and also URL
if (dir is not None):
file = os.path.join(dir,file)
if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))):
return
uri = self.mapper.dst_to_src(file)
if (uri is None):
raise Exception("Internal error, mapping failed")
file_stat=os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e)))
return
timestamp = file_stat.st_mtime #UTC
r = Resource(uri=uri,timestamp=timestamp)
if (self.set_path):
# add full local path
r.path=file
if (self.set_md5):
# add md5
r.md5=compute_md5_for_file(file)
if (self.set_length):
# add length
r.length=file_stat.st_size
resource_list.add(r)

27
resync/test/test_client.py
View File

@ -28,12 +28,8 @@ class TestClient(unittest.TestCase):
def test01_make_resource_list_empty(self):
c = Client()
# No mapping is error
#
def wrap_resource_list_property_call(c):
# do this because assertRaises( ClientFatalError, c.resource_list ) doesn't work
return(c.resource_list)
self.assertRaises( ClientFatalError, wrap_resource_list_property_call, c )
# No mapping is an error
self.assertRaises( ClientFatalError, c.build_resource_list )
def test02_bad_source_uri(self):
c = Client()
@ -81,6 +77,25 @@ class TestClient(unittest.TestCase):
c.parse_document()
self.assertTrue( re.search(r'Parsed changedump document with 3 entries',capturer.result) )
def test40_write_resource_list(self):
c = Client()
c.set_mappings( ['http://example.org/','resync/test/testdata'] )
# with no explicit paths seting the mapping will be used
with capture_stdout() as capturer:
c.write_resource_list()
self.assertTrue( re.search(r'<rs:md capability="resourcelist" modified="', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir2/file_x</loc>', capturer.result ) )
# with an explicit paths setting only the specified paths will be included
with capture_stdout() as capturer:
c.write_resource_list(paths='resync/test/testdata/dir1')
self.assertTrue( re.search(r'<rs:md capability="resourcelist" modified="', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>2012-07-25T17:13:46Z</lastmod><rs:md length="20" /></url>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>2001-09-09T01:46:40Z</lastmod><rs:md length="45" /></url>', capturer.result ) )
self.assertFalse( re.search(r'dir2', capturer.result ) )
if __name__ == '__main__':
suite = unittest.TestLoader().loadTestsFromTestCase(TestClient)
unittest.TextTestRunner(verbosity=2).run(suite)

108
resync/test/test_resource_list_builder.py
View File

@ -15,40 +15,104 @@ class TestResourceListBuilder(unittest.TestCase):
os.utime( "resync/test/testdata/dir1/file_a", (0, 1343236426 ) )
os.utime( "resync/test/testdata/dir1/file_b", (0, 1000000000 ) )
def test1_simple_output(self):
def test01_simple_scan(self):
rlb = ResourceListBuilder()
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
rl = rlb.from_disk()
rl.md['modified']=None #don't write so we can test output easily
self.assertEqual(rl.as_xml(),'<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/"><rs:md capability="resourcelist" /><url><loc>http://example.org/t/file_a</loc><lastmod>2012-07-25T17:13:46Z</lastmod><rs:md length=\"20\" /></url><url><loc>http://example.org/t/file_b</loc><lastmod>2001-09-09T01:46:40Z</lastmod><rs:md length=\"45\" /></url></urlset>' )
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, 20 )
self.assertEqual( r.path, None )
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, 45 )
self.assertEqual( r.path, None )
def test2_pretty_output(self):
rlb = ResourceListBuilder()
def test02_no_length(self):
rlb = ResourceListBuilder(set_length=False)
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
rl = rlb.from_disk()
rl.md['modified']=None #don't write so we can test output easily
rl.pretty_xml=True
self.assertEqual(rl.as_xml(),'<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\n<rs:md capability="resourcelist" />\n<url><loc>http://example.org/t/file_a</loc><lastmod>2012-07-25T17:13:46Z</lastmod><rs:md length=\"20\" /></url>\n<url><loc>http://example.org/t/file_b</loc><lastmod>2001-09-09T01:46:40Z</lastmod><rs:md length=\"45\" /></url>\n</urlset>' )
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, None )
self.assertEqual( r.path, None )
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, None )
self.assertEqual( r.path, None )
def test3_with_md5(self):
rlb = ResourceListBuilder(do_md5=True)
def test03_set_md5(self):
rlb = ResourceListBuilder(set_md5=True)
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
rl = rlb.from_disk()
xml = rl.as_xml()
self.assertNotEqual( None, re.search('<loc>http://example.org/t/file_a</loc><lastmod>[\w\:\-]+Z</lastmod><rs:md hash=\"md5:a/Jv1mYBtSjS4LR\+qoft/Q==\" length=\"20\" />',xml) ) #must escape + in md5
self.assertNotEqual( None, re.search('<loc>http://example.org/t/file_b</loc><lastmod>[\w\:\-]+Z</lastmod><rs:md hash=\"md5:RS5Uva4WJqxdbnvoGzneIQ==\" length=\"45\" />',xml) )
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' )
self.assertEqual( r.length, 20 )
self.assertEqual( r.path, None )
r = rli.next()
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==' )
self.assertEqual( r.length, 45 )
self.assertEqual( r.path, None )
def test4_data(self):
rlb = ResourceListBuilder(do_md5=True)
def test04_data(self):
rlb = ResourceListBuilder(set_path=True,set_md5=True)
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
rl = rlb.from_disk(set_path=True)
rl = rlb.from_disk()
self.assertEqual( len(rl), 2)
r1 = rl.resources.get('http://example.org/t/file_a')
self.assertTrue( r1 is not None )
self.assertEqual( r1.uri, 'http://example.org/t/file_a' )
self.assertEqual( r1.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r1.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' )
self.assertEqual( r1.path, 'resync/test/testdata/dir1/file_a' )
r = rl.resources.get('http://example.org/t/file_a')
self.assertTrue( r is not None )
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' )
self.assertEqual( r.path, 'resync/test/testdata/dir1/file_a' )
def test05_from_disk_paths(self):
rlb = ResourceListBuilder()
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
# no path, should get no resources
rl = rlb.from_disk(paths=[])
self.assertEqual( len(rl), 0)
# full path, 2 resources
rl = rlb.from_disk(paths=['resync/test/testdata/dir1'])
self.assertEqual( len(rl), 2)
# new object with mapper covering larger space of disk
rlb = ResourceListBuilder(set_path=True)
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata'])
# same path with 2 resources
rl = rlb.from_disk(paths=['resync/test/testdata/dir1'])
self.assertEqual( len(rl), 2)
# same path with 2 resources
rl = rlb.from_disk(paths=['resync/test/testdata/dir1','resync/test/testdata/dir2'])
self.assertEqual( len(rl), 3)
# path that is just a single file
rl = rlb.from_disk(paths=['resync/test/testdata/dir1/file_a'])
self.assertEqual( len(rl), 1)
rli = iter(rl)
r = rli.next()
self.assertTrue( r is not None )
self.assertEqual( r.uri, 'http://example.org/t/dir1/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, 20 )
self.assertEqual( r.path, 'resync/test/testdata/dir1/file_a' )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceListBuilder)