gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working towards real dump implementation (not finished)

Browse Source
This commit is contained in:
Simeon Warner 2014-06-19 09:23:48 -04:00
parent 874f8ba891
commit 0ebf87f174
3 changed files with 112 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
resync/dump.py
View File

@ -2,7 +2,7 @@
import os.path
from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
from sitemap import Sitemap
from resync.resource_dump_manifest import ResourceDumpManifest
class DumpError(Exception):
pass
@ -10,48 +10,59 @@ class DumpError(Exception):
class Dump(object):
"""Dump of content for a Resource Dump or Change Dump
The resource_list must be comprised of Resource objects
The resources itearable must be comprised of Resource objects
which have the path attributes set to indicate the local
location of the copies of the resources.
rl = ResourceList()
# ... add items by whatever means, may have >50k items and/or
# >100MB total size of files ...
d = Dump(rl)
d.write()
"""
def __init__(self, format=None, compress=True):
def __init__(self, resources=None, format=None, compress=True):
self.resources = resources
self.format = ('zip' if (format is None) else format)
self.compress = compress
self.max_size = 100*1024*1024 #100MB
self.max_files = 50000
self.path_prefix = None
def write(self, resource_list=None, dumpfile=None):
"""Wrapper to Write a dump file"""
self.check_files(resource_list)
if (self.format == 'zip'):
self.write_zip(resource_list,dumpfile)
elif (self.format == 'warc'):
self.write_warc(resource_list,dumpfile)
else:
raise DumpError("Unknown dump format '%s'" % (self.format))
def write(self, dumpfile=None):
"""Write one or more dump files to complete this dump"""
self.check_files()
n=0
for manifest in self.partition_dumps():
if (self.format == 'zip'):
self.write_zip(self.resources,dumpfile)
elif (self.format == 'warc'):
self.write_warc(self.resources,dumpfile)
else:
raise DumpError("Unknown dump format requested (%s)" % (self.format))
n+=1
print "Wrote %d dump files" % (n)
def write_zip(self, resource_list=None, dumpfile=None):
def write_zip(self, resources=None, dumpfile=None):
"""Write a ZIP format dump file"""
compression = ( ZIP_DEFLATED if self.compress else ZIP_STORED )
zf = ZipFile(dumpfile, mode="w", compression=compression, allowZip64=True)
# Write resource_list first
s = Sitemap()
# Write resources first
rdm = ResourceDumpManifest(resources=resources)
real_path = {}
for resource in resource_list:
for resource in resources:
archive_path = self.archive_path(resource.path)
real_path[archive_path] = resource.path
resource.path = archive_path
zf.writestr('manifest.xml',s.resources_as_xml(resource_list))
# Add all files in the resource_list
for resource in resource_list:
zf.writestr('manifest.xml',rdm.as_xml())
# Add all files in the resources
for resource in resources:
zf.write(real_path[resource.path], arcname=resource.path)
zf.close()
zipsize = os.path.getsize(dumpfile)
print "Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize)
def write_warc(self, resource_list=None, dumpfile=None):
def write_warc(self, resources=None, dumpfile=None):
"""Write a WARC dump file"""
# Load library late as we want to be able to run rest of code
# without this installed
@ -60,8 +71,8 @@ class Dump(object):
except:
raise DumpError("Failed to load WARC library")
wf = WARCFile(dumpfile, mode="w", compress=self.compress)
# Add all files in the resource_list
for resource in resource_list:
# Add all files in the resources
for resource in resources:
wh = WARCHeader({})
wh.url = resource.uri
wh.ip_address = None
@ -75,17 +86,21 @@ class Dump(object):
warcsize = os.path.getsize(dumpfile)
print "Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize)
def check_files(self,resource_list):
"""Go though and check all files in resource_list, add up size, find common path
def check_files(self,set_length=True,check_length=True):
"""Go though and check all files in self.resources, add up size, and find
longest common path that can be used when writing the dump file. Saved in
self.path_prefix.
Will find the longest common path prefix that can be used when writing the
dump file. Saved in self.path_prefix.
Parameters set_length and check_lenght control control whether then set_length
attribute should be set from the file size if not specified, and whether any
length specified should be checked. By default both are True. In any event, the
total size calculated is the size of files on disk.
"""
if (len(resource_list) > self.max_files):
raise DumpError("Number of files to dump (%d) exceeds maximum (%d)" % (len(resource_list),self.max_files))
if (len(self.resources) > self.max_files):
raise DumpError("Number of files to dump (%d) exceeds maximum (%d)" % (len(resources),self.max_files))
total_size = 0 #total size of all files in bytes
path_prefix = None
for resource in resource_list:
for resource in self.resources:
if (resource.path is None):
#explicit test because exception raised by getsize otherwise confusing
raise DumpError("No file path defined for resource %s" % resource.uri)
@ -93,12 +108,32 @@ class Dump(object):
path_prefix = os.path.dirname(resource.path)
else:
path_prefix = os.path.commonprefix( [ path_prefix, os.path.dirname(resource.path) ])
total_size += os.path.getsize(resource.path)
size = os.path.getsize(resource.path)
if (resource.length is not None):
if (check_length and resource.length!=size):
raise DumpError("Size of resource %s is %d on disk, not %d as specified" %
(resource.uri, size, resource.length) )
elif (set_length):
resource.length = size
total_size += size
self.path_prefix = path_prefix
self.total_size = total_size
print "Total size of files to include in dump %d bytes" % (total_size)
if (total_size > self.max_size):
raise DumpError("Size of files to dump (%d) exceeds maximum (%d)" % (total_size,self.max_size))
return True
def partition_dumps(self):
"""Return a set of manifest object that parition the dumps
"""
manifests=[]
if (self.total_size<=self.max_size and
self.total_files<=self.max_files):
# No need to partition into multiple files
pass #FIXME - fill in
else:
pass #FIXME - fill in
return manifests
def archive_path(self,real_path):
"""Return the archive path for file with real_path

10
resync/resource_dump.py
View File

@ -17,9 +17,19 @@ class ResourceDump(ResourceList):
A Resource Dump comprises a set of content packages that are
the resources listed. Properties similar to a ResourceList
and implemented as a sub-class of ResourceList.
See the Dump class for details of how to create a Resource
Dump from a ResourceList.
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None):
super(ResourceDump, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
mapper=mapper)
self.capability_name='resourcedump'
def write(self, basename="/tmp/resource_dump.xml", dumpfile=None):
"""Write out a Resource Dump document and optionally also the actual dump files
Dump files will be written if the dumpfile argument is set to a base filename.
"""
super(ResourceDump,self).write(basename)

42
resync/test/test_dump.py
View File

@ -1,20 +1,52 @@
import unittest
from resync.dump import Dump, DumpError
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
from resync.resource import Resource
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.change_dump_manifest import ChangeDumpManifest
class TestDump(unittest.TestCase):
def test00_dump_creation(self):
i=ResourceList()
i.add( Resource('http://ex.org/a', length=1, path='resync/test/testdata/a') )
i.add( Resource('http://ex.org/b', length=2, path='resync/test/testdata/b') )
def test00_dump_zip_resource_list(self):
rl=ResourceDumpManifest()
rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') )
rl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b') )
d=Dump()
d.check_files(resource_list=i)
d.write_zip(rl,"test00_dump.zip")
def test01_dump_zip_change_list(self):
cl=ChangeDumpManifest()
cl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a', change="updated") )
cl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b', change="updated") )
d=Dump()
d.write_zip(cl,"test00_dump.zip")
def test02_dump_check_files(self):
cl=ChangeList()
cl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a', change="updated") )
cl.add( Resource('http://ex.org/b', length=21, path='resync/test/testdata/b', change="updated") )
d=Dump(resources=cl)
self.assertTrue(d.check_files())
self.assertEqual(d.total_size, 28)
#FIXME -- need some code to actually write and read dump
def test10_no_path(self):
rl=ResourceList()
rl.add( Resource('http://ex.org/a', length=7, path='resync/test/testdata/a') )
rl.add( Resource('http://ex.org/b', length=21 ) )
d=Dump(rl)
self.assertRaises( DumpError, d.check_files )
def test11_bad_size(self):
rl=ResourceList()
rl.add( Resource('http://ex.org/a', length=9999, path='resync/test/testdata/a') )
d=Dump(rl)
self.assertTrue( d.check_files(check_length=False) )
self.assertRaises( DumpError, d.check_files )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDump)
unittest.TextTestRunner(verbosity=2).run(suite)