Fix writing of multi file resource list, add up relation to component sitemaps, tidy use of pretty_xml
This commit is contained in:
Simeon Warner
parent
d791d54787
commit
d433202095
@ -56,6 +56,7 @@ class Client(object):
|
||||
self.max_sitemap_entries = None
|
||||
self.ignore_failures = False
|
||||
self.status_file = '.resync-client-status.cfg'
|
||||
self.pretty_xml = True
|
||||
|
||||
def set_mappings(self,mappings):
|
||||
"""Build and set Mapper object based on input mappings"""
|
||||
@ -82,17 +83,24 @@ class Client(object):
|
||||
|
||||
@property
|
||||
def resource_list(self):
|
||||
"""Return resource_list on disk based on current mappings
|
||||
"""Return a resource list for files on disk based on current mappings
|
||||
|
||||
Return resource_list. Uses existing self.mapper settings.
|
||||
"""
|
||||
### 0. Sanity checks
|
||||
# 0. Sanity checks
|
||||
if (len(self.mapper)<1):
|
||||
raise ClientFatalError("No source to destination mapping specified")
|
||||
### 1. Build from disk
|
||||
# 1. Build from disk
|
||||
rlb = ResourceListBuilder(do_md5=self.checksum,mapper=self.mapper)
|
||||
rlb.add_exclude_files(self.exclude_patterns)
|
||||
return( rlb.from_disk() )
|
||||
rl = rlb.from_disk()
|
||||
# 2. Set defaults and overrides
|
||||
rl.allow_multifile = self.allow_multifile
|
||||
rl.pretty_xml = self.pretty_xml
|
||||
rl.mapper = self.mapper
|
||||
if (self.max_sitemap_entries is not None):
|
||||
rl.max_sitemap_entries = self.max_sitemap_entries
|
||||
return(rl)
|
||||
|
||||
def log_event(self, change):
|
||||
"""Log a Resource object as an event for automated analysis"""
|
||||
@ -484,19 +492,17 @@ class Client(object):
|
||||
|
||||
def write_resource_list(self,outfile=None,links=None,dump=None):
|
||||
"""Write a resource list sitemap for files on local disk
|
||||
based on the base_path->base_uri mappings.
|
||||
|
||||
Set of resources included is based on the mappings. Optionally
|
||||
links can be added. Output will be to stdout unless outfile
|
||||
is specified.
|
||||
"""
|
||||
rl = self.resource_list
|
||||
rl.ln = links
|
||||
kwargs = { 'pretty_xml': True,
|
||||
'allow_multifile': self.allow_multifile,
|
||||
'mapper' : self.mapper }
|
||||
if (self.max_sitemap_entries is not None):
|
||||
kwargs['max_sitemap_entries'] = self.max_sitemap_entries
|
||||
if (outfile is None):
|
||||
print rl.as_xml(**kwargs)
|
||||
print rl.as_xml()
|
||||
else:
|
||||
rl.write(basename=outfile,**kwargs)
|
||||
rl.write(basename=outfile)
|
||||
self.write_dump_if_requested(rl,dump)
|
||||
|
||||
def write_change_list(self,outfile=None,ref_sitemap=None,newref_sitemap=None,
|
||||
@ -518,39 +524,39 @@ class Client(object):
|
||||
cl.add_changed_resources( deleted, change='deleted' )
|
||||
cl.add_changed_resources( created, change='created' )
|
||||
# 4. Write out change list
|
||||
kwargs = { 'pretty_xml': True,
|
||||
'mapper' : self.mapper }
|
||||
cl.mapper = self.mapper
|
||||
cl.pretty_xml = self.pretty_xml
|
||||
if (self.max_sitemap_entries is not None):
|
||||
kwargs['max_sitemap_entries'] = self.max_sitemap_entries
|
||||
cl.max_sitemap_entries = self.max_sitemap_entries
|
||||
if (outfile is None):
|
||||
print cl.as_xml(**kwargs)
|
||||
print cl.as_xml()
|
||||
else:
|
||||
cl.write(basename=outfile,**kwargs)
|
||||
cl.write(basename=outfile)
|
||||
self.write_dump_if_requested(cl,dump)
|
||||
|
||||
def write_capability_list(self,capabilities=None,outfile=None,links=None):
|
||||
"""Write a Capability List to outfile or STDOUT"""
|
||||
capl = CapabilityList(ln=links)
|
||||
capl.pretty_xml = self.pretty_xml
|
||||
if (capabilities is not None):
|
||||
for name in capabilities.keys():
|
||||
capl.add_capability(name=name, uri=capabilities[name])
|
||||
kwargs = { 'pretty_xml': True }
|
||||
if (outfile is None):
|
||||
print capl.as_xml(**kwargs)
|
||||
print capl.as_xml()
|
||||
else:
|
||||
capl.write(basename=outfile,**kwargs)
|
||||
capl.write(basename=outfile)
|
||||
|
||||
def write_capability_list_index(self,capability_lists=None,outfile=None,links=None):
|
||||
"""Write a Capability List to outfile or STDOUT"""
|
||||
capli = CapabilityListIndex(ln=links)
|
||||
capli.pretty_xml = self.pretty_xml
|
||||
if (capability_lists is not None):
|
||||
for uri in capability_lists:
|
||||
capli.add_capability_list(uri)
|
||||
kwargs = { 'pretty_xml': True }
|
||||
if (outfile is None):
|
||||
print capli.as_xml(**kwargs)
|
||||
print capli.as_xml()
|
||||
else:
|
||||
capli.write(basename=outfile,**kwargs)
|
||||
capli.write(basename=outfile)
|
||||
|
||||
def write_dump_if_requested(self,resource_list,dump):
|
||||
if (dump is None):
|
||||
|
||||
@ -26,6 +26,7 @@ class ListBase(ResourceContainer):
|
||||
self.capability_name = 'unknown'
|
||||
self.capability_md = 'unknown'
|
||||
self.sitemapindex = False
|
||||
self.pretty_xml = False
|
||||
#
|
||||
self.logger = logging.getLogger('list_base')
|
||||
self.bytes_read = 0
|
||||
@ -41,15 +42,15 @@ class ListBase(ResourceContainer):
|
||||
|
||||
##### INPUT #####
|
||||
|
||||
def read(self,uri=None,**kwargs):
|
||||
def read(self,uri=None):
|
||||
"""Default case is just to parse document at this URI
|
||||
|
||||
Intention is that the read() method may be overridden to support reading
|
||||
of compound documents in more then one sitemapindex/sitemap.
|
||||
"""
|
||||
self.parse(uri=uri,**kwargs)
|
||||
self.parse(uri=uri)
|
||||
|
||||
def parse(self,uri=None,fh=None,**kwargs):
|
||||
def parse(self,uri=None,fh=None):
|
||||
"""Parse a single XML document for this list
|
||||
|
||||
Does not handle the case of sitemapindex+sitemaps ResourceList
|
||||
@ -61,29 +62,35 @@ class ListBase(ResourceContainer):
|
||||
raise Exception("Failed to load sitemap/sitemapindex from %s (%s)" % (uri,str(e)))
|
||||
if (fh is None):
|
||||
raise Exception("Nothing to parse")
|
||||
s = Sitemap(**kwargs)
|
||||
s = self.new_sitemap()
|
||||
s.parse_xml(fh=fh,resources=self,capability=self.capability_md,sitemapindex=False)
|
||||
self.parsed_index = s.parsed_index
|
||||
|
||||
##### OUTPUT #####
|
||||
|
||||
def as_xml(self,**kwargs):
|
||||
def as_xml(self):
|
||||
"""Return XML serialization of this list
|
||||
|
||||
This code does not support the case where the list is too big for
|
||||
a single XML document.
|
||||
"""
|
||||
self.default_capability_and_modified()
|
||||
s = Sitemap(**kwargs)
|
||||
s = self.new_sitemap()
|
||||
return s.resources_as_xml(self,sitemapindex=self.sitemapindex)
|
||||
|
||||
def write(self,basename="/tmp/resynclist.xml",**kwargs):
|
||||
def write(self,basename="/tmp/resynclist.xml"):
|
||||
"""Write a single sitemap or sitemapindex XML document
|
||||
|
||||
Must be overridden to support multi-file lists.
|
||||
"""
|
||||
self.default_capability_and_modified()
|
||||
fh = open(basename,'w')
|
||||
s = Sitemap(**kwargs)
|
||||
s = self.new_sitemap()
|
||||
s.resources_as_xml(self,fh=fh,sitemapindex=self.sitemapindex)
|
||||
fh.close()
|
||||
|
||||
##### UTILITY #####
|
||||
|
||||
def new_sitemap(self):
|
||||
"""Create new Sitemap object with default settings"""
|
||||
return Sitemap(pretty_xml=self.pretty_xml)
|
||||
|
||||
@ -36,67 +36,7 @@ class ListBaseWithIndex(ListBase):
|
||||
self.num_files = 0 # Number of files read
|
||||
self.bytes_read = 0 # Aggregate of content_length values
|
||||
|
||||
##### General sitemap methods that also handle sitemapindexes #####
|
||||
|
||||
def write(self, basename='/tmp/sitemap.xml', **kwargs):
|
||||
"""Write one or a set of sitemap files to disk
|
||||
|
||||
resources is a ResourceContainer that may be an ResourceList or
|
||||
a ChangeList. This may be a generator so data is read as needed
|
||||
and length is determined at the end.
|
||||
|
||||
basename is used as the name of the single sitemap file or the
|
||||
sitemapindex for a set of sitemap files.
|
||||
|
||||
Uses self.max_sitemap_entries to determine whether the resource_list can
|
||||
be written as one sitemap. If there are more entries and
|
||||
self.allow_multifile is set true then a set of sitemap files,
|
||||
with an sitemapindex, will be written.
|
||||
"""
|
||||
# Access resources through iterator only
|
||||
resources_iter = iter(self.resources)
|
||||
( chunk, next ) = self.get_resources_chunk(resources_iter)
|
||||
s = Sitemap(**kwargs)
|
||||
if (next is not None):
|
||||
# Have more than self.max_sitemap_entries => sitemapindex
|
||||
if (not self.allow_multifile):
|
||||
raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled")
|
||||
# Work out how to name the sitemaps, attempt to add %05d before ".xml$", else append
|
||||
sitemap_prefix = basename
|
||||
sitemap_suffix = '.xml'
|
||||
if (basename[-4:] == '.xml'):
|
||||
sitemap_prefix = basename[:-4]
|
||||
# Use iterator over all resources and count off sets of
|
||||
# max_sitemap_entries to go into each sitemap, store the
|
||||
# names of the sitemaps as we go
|
||||
sitemaps=ListBase()
|
||||
while (len(chunk)>0):
|
||||
file = sitemap_prefix + ( "%05d" % (len(sitemaps)) ) + sitemap_suffix
|
||||
self.logger.info("Writing sitemap %s..." % (file))
|
||||
f = open(file, 'w')
|
||||
s.resources_as_xml(chunk, fh=f)
|
||||
f.close()
|
||||
# Record information about this sitemap for index
|
||||
r = Resource( uri = self.mapper.dst_to_src(file),
|
||||
path = file,
|
||||
timestamp = os.stat(file).st_mtime,
|
||||
md5 = compute_md5_for_file(file) )
|
||||
sitemaps.add(r)
|
||||
# Get next chunk
|
||||
( chunk, next ) = self.get_resources_chunk(resources_iter,next)
|
||||
self.logger.info("Wrote %d sitemaps" % (len(sitemaps)))
|
||||
f = open(basename, 'w')
|
||||
self.logger.info("Writing sitemapindex %s..." % (basename))
|
||||
s.resources_as_xml(resources=sitemaps,sitemapindex=True,fh=f)
|
||||
f.close()
|
||||
self.logger.info("Wrote sitemapindex %s" % (basename))
|
||||
else:
|
||||
f = open(basename, 'w')
|
||||
self.logger.info("Writing sitemap %s..." % (basename))
|
||||
s.resources_as_xml(chunk, fh=f)
|
||||
f.close()
|
||||
self.logger.info("Wrote sitemap %s" % (basename))
|
||||
|
||||
##### INPUT #####
|
||||
|
||||
def read(self, uri=None, resources=None, capability=None, index_only=False):
|
||||
"""Read sitemap from a URI including handling sitemapindexes
|
||||
@ -124,7 +64,7 @@ class ListBaseWithIndex(ListBase):
|
||||
self.logger.debug( "Read ????? bytes from %s" % (uri) )
|
||||
pass
|
||||
self.logger.info( "Read sitemap/sitemapindex from %s" % (uri) )
|
||||
s = Sitemap()
|
||||
s = self.new_sitemap()
|
||||
s.parse_xml(fh=fh,resources=self,capability='resourcelist')
|
||||
# what did we read? sitemap or sitemapindex?
|
||||
if (s.parsed_index):
|
||||
@ -150,6 +90,8 @@ class ListBaseWithIndex(ListBase):
|
||||
|
||||
def read_component_sitemap(self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
|
||||
"""Read a component sitemap of a Resource List with index
|
||||
|
||||
Each component must be a sitemap with the
|
||||
"""
|
||||
if (sitemapindex_is_file):
|
||||
if (not self.is_file_uri(sitemap_uri)):
|
||||
@ -176,19 +118,110 @@ class ListBaseWithIndex(ListBase):
|
||||
# If we don't get a length then c'est la vie
|
||||
pass
|
||||
self.logger.info( "Reading sitemap from %s (%d bytes)" % (sitemap_uri,self.content_length) )
|
||||
sitemap.parse_xml( fh=fh, resources=self.resources, sitemapindex=False )
|
||||
component = sitemap.parse_xml( fh=fh, sitemapindex=False )
|
||||
# Copy resources into self, check any metadata
|
||||
for r in component:
|
||||
self.resources.add(r)
|
||||
# FIXME - if rel="up" check it goes to correct place
|
||||
# FIXME - check capability
|
||||
|
||||
##### OUTPUT #####
|
||||
|
||||
def index_as_xml(self,**kwargs):
|
||||
def as_xml(self):
|
||||
"""Return XML serialization of this list
|
||||
|
||||
A single XML serailization does not make sense in the case that the list
|
||||
resources is more than is allowed in a single sitemap so will raise an
|
||||
exception if that is the case. Otherwise passes to superclass method.
|
||||
"""
|
||||
if (self.max_sitemap_entries is not None):
|
||||
if (len(self)>self.max_sitemap_entries):
|
||||
raise ListBaseIndexError("Attempt to write single XLM string for list with %d entries when max_sitemap_entries is set to %d" % (len(self),self.max_sitemap_entries))
|
||||
return super(ListBaseWithIndex, self).as_xml()
|
||||
|
||||
def write(self, basename='/tmp/sitemap.xml'):
|
||||
"""Write one or a set of sitemap files to disk
|
||||
|
||||
resources is a ResourceContainer that may be an ResourceList or
|
||||
a ChangeList. This may be a generator so data is read as needed
|
||||
and length is determined at the end.
|
||||
|
||||
basename is used as the name of the single sitemap file or the
|
||||
sitemapindex for a set of sitemap files.
|
||||
|
||||
Uses self.max_sitemap_entries to determine whether the resource_list can
|
||||
be written as one sitemap. If there are more entries and
|
||||
self.allow_multifile is set true then a set of sitemap files,
|
||||
with an sitemapindex, will be written.
|
||||
"""
|
||||
# Access resources through iterator only
|
||||
resources_iter = iter(self.resources)
|
||||
( chunk, next ) = self.get_resources_chunk(resources_iter)
|
||||
s = self.new_sitemap()
|
||||
if (next is not None):
|
||||
# Have more than self.max_sitemap_entries => sitemapindex
|
||||
if (not self.allow_multifile):
|
||||
raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled")
|
||||
# Work out how to name the sitemaps, attempt to add %05d before ".xml$", else append
|
||||
sitemap_prefix = basename
|
||||
sitemap_suffix = '.xml'
|
||||
if (basename[-4:] == '.xml'):
|
||||
sitemap_prefix = basename[:-4]
|
||||
# Work out URI of sitemapindex so that we can link up to
|
||||
# it from the individual sitemap files
|
||||
try:
|
||||
index_uri = self.mapper.dst_to_src(basename)
|
||||
except MapperError as e:
|
||||
raise ListBaseIndexError("Cannot map sitemapindex filename to URI (%s)" % str(e))
|
||||
# Use iterator over all resources and count off sets of
|
||||
# max_sitemap_entries to go into each sitemap, store the
|
||||
# names of the sitemaps as we go
|
||||
index=ListBase()
|
||||
index.capability_name = self.capability_name
|
||||
index.capability_md = self.capability_md
|
||||
index.default_capability_and_modified()
|
||||
while (len(chunk)>0):
|
||||
file = sitemap_prefix + ( "%05d" % (len(index)) ) + sitemap_suffix
|
||||
# Check that we can map the filename of this sitemap into
|
||||
# URI space for the sitemapindex
|
||||
try:
|
||||
uri = self.mapper.dst_to_src(file)
|
||||
except MapperError as e:
|
||||
raise ListBaseIndexError("Cannot map sitemap filename to URI (%s)" % str(e))
|
||||
self.logger.info("Writing sitemap %s..." % (file))
|
||||
f = open(file, 'w')
|
||||
chunk.ln.append({'rel': 'up', 'href': index_uri})
|
||||
s.resources_as_xml(chunk, fh=f)
|
||||
f.close()
|
||||
# Record information about this sitemap for index
|
||||
r = Resource( uri = uri, path = file,
|
||||
timestamp = os.stat(file).st_mtime,
|
||||
md5 = compute_md5_for_file(file) )
|
||||
index.add(r)
|
||||
# Get next chunk
|
||||
( chunk, next ) = self.get_resources_chunk(resources_iter,next)
|
||||
self.logger.info("Wrote %d sitemaps" % (len(index)))
|
||||
f = open(basename, 'w')
|
||||
self.logger.info("Writing sitemapindex %s..." % (basename))
|
||||
s.resources_as_xml(index,sitemapindex=True,fh=f)
|
||||
f.close()
|
||||
self.logger.info("Wrote sitemapindex %s" % (basename))
|
||||
else:
|
||||
f = open(basename, 'w')
|
||||
self.logger.info("Writing sitemap %s..." % (basename))
|
||||
s.resources_as_xml(chunk, fh=f)
|
||||
f.close()
|
||||
self.logger.info("Wrote sitemap %s" % (basename))
|
||||
|
||||
def index_as_xml(self):
|
||||
"""Return XML serialization of this list taken to be sitemapindex entries
|
||||
|
||||
"""
|
||||
self.default_capability_and_modified()
|
||||
s = Sitemap(**kwargs)
|
||||
s = self.new_sitemap()
|
||||
return s.resources_as_xml(self,sitemapindex=True)
|
||||
|
||||
|
||||
##### Utility #####
|
||||
##### Utility #####
|
||||
|
||||
def get_resources_chunk(self, resource_iter, first=None):
|
||||
"""Return next chunk of resources from resource_iter, and next item
|
||||
|
||||
@ -63,7 +63,9 @@ class Sitemap(object):
|
||||
"""Write or return XML for a set of resources in sitemap format
|
||||
|
||||
Arguments:
|
||||
- resources - either an iterable or iterator of Resource objects.
|
||||
- resources - either an iterable or iterator of Resource objects;
|
||||
if there an md attribute this will go to <rs:md>
|
||||
if there an ln attribute this will go to <rs:ln>
|
||||
- sitemapindex - set True to write sitemapindex instead of sitemap
|
||||
- fh - write to filehandle fh instead of returning string
|
||||
"""
|
||||
|
||||
@ -27,7 +27,8 @@ class TestResourceListBuilder(unittest.TestCase):
|
||||
rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1'])
|
||||
rl = rlb.from_disk()
|
||||
rl.md['modified']=None #don't write so we can test output easily
|
||||
self.assertEqual(rl.as_xml(pretty_xml=True),'<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\n<rs:md capability="resourcelist" />\n<url><loc>http://example.org/t/file_a</loc><lastmod>2012-07-25T17:13:46Z</lastmod><rs:md length=\"20\" /></url>\n<url><loc>http://example.org/t/file_b</loc><lastmod>2001-09-09T01:46:40Z</lastmod><rs:md length=\"45\" /></url>\n</urlset>' )
|
||||
rl.pretty_xml=True
|
||||
self.assertEqual(rl.as_xml(),'<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\n<rs:md capability="resourcelist" />\n<url><loc>http://example.org/t/file_a</loc><lastmod>2012-07-25T17:13:46Z</lastmod><rs:md length=\"20\" /></url>\n<url><loc>http://example.org/t/file_b</loc><lastmod>2001-09-09T01:46:40Z</lastmod><rs:md length=\"45\" /></url>\n</urlset>' )
|
||||
|
||||
def test3_with_md5(self):
|
||||
rlb = ResourceListBuilder(do_md5=True)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
