diff --git a/resync/client.py b/resync/client.py index dbb4d30..be94a38 100644 --- a/resync/client.py +++ b/resync/client.py @@ -56,6 +56,7 @@ class Client(object): self.max_sitemap_entries = None self.ignore_failures = False self.status_file = '.resync-client-status.cfg' + self.pretty_xml = True def set_mappings(self,mappings): """Build and set Mapper object based on input mappings""" @@ -82,17 +83,24 @@ class Client(object): @property def resource_list(self): - """Return resource_list on disk based on current mappings + """Return a resource list for files on disk based on current mappings Return resource_list. Uses existing self.mapper settings. """ - ### 0. Sanity checks + # 0. Sanity checks if (len(self.mapper)<1): raise ClientFatalError("No source to destination mapping specified") - ### 1. Build from disk + # 1. Build from disk rlb = ResourceListBuilder(do_md5=self.checksum,mapper=self.mapper) rlb.add_exclude_files(self.exclude_patterns) - return( rlb.from_disk() ) + rl = rlb.from_disk() + # 2. Set defaults and overrides + rl.allow_multifile = self.allow_multifile + rl.pretty_xml = self.pretty_xml + rl.mapper = self.mapper + if (self.max_sitemap_entries is not None): + rl.max_sitemap_entries = self.max_sitemap_entries + return(rl) def log_event(self, change): """Log a Resource object as an event for automated analysis""" @@ -484,19 +492,17 @@ class Client(object): def write_resource_list(self,outfile=None,links=None,dump=None): """Write a resource list sitemap for files on local disk - based on the base_path->base_uri mappings. + + Set of resources included is based on the mappings. Optionally + links can be added. Output will be to stdout unless outfile + is specified. """ rl = self.resource_list rl.ln = links - kwargs = { 'pretty_xml': True, - 'allow_multifile': self.allow_multifile, - 'mapper' : self.mapper } - if (self.max_sitemap_entries is not None): - kwargs['max_sitemap_entries'] = self.max_sitemap_entries if (outfile is None): - print rl.as_xml(**kwargs) + print rl.as_xml() else: - rl.write(basename=outfile,**kwargs) + rl.write(basename=outfile) self.write_dump_if_requested(rl,dump) def write_change_list(self,outfile=None,ref_sitemap=None,newref_sitemap=None, @@ -518,39 +524,39 @@ class Client(object): cl.add_changed_resources( deleted, change='deleted' ) cl.add_changed_resources( created, change='created' ) # 4. Write out change list - kwargs = { 'pretty_xml': True, - 'mapper' : self.mapper } + cl.mapper = self.mapper + cl.pretty_xml = self.pretty_xml if (self.max_sitemap_entries is not None): - kwargs['max_sitemap_entries'] = self.max_sitemap_entries + cl.max_sitemap_entries = self.max_sitemap_entries if (outfile is None): - print cl.as_xml(**kwargs) + print cl.as_xml() else: - cl.write(basename=outfile,**kwargs) + cl.write(basename=outfile) self.write_dump_if_requested(cl,dump) def write_capability_list(self,capabilities=None,outfile=None,links=None): """Write a Capability List to outfile or STDOUT""" capl = CapabilityList(ln=links) + capl.pretty_xml = self.pretty_xml if (capabilities is not None): for name in capabilities.keys(): capl.add_capability(name=name, uri=capabilities[name]) - kwargs = { 'pretty_xml': True } if (outfile is None): - print capl.as_xml(**kwargs) + print capl.as_xml() else: - capl.write(basename=outfile,**kwargs) + capl.write(basename=outfile) def write_capability_list_index(self,capability_lists=None,outfile=None,links=None): """Write a Capability List to outfile or STDOUT""" capli = CapabilityListIndex(ln=links) + capli.pretty_xml = self.pretty_xml if (capability_lists is not None): for uri in capability_lists: capli.add_capability_list(uri) - kwargs = { 'pretty_xml': True } if (outfile is None): - print capli.as_xml(**kwargs) + print capli.as_xml() else: - capli.write(basename=outfile,**kwargs) + capli.write(basename=outfile) def write_dump_if_requested(self,resource_list,dump): if (dump is None): diff --git a/resync/list_base.py b/resync/list_base.py index df2e772..226e985 100644 --- a/resync/list_base.py +++ b/resync/list_base.py @@ -26,6 +26,7 @@ class ListBase(ResourceContainer): self.capability_name = 'unknown' self.capability_md = 'unknown' self.sitemapindex = False + self.pretty_xml = False # self.logger = logging.getLogger('list_base') self.bytes_read = 0 @@ -41,15 +42,15 @@ class ListBase(ResourceContainer): ##### INPUT ##### - def read(self,uri=None,**kwargs): + def read(self,uri=None): """Default case is just to parse document at this URI Intention is that the read() method may be overridden to support reading of compound documents in more then one sitemapindex/sitemap. """ - self.parse(uri=uri,**kwargs) + self.parse(uri=uri) - def parse(self,uri=None,fh=None,**kwargs): + def parse(self,uri=None,fh=None): """Parse a single XML document for this list Does not handle the case of sitemapindex+sitemaps ResourceList @@ -61,29 +62,35 @@ class ListBase(ResourceContainer): raise Exception("Failed to load sitemap/sitemapindex from %s (%s)" % (uri,str(e))) if (fh is None): raise Exception("Nothing to parse") - s = Sitemap(**kwargs) + s = self.new_sitemap() s.parse_xml(fh=fh,resources=self,capability=self.capability_md,sitemapindex=False) self.parsed_index = s.parsed_index ##### OUTPUT ##### - def as_xml(self,**kwargs): + def as_xml(self): """Return XML serialization of this list This code does not support the case where the list is too big for a single XML document. """ self.default_capability_and_modified() - s = Sitemap(**kwargs) + s = self.new_sitemap() return s.resources_as_xml(self,sitemapindex=self.sitemapindex) - def write(self,basename="/tmp/resynclist.xml",**kwargs): + def write(self,basename="/tmp/resynclist.xml"): """Write a single sitemap or sitemapindex XML document Must be overridden to support multi-file lists. """ self.default_capability_and_modified() fh = open(basename,'w') - s = Sitemap(**kwargs) + s = self.new_sitemap() s.resources_as_xml(self,fh=fh,sitemapindex=self.sitemapindex) fh.close() + + ##### UTILITY ##### + + def new_sitemap(self): + """Create new Sitemap object with default settings""" + return Sitemap(pretty_xml=self.pretty_xml) diff --git a/resync/list_base_with_index.py b/resync/list_base_with_index.py index 59c9678..c706af2 100644 --- a/resync/list_base_with_index.py +++ b/resync/list_base_with_index.py @@ -36,67 +36,7 @@ class ListBaseWithIndex(ListBase): self.num_files = 0 # Number of files read self.bytes_read = 0 # Aggregate of content_length values - ##### General sitemap methods that also handle sitemapindexes ##### - - def write(self, basename='/tmp/sitemap.xml', **kwargs): - """Write one or a set of sitemap files to disk - - resources is a ResourceContainer that may be an ResourceList or - a ChangeList. This may be a generator so data is read as needed - and length is determined at the end. - - basename is used as the name of the single sitemap file or the - sitemapindex for a set of sitemap files. - - Uses self.max_sitemap_entries to determine whether the resource_list can - be written as one sitemap. If there are more entries and - self.allow_multifile is set true then a set of sitemap files, - with an sitemapindex, will be written. - """ - # Access resources through iterator only - resources_iter = iter(self.resources) - ( chunk, next ) = self.get_resources_chunk(resources_iter) - s = Sitemap(**kwargs) - if (next is not None): - # Have more than self.max_sitemap_entries => sitemapindex - if (not self.allow_multifile): - raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled") - # Work out how to name the sitemaps, attempt to add %05d before ".xml$", else append - sitemap_prefix = basename - sitemap_suffix = '.xml' - if (basename[-4:] == '.xml'): - sitemap_prefix = basename[:-4] - # Use iterator over all resources and count off sets of - # max_sitemap_entries to go into each sitemap, store the - # names of the sitemaps as we go - sitemaps=ListBase() - while (len(chunk)>0): - file = sitemap_prefix + ( "%05d" % (len(sitemaps)) ) + sitemap_suffix - self.logger.info("Writing sitemap %s..." % (file)) - f = open(file, 'w') - s.resources_as_xml(chunk, fh=f) - f.close() - # Record information about this sitemap for index - r = Resource( uri = self.mapper.dst_to_src(file), - path = file, - timestamp = os.stat(file).st_mtime, - md5 = compute_md5_for_file(file) ) - sitemaps.add(r) - # Get next chunk - ( chunk, next ) = self.get_resources_chunk(resources_iter,next) - self.logger.info("Wrote %d sitemaps" % (len(sitemaps))) - f = open(basename, 'w') - self.logger.info("Writing sitemapindex %s..." % (basename)) - s.resources_as_xml(resources=sitemaps,sitemapindex=True,fh=f) - f.close() - self.logger.info("Wrote sitemapindex %s" % (basename)) - else: - f = open(basename, 'w') - self.logger.info("Writing sitemap %s..." % (basename)) - s.resources_as_xml(chunk, fh=f) - f.close() - self.logger.info("Wrote sitemap %s" % (basename)) - + ##### INPUT ##### def read(self, uri=None, resources=None, capability=None, index_only=False): """Read sitemap from a URI including handling sitemapindexes @@ -124,7 +64,7 @@ class ListBaseWithIndex(ListBase): self.logger.debug( "Read ????? bytes from %s" % (uri) ) pass self.logger.info( "Read sitemap/sitemapindex from %s" % (uri) ) - s = Sitemap() + s = self.new_sitemap() s.parse_xml(fh=fh,resources=self,capability='resourcelist') # what did we read? sitemap or sitemapindex? if (s.parsed_index): @@ -150,6 +90,8 @@ class ListBaseWithIndex(ListBase): def read_component_sitemap(self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file): """Read a component sitemap of a Resource List with index + + Each component must be a sitemap with the """ if (sitemapindex_is_file): if (not self.is_file_uri(sitemap_uri)): @@ -176,19 +118,110 @@ class ListBaseWithIndex(ListBase): # If we don't get a length then c'est la vie pass self.logger.info( "Reading sitemap from %s (%d bytes)" % (sitemap_uri,self.content_length) ) - sitemap.parse_xml( fh=fh, resources=self.resources, sitemapindex=False ) + component = sitemap.parse_xml( fh=fh, sitemapindex=False ) + # Copy resources into self, check any metadata + for r in component: + self.resources.add(r) + # FIXME - if rel="up" check it goes to correct place + # FIXME - check capability + ##### OUTPUT ##### - def index_as_xml(self,**kwargs): + def as_xml(self): + """Return XML serialization of this list + + A single XML serailization does not make sense in the case that the list + resources is more than is allowed in a single sitemap so will raise an + exception if that is the case. Otherwise passes to superclass method. + """ + if (self.max_sitemap_entries is not None): + if (len(self)>self.max_sitemap_entries): + raise ListBaseIndexError("Attempt to write single XLM string for list with %d entries when max_sitemap_entries is set to %d" % (len(self),self.max_sitemap_entries)) + return super(ListBaseWithIndex, self).as_xml() + + def write(self, basename='/tmp/sitemap.xml'): + """Write one or a set of sitemap files to disk + + resources is a ResourceContainer that may be an ResourceList or + a ChangeList. This may be a generator so data is read as needed + and length is determined at the end. + + basename is used as the name of the single sitemap file or the + sitemapindex for a set of sitemap files. + + Uses self.max_sitemap_entries to determine whether the resource_list can + be written as one sitemap. If there are more entries and + self.allow_multifile is set true then a set of sitemap files, + with an sitemapindex, will be written. + """ + # Access resources through iterator only + resources_iter = iter(self.resources) + ( chunk, next ) = self.get_resources_chunk(resources_iter) + s = self.new_sitemap() + if (next is not None): + # Have more than self.max_sitemap_entries => sitemapindex + if (not self.allow_multifile): + raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled") + # Work out how to name the sitemaps, attempt to add %05d before ".xml$", else append + sitemap_prefix = basename + sitemap_suffix = '.xml' + if (basename[-4:] == '.xml'): + sitemap_prefix = basename[:-4] + # Work out URI of sitemapindex so that we can link up to + # it from the individual sitemap files + try: + index_uri = self.mapper.dst_to_src(basename) + except MapperError as e: + raise ListBaseIndexError("Cannot map sitemapindex filename to URI (%s)" % str(e)) + # Use iterator over all resources and count off sets of + # max_sitemap_entries to go into each sitemap, store the + # names of the sitemaps as we go + index=ListBase() + index.capability_name = self.capability_name + index.capability_md = self.capability_md + index.default_capability_and_modified() + while (len(chunk)>0): + file = sitemap_prefix + ( "%05d" % (len(index)) ) + sitemap_suffix + # Check that we can map the filename of this sitemap into + # URI space for the sitemapindex + try: + uri = self.mapper.dst_to_src(file) + except MapperError as e: + raise ListBaseIndexError("Cannot map sitemap filename to URI (%s)" % str(e)) + self.logger.info("Writing sitemap %s..." % (file)) + f = open(file, 'w') + chunk.ln.append({'rel': 'up', 'href': index_uri}) + s.resources_as_xml(chunk, fh=f) + f.close() + # Record information about this sitemap for index + r = Resource( uri = uri, path = file, + timestamp = os.stat(file).st_mtime, + md5 = compute_md5_for_file(file) ) + index.add(r) + # Get next chunk + ( chunk, next ) = self.get_resources_chunk(resources_iter,next) + self.logger.info("Wrote %d sitemaps" % (len(index))) + f = open(basename, 'w') + self.logger.info("Writing sitemapindex %s..." % (basename)) + s.resources_as_xml(index,sitemapindex=True,fh=f) + f.close() + self.logger.info("Wrote sitemapindex %s" % (basename)) + else: + f = open(basename, 'w') + self.logger.info("Writing sitemap %s..." % (basename)) + s.resources_as_xml(chunk, fh=f) + f.close() + self.logger.info("Wrote sitemap %s" % (basename)) + + def index_as_xml(self): """Return XML serialization of this list taken to be sitemapindex entries """ self.default_capability_and_modified() - s = Sitemap(**kwargs) + s = self.new_sitemap() return s.resources_as_xml(self,sitemapindex=True) - - ##### Utility ##### + ##### Utility ##### def get_resources_chunk(self, resource_iter, first=None): """Return next chunk of resources from resource_iter, and next item diff --git a/resync/sitemap.py b/resync/sitemap.py index 5ffec44..face3b4 100644 --- a/resync/sitemap.py +++ b/resync/sitemap.py @@ -63,7 +63,9 @@ class Sitemap(object): """Write or return XML for a set of resources in sitemap format Arguments: - - resources - either an iterable or iterator of Resource objects. + - resources - either an iterable or iterator of Resource objects; + if there an md attribute this will go to + if there an ln attribute this will go to - sitemapindex - set True to write sitemapindex instead of sitemap - fh - write to filehandle fh instead of returning string """ diff --git a/resync/test/test_resource_list_builder.py b/resync/test/test_resource_list_builder.py index 2da6e72..58a3c34 100644 --- a/resync/test/test_resource_list_builder.py +++ b/resync/test/test_resource_list_builder.py @@ -27,7 +27,8 @@ class TestResourceListBuilder(unittest.TestCase): rlb.mapper = Mapper(['http://example.org/t','resync/test/testdata/dir1']) rl = rlb.from_disk() rl.md['modified']=None #don't write so we can test output easily - self.assertEqual(rl.as_xml(pretty_xml=True),'\n\n\nhttp://example.org/t/file_a2012-07-25T17:13:46Z\nhttp://example.org/t/file_b2001-09-09T01:46:40Z\n' ) + rl.pretty_xml=True + self.assertEqual(rl.as_xml(),'\n\n\nhttp://example.org/t/file_a2012-07-25T17:13:46Z\nhttp://example.org/t/file_b2001-09-09T01:46:40Z\n' ) def test3_with_md5(self): rlb = ResourceListBuilder(do_md5=True)