diff --git a/resync/sitemap.py b/resync/sitemap.py index 1610d8f..0cf5271 100644 --- a/resync/sitemap.py +++ b/resync/sitemap.py @@ -3,7 +3,6 @@ import io import logging import os -import re import sys from defusedxml.ElementTree import parse from xml.etree.ElementTree import ElementTree, Element, tostring @@ -35,7 +34,7 @@ class SitemapIndexError(Exception): self.message = message self.etree = etree - def __repr__(self): + def __str__(self): """Return just the message attribute.""" return(self.message) @@ -93,6 +92,13 @@ class Sitemap(object): self.md_att_keys = ['md_at', 'capability', 'change', 'datetime', 'md_completed', 'md_from', 'hash', 'length', 'path', 'mime_type', 'md_until'] + # capabilities + self.capabilities = ['resourcelist', 'changelist', 'resourcedump', + 'changedump', 'resourcedump-manifest', + 'changedump-manifest', 'capabilitylist', + 'description', + 'resourcelist-archive', 'resourcedump-archive', + 'changelist-archive', 'changedump-archive'] if self.spec_1_0: self.md_att_keys.remove('datetime') @@ -198,17 +204,13 @@ class Sitemap(object): self.resources_created = 0 seen_top_level_md = False for e in list(etree.getroot()): - # look for and , first ends - # then look for resources in blocks + # look for and , first / ends + # then look for resources in / blocks. + # ignore any elements we don't recognize if (e.tag == resource_tag): in_preamble = False # any later rs:md or rs:ln is error r = self.resource_from_etree(e, self.resource_class) - try: - resources.add(r) - except SitemapDupeError: - self.logger.warning( - "dupe of: %s (lastmod=%s)" % - (r.uri, r.lastmod)) + resources.add(r) self.resources_created += 1 elif (e.tag == "{" + RS_NS + "}md"): if (in_preamble): @@ -227,9 +229,6 @@ class Sitemap(object): else: raise SitemapParseError( "Found after first in sitemap") - else: - # element we don't recognize, ignore - pass # check that we read to right capability document if (capability is not None): if ('capability' not in resources.md): @@ -288,19 +287,11 @@ class Sitemap(object): """Return string for the resource as part of an XML sitemap. Returns a string with the XML snippet representing the resource, - without any XML declaration. + without any XML declaration. (So much simpler now only Python 3.x + supported, see earlier versions for 2.6, 2.7 etc.) """ e = self.resource_etree_element(resource) - if (sys.version_info >= (3, 0)): - # python3.x - return(tostring(e, encoding='unicode', method='xml')) - elif (sys.version_info >= (2, 7)): - s = tostring(e, encoding='UTF-8', method='xml') - else: - # must not specify method='xml' in python2.6 - s = tostring(e, encoding='UTF-8') - # Chop off XML declaration that is added in 2.x... sigh - return(s.replace("\n", '')) + return(tostring(e, encoding='unicode', method='xml')) def resource_from_etree(self, etree, resource_class): """Construct a Resource from an etree. @@ -382,27 +373,20 @@ class Sitemap(object): val = md_element.attrib.get(xml_att, None) if (val is not None): md[att] = val - # capability. Allow this to be missing but do a very simple syntax - # check on plausible values if present - if ('capability' in md): - if (re.match(r"^[\w\-]+$", md['capability']) is None): - raise SitemapParseError( - "Bad capability name '%s' in %s" % - (capability, context)) - # change should be one of defined values - if ('change' in md): - if (md['change'] not in ['created', 'updated', 'deleted']): - self.logger.warning( - "Bad change attribute in for %s" % - (context)) - # length should be an integer + # capability. Allow this to be missing or a new value but warn if it + # isn't recognized + if ('capability' in md and md['capability'] not in self.capabilities): + self.logger.warning("Unknown capability name '%s' in %s" % (md['capability'], context)) + # change must be one of defined values + if ('change' in md and md['change'] not in ['created', 'updated', 'deleted']): + raise SitemapParseError("Bad change attribute in for %s" % (context)) + # length must be an integer if ('length' in md): try: md['length'] = int(md['length']) except ValueError as e: - raise SitemapParseError( - "Invalid length element in for %s" % - (context)) + raise SitemapParseError("Invalid length element in for %s" % + (context)) return(md) def ln_from_etree(self, ln_element, context=''): @@ -423,9 +407,8 @@ class Sitemap(object): # now do some checks and conversions... # href (MANDATORY) if ('href' not in ln): - raise SitemapParseError( - "Missing href in in %s" % - (context)) + raise SitemapParseError("Missing href in in %s" % + (context)) # rel (MANDATORY) if ('rel' not in ln): raise SitemapParseError("Missing rel in in %s" % (context)) @@ -434,9 +417,8 @@ class Sitemap(object): try: ln['length'] = int(ln['length']) except ValueError as e: - raise SitemapParseError( - "Invalid length attribute value in for %s" % - (context)) + raise SitemapParseError("Invalid length attribute value in for %s" % + (context)) # pri - priority, must be a number between 1 and 999999 if ('pri' in ln): try: diff --git a/tests/test_sitemap.py b/tests/test_sitemap.py index 5e393e0..55582ee 100644 --- a/tests/test_sitemap.py +++ b/tests/test_sitemap.py @@ -1,27 +1,24 @@ +"""Test for resync.sitemap.""" + +import io +from testfixtures import LogCapture import re import sys import unittest -try: # python2 - # Must try this first as io also exists in python2 - # but in the wrong one! - import StringIO as io -except ImportError: # python3 - import io +import xml.etree.ElementTree # for xml.etree.ElementTree.ParseError +from defusedxml.ElementTree import parse from resync.resource import Resource from resync.resource_list import ResourceList from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError -# etree gives ParseError in 2.7,3.x; ExpatError in 2.6 -etree_error_class = None -if (sys.version_info < (2, 7)): - from xml.parsers.expat import ExpatError - etree_error_class = ExpatError -else: - # In python3 this seems only to work with the full class name?? - # from xml.etree.ElementTree import ParseError - import xml.etree.ElementTree - etree_error_class = xml.etree.ElementTree.ParseError + +class TestSitemapIndexError(unittest.TestCase): + + def test_str(self): + """Test str(...) gives just message part.""" + err = SitemapIndexError("howdy", "this should be the etree") + self.assertEqual(str(err), "howdy") class TestSitemap(unittest.TestCase): @@ -115,7 +112,8 @@ class TestSitemap(unittest.TestCase): i = iter(m) self.assertEqual(Sitemap().resources_as_xml(i), "\na2001-01-01T00:00:00Zb2002-02-02T00:00:00Z") - def test_10_sitemap(self): + def test_10_parse_xml(self): + """Test parse_xml method with string XML.""" xml = '\n\ \ http://e.com/a2012-03-14T18:37:36Z\ @@ -130,8 +128,7 @@ class TestSitemap(unittest.TestCase): self.assertEqual(r.lastmod, '2012-03-14T18:37:36Z') self.assertEqual(r.length, 12) self.assertEqual(r.md5, 'Q2hlY2sgSW50ZWdyaXR5IQ==') - - def test_11_parse_2(self): + # ..another xml = '\n\ \ /tmp/rs_test/src/file_a2012-03-14T18:37:36Z\ @@ -142,6 +139,54 @@ class TestSitemap(unittest.TestCase): self.assertFalse(s.parsed_index, 'was a sitemap') self.assertEqual(s.resources_created, 2, 'got 2 resources') + def test_11_parse_xml_error(self): + """Test exceptiona from parse_xml method.""" + # bad params + s = Sitemap() + self.assertRaises(ValueError, s.parse_xml) + # got a sitemap when told to expect and indexp + xml = '\n\ +\ +' + self.assertRaises(SitemapIndexError, s.parse_xml, fh=io.StringIO(xml), sitemapindex=True) + # dupe entries DO NOT create an error + xml = '\n\ +\ +/mouse2020-12-21T00:00:00Z\ +/mouse2020-12-21T00:00:00Z\ +' + s = Sitemap() + i = s.parse_xml(fh=io.StringIO(xml)) + self.assertEqual(len(i.resources), 2) + # preamble rs:md after is error + xml = '\n\ +\ +/frog2020-12-21T00:01:00Z\ +\ +/toad2020-12-21T00:02:00Z\ +' + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + # preamble rs:ln after is also error + xml = '\n\ +\ +/wills2020-12-21T00:01:00Z\ +\ +' + s = Sitemap() + self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml)) + # but random unknown junk should be ignored... + xml = '\n\ +\ +beetle\ +\ +fly\ +/whale2020-12-21T00:01:00Z\ +ant\ +' + s = Sitemap() + i = s.parse_xml(fh=io.StringIO(xml)) + self.assertEqual(len(i.resources), 1) + def test_12_parse_multi_loc(self): xml_start = '\n\ \ @@ -192,9 +237,9 @@ class TestSitemap(unittest.TestCase): def test_15_parse_illformed(self): s = Sitemap() # ExpatError in python2.6, ParserError in 2.7,3.x - self.assertRaises(etree_error_class, s.parse_xml, + self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml, io.StringIO('not xml')) - self.assertRaises(etree_error_class, s.parse_xml, + self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml, io.StringIO('something')) def test_16_parse_valid_xml_but_other(self): @@ -326,3 +371,65 @@ class TestSitemap(unittest.TestCase): r2 = next(i) self.assertEqual(r2.uri, '/tmp/rs_test/src/file_b') self.assertEqual(r2.change, None) + + def test_31_resource_from_etree(self): + """Test resource_from_etree method.""" + # multiple + xml = ''' +\ +a_name +another_name_oops +''' + et = parse(io.StringIO(xml)) + self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource) + # no + xml = ''' +\ + +''' + et = parse(io.StringIO(xml)) + self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource) + # muktiple not allowed + xml = ''' +\ +a_name + + +''' + et = parse(io.StringIO(xml)) + self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource) + # warn is hash invalid + with LogCapture() as lc: + xml = ''' +\ +a_name + +''' + et = parse(io.StringIO(xml)) + Sitemap().resource_from_etree(et, Resource) + self.assertIn('Ignored unsupported hash type (UNKNOWN-TYPE)', lc.records[-1].msg) + + def test_32_md_from_etree(self): + """Test md_from_etree method.""" + # Warning for unknwon capability + with LogCapture() as lc: + xml = ''' + +''' + et = parse(io.StringIO(xml)).getroot() + Sitemap().md_from_etree(et) + self.assertIn("Unknown capability name 'WHY'", lc.records[-1].msg) + # Bad value for change is an error + with LogCapture() as lc: + xml = ''' + +''' + et = parse(io.StringIO(xml)).getroot() + self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et) + # length must be an integer + with LogCapture() as lc: + xml = ''' + +''' + et = parse(io.StringIO(xml)).getroot() + self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et)