Tidy and add tests
This commit is contained in:
Simeon Warner
parent
e3df46b37d
commit
42d9269754
@ -3,7 +3,6 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from defusedxml.ElementTree import parse
|
||||
from xml.etree.ElementTree import ElementTree, Element, tostring
|
||||
@ -35,7 +34,7 @@ class SitemapIndexError(Exception):
|
||||
self.message = message
|
||||
self.etree = etree
|
||||
|
||||
def __repr__(self):
|
||||
def __str__(self):
|
||||
"""Return just the message attribute."""
|
||||
return(self.message)
|
||||
|
||||
@ -93,6 +92,13 @@ class Sitemap(object):
|
||||
self.md_att_keys = ['md_at', 'capability', 'change', 'datetime',
|
||||
'md_completed', 'md_from', 'hash', 'length',
|
||||
'path', 'mime_type', 'md_until']
|
||||
# capabilities
|
||||
self.capabilities = ['resourcelist', 'changelist', 'resourcedump',
|
||||
'changedump', 'resourcedump-manifest',
|
||||
'changedump-manifest', 'capabilitylist',
|
||||
'description',
|
||||
'resourcelist-archive', 'resourcedump-archive',
|
||||
'changelist-archive', 'changedump-archive']
|
||||
if self.spec_1_0:
|
||||
self.md_att_keys.remove('datetime')
|
||||
|
||||
@ -198,17 +204,13 @@ class Sitemap(object):
|
||||
self.resources_created = 0
|
||||
seen_top_level_md = False
|
||||
for e in list(etree.getroot()):
|
||||
# look for <rs:md> and <rs:ln>, first <url> ends
|
||||
# then look for resources in <url> blocks
|
||||
# look for <rs:md> and <rs:ln>, first <url>/<sitemap> ends
|
||||
# then look for resources in <url>/<sitemap> blocks.
|
||||
# ignore any elements we don't recognize
|
||||
if (e.tag == resource_tag):
|
||||
in_preamble = False # any later rs:md or rs:ln is error
|
||||
r = self.resource_from_etree(e, self.resource_class)
|
||||
try:
|
||||
resources.add(r)
|
||||
except SitemapDupeError:
|
||||
self.logger.warning(
|
||||
"dupe of: %s (lastmod=%s)" %
|
||||
(r.uri, r.lastmod))
|
||||
resources.add(r)
|
||||
self.resources_created += 1
|
||||
elif (e.tag == "{" + RS_NS + "}md"):
|
||||
if (in_preamble):
|
||||
@ -227,9 +229,6 @@ class Sitemap(object):
|
||||
else:
|
||||
raise SitemapParseError(
|
||||
"Found <rs:ln> after first <url> in sitemap")
|
||||
else:
|
||||
# element we don't recognize, ignore
|
||||
pass
|
||||
# check that we read to right capability document
|
||||
if (capability is not None):
|
||||
if ('capability' not in resources.md):
|
||||
@ -288,19 +287,11 @@ class Sitemap(object):
|
||||
"""Return string for the resource as part of an XML sitemap.
|
||||
|
||||
Returns a string with the XML snippet representing the resource,
|
||||
without any XML declaration.
|
||||
without any XML declaration. (So much simpler now only Python 3.x
|
||||
supported, see earlier versions for 2.6, 2.7 etc.)
|
||||
"""
|
||||
e = self.resource_etree_element(resource)
|
||||
if (sys.version_info >= (3, 0)):
|
||||
# python3.x
|
||||
return(tostring(e, encoding='unicode', method='xml'))
|
||||
elif (sys.version_info >= (2, 7)):
|
||||
s = tostring(e, encoding='UTF-8', method='xml')
|
||||
else:
|
||||
# must not specify method='xml' in python2.6
|
||||
s = tostring(e, encoding='UTF-8')
|
||||
# Chop off XML declaration that is added in 2.x... sigh
|
||||
return(s.replace("<?xml version='1.0' encoding='UTF-8'?>\n", ''))
|
||||
return(tostring(e, encoding='unicode', method='xml'))
|
||||
|
||||
def resource_from_etree(self, etree, resource_class):
|
||||
"""Construct a Resource from an etree.
|
||||
@ -382,27 +373,20 @@ class Sitemap(object):
|
||||
val = md_element.attrib.get(xml_att, None)
|
||||
if (val is not None):
|
||||
md[att] = val
|
||||
# capability. Allow this to be missing but do a very simple syntax
|
||||
# check on plausible values if present
|
||||
if ('capability' in md):
|
||||
if (re.match(r"^[\w\-]+$", md['capability']) is None):
|
||||
raise SitemapParseError(
|
||||
"Bad capability name '%s' in %s" %
|
||||
(capability, context))
|
||||
# change should be one of defined values
|
||||
if ('change' in md):
|
||||
if (md['change'] not in ['created', 'updated', 'deleted']):
|
||||
self.logger.warning(
|
||||
"Bad change attribute in <rs:md> for %s" %
|
||||
(context))
|
||||
# length should be an integer
|
||||
# capability. Allow this to be missing or a new value but warn if it
|
||||
# isn't recognized
|
||||
if ('capability' in md and md['capability'] not in self.capabilities):
|
||||
self.logger.warning("Unknown capability name '%s' in %s" % (md['capability'], context))
|
||||
# change must be one of defined values
|
||||
if ('change' in md and md['change'] not in ['created', 'updated', 'deleted']):
|
||||
raise SitemapParseError("Bad change attribute in <rs:md> for %s" % (context))
|
||||
# length must be an integer
|
||||
if ('length' in md):
|
||||
try:
|
||||
md['length'] = int(md['length'])
|
||||
except ValueError as e:
|
||||
raise SitemapParseError(
|
||||
"Invalid length element in <rs:md> for %s" %
|
||||
(context))
|
||||
raise SitemapParseError("Invalid length element in <rs:md> for %s" %
|
||||
(context))
|
||||
return(md)
|
||||
|
||||
def ln_from_etree(self, ln_element, context=''):
|
||||
@ -423,9 +407,8 @@ class Sitemap(object):
|
||||
# now do some checks and conversions...
|
||||
# href (MANDATORY)
|
||||
if ('href' not in ln):
|
||||
raise SitemapParseError(
|
||||
"Missing href in <rs:ln> in %s" %
|
||||
(context))
|
||||
raise SitemapParseError("Missing href in <rs:ln> in %s" %
|
||||
(context))
|
||||
# rel (MANDATORY)
|
||||
if ('rel' not in ln):
|
||||
raise SitemapParseError("Missing rel in <rs:ln> in %s" % (context))
|
||||
@ -434,9 +417,8 @@ class Sitemap(object):
|
||||
try:
|
||||
ln['length'] = int(ln['length'])
|
||||
except ValueError as e:
|
||||
raise SitemapParseError(
|
||||
"Invalid length attribute value in <rs:ln> for %s" %
|
||||
(context))
|
||||
raise SitemapParseError("Invalid length attribute value in <rs:ln> for %s" %
|
||||
(context))
|
||||
# pri - priority, must be a number between 1 and 999999
|
||||
if ('pri' in ln):
|
||||
try:
|
||||
|
||||
@ -1,27 +1,24 @@
|
||||
"""Test for resync.sitemap."""
|
||||
|
||||
import io
|
||||
from testfixtures import LogCapture
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
try: # python2
|
||||
# Must try this first as io also exists in python2
|
||||
# but in the wrong one!
|
||||
import StringIO as io
|
||||
except ImportError: # python3
|
||||
import io
|
||||
import xml.etree.ElementTree # for xml.etree.ElementTree.ParseError
|
||||
from defusedxml.ElementTree import parse
|
||||
|
||||
from resync.resource import Resource
|
||||
from resync.resource_list import ResourceList
|
||||
from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError
|
||||
|
||||
# etree gives ParseError in 2.7,3.x; ExpatError in 2.6
|
||||
etree_error_class = None
|
||||
if (sys.version_info < (2, 7)):
|
||||
from xml.parsers.expat import ExpatError
|
||||
etree_error_class = ExpatError
|
||||
else:
|
||||
# In python3 this seems only to work with the full class name??
|
||||
# from xml.etree.ElementTree import ParseError
|
||||
import xml.etree.ElementTree
|
||||
etree_error_class = xml.etree.ElementTree.ParseError
|
||||
|
||||
class TestSitemapIndexError(unittest.TestCase):
|
||||
|
||||
def test_str(self):
|
||||
"""Test str(...) gives just message part."""
|
||||
err = SitemapIndexError("howdy", "this should be the etree")
|
||||
self.assertEqual(str(err), "howdy")
|
||||
|
||||
|
||||
class TestSitemap(unittest.TestCase):
|
||||
@ -115,7 +112,8 @@ class TestSitemap(unittest.TestCase):
|
||||
i = iter(m)
|
||||
self.assertEqual(Sitemap().resources_as_xml(i), "<?xml version='1.0' encoding='UTF-8'?>\n<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:rs=\"http://www.openarchives.org/rs/terms/\"><url><loc>a</loc><lastmod>2001-01-01T00:00:00Z</lastmod><rs:md length=\"1234\" /></url><url><loc>b</loc><lastmod>2002-02-02T00:00:00Z</lastmod><rs:md length=\"56789\" /></url></urlset>")
|
||||
|
||||
def test_10_sitemap(self):
|
||||
def test_10_parse_xml(self):
|
||||
"""Test parse_xml method with string XML."""
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<url><loc>http://e.com/a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md hash="md5:Q2hlY2sgSW50ZWdyaXR5IQ==" length=\"12\" /></url>\
|
||||
@ -130,8 +128,7 @@ class TestSitemap(unittest.TestCase):
|
||||
self.assertEqual(r.lastmod, '2012-03-14T18:37:36Z')
|
||||
self.assertEqual(r.length, 12)
|
||||
self.assertEqual(r.md5, 'Q2hlY2sgSW50ZWdyaXR5IQ==')
|
||||
|
||||
def test_11_parse_2(self):
|
||||
# ..another
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<url><loc>/tmp/rs_test/src/file_a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length=\"12\" /></url>\
|
||||
@ -142,6 +139,54 @@ class TestSitemap(unittest.TestCase):
|
||||
self.assertFalse(s.parsed_index, 'was a sitemap')
|
||||
self.assertEqual(s.resources_created, 2, 'got 2 resources')
|
||||
|
||||
def test_11_parse_xml_error(self):
|
||||
"""Test exceptiona from parse_xml method."""
|
||||
# bad params
|
||||
s = Sitemap()
|
||||
self.assertRaises(ValueError, s.parse_xml)
|
||||
# got a sitemap when told to expect and indexp
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
</urlset>'
|
||||
self.assertRaises(SitemapIndexError, s.parse_xml, fh=io.StringIO(xml), sitemapindex=True)
|
||||
# dupe entries DO NOT create an error
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<url><loc>/mouse</loc><lastmod>2020-12-21T00:00:00Z</lastmod><rs:md length=\"12\" /></url>\
|
||||
<url><loc>/mouse</loc><lastmod>2020-12-21T00:00:00Z</lastmod><rs:md length=\"12\" /></url>\
|
||||
</urlset>'
|
||||
s = Sitemap()
|
||||
i = s.parse_xml(fh=io.StringIO(xml))
|
||||
self.assertEqual(len(i.resources), 2)
|
||||
# preamble rs:md after <url> is error
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<url><loc>/frog</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
|
||||
<rs:md capability=\"resourcelist\"/>\
|
||||
<url><loc>/toad</loc><lastmod>2020-12-21T00:02:00Z</lastmod><rs:md length=\"8\" /></url>\
|
||||
</urlset>'
|
||||
self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
|
||||
# preamble rs:ln after <url> is also error
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<url><loc>/wills</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
|
||||
<rs:ln rel="up" href="http://example.com/resourcesync_description.xml"/>\
|
||||
</urlset>'
|
||||
s = Sitemap()
|
||||
self.assertRaises(SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
|
||||
# but random unknown junk should be ignored...
|
||||
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<junk1>beetle</junk1>\
|
||||
<rs:md capability=\"resourcelist\"/>\
|
||||
<junk2>fly</junk2>\
|
||||
<url><loc>/whale</loc><lastmod>2020-12-21T00:01:00Z</lastmod><rs:md length=\"5\" /></url>\
|
||||
<junk3>ant</junk3>\
|
||||
</urlset>'
|
||||
s = Sitemap()
|
||||
i = s.parse_xml(fh=io.StringIO(xml))
|
||||
self.assertEqual(len(i.resources), 1)
|
||||
|
||||
def test_12_parse_multi_loc(self):
|
||||
xml_start = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
@ -192,9 +237,9 @@ class TestSitemap(unittest.TestCase):
|
||||
def test_15_parse_illformed(self):
|
||||
s = Sitemap()
|
||||
# ExpatError in python2.6, ParserError in 2.7,3.x
|
||||
self.assertRaises(etree_error_class, s.parse_xml,
|
||||
self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml,
|
||||
io.StringIO('not xml'))
|
||||
self.assertRaises(etree_error_class, s.parse_xml,
|
||||
self.assertRaises(xml.etree.ElementTree.ParseError, s.parse_xml,
|
||||
io.StringIO('<urlset><url>something</urlset>'))
|
||||
|
||||
def test_16_parse_valid_xml_but_other(self):
|
||||
@ -326,3 +371,65 @@ class TestSitemap(unittest.TestCase):
|
||||
r2 = next(i)
|
||||
self.assertEqual(r2.uri, '/tmp/rs_test/src/file_b')
|
||||
self.assertEqual(r2.change, None)
|
||||
|
||||
def test_31_resource_from_etree(self):
|
||||
"""Test resource_from_etree method."""
|
||||
# multiple <loc>
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<loc>a_name</loc>
|
||||
<loc>another_name_oops</loc>
|
||||
</url>'''
|
||||
et = parse(io.StringIO(xml))
|
||||
self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
|
||||
# no <loc>
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<no_loc_element />
|
||||
</url>'''
|
||||
et = parse(io.StringIO(xml))
|
||||
self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
|
||||
# muktiple <rs:md> not allowed
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<loc>a_name</loc>
|
||||
<rs:md type="text/plain" />
|
||||
<rs:md something="123" />
|
||||
</url>'''
|
||||
et = parse(io.StringIO(xml))
|
||||
self.assertRaises(SitemapParseError, Sitemap().resource_from_etree, et, Resource)
|
||||
# warn is hash invalid
|
||||
with LogCapture() as lc:
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
|
||||
<loc>a_name</loc>
|
||||
<rs:md hash="UNKNOWN-TYPE:blah" />
|
||||
</url>'''
|
||||
et = parse(io.StringIO(xml))
|
||||
Sitemap().resource_from_etree(et, Resource)
|
||||
self.assertIn('Ignored unsupported hash type (UNKNOWN-TYPE)', lc.records[-1].msg)
|
||||
|
||||
def test_32_md_from_etree(self):
|
||||
"""Test md_from_etree method."""
|
||||
# Warning for unknwon capability
|
||||
with LogCapture() as lc:
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" capability="WHY" />
|
||||
'''
|
||||
et = parse(io.StringIO(xml)).getroot()
|
||||
Sitemap().md_from_etree(et)
|
||||
self.assertIn("Unknown capability name 'WHY'", lc.records[-1].msg)
|
||||
# Bad value for change is an error
|
||||
with LogCapture() as lc:
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" change="BAD" />
|
||||
'''
|
||||
et = parse(io.StringIO(xml)).getroot()
|
||||
self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et)
|
||||
# length must be an integer
|
||||
with LogCapture() as lc:
|
||||
xml = '''<?xml version=\'1.0\' encoding=\'UTF-8\'?>
|
||||
<rs:md xmlns:rs="http://www.openarchives.org/rs/terms/" length="short" />
|
||||
'''
|
||||
et = parse(io.StringIO(xml)).getroot()
|
||||
self.assertRaises(SitemapParseError, Sitemap().md_from_etree, et)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
