gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First pass at python3 compatibility

Browse Source
This commit is contained in:
Simeon Warner 2016-03-04 16:55:10 -05:00
parent 5f6bae8bf3
commit 28dd1bd7d9
40 changed files with 432 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,6 +2,7 @@
build
dist
MANIFEST
.eggs
# Testing
.coverage
htmlcov
@ -18,5 +19,6 @@ resync.egg-info
.DS_store
*.pyc
*~
.cache
rc
re

21
resync/__init__.py
View File

@ -1,16 +1,15 @@
from _version import __version__
from resync._version import __version__
"""Enable easy import for core classes, e.g.
from resync import Resource
from import Resource
"""
from source_description import SourceDescription
from capability_list import CapabilityList
from resource_list import ResourceList
from change_list import ChangeList
from resource_dump import ResourceDump
from resource_dump_manifest import ResourceDumpManifest
from change_dump import ChangeDump
from resync.source_description import SourceDescription
from resync.capability_list import CapabilityList
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
from resync.resource_dump import ResourceDump
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.change_dump import ChangeDump
from archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive
from resource import Resource
from resync.archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive
from resync.resource import Resource

11
resync/archives.py
View File

@ -9,11 +9,14 @@ Resource Dump Archive, and Change Dump Archive.
"""
import collections
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from list_base_with_index import ListBaseWithIndex
from resource import Resource
from sitemap import Sitemap
from .list_base_with_index import ListBaseWithIndex
from .resource import Resource
from .sitemap import Sitemap
class ResourceListArchive(ListBaseWithIndex):
"""Class representing an Change List Archive"""

8
resync/capability_list.py
View File

@ -7,10 +7,10 @@ and links like other lists.
import collections
from resource import Resource
from resource_set import ResourceSet
from list_base import ListBase
from sitemap import Sitemap
from .resource import Resource
from .resource_set import ResourceSet
from .list_base import ListBase
from .sitemap import Sitemap
class CapabilitySet(ResourceSet):
"""Class for storage of resources in a Capability List

2
resync/change_dump.py
View File

@ -10,7 +10,7 @@ Described in specification at:
http://www.openarchives.org/rs/resourcesync#ChangeDump
"""
from resource_list import ResourceList
from .resource_list import ResourceList
class ChangeDump(ResourceList):
"""Class representing an Change Dump

2
resync/change_dump_manifest.py
View File

@ -11,7 +11,7 @@ Described in specification at:
http://www.openarchives.org/rs/resourcesync#ChangeDumpManifest
"""
from change_list import ChangeList
from .change_list import ChangeList
class ChangeDumpManifest(ChangeList):
"""Class representing a Change Dump Manifest

11
resync/change_list.py
View File

@ -16,11 +16,14 @@ particular resource.
"""
import collections
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from list_base_with_index import ListBaseWithIndex
from resource import Resource,ChangeTypeError
from sitemap import Sitemap
from .list_base_with_index import ListBaseWithIndex
from .resource import Resource,ChangeTypeError
from .sitemap import Sitemap
class ChangeList(ListBaseWithIndex):
"""Class representing an Change List"""

106
resync/client.py
View File

@ -1,8 +1,12 @@
"""ResourceSync client implementation"""
import sys
import urllib
import urlparse
try: #python3
from urllib.request import urlopen, urlretrieve
from urllib.parse import urlparse, urlunparse
except ImportError: #python2
from urllib import urlopen,urlretrieve
from urlparse import urlparse, urlunparse
import os.path
import datetime
import distutils.dir_util
@ -11,20 +15,26 @@ import time
import logging
import requests
from resync.resource_list_builder import ResourceListBuilder
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
from resync.capability_list import CapabilityList
from resync.source_description import SourceDescription
from resync.mapper import Mapper
from resync.sitemap import Sitemap
from resync.dump import Dump
from resync.resource import Resource
from resync.url_authority import UrlAuthority
from resync.utils import compute_md5_for_file
from resync.client_state import ClientState
from resync.list_base_with_index import ListBaseIndexError
from w3c_datetime import str_to_datetime,datetime_to_str
from .resource_list_builder import ResourceListBuilder
from .resource_list import ResourceList
from .change_list import ChangeList
from .capability_list import CapabilityList
from .source_description import SourceDescription
from .mapper import Mapper
from .sitemap import Sitemap
from .dump import Dump
from .resource import Resource
from .url_authority import UrlAuthority
from .utils import compute_md5_for_file
from .client_state import ClientState
from .list_base_with_index import ListBaseIndexError
from .w3c_datetime import str_to_datetime,datetime_to_str
def url_or_file_open(uri):
"""Wrapper around urlopen() to prepend file: if no scheme provided"""
if (not re.match(r'''\w+:''',uri)):
uri = 'file:'+uri
return(urlopen(uri))
class ClientFatalError(Exception):
"""Non-recoverable error in client, should include message to user"""
@ -206,7 +216,7 @@ class Client(object):
self.logger.debug("Completed %s" % (action))
def incremental(self, allow_deletion=False, change_list_uri=None, from_datetime=None):
"""Incremental synchronization
"""Incremental synchronization
Use Change List to do incremental sync
"""
@ -352,7 +362,7 @@ class Client(object):
else:
# 1. GET
try:
urllib.urlretrieve(resource.uri,file)
urlretrieve(resource.uri,file)
num_updated+=1
except IOError as e:
msg = "Failed to GET %s -- %s" % (resource.uri,str(e))
@ -421,14 +431,14 @@ class Client(object):
s=Sitemap()
self.logger.info("Reading sitemap(s) from %s ..." % (self.sitemap))
try:
list = s.parse_xml(urllib.urlopen(self.sitemap))
list = s.parse_xml(url_or_file_open(self.sitemap))
except IOError as e:
raise ClientFatalError("Cannot read document (%s)" % str(e))
num_entries = len(list.resources)
capability = '(unknown capability)'
if ('capability' in list.md):
capability = list.md['capability']
print "Parsed %s document with %d entries" % (capability,num_entries)
print("Parsed %s document with %d entries" % (capability,num_entries))
if (self.verbose):
to_show = 100
override_str = ' (override with --max-sitemap-entries)'
@ -436,10 +446,10 @@ class Client(object):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries>to_show):
print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
print("Showing first %d entries sorted by URI%s..." % (to_show,override_str))
n=0
for resource in list:
print '[%d] %s' % (n,str(resource))
print('[%d] %s' % (n,str(resource)))
n+=1
if ( n >= to_show ):
break
@ -453,12 +463,12 @@ class Client(object):
uri = None
if (self.sitemap_name is not None):
uri = self.sitemap
print "Taking location from --sitemap option"
print("Taking location from --sitemap option")
acceptable_capabilities = None #ie. any
elif (len(self.mapper)>0):
pu = urlparse.urlparse(self.mapper.default_src_uri())
uri = urlparse.urlunparse( [ pu[0], pu[1], '/.well-known/resourcesync', '', '', '' ] )
print "Will look for discovery information based on mappings"
pu = urlparse(self.mapper.default_src_uri())
uri = urlunparse( [ pu[0], pu[1], '/.well-known/resourcesync', '', '', '' ] )
print("Will look for discovery information based on mappings")
acceptable_capabilities = [ 'capabilitylist', 'capabilitylistindex' ]
else:
raise ClientFatalError("Neither explicit sitemap nor mapping specified")
@ -466,7 +476,7 @@ class Client(object):
inp = None
checks = None
while (inp!='q'):
print
print()
if (inp=='b'):
if (len(history)<2):
break #can't do this, exit
@ -475,7 +485,7 @@ class Client(object):
acceptable_capabilities=None
history.append(uri)
(uri,checks,acceptable_capabilities,inp) = self.explore_uri(uri,checks,acceptable_capabilities,len(history)>1)
print "--explore done, bye..."
print("--explore done, bye...")
def explore_uri(self, uri, checks, caps, show_back=True):
"""Interactive exploration of document at uri
@ -483,20 +493,20 @@ class Client(object):
Will flag warnings if the document is not of type listed in caps
"""
s=Sitemap()
print "Reading %s" % (uri)
print("Reading %s" % (uri))
options={}
capability=None
try:
if (caps=='resource'):
self.explore_show_head(uri,check_headers=checks)
else:
list = s.parse_xml(urllib.urlopen(uri))
list = s.parse_xml(url_or_file_open(uri))
(options,capability)=self.explore_show_summary(list,s.parsed_index,caps)
except IOError as e:
print "Cannot read %s (%s)\nGoing back" % (uri,str(e))
print("Cannot read %s (%s)\nGoing back" % (uri,str(e)))
return('','','','b')
except Exception as e:
print "Cannot parse %s (%s)\nGoing back" % (uri,str(e))
print("Cannot parse %s (%s)\nGoing back" % (uri,str(e)))
return('','','','b')
while (True):
# don't offer number option for no resources/capabilities
@ -537,9 +547,9 @@ class Client(object):
capability = list.md['capability']
if (parsed_index):
capability += 'index'
print "Parsed %s document with %d entries:" % (capability,num_entries)
print("Parsed %s document with %d entries:" % (capability,num_entries))
if (caps is not None and capability not in caps):
print "WARNING - expected a %s document" % (','.join(caps))
print("WARNING - expected a %s document" % (','.join(caps)))
to_show = num_entries
if (num_entries>21):
to_show = 20
@ -556,22 +566,22 @@ class Client(object):
n=0
if ('up' in list.ln):
options['up']=list.ln['up']
print "[%s] %s" % ('up',list.ln['up'].uri)
print("[%s] %s" % ('up',list.ln['up'].uri))
for r in list.resources:
if (n>=to_show):
print "(not showing remaining %d entries)" % (num_entries-n)
print("(not showing remaining %d entries)" % (num_entries-n))
break
n+=1
options[str(n)]=r
print "[%d] %s" % (n,r.uri)
print("[%d] %s" % (n,r.uri))
if (r.capability is not None):
warning = ''
if (r.capability not in entry_caps):
warning = " (EXPECTED %s)" % (' or '.join(entry_caps))
print " %s%s" % (r.capability,warning)
print(" %s%s" % (r.capability,warning))
elif (len(entry_caps)==1):
r.capability=entry_caps[0]
print " capability not specified, should be %s" % (r.capability)
print(" capability not specified, should be %s" % (r.capability))
return(options,capability)
def explore_show_head(self,uri,check_headers=None):
@ -580,9 +590,9 @@ class Client(object):
Will also check headers against any values specified in
check_headers.
"""
print "HEAD %s" % (uri)
print("HEAD %s" % (uri))
response = requests.head(uri)
print " status: %s" % (response.status_code)
print(" status: %s" % (response.status_code))
# generate normalized lastmod
# if ('last-modified' in response.headers):
# response.headers.add['lastmod'] = datetime_to_str(str_to_datetime(response.headers['last-modified']))
@ -596,7 +606,7 @@ class Client(object):
check_str=' MATCHES EXPECTED VALUE'
else:
check_STR=' EXPECTED %s' % (check_headers[header])
print " %s: %s%s" % (header, response.headers[header], check_str)
print(" %s: %s%s" % (header, response.headers[header], check_str))
def write_resource_list(self,paths=None,outfile=None,links=None,dump=None):
"""Write a Resource List or a Resource Dump for files on local disk
@ -620,7 +630,7 @@ class Client(object):
else:
if (outfile is None):
try:
print rl.as_xml()
print(rl.as_xml())
except ListBaseIndexError as e:
raise ClientFatalError("%s. Use --output option to specify base name for output files." % str(e))
else:
@ -657,7 +667,7 @@ class Client(object):
if (self.max_sitemap_entries is not None):
cl.max_sitemap_entries = self.max_sitemap_entries
if (outfile is None):
print cl.as_xml()
print(cl.as_xml())
else:
cl.write(basename=outfile)
self.write_dump_if_requested(cl,dump)
@ -670,7 +680,7 @@ class Client(object):
for name in capabilities.keys():
capl.add_capability(name=name, uri=capabilities[name])
if (outfile is None):
print capl.as_xml()
print(capl.as_xml())
else:
capl.write(basename=outfile)
@ -682,7 +692,7 @@ class Client(object):
for uri in capability_lists:
rsd.add_capability_list(uri)
if (outfile is None):
print rsd.as_xml()
print(rsd.as_xml())
else:
rsd.write(basename=outfile)
@ -710,10 +720,10 @@ class Client(object):
to_show = self.max_sitemap_entries
override_str = ''
if (num_entries>to_show):
print "Showing first %d entries sorted by URI%s..." % (to_show,override_str)
print("Showing first %d entries sorted by URI%s..." % (to_show,override_str))
n=0
for r in rl.resources:
print r
print(r)
n+=1
if ( n >= to_show ):
break

7
resync/client_state.py
View File

@ -6,15 +6,16 @@ of the last change seen.
"""
import sys
import urllib
import urlparse
import os.path
import datetime
import distutils.dir_util
import re
import time
import logging
import ConfigParser
try: #python3
from configparser import ConfigParser
except ImportError: #python2
import ConfigParser
class ClientState(object):

77
resync/explorer.py
View File

@ -7,7 +7,10 @@ look for and interpret capabilities.
import sys
import urllib
import urlparse
try: #python3
from urllib.parse import urlparse, urlunparse, urljoin
except ImportError: #python2
from urlparse import urlparse, urlunparse, urljoin
import os.path
import datetime
import distutils.dir_util
@ -16,12 +19,12 @@ import time
import logging
import requests
from resync.mapper import Mapper
from resync.sitemap import Sitemap
from resync.client import Client,ClientFatalError
from resync.client_state import ClientState
from resync.resource import Resource
from w3c_datetime import str_to_datetime,datetime_to_str
from .mapper import Mapper
from .sitemap import Sitemap
from .client import Client,ClientFatalError
from .client_state import ClientState
from .resource import Resource
from .w3c_datetime import str_to_datetime,datetime_to_str
class XResource(object):
"""Information about a resource for the explorer
@ -36,7 +39,7 @@ class XResource(object):
context of base_uri, store the resulting full URI
"""
def __init__(self, uri, acceptable_capabilities=None, checks=None, context=None):
self.uri=urlparse.urljoin(context,uri)
self.uri=urljoin(context,uri)
self.acceptable_capabilities=acceptable_capabilities
self.checks=checks
@ -69,15 +72,15 @@ class Explorer(Client):
# capabilities
starts = []
if (self.sitemap_name is not None):
print "Starting from explicit --sitemap %s" % (uri)
print("Starting from explicit --sitemap %s" % (uri))
starts.append( XResource(uri) )
elif (len(self.mapper)>0):
uri = self.mapper.default_src_uri()
(scheme, netloc, path, params, query, fragment) = urlparse.urlparse(uri)
(scheme, netloc, path, params, query, fragment) = urlparse(uri)
if (not scheme and not netloc):
if (os.path.isdir(path)):
# have a dir, look for 'likely' file names
print "Looking for capability documents in local directory %s" % (path)
print("Looking for capability documents in local directory %s" % (path))
for name in ['resourcesync','capabilities.xml',
'resourcelist.xml','changelist.xml']:
file = os.path.join(path,name)
@ -88,20 +91,20 @@ class Explorer(Client):
(path) )
else:
# local file, might be anything (or not exist)
print "Starting from local file %s" % (path)
print("Starting from local file %s" % (path))
starts.append( XResource(path) )
else:
# remote, can't tell whether we have a sitemap or a server name or something
# else, build list of options depending on whether there is a path and whether
# there is an extension/name
well_known = urlparse.urlunparse( [ scheme,netloc,'/.well-known/resourcesync','','','' ] )
well_known = urlunparse( [ scheme,netloc,'/.well-known/resourcesync','','','' ] )
if (not path):
# root, just look for .well-known
starts.append( XResource(well_known, ['capabilitylist','capabilitylistindex']) )
else:
starts.append( XResource(uri) )
starts.append( XResource(well_known, ['capabilitylist','capabilitylistindex']) )
print "Looking for discovery information based on mappings"
print("Looking for discovery information based on mappings")
else:
raise ClientFatalError("No source information (server base uri or capability uri) specified, use -h for help")
#
@ -121,7 +124,7 @@ class Explorer(Client):
history.append( new_xr )
except ExplorerQuit:
pass # expected way to exit
print "\nresync-explorer done, bye...\n"
print("\nresync-explorer done, bye...\n")
def explore_uri(self, explorer_resource, show_back=True):
"""INTERACTIVE exploration of capabilities document(s) starting at a given URI
@ -131,7 +134,7 @@ class Explorer(Client):
uri = explorer_resource.uri
caps = explorer_resource.acceptable_capabilities
checks = explorer_resource.checks
print "Reading %s" % (uri)
print("Reading %s" % (uri))
options={}
capability=None
try:
@ -143,9 +146,9 @@ class Explorer(Client):
list = s.parse_xml(urllib.urlopen(uri))
(options,capability)=self.explore_show_summary(list,s.parsed_index,caps,context=uri)
except IOError as e:
print "Cannot read %s (%s)" % (uri,str(e))
print("Cannot read %s (%s)" % (uri,str(e)))
except Exception as e:
print "Cannot parse %s (%s)" % (uri,str(e))
print("Cannot parse %s (%s)" % (uri,str(e)))
#
# Loop until we have some valide input
#
@ -206,15 +209,15 @@ class Explorer(Client):
capability = list.md['capability']
if (index):
capability += 'index'
print "Parsed %s document with %d entries:" % (capability,num_entries)
print("Parsed %s document with %d entries:" % (capability,num_entries))
if (expected is not None and capability not in expected):
print "WARNING - expected a %s document" % (','.join(expected))
print("WARNING - expected a %s document" % (','.join(expected)))
if (capability not in ['description','descriptionoindex','capabilitylist',
'resourcelist','resourcelistindex','changelist','changelistindex',
'resourcedump','resourcedumpindex','changedump','changedumpindex',
'resourcelist-archive', 'resourcedump-archive',
'changelist-archive', 'changedump-archive']):
print "WARNING - unknown %s document type" % (capability)
print("WARNING - unknown %s document type" % (capability))
to_show = num_entries
if (num_entries>21):
to_show = 20
@ -227,48 +230,48 @@ class Explorer(Client):
if (ln_describedby):
if ('href' in ln_describedby):
uri = ln_describedby['href']
print "[%s] rel='describedby' link to %s" % ('d',uri)
print("[%s] rel='describedby' link to %s" % ('d',uri))
uri = self.expand_relative_uri(context,uri)
options['d']=Resource(uri,capability='resource')
else:
print "WARNING - describedby link with no href, ignoring"
print("WARNING - describedby link with no href, ignoring")
ln_up = list.link('up')
if (ln_up):
if ('href' in ln_up):
uri = ln_up['href']
print "[%s] rel='up' link to %s" % ('u',uri)
print("[%s] rel='up' link to %s" % ('u',uri))
uri = self.expand_relative_uri(context,uri)
options['u']=Resource(uri)
else:
print "WARNING - up link with no href, ignoring"
print("WARNING - up link with no href, ignoring")
ln_index = list.link('index')
if (ln_index):
if ('href' in ln_index):
uri = ln_index['href']
print "[%s] rel='index' link to %s" % ('i',uri)
print("[%s] rel='index' link to %s" % ('i',uri))
uri = self.expand_relative_uri(context,uri)
options['i']=Resource(uri)
else:
print "WARNING - index link with no href, ignoring"
print("WARNING - index link with no href, ignoring")
# Show listed resources as numbered options
for r in list.resources:
if (n>=to_show):
print "(not showing remaining %d entries)" % (num_entries-n)
print("(not showing remaining %d entries)" % (num_entries-n))
break
n+=1
options[str(n)]=r
print "[%d] %s" % (n,r.uri)
print("[%d] %s" % (n,r.uri))
if (self.verbose):
print " " + str(r)
print(" " + str(r))
r.uri = self.expand_relative_uri(context,r.uri)
if (r.capability is not None):
warning = ''
if (r.capability not in entry_caps):
warning = " (EXPECTED %s)" % (' or '.join(entry_caps))
print " %s%s" % (r.capability,warning)
print(" %s%s" % (r.capability,warning))
elif (len(entry_caps)==1):
r.capability=entry_caps[0]
print " capability not specified, should be %s" % (r.capability)
print(" capability not specified, should be %s" % (r.capability))
return(options,capability)
def explore_show_head(self,uri,check_headers=None):
@ -277,14 +280,14 @@ class Explorer(Client):
Will also check headers against any values specified in
check_headers.
"""
print "HEAD %s" % (uri)
print("HEAD %s" % (uri))
if (re.match(r'^\w+:', uri)):
# Looks like a URI
response = requests.head(uri)
else:
# Mock up response if we have a local file
response = self.head_on_file(uri)
print " status: %s" % (response.status_code)
print(" status: %s" % (response.status_code))
if (response.status_code=='200'):
# generate normalized lastmod
# if ('last-modified' in response.headers):
@ -299,7 +302,7 @@ class Explorer(Client):
check_str=' MATCHES EXPECTED VALUE'
else:
check_str=' EXPECTED %s' % (check_headers[header])
print " %s: %s%s" % (header, response.headers[header], check_str)
print(" %s: %s%s" % (header, response.headers[header], check_str))
def head_on_file(self,file):
"""Mock up requests.head(..) response on local file
@ -341,8 +344,8 @@ class Explorer(Client):
Prints warning if expansion happens
"""
full_uri = urlparse.urljoin(context,uri)
full_uri = urljoin(context,uri)
if (full_uri != uri):
print " WARNING - expanded relative URI to %s" % (full_uri)
print(" WARNING - expanded relative URI to %s" % (full_uri))
uri = full_uri
return(uri)

20
resync/list_base.py
View File

@ -10,12 +10,20 @@ import os
from datetime import datetime
import re
import sys
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import logging
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from resource_container import ResourceContainer
from sitemap import Sitemap
from .resource_container import ResourceContainer
from .sitemap import Sitemap
class ListBase(ResourceContainer):
"""Class that adds Sitemap based IO to ResourceContainer
@ -88,7 +96,7 @@ class ListBase(ResourceContainer):
except IOError as e:
raise Exception("Failed to load sitemap/sitemapindex from %s (%s)" % (uri,str(e)))
elif (str is not None):
fh=StringIO.StringIO(str)
fh=io.StringIO(str)
if (fh is None):
raise Exception("Nothing to parse")
s = self.new_sitemap()
@ -113,7 +121,7 @@ class ListBase(ResourceContainer):
Must be overridden to support multi-file lists.
"""
self.default_capability()
fh = open(basename,'w')
fh = open(basename,'wb')
s = self.new_sitemap()
s.resources_as_xml(self,fh=fh,sitemapindex=self.sitemapindex)
fh.close()

29
resync/list_base_with_index.py
View File

@ -11,14 +11,17 @@ from datetime import datetime
import re
import sys
import itertools
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from list_base import ListBase
from resource import Resource
from sitemap import Sitemap
from mapper import Mapper, MapperError
from url_authority import UrlAuthority
from utils import compute_md5_for_file
from .list_base import ListBase
from .resource import Resource
from .sitemap import Sitemap
from .mapper import Mapper, MapperError
from .url_authority import UrlAuthority
from .utils import compute_md5_for_file
class ListBaseIndexError(Exception):
"""Exception for problems with sitemapindexes"""
@ -250,9 +253,9 @@ class ListBaseWithIndex(ListBase):
"""
# Access resources through iterator only
resources_iter = iter(self.resources)
( chunk, next ) = self.get_resources_chunk(resources_iter)
( chunk, nxt ) = self.get_resources_chunk(resources_iter)
s = self.new_sitemap()
if (next is not None):
if (nxt is not None):
# Have more than self.max_sitemap_entries => sitemapindex
if (not self.allow_multifile):
raise ListBaseIndexError("Too many entries for a single sitemap but multifile disabled")
@ -289,7 +292,7 @@ class ListBaseWithIndex(ListBase):
md5 = compute_md5_for_file(file) )
index.add(r)
# Get next chunk
( chunk, next ) = self.get_resources_chunk(resources_iter,next)
( chunk, nxt ) = self.get_resources_chunk(resources_iter,nxt)
self.logger.info("Wrote %d sitemaps" % (len(index)))
f = open(basename, 'w')
self.logger.info("Writing sitemapindex %s..." % (basename))
@ -335,10 +338,10 @@ class ListBaseWithIndex(ListBase):
break
# Get next to see whether there are more resources
try:
next = resource_iter.next()
nxt = next(resource_iter)
except StopIteration:
next = None
return(chunk,next)
nxt = None
return(chunk,nxt)
def part_name(self, basename='/tmp/sitemap.xml', part_number=0):
"""Name (file or URI) for one component sitemap

9
resync/mapper.py
View File

@ -2,7 +2,10 @@
import os
import os.path
import re
import urlparse
try: #python3
from urllib.parse import urlparse
except ImportError: #python2
from urlparse import urlparse
import logging
class MapperError(Exception):
@ -104,7 +107,7 @@ class Mapper():
In the case that uri is already a local path then the same path
is returned.
"""
(scheme, netloc, path, params, query, fragment) = urlparse.urlparse( uri )
(scheme, netloc, path, params, query, fragment) = urlparse( uri )
if (netloc == ''):
return(uri)
path = '/'.join([netloc,path])
@ -171,7 +174,7 @@ class Map:
Applies only to local source. Returns True if the paths for source and
destination are the same, or if one is a component of the other path.
"""
(scheme, netloc, path, params, query, fragment) = urlparse.urlparse( self.src_uri )
(scheme, netloc, path, params, query, fragment) = urlparse( self.src_uri )
if (scheme != ''):
return(False)
s = os.path.normpath(self.src_uri)

11
resync/resource.py
View File

@ -43,9 +43,13 @@ object that contains such information.
"""
import re
from urlparse import urlparse
try: #python3
from urllib.parse import urlparse
except: #python2
from urlparse import urlparse
from posixpath import basename
from w3c_datetime import str_to_datetime, datetime_to_str
from .w3c_datetime import str_to_datetime, datetime_to_str
class ChangeTypeError(Exception):
@ -244,7 +248,8 @@ class Resource(object):
"""Provide access to the complete hash string
The hash string may have zero or more hash values with
appropriate prefixes
appropriate prefixes. All hash values are assumed to be
strings
"""
hashvals = []
if (self.md5 is not None):

2
resync/resource_container.py
View File

@ -11,7 +11,7 @@ only the data storage and manipulation, the ListBase class
adds IO.
"""
import collections
from w3c_datetime import datetime_to_str
from .w3c_datetime import datetime_to_str
class ResourceContainer(object):
"""Class containing resource-like objects

2
resync/resource_dump.py
View File

@ -9,7 +9,7 @@ Described in specification at:
http://www.openarchives.org/rs/resourcesync#ResourceDump
"""
from resource_list import ResourceList
from .resource_list import ResourceList
class ResourceDump(ResourceList):
"""Class representing an Resource Dump

2
resync/resource_dump_manifest.py
View File

@ -9,7 +9,7 @@ Described in specification at:
http://www.openarchives.org/rs/resourcesync#ResourceDumpManifest
"""
from resource_list import ResourceList
from .resource_list import ResourceList
class ResourceDumpManifest(ResourceList):
"""Class representing a Resource Dump Manifest

13
resync/resource_list.py
View File

@ -22,12 +22,15 @@ import os
from datetime import datetime
import re
import sys
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from list_base_with_index import ListBaseWithIndex
from sitemap import Sitemap
from mapper import Mapper, MapperError
from url_authority import UrlAuthority
from .list_base_with_index import ListBaseWithIndex
from .sitemap import Sitemap
from .mapper import Mapper, MapperError
from .url_authority import UrlAuthority
class ResourceListDict(dict):

15
resync/resource_list_builder.py
View File

@ -18,14 +18,17 @@ import os.path
import re
import time
import logging
from urllib import URLopener
try: #python3
from urllib.request import URLopener
except ImportError: #python2
from urllib import URLopener
from defusedxml.ElementTree import parse
from resource import Resource
from resource_list import ResourceList
from sitemap import Sitemap
from utils import compute_md5_for_file
from w3c_datetime import datetime_to_str
from .resource import Resource
from .resource_list import ResourceList
from .sitemap import Sitemap
from .utils import compute_md5_for_file
from .w3c_datetime import datetime_to_str
class ResourceListBuilder():

36
resync/sitemap.py
View File

@ -6,10 +6,15 @@ import sys
import logging
from defusedxml.ElementTree import parse
from xml.etree.ElementTree import ElementTree, Element, tostring
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resource import Resource
from resource_container import ResourceContainer
from .resource import Resource
from .resource_container import ResourceContainer
SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9'
RS_NS = 'http://www.openarchives.org/rs/terms/'
@ -101,12 +106,14 @@ class Sitemap(object):
tree = ElementTree(root);
xml_buf=None
if (fh is None):
xml_buf=StringIO.StringIO()
xml_buf=io.StringIO()
fh=xml_buf
if (sys.version_info < (2,7)):
tree.write(fh,encoding='UTF-8')
else:
elif (sys.version_info < (3,0)):
tree.write(fh,encoding='UTF-8',xml_declaration=True,method='xml')
else:
tree.write(fh,encoding="unicode",xml_declaration=True,method='xml')
if (xml_buf is not None):
return(xml_buf.getvalue())
@ -241,16 +248,23 @@ class Sitemap(object):
e.tail="\n"
return(e)
def resource_as_xml(self,resource,indent=' '):
"""Return string for the the resource as part of an XML sitemap
def resource_as_xml(self,resource):
"""Return string for the resource as part of an XML sitemap
Returns a string with the XML snippet representing the resource,
without any XML declaration.
"""
e = self.resource_etree_element(resource)
if (sys.version_info < (2,7)):
#must not specify method='xml' in python2.6
return(tostring(e, encoding='UTF-8'))
if (sys.version_info > (3,0)):
# python3.x
return(tostring(e, encoding='unicode', method='xml'))
elif (sys.version_info >= (2,7)):
s = tostring(e, encoding='UTF-8', method='xml')
else:
return(tostring(e, encoding='UTF-8', method='xml'))
#must not specify method='xml' in python2.6
s = tostring(e, encoding='UTF-8')
# Chop off XML declaration that is added in 2.x... sigh
return(s.replace("<?xml version='1.0' encoding='UTF-8'?>\n",''))
def resource_from_etree(self, etree, resource_class):
"""Construct a Resource from an etree

6
resync/source_description.py
View File

@ -23,9 +23,9 @@ See: http://www.openarchives.org/rs/resourcesync#SourceDesc
import collections
from resource import Resource
from resource_set import ResourceSet
from list_base_with_index import ListBaseWithIndex
from resync.resource import Resource
from resync.resource_set import ResourceSet
from resync.list_base_with_index import ListBaseWithIndex
class SourceDescription(ListBaseWithIndex):
"""Class representing the set of Capability Lists supported

9
resync/url_authority.py
View File

@ -24,7 +24,10 @@ Example use:
# will be false
"""
import urlparse
try: #python3
from urllib.parse import urlparse
except ImportError: #python2
from urlparse import urlparse
import os.path
class UrlAuthority(object):
@ -42,7 +45,7 @@ class UrlAuthority(object):
def set_master(self, url):
"""Set the master url that this object works with"""
m = urlparse.urlparse(url)
m = urlparse(url)
self.master_scheme=m.scheme
self.master_netloc=m.netloc
self.master_path=os.path.dirname(m.path)
@ -54,7 +57,7 @@ class UrlAuthority(object):
just that the server names match or the query url is a
sub-domain of the master
"""
s = urlparse.urlparse(url)
s = urlparse(url)
if (s.scheme != self.master_scheme):
return(False)
if (s.netloc != self.master_netloc):

18
resync/utils.py
View File

@ -45,20 +45,26 @@ import base64
import hashlib
def compute_md5_for_string(string):
"""Compute MD5 digest over some string payload"""
return base64.b64encode(hashlib.md5(string).digest())
"""Compute MD5 digest over some string or byte payload
Returns a string containing the digest.
"""
if (type(string) != 'bytes'):
string = string.encode('utf-8') #make bytes
return base64.b64encode(hashlib.md5(string).digest()).decode('utf-8')
def compute_md5_for_file(file, block_size=2**14):
"""Compute MD5 digest for a file
Optional block_size parameter controls memory used to do MD5 calculation.
This should be a multiple of 128 bytes.
Returns a string containing the digest. Optional block_size parameter
controls memory used to do MD5 calculation. This should be a multiple
of 128 bytes.
"""
f = open(file, 'r')
f = open(file, 'rb')
md5 = hashlib.md5()
while True:
data = f.read(block_size)
if not data:
break
md5.update(data)
return base64.b64encode(md5.digest())
return base64.b64encode(md5.digest()).decode('utf-8')

16
tests/test_change_dump.py
View File

@ -1,6 +1,12 @@
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import re
from resync.resource import Resource
from resync.change_dump import ChangeDump
from resync.sitemap import SitemapParseError
@ -12,7 +18,7 @@ class TestChangeDump(unittest.TestCase):
rd.add( Resource('a.zip',timestamp=1) )
rd.add( Resource('b.zip',timestamp=2) )
xml = rd.as_xml()
print xml
#print(xml)
self.assertTrue( re.search(r'<rs:md .*capability="changedump"', xml), 'XML has capability' )
self.assertTrue( re.search(r'<url><loc>a.zip</loc><lastmod>1970-01-01T00:00:01Z</lastmod></url>', xml), 'XML has resource a' )
@ -24,7 +30,7 @@ class TestChangeDump(unittest.TestCase):
<url><loc>http://example.com/b.zip</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="56789" /></url>\
</urlset>'
rd=ChangeDump()
rd.parse(fh=StringIO.StringIO(xml))
rd.parse(fh=io.StringIO(xml))
self.assertEqual( len(rd.resources), 2, 'got 2 resource dumps')
self.assertEqual( rd.md['capability'], 'changedump', 'capability set' )
self.assertEqual( rd.md_from, '2013-01-01' )
@ -41,7 +47,7 @@ class TestChangeDump(unittest.TestCase):
<url><loc>http://example.com/a.zip</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="12" /></url>\
</urlset>'
rd=ChangeDump()
self.assertRaises( SitemapParseError, rd.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) )
def test12_parse_bad_capability(self):
# the <rs:md capability="bad_capability".. should give error
@ -51,7 +57,7 @@ class TestChangeDump(unittest.TestCase):
<url><loc>http://example.com/bad_res_1</loc><lastmod>2012-03-14T18:37:36Z</lastmod></url>\
</urlset>'
rd=ChangeDump()
self.assertRaises( SitemapParseError, rd.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestChangeDump)

18
tests/test_change_list.py
View File

@ -1,6 +1,12 @@
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import re
from resync.resource import Resource
from resync.change_list import ChangeList
from resync.resource_list import ResourceList
@ -60,11 +66,11 @@ class TestChangeList(unittest.TestCase):
changes.add_changed_resources( added, change='created' )
self.assertEqual(len(changes), 2, "2 things added")
i = iter(changes)
first = i.next()
first = next(i)
self.assertEqual(first.uri, 'a', "changes[0].uri=a")
self.assertEqual(first.timestamp, 1, "changes[0].timestamp=1")
self.assertEqual(first.change, 'created') #, "changes[0].change=createdd")
second = i.next()
second = next(i)
self.assertEqual(second.timestamp, 4, "changes[1].timestamp=4")
self.assertEqual(second.change, 'created', "changes[1].change=createdd")
# Now add some with updated (one same, one diff)
@ -103,7 +109,7 @@ class TestChangeList(unittest.TestCase):
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md change="deleted" length="32" /></url>\
</urlset>'
cl=ChangeList()
cl.parse(fh=StringIO.StringIO(xml))
cl.parse(fh=io.StringIO(xml))
self.assertEqual( len(cl.resources), 2, 'got 2 resources')
self.assertEqual( cl.md['capability'], 'changelist', 'capability set' )
self.assertEqual( cl.md['md_from'], '2013-01-01' )
@ -115,7 +121,7 @@ class TestChangeList(unittest.TestCase):
<url><loc>http://example.com/res1</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md change="updated"/></url>\
</urlset>'
cl=ChangeList()
self.assertRaises( SitemapParseError, cl.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, cl.parse, fh=io.StringIO(xml) )
def test32_parse_bad_capability(self):
# the <rs:md capability="bad_capability".. should give error
@ -125,7 +131,7 @@ class TestChangeList(unittest.TestCase):
<url><loc>http://example.com/bad_res_1</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md change="updated"/></url>\
</urlset>'
cl=ChangeList()
self.assertRaises( SitemapParseError, cl.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, cl.parse, fh=io.StringIO(xml) )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestChangeList)

10
tests/test_client.py
View File

@ -1,7 +1,13 @@
import unittest
import re
import logging
import sys, StringIO, contextlib
import sys, contextlib
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resync.client import Client, ClientFatalError
@ -12,7 +18,7 @@ class Data(object):
@contextlib.contextmanager
def capture_stdout():
old = sys.stdout
capturer = StringIO.StringIO()
capturer = io.StringIO()
sys.stdout = capturer
data = Data()
yield data

13
tests/test_client_link_options.py
View File

@ -1,6 +1,11 @@
import sys
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import BytesIO as io
except ImportError: #python3
import io
sys.path.insert(0, '.')
@ -23,7 +28,7 @@ class TestClientLinkOptions(unittest.TestCase):
xml = run_resync(['--resourcelist',
'http://example.org/t','tests/testdata/dir1'])
rl = ResourceList()
rl.parse(fh=StringIO.StringIO(xml))
rl.parse(fh=io.BytesIO(xml))
self.assertEqual( len(rl), 2 )
self.assertEqual( rl.link('describedby'), None )
@ -34,7 +39,7 @@ class TestClientLinkOptions(unittest.TestCase):
'--capabilitylist-link=c',
'http://example.org/t','tests/testdata/dir1'])
rl = ResourceList()
rl.parse(fh=StringIO.StringIO(xml))
rl.parse(fh=io.BytesIO(xml))
self.assertEqual( len(rl), 2 )
self.assertNotEqual( rl.link('describedby'), None )
self.assertEqual( rl.link('describedby')['href'], 'a' )
@ -47,7 +52,7 @@ class TestClientLinkOptions(unittest.TestCase):
'--sourcedescription-link=b',
'--capabilitylist-link=c' ]) #will be ignored
capl = CapabilityList()
capl.parse(fh=StringIO.StringIO(xml))
capl.parse(fh=io.BytesIO(xml))
self.assertEqual( len(capl), 2 )
self.assertNotEqual( capl.link('describedby'), None )
self.assertEqual( capl.link('describedby')['href'], 'a' )

4
tests/test_dump.py
View File

@ -72,7 +72,7 @@ class TestDump(unittest.TestCase):
def test03_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
for letter in map(chr,range(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
fname='tests/testdata/a_to_z/%s' % (letter)
rl.add( Resource(uri, path=fname) )
@ -118,7 +118,7 @@ class TestDump(unittest.TestCase):
def test04_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
for letter in map(chr,range(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
fname='tests/testdata/a_to_z/%s' % (letter)
rl.add( Resource(uri, path=fname) )

4
tests/test_examples_from_spec.py
View File

@ -812,8 +812,8 @@ class TestExamplesFromSpec(unittest.TestCase):
ib=iter(bb)
try:
while (1):
self._assert_xml_elements_equal( self._xml_reorder_attributes(ia.next()),
self._xml_reorder_attributes(ib.next()),
self._assert_xml_elements_equal( self._xml_reorder_attributes(next(ia)),
self._xml_reorder_attributes(next(ib)),
context )
except StopIteration:
# all is good provided there were the same number of elements

10
tests/test_explorer.py
View File

@ -1,7 +1,13 @@
import unittest
import re
import logging
import sys, StringIO, contextlib
import sys, contextlib
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resync.client import Client, ClientFatalError
from resync.capability_list import CapabilityList
@ -15,7 +21,7 @@ class Data(object):
@contextlib.contextmanager
def capture_stdout():
old = sys.stdout
capturer = StringIO.StringIO()
capturer = io.StringIO()
sys.stdout = capturer
data = Data()
yield data

17
tests/test_list_base.py
View File

@ -1,6 +1,12 @@
import sys
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resync.resource import Resource
from resync.list_base import ListBase
from resync.sitemap import Sitemap, SitemapIndexError
@ -31,18 +37,19 @@ class TestListBase(unittest.TestCase):
lb.add( Resource(uri='b',lastmod='2002-02-02',length=56789) )
lb.add( Resource(uri='c',lastmod='2003-03-03',length=0) )
lb.md['from']=None #avoid now being added
#print lb
self.assertEqual( lb.as_xml(), '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/"><rs:md capability="unknown" /><url><loc>a</loc><lastmod>2001-01-01T00:00:00Z</lastmod><rs:md length="1234" /></url><url><loc>b</loc><lastmod>2002-02-02T00:00:00Z</lastmod><rs:md length="56789" /></url><url><loc>c</loc><lastmod>2003-03-03T00:00:00Z</lastmod><rs:md length="0" /></url></urlset>' )
print(lb)
x = lb.as_xml()
self.assertEqual( x, '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/"><rs:md capability="unknown" /><url><loc>a</loc><lastmod>2001-01-01T00:00:00Z</lastmod><rs:md length="1234" /></url><url><loc>b</loc><lastmod>2002-02-02T00:00:00Z</lastmod><rs:md length="56789" /></url><url><loc>c</loc><lastmod>2003-03-03T00:00:00Z</lastmod><rs:md length="0" /></url></urlset>' )
def test_11_parse_2(self):
xml='<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
xml = '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
<rs:md capability="unknown" from="2013-02-12T14:09:00Z" />\
<url><loc>/tmp/rs_test/src/file_a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="12" /></url>\
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="32" /></url>\
</urlset>'
lb=ListBase()
lb.parse(fh=StringIO.StringIO(xml))
lb.parse(fh=io.StringIO(xml))
self.assertEqual( len(lb.resources), 2, 'got 2 resources')
if __name__ == '__main__':

10
tests/test_list_base_with_index.py
View File

@ -1,6 +1,12 @@
import sys
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resync.resource import Resource
from resync.list_base_with_index import ListBaseWithIndex, ListBaseIndexError
from resync.sitemap import Sitemap, SitemapIndexError
@ -51,7 +57,7 @@ class TestListBaseWithIndex(unittest.TestCase):
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="32" /></url>\
</urlset>'
lb=ListBaseWithIndex()
lb.parse(fh=StringIO.StringIO(xml))
lb.parse(fh=io.StringIO(xml))
self.assertEqual( len(lb.resources), 2, 'got 2 resources')
if __name__ == '__main__':

14
tests/test_resource_container.py
View File

@ -33,8 +33,8 @@ class TestResourceContainer(unittest.TestCase):
rc.prune_before(3)
self.assertEqual( len(rc.resources), 2 )
i = iter(rc)
self.assertEqual( i.next().uri, 'c' )
self.assertEqual( i.next().uri, 'd' )
self.assertEqual( next(i).uri, 'c' )
self.assertEqual( next(i).uri, 'd' )
# put some more back out of order
rc.resources.append( Resource('a',timestamp=1) )
rc.resources.append( Resource('b',timestamp=2) )
@ -42,8 +42,8 @@ class TestResourceContainer(unittest.TestCase):
rc.prune_before(3.5)
self.assertEqual( len(rc.resources), 2 )
i = iter(rc)
self.assertEqual( i.next().uri, 'd' )
self.assertEqual( i.next().uri, 'e' )
self.assertEqual( next(i).uri, 'd' )
self.assertEqual( next(i).uri, 'e' )
def test04_prune_dupes(self):
rc = ResourceContainer()
@ -60,13 +60,13 @@ class TestResourceContainer(unittest.TestCase):
rc.prune_dupes()
self.assertEqual( len(rc.resources), 3 )
i = iter(rc)
i1 = i.next()
i1 = next(i)
self.assertEqual( i1.uri, 'a' )
self.assertEqual( i1.change, 'updated' )
i2 = i.next()
i2 = next(i)
self.assertEqual( i2.uri, 'c' )
self.assertEqual( i2.change, 'deleted' )
i3 = i.next()
i3 = next(i)
self.assertEqual( i3.uri, 'd' )
self.assertEqual( i3.change, 'created' )

15
tests/test_resource_dump.py
View File

@ -1,6 +1,12 @@
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import re
from resync.resource import Resource
from resync.resource_dump import ResourceDump
from resync.sitemap import SitemapParseError
@ -12,7 +18,6 @@ class TestResourceDump(unittest.TestCase):
rd.add( Resource('a.zip',timestamp=1) )
rd.add( Resource('b.zip',timestamp=2) )
xml = rd.as_xml()
print xml
self.assertTrue( re.search(r'<rs:md .*capability="resourcedump"', xml), 'XML has capability' )
self.assertTrue( re.search(r'<url><loc>a.zip</loc><lastmod>1970-01-01T00:00:01Z</lastmod></url>', xml), 'XML has resource a' )
@ -24,7 +29,7 @@ class TestResourceDump(unittest.TestCase):
<url><loc>http://example.com/b.zip</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="56789" /></url>\
</urlset>'
rd=ResourceDump()
rd.parse(fh=StringIO.StringIO(xml))
rd.parse(fh=io.StringIO(xml))
self.assertEqual( len(rd.resources), 2, 'got 2 resource dumps')
self.assertEqual( rd.capability, 'resourcedump', 'capability set' )
self.assertEqual( rd.md_at, '2013-01-01' )
@ -41,7 +46,7 @@ class TestResourceDump(unittest.TestCase):
<url><loc>http://example.com/a.zip</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="12" /></url>\
</urlset>'
rd=ResourceDump()
self.assertRaises( SitemapParseError, rd.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) )
def test12_parse_bad_capability(self):
# the <rs:md capability="bad_capability".. should give error
@ -51,7 +56,7 @@ class TestResourceDump(unittest.TestCase):
<url><loc>http://example.com/bad_res_1</loc><lastmod>2012-03-14T18:37:36Z</lastmod></url>\
</urlset>'
rd=ResourceDump()
self.assertRaises( SitemapParseError, rd.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rd.parse, fh=io.StringIO(xml) )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceDump)

15
tests/test_resource_dump_manifest.py
View File

@ -1,6 +1,12 @@
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import re
from resync.resource import Resource
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.sitemap import SitemapParseError
@ -12,7 +18,6 @@ class TestResourceDumpManifest(unittest.TestCase):
rdm.add( Resource('a.zip',timestamp=1) )
rdm.add( Resource('b.zip',timestamp=2) )
xml = rdm.as_xml()
print xml
self.assertTrue( re.search(r'<rs:md .*capability="resourcedump-manifest"', xml), 'XML has capability' )
self.assertTrue( re.search(r'<url><loc>a.zip</loc><lastmod>1970-01-01T00:00:01Z</lastmod></url>', xml), 'XML has resource a' )
@ -24,7 +29,7 @@ class TestResourceDumpManifest(unittest.TestCase):
<url><loc>http://example.com/res2</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="32" path="/res2"/></url>\
</urlset>'
rdm=ResourceDumpManifest()
rdm.parse(fh=StringIO.StringIO(xml))
rdm.parse(fh=io.StringIO(xml))
self.assertEqual( len(rdm.resources), 2, 'got 2 resource dumps')
self.assertEqual( rdm.capability, 'resourcedump-manifest', 'capability set' )
self.assertEqual( rdm.md_at, '2013-08-08' )
@ -43,7 +48,7 @@ class TestResourceDumpManifest(unittest.TestCase):
<url><loc>http://example.com/res2</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="32" path="/res2"/></url>\
</urlset>'
rdm=ResourceDumpManifest()
self.assertRaises( SitemapParseError, rdm.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rdm.parse, fh=io.StringIO(xml) )
def test12_parse_bad_capability(self):
# the <rs:md capability="bad_capability".. should give error
@ -53,7 +58,7 @@ class TestResourceDumpManifest(unittest.TestCase):
<url><loc>http://example.com/a.zip</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="12" /></url>\
</urlset>'
rdm=ResourceDumpManifest()
self.assertRaises( SitemapParseError, rdm.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rdm.parse, fh=io.StringIO(xml) )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceDumpManifest)

31
tests/test_resource_list.py
View File

@ -1,6 +1,12 @@
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import re
from resync.resource import Resource
from resync.resource_list import ResourceList, ResourceListDupeError
from resync.sitemap import SitemapParseError
@ -17,8 +23,8 @@ class TestResourceList(unittest.TestCase):
( same, changed, deleted, added ) = dst.compare(src)
self.assertEqual( len(same), 2, "2 things unchanged" )
i = iter(same)
self.assertEqual( i.next().uri, 'a', "first was a" )
self.assertEqual( i.next().uri, 'b', "second was b" )
self.assertEqual( next(i).uri, 'a', "first was a" )
self.assertEqual( next(i).uri, 'b', "second was b" )
self.assertEqual( len(changed), 0, "nothing changed" )
self.assertEqual( len(deleted), 0, "nothing deleted" )
self.assertEqual( len(added), 0, "nothing added" )
@ -34,8 +40,8 @@ class TestResourceList(unittest.TestCase):
self.assertEqual( len(same), 0, "0 things unchanged" )
self.assertEqual( len(changed), 2, "2 things changed" )
i = iter(changed)
self.assertEqual( i.next().uri, 'a', "first was a" )
self.assertEqual( i.next().uri, 'b', "second was b" )
self.assertEqual( next(i).uri, 'a', "first was a" )
self.assertEqual( next(i).uri, 'b', "second was b" )
self.assertEqual( len(deleted), 0, "nothing deleted" )
self.assertEqual( len(added), 0, "nothing added" )
@ -53,8 +59,8 @@ class TestResourceList(unittest.TestCase):
self.assertEqual( len(changed), 0, "nothing changed" )
self.assertEqual( len(deleted), 2, "c and d deleted" )
i = iter(deleted)
self.assertEqual( i.next().uri, 'c', "first was c" )
self.assertEqual( i.next().uri, 'd', "second was d" )
self.assertEqual( next(i).uri, 'c', "first was c" )
self.assertEqual( next(i).uri, 'd', "second was d" )
self.assertEqual( len(added), 0, "nothing added" )
def test04_added(self):
@ -72,8 +78,8 @@ class TestResourceList(unittest.TestCase):
self.assertEqual( len(deleted), 0, "nothing deleted" )
self.assertEqual( len(added), 2, "b and d added" )
i = iter(added)
self.assertEqual( i.next().uri, 'b', "first was b" )
self.assertEqual( i.next().uri, 'd', "second was d" )
self.assertEqual( next(i).uri, 'b', "first was b" )
self.assertEqual( next(i).uri, 'd', "second was d" )
def test05_add(self):
r1 = Resource(uri='a',length=1)
@ -131,7 +137,6 @@ class TestResourceList(unittest.TestCase):
rl.add( Resource('a',timestamp=1) )
rl.add( Resource('b',timestamp=2) )
xml = rl.as_xml()
print xml
self.assertTrue( re.search(r'<rs:md .*capability="resourcelist"', xml), 'XML has capability' )
self.assertTrue( re.search(r'<url><loc>a</loc><lastmod>1970-01-01T00:00:01Z</lastmod></url>', xml), 'XML has resource a' )
@ -143,7 +148,7 @@ class TestResourceList(unittest.TestCase):
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length="32" /></url>\
</urlset>'
rl=ResourceList()
rl.parse(fh=StringIO.StringIO(xml))
rl.parse(fh=io.StringIO(xml))
self.assertEqual( len(rl.resources), 2, 'got 2 resources')
self.assertEqual( rl.md['capability'], 'resourcelist', 'capability set' )
self.assertEqual( rl.md_at, '2013-08-07' )
@ -155,7 +160,7 @@ class TestResourceList(unittest.TestCase):
<url><loc>http://example.com/res1</loc><lastmod>2012-03-14T18:37:36Z</lastmod></url>\
</urlset>'
rl=ResourceList()
rl.parse(fh=StringIO.StringIO(xml))
rl.parse(fh=io.StringIO(xml))
self.assertEqual( len(rl.resources), 1, 'got 1 resource')
self.assertEqual( rl.md['capability'], 'resourcelist', 'capability set by reading routine' )
self.assertFalse( 'from' in rl.md )
@ -168,7 +173,7 @@ class TestResourceList(unittest.TestCase):
<url><loc>http://example.com/bad_res_1</loc><lastmod>2012-03-14T18:37:36Z</lastmod></url>\
</urlset>'
rl=ResourceList()
self.assertRaises( SitemapParseError, rl.parse, fh=StringIO.StringIO(xml) )
self.assertRaises( SitemapParseError, rl.parse, fh=io.StringIO(xml) )
if __name__ == '__main__':
suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestResourceList)

14
tests/test_resource_list_builder.py
View File

@ -21,13 +21,13 @@ class TestResourceListBuilder(unittest.TestCase):
rl = rlb.from_disk()
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, 20 )
self.assertEqual( r.path, None )
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, None )
@ -43,13 +43,13 @@ class TestResourceListBuilder(unittest.TestCase):
rl = rlb.from_disk()
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, None )
self.assertEqual( r.length, None )
self.assertEqual( r.path, None )
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, None )
@ -62,13 +62,13 @@ class TestResourceListBuilder(unittest.TestCase):
rl = rlb.from_disk()
self.assertEqual( len(rl), 2 )
rli = iter(rl)
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )
self.assertEqual( r.md5, 'a/Jv1mYBtSjS4LR+qoft/Q==' )
self.assertEqual( r.length, 20 )
self.assertEqual( r.path, None )
r = rli.next()
r = next(rli)
self.assertEqual( r.uri, 'http://example.org/t/file_b' )
self.assertEqual( r.lastmod, '2001-09-09T01:46:40Z' )
self.assertEqual( r.md5, 'RS5Uva4WJqxdbnvoGzneIQ==' )
@ -109,7 +109,7 @@ class TestResourceListBuilder(unittest.TestCase):
rl = rlb.from_disk(paths=['tests/testdata/dir1/file_a'])
self.assertEqual( len(rl), 1)
rli = iter(rl)
r = rli.next()
r = next(rli)
self.assertTrue( r is not None )
self.assertEqual( r.uri, 'http://example.org/t/dir1/file_a' )
self.assertEqual( r.lastmod, '2012-07-25T17:13:46Z' )

21
tests/test_resource_list_multifile.py
View File

@ -1,6 +1,5 @@
import sys
import unittest
import StringIO
import tempfile
import os.path
import shutil
@ -66,14 +65,14 @@ class TestResourceListMultifile(unittest.TestCase):
self.assertEquals( rl1.capability, 'resourcelist' )
self.assertFalse( rl1.sitemapindex )
i = iter(rl1)
self.assertEquals( i.next().uri, 'http://localhost/a' )
self.assertEquals( i.next().uri, 'http://localhost/b' )
self.assertEquals( next(i).uri, 'http://localhost/a' )
self.assertEquals( next(i).uri, 'http://localhost/b' )
rl2 = ResourceList()
rl2.read( os.path.join(tempdir,'sitemap00001.xml') )
self.assertEquals( len(rl2), 2 )
i = iter(rl2)
self.assertEquals( i.next().uri, 'http://localhost/c' )
self.assertEquals( i.next().uri, 'http://localhost/d' )
self.assertEquals( next(i).uri, 'http://localhost/c' )
self.assertEquals( next(i).uri, 'http://localhost/d' )
# check the sitemapindex (read just as index)
rli = ResourceList()
rli.read( os.path.join(tempdir,'sitemap.xml'), index_only=True )
@ -81,8 +80,8 @@ class TestResourceListMultifile(unittest.TestCase):
i = iter(rli)
self.assertEquals( rli.capability, 'resourcelist' )
self.assertTrue( rli.sitemapindex )
self.assertEquals( i.next().uri, 'http://localhost/sitemap00000.xml' )
self.assertEquals( i.next().uri, 'http://localhost/sitemap00001.xml' )
self.assertEquals( next(i).uri, 'http://localhost/sitemap00000.xml' )
self.assertEquals( next(i).uri, 'http://localhost/sitemap00001.xml' )
# check the sitemapindex and components
rli = ResourceList( mapper=rl.mapper )
rli.read( os.path.join(tempdir,'sitemap.xml') )
@ -90,10 +89,10 @@ class TestResourceListMultifile(unittest.TestCase):
self.assertEquals( rli.capability, 'resourcelist' )
self.assertFalse( rli.sitemapindex )
i = iter(rli)
self.assertEquals( i.next().uri, 'http://localhost/a' )
self.assertEquals( i.next().uri, 'http://localhost/b' )
self.assertEquals( i.next().uri, 'http://localhost/c' )
self.assertEquals( i.next().uri, 'http://localhost/d' )
self.assertEquals( next(i).uri, 'http://localhost/a' )
self.assertEquals( next(i).uri, 'http://localhost/b' )
self.assertEquals( next(i).uri, 'http://localhost/c' )
self.assertEquals( next(i).uri, 'http://localhost/d' )
# cleanup tempdir
shutil.rmtree(tempdir)

8
tests/test_resource_set.py
View File

@ -21,10 +21,10 @@ class TestResourceSet(unittest.TestCase):
rs.add( Resource('a3') )
rs.add( Resource('a1') )
i = iter(rs)
self.assertEqual( i.next().uri, 'a1' )
self.assertEqual( i.next().uri, 'a2' )
self.assertEqual( i.next().uri, 'a3' )
self.assertRaises( StopIteration, i.next )
self.assertEqual( next(i).uri, 'a1' )
self.assertEqual( next(i).uri, 'a2' )
self.assertEqual( next(i).uri, 'a3' )
self.assertRaises( StopIteration, next, i )
def test03_dupe(self):
rs = ResourceSet()

92
tests/test_sitemap.py
View File

@ -1,39 +1,47 @@
import sys
import unittest
import StringIO
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
from resync.resource import Resource
from resync.resource_list import ResourceList
from resync.sitemap import Sitemap, SitemapIndexError, SitemapParseError
# etree gives ParseError in 2.7, ExpatError in 2.6
# etree gives ParseError in 2.7,3.x; ExpatError in 2.6
etree_error_class = None
if (sys.version_info < (2,7)):
from xml.parsers.expat import ExpatError
etree_error_class = ExpatError
else:
from xml.etree.ElementTree import ParseError
etree_error_class = ParseError
#In python3 this seems only to work with the full class name??
#from xml.etree.ElementTree import ParseError
import xml.etree.ElementTree
etree_error_class = xml.etree.ElementTree.ParseError
class TestSitemap(unittest.TestCase):
def test_01_resource_str(self):
r1 = Resource('a3')
r1.lastmod='2012-01-11T01:02:03Z'
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>a3</loc><lastmod>2012-01-11T01:02:03Z</lastmod></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>a3</loc><lastmod>2012-01-11T01:02:03Z</lastmod></url>" )
def test_02_resource_str(self):
r1 = Resource('3b',1234.1,9999,'ab54de')
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>3b</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>3b</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /></url>" )
def test_03_resource_str_hashes(self):
r1 = Resource('03hashes',1234.1)
r1.md5 = 'aaa'
r1.sha1 = 'bbb'
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-1:bbb\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-1:bbb\" /></url>" )
r1.sha256 = 'ccc'
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-1:bbb sha-256:ccc\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-1:bbb sha-256:ccc\" /></url>" )
r1.sha1 = None
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-256:ccc\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>03hashes</loc><lastmod>1970-01-01T00:20:34.100000Z</lastmod><rs:md hash=\"md5:aaa sha-256:ccc\" /></url>" )
def test_04_resource_str(self):
r1 = Resource(uri='4a',lastmod="2013-01-02",length=9999,md5='ab54de')
@ -41,11 +49,11 @@ class TestSitemap(unittest.TestCase):
'pri':'1',
'href':'http://mirror1.example.com/res1',
'modified':'2013-01-02T18:00:00Z' }]
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>4a</loc><lastmod>2013-01-02T00:00:00Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /><rs:ln href=\"http://mirror1.example.com/res1\" modified=\"2013-01-02T18:00:00Z\" pri=\"1\" rel=\"duplicate\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>4a</loc><lastmod>2013-01-02T00:00:00Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /><rs:ln href=\"http://mirror1.example.com/res1\" modified=\"2013-01-02T18:00:00Z\" pri=\"1\" rel=\"duplicate\" /></url>" )
# add another two rs:ln's
r1.ln.append( { 'rel':'num2' } )
r1.ln.append( { 'rel':'num3' } )
self.assertEqual( Sitemap().resource_as_xml(r1), "<?xml version='1.0' encoding='UTF-8'?>\n<url><loc>4a</loc><lastmod>2013-01-02T00:00:00Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /><rs:ln href=\"http://mirror1.example.com/res1\" modified=\"2013-01-02T18:00:00Z\" pri=\"1\" rel=\"duplicate\" /><rs:ln rel=\"num2\" /><rs:ln rel=\"num3\" /></url>" )
self.assertEqual( Sitemap().resource_as_xml(r1), "<url><loc>4a</loc><lastmod>2013-01-02T00:00:00Z</lastmod><rs:md hash=\"md5:ab54de\" length=\"9999\" /><rs:ln href=\"http://mirror1.example.com/res1\" modified=\"2013-01-02T18:00:00Z\" pri=\"1\" rel=\"duplicate\" /><rs:ln rel=\"num2\" /><rs:ln rel=\"num3\" /></url>" )
def test_08_print(self):
r1 = Resource(uri='a',lastmod='2001-01-01',length=1234)
@ -73,7 +81,7 @@ class TestSitemap(unittest.TestCase):
<url><loc>http://e.com/a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md hash="md5:Q2hlY2sgSW50ZWdyaXR5IQ==" length=\"12\" /></url>\
</urlset>'
s=Sitemap()
i=s.parse_xml(fh=StringIO.StringIO(xml))
i=s.parse_xml(fh=io.StringIO(xml))
self.assertFalse( s.parsed_index, 'was a sitemap')
self.assertEqual( s.resources_created, 1, 'got 1 resources')
for r in i.resources:
@ -90,7 +98,7 @@ class TestSitemap(unittest.TestCase):
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md length=\"32\" /></url>\
</urlset>'
s=Sitemap()
i=s.parse_xml(fh=StringIO.StringIO(xml))
i=s.parse_xml(fh=io.StringIO(xml))
self.assertFalse( s.parsed_index, 'was a sitemap')
self.assertEqual( s.resources_created, 2, 'got 2 resources')
@ -102,13 +110,13 @@ class TestSitemap(unittest.TestCase):
s=Sitemap()
two_loc='<loc>/tmp/rs_test/src/file_a</loc><loc>/tmp/rs_test/src/file_b</loc>'
self.assertRaises( SitemapParseError, s.parse_xml,
StringIO.StringIO(xml_start+two_loc+xml_end))
io.StringIO(xml_start+two_loc+xml_end))
mt_loc='<loc></loc>'
self.assertRaises( SitemapParseError, s.parse_xml,
StringIO.StringIO(xml_start+mt_loc+xml_end))
io.StringIO(xml_start+mt_loc+xml_end))
mt_loc_att='<loc att="value"/>'
self.assertRaises( SitemapParseError, s.parse_xml,
StringIO.StringIO(xml_start+mt_loc_att+xml_end))
io.StringIO(xml_start+mt_loc_att+xml_end))
def test_13_parse_multi_lastmod(self):
xml_start='<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
@ -118,17 +126,17 @@ class TestSitemap(unittest.TestCase):
s=Sitemap()
two_lastmod='<lastmod>2013-01-01</lastmod><lastmod>2013-01-02</lastmod>'
self.assertRaises( SitemapParseError, s.parse_xml,
StringIO.StringIO(xml_start+two_lastmod+xml_end))
io.StringIO(xml_start+two_lastmod+xml_end))
# While it not ideal to omit, <lastmod> is not required and
# thus either empty lastmod or lastmod with just an attribute
# and no content are not ambiguous and thus should be accepted
# with resulting None for resource.lastmod
mt_lastmod='<lastmod></lastmod>'
i=s.parse_xml(fh=StringIO.StringIO(xml_start+mt_lastmod+xml_end))
i=s.parse_xml(fh=io.StringIO(xml_start+mt_lastmod+xml_end))
self.assertEqual( s.resources_created, 1 )
self.assertEqual( i.resources[0].lastmod, None )
mt_lastmod_att='<lastmod att="value"/>'
i=s.parse_xml(fh=StringIO.StringIO(xml_start+mt_lastmod_att+xml_end))
i=s.parse_xml(fh=io.StringIO(xml_start+mt_lastmod_att+xml_end))
self.assertEqual( s.resources_created, 1 )
self.assertEqual( i.resources[0].lastmod, None )
@ -139,22 +147,22 @@ class TestSitemap(unittest.TestCase):
s=Sitemap()
two_md='<rs:md capability="resourcelist"/><rs:md capability="resourcelist"/>'
self.assertRaises( SitemapParseError, s.parse_xml,
StringIO.StringIO(xml_start+two_md+xml_end))
io.StringIO(xml_start+two_md+xml_end))
def test_15_parse_illformed(self):
s=Sitemap()
# ExpatError in python2.6, ParserError in 2.7
self.assertRaises( etree_error_class, s.parse_xml, StringIO.StringIO('not xml') )
self.assertRaises( etree_error_class, s.parse_xml, StringIO.StringIO('<urlset><url>something</urlset>') )
# ExpatError in python2.6, ParserError in 2.7,3.x
self.assertRaises( etree_error_class, s.parse_xml, io.StringIO('not xml') )
self.assertRaises( etree_error_class, s.parse_xml, io.StringIO('<urlset><url>something</urlset>') )
def test_16_parse_valid_xml_but_other(self):
s=Sitemap()
self.assertRaises( SitemapParseError, s.parse_xml, StringIO.StringIO('<urlset xmlns="http://example.org/other_namespace"> </urlset>') )
self.assertRaises( SitemapParseError, s.parse_xml, StringIO.StringIO('<other xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </other>') )
self.assertRaises( SitemapParseError, s.parse_xml, io.StringIO('<urlset xmlns="http://example.org/other_namespace"> </urlset>') )
self.assertRaises( SitemapParseError, s.parse_xml, io.StringIO('<other xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </other>') )
def test_17_parse_sitemapindex_as_sitemap(self):
s=Sitemap()
self.assertRaises( SitemapIndexError, s.parse_xml, StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>'), sitemapindex=False )
self.assertRaises( SitemapIndexError, s.parse_xml, io.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>'), sitemapindex=False )
def test_18_parse_with_rs_ln_on_resource(self):
xml='<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
@ -173,12 +181,12 @@ class TestSitemap(unittest.TestCase):
</url>\
</urlset>'
s=Sitemap()
rc=s.parse_xml(fh=StringIO.StringIO(xml))
rc=s.parse_xml(fh=io.StringIO(xml))
self.assertFalse( s.parsed_index, 'was a sitemap')
self.assertEqual( s.resources_created, 2, 'got 2 resources')
i = iter(rc)
r1 = i.next()
r2 = i.next()
r1 = next(i)
r2 = next(i)
self.assertEqual( r1.uri, 'http://example.com/file_a' )
self.assertEqual( r1.ln[0]['rel'], 'duplicate' )
self.assertEqual( r1.ln[0]['href'], 'http://mirror1.example.com/res1' )
@ -196,39 +204,39 @@ class TestSitemap(unittest.TestCase):
#
# missing href
xml=xmlstart+'<rs:ln rel="duplicate"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
# missing rel
xml=xmlstart+'<rs:ln href="http://example.com/"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
# bad length
xml=xmlstart+'<rs:ln rel="duplicate" href="http://example.com/" length="a"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
# bad pri
xml=xmlstart+'<rs:ln rel="duplicate" href="http://example.com/" pri="fff"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
xml=xmlstart+'<rs:ln rel="duplicate" href="http://example.com/" pri="0"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
xml=xmlstart+'<rs:ln rel="duplicate" href="http://example.com/" pri="1000000"/>'+xmlend
self.assertRaises( SitemapParseError, s.parse_xml, fh=StringIO.StringIO(xml))
self.assertRaises( SitemapParseError, s.parse_xml, fh=io.StringIO(xml))
# and finally OK with errors fixes
xml=xmlstart+'<rs:ln rel="duplicate" href="http://example.com/" length="12345" pri="1" other="whatever"/>'+xmlend
rc = s.parse_xml(fh=StringIO.StringIO(xml))
rc = s.parse_xml(fh=io.StringIO(xml))
self.assertEqual( len(rc.resources), 1, 'good at last, extra attribute ignored' )
def test_20_parse_sitemapindex_empty(self):
s=Sitemap()
si = s.parse_xml( fh=StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>'), sitemapindex=True )
si = s.parse_xml( fh=io.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>'), sitemapindex=True )
self.assertTrue( s.parsed_index, 'was a sitemapindex')
self.assertEqual( len(si.resources), 0, '0 sitemaps')
def test_21_parse_sitemapindex(self):
s=Sitemap()
si = s.parse_xml( fh=StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>aaa</loc></sitemap><sitemap><loc>bbb</loc></sitemap></sitemapindex>'), sitemapindex=True )
si = s.parse_xml( fh=io.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>aaa</loc></sitemap><sitemap><loc>bbb</loc></sitemap></sitemapindex>'), sitemapindex=True )
self.assertEqual( len(si.resources), 2, '2 sitemaps')
sms = sorted(si.uris())
self.assertEqual( sms, ['aaa','bbb'] )
# add a couple more
s.parse_xml( fh=StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>cc</loc></sitemap><sitemap><loc>dd</loc></sitemap></sitemapindex>'), resources=si )
s.parse_xml( fh=io.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>cc</loc></sitemap><sitemap><loc>dd</loc></sitemap></sitemapindex>'), resources=si )
self.assertTrue( s.parsed_index, 'was a sitemapindex')
self.assertEqual( len(si.resources), 4, '4 sitemaps total')
sms = sorted(si.uris())
@ -259,13 +267,13 @@ class TestSitemap(unittest.TestCase):
</urlset>'
s=Sitemap()
s.resource_class=Resource
c=s.parse_xml(fh=StringIO.StringIO(xml))
c=s.parse_xml(fh=io.StringIO(xml))
self.assertEqual( s.resources_created, 2, 'got 2 resources')
i = iter(c)
r1 = i.next()
r1 = next(i)
self.assertEqual( r1.uri, '/tmp/rs_test/src/file_a' )
self.assertEqual( r1.change, 'updated' )
r2 = i.next()
r2 = next(i)
self.assertEqual( r2.uri, '/tmp/rs_test/src/file_b' )
self.assertEqual( r2.change, None )