gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use resource_list/change_list instead of resourcelist/changelist

Browse Source
This commit is contained in:
Simeon Warner 2013-01-21 14:11:54 -05:00
parent 172f509f60
commit ce19a0df81
16 changed files with 229 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.txt
View File

@ -11,7 +11,7 @@ Typical client usage:
Typical library usage in a source:
```python
from resync.resourcelist import ResourceList
from resync.resource_list import ResourceList
from resync.resource import Resource
from resync.sitemap import Sitemap

2
docs/examples/01_make_resourcelist.py
View File

@ -1,4 +1,4 @@
from resync.resourcelist import ResourceList
from resync.resource_list import ResourceList
from resync.resource import Resource
from resync.sitemap import Sitemap

8
resync/changelist.py → resync/change_list.py
View File

@ -6,11 +6,11 @@ time, and also metadata about a change that may have occurred
to bring the resource to that states. These descriptions
are Resource objects.
Different from an resourcelist, a changelist may include multiple
descriptions for the same resource. The changelist is ordered
Different from an resource_list, a change_list may include multiple
descriptions for the same resource. The change_list is ordered
from first entry to last entry.
Different from an resourcelist, dereference by a URI yields a
Different from an resource_list, dereference by a URI yields a
ChangeList containing descriptions pertaining to that
particular resource.
"""
@ -29,7 +29,7 @@ class ChangeList(ResourceContainer):
super(ChangeList, self).__init__(resources, capabilities)
def __len__(self):
"""Number of entries in this changelist"""
"""Number of entries in this change_list"""
return(len(self.resources))
def add(self, resource):

174
resync/client.py
View File

@ -10,9 +10,9 @@ import time
import logging
import ConfigParser
from resync.resourcelist_builder import ResourceListBuilder
from resync.resourcelist import ResourceList
from resync.changelist import ChangeList
from resync.resource_list_builder import ResourceListBuilder
from resync.resource_list import ResourceList
from resync.change_list import ChangeList
from resync.mapper import Mapper
from resync.sitemap import Sitemap
from resync.dump import Dump
@ -40,7 +40,7 @@ class Client(object):
self.dryrun = dryrun
self.logger = logging.getLogger('client')
self.mapper = None
self.resourcelist_name = 'resourcelist.xml'
self.resource_list_name = 'resource_list.xml'
self.dump_format = None
self.exclude_patterns = []
self.allow_multifile = True
@ -60,8 +60,8 @@ class Client(object):
"""Build and set Mapper object based on input mappings"""
self.mapper = Mapper(mappings)
def sitemap_changelist_uri(self,basename):
"""Get full URI (filepath) for sitemap/changelist based on basename"""
def sitemap_change_list_uri(self,basename):
"""Get full URI (filepath) for sitemap/change_list based on basename"""
if (re.match(r"\w+:",basename)):
# looks like URI
return(basename)
@ -75,13 +75,13 @@ class Client(object):
@property
def sitemap(self):
"""Return the sitemap URI based on maps or explicit settings"""
return(self.sitemap_changelist_uri(self.resourcelist_name))
return(self.sitemap_change_list_uri(self.resource_list_name))
@property
def resourcelist(self):
"""Return resourcelist on disk based on current mappings
def resource_list(self):
"""Return resource_list on disk based on current mappings
Return resourcelist. Uses existing self.mapper settings.
Return resource_list. Uses existing self.mapper settings.
"""
### 0. Sanity checks
if (len(self.mappings)<1):
@ -107,26 +107,26 @@ class Client(object):
if (len(self.mappings)<1):
raise ClientFatalError("No source to destination mapping specified")
### 1. Get inventories from both src and dst
# 1.a source resourcelist
# 1.a source resource_list
ib = ResourceListBuilder(mapper=self.mapper)
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_resourcelist = src_sitemap.read(uri=self.sitemap)
src_resource_list = src_sitemap.read(uri=self.sitemap)
self.logger.debug("Finished reading sitemap")
except Exception as e:
raise ClientFatalError("Can't read source resourcelist from %s (%s)" % (self.sitemap,str(e)))
self.logger.info("Read source resourcelist, %d resources listed" % (len(src_resourcelist)))
if (len(src_resourcelist)==0):
raise ClientFatalError("Can't read source resource_list from %s (%s)" % (self.sitemap,str(e)))
self.logger.info("Read source resource_list, %d resources listed" % (len(src_resource_list)))
if (len(src_resource_list)==0):
raise ClientFatalError("Aborting as there are no resources to sync")
if (self.checksum and not src_resourcelist.has_md5()):
if (self.checksum and not src_resource_list.has_md5()):
self.checksum=False
self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
# 1.b destination resourcelist mapped back to source URIs
self.logger.info("Not calculating checksums on destination as not present in source resource_list")
# 1.b destination resource_list mapped back to source URIs
ib.do_md5=self.checksum
dst_resourcelist = ib.from_disk()
### 2. Compare these resourcelists respecting any comparison options
(same,updated,deleted,created)=dst_resourcelist.compare(src_resourcelist)
dst_resource_list = ib.from_disk()
### 2. Compare these resource_lists respecting any comparison options
(same,updated,deleted,created)=dst_resource_list.compare(src_resource_list)
### 3. Report status and planned actions
status = " IN SYNC "
if (len(updated)>0 or len(deleted)>0 or len(created)>0):
@ -138,7 +138,7 @@ class Client(object):
return
### 4. Check that sitemap has authority over URIs listed
uauth = UrlAuthority(self.sitemap)
for resource in src_resourcelist:
for resource in src_resource_list:
if (not uauth.has_authority_over(resource.uri)):
if (self.noauth):
#self.logger.info("Sitemap (%s) mentions resource at a location it does not have authority over (%s)" % (self.sitemap,resource.uri))
@ -162,7 +162,7 @@ class Client(object):
self.delete_resource(resource,file,allow_deletion)
### 6. For sync reset any incremental status for site
if (not audit_only):
links = self.extract_links(src_resourcelist)
links = self.extract_links(src_resource_list)
if ('next' in links):
self.write_incremental_status(self.sitemap,links['next'])
self.logger.info("Written config with next incremental at %s" % (links['next']))
@ -170,7 +170,7 @@ class Client(object):
self.write_incremental_status(self.sitemap)
self.logger.debug("Completed "+action)
def incremental(self, allow_deletion=False, changelist_uri=None):
def incremental(self, allow_deletion=False, change_list_uri=None):
"""Incremental synchronization"""
self.logger.debug("Starting incremental sync")
### 0. Sanity checks
@ -178,63 +178,63 @@ class Client(object):
raise ClientFatalError("No source to destination mapping specified")
# Get current config
inc_config_next=self.read_incremental_status(self.sitemap)
### 1. Get URI of changelist, from sitemap or explicit
### 1. Get URI of change_list, from sitemap or explicit
if (inc_config_next is not None):
# We have config from last run for this site
changelist = inc_config_next
self.logger.info("ChangeList location from last incremental run %s" % (changelist))
elif (changelist_uri):
change_list = inc_config_next
self.logger.info("ChangeList location from last incremental run %s" % (change_list))
elif (change_list_uri):
# Translate as necessary using maps
changelist = self.sitemap_changelist_uri(changelist_uri)
change_list = self.sitemap_change_list_uri(change_list_uri)
else:
# Get sitemap
try:
self.logger.info("Reading sitemap %s" % (self.sitemap))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_resourcelist = src_sitemap.read(uri=self.sitemap, index_only=True)
src_resource_list = src_sitemap.read(uri=self.sitemap, index_only=True)
self.logger.debug("Finished reading sitemap/sitemapindex")
except Exception as e:
raise ClientFatalError("Can't read source sitemap from %s (%s)" % (self.sitemap,str(e)))
# Extract changelist location
# Extract change_list location
# FIXME - need to completely rework the way we handle/store capabilities
links = self.extract_links(src_resourcelist)
links = self.extract_links(src_resource_list)
if ('current' not in links):
raise ClientFatalError("Failed to extract changelist location from sitemap %s" % (self.sitemap))
changelist = links['current']
### 2. Read changelist from source
raise ClientFatalError("Failed to extract change_list location from sitemap %s" % (self.sitemap))
change_list = links['current']
### 2. Read change_list from source
ib = ResourceListBuilder(mapper=self.mapper)
try:
self.logger.info("Reading changelist %s" % (changelist))
self.logger.info("Reading change_list %s" % (change_list))
src_sitemap = Sitemap(allow_multifile=self.allow_multifile, mapper=self.mapper)
src_changelist = src_sitemap.read(uri=changelist, changelist=True)
self.logger.debug("Finished reading changelist")
src_change_list = src_sitemap.read(uri=change_list, change_list=True)
self.logger.debug("Finished reading change_list")
except Exception as e:
raise ClientFatalError("Can't read source changelist from %s (%s)" % (changelist,str(e)))
self.logger.info("Read source changelist, %d resources listed" % (len(src_changelist)))
#if (len(src_changelist)==0):
raise ClientFatalError("Can't read source change_list from %s (%s)" % (change_list,str(e)))
self.logger.info("Read source change_list, %d resources listed" % (len(src_change_list)))
#if (len(src_change_list)==0):
# raise ClientFatalError("Aborting as there are no resources to sync")
if (self.checksum and not src_changelist.has_md5()):
if (self.checksum and not src_change_list.has_md5()):
self.checksum=False
self.logger.info("Not calculating checksums on destination as not present in source resourcelist")
self.logger.info("Not calculating checksums on destination as not present in source resource_list")
### 3. Check that sitemap has authority over URIs listed
# FIXME - What does authority mean for changelist? Here use both the
# changelist URI and, if we used it, the sitemap URI
uauth_cs = UrlAuthority(changelist)
if (not changelist_uri):
# FIXME - What does authority mean for change_list? Here use both the
# change_list URI and, if we used it, the sitemap URI
uauth_cs = UrlAuthority(change_list)
if (not change_list_uri):
uauth_sm = UrlAuthority(self.sitemap)
for resource in src_changelist:
for resource in src_change_list:
if (not uauth_cs.has_authority_over(resource.uri) and
(changelist_uri or not uauth_sm.has_authority_over(resource.uri))):
(change_list_uri or not uauth_sm.has_authority_over(resource.uri))):
if (self.noauth):
#self.logger.info("ChangeList (%s) mentions resource at a location it does not have authority over (%s)" % (changelist,resource.uri))
#self.logger.info("ChangeList (%s) mentions resource at a location it does not have authority over (%s)" % (change_list,resource.uri))
pass
else:
raise ClientFatalError("Aborting as changelist (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (changelist,resource.uri))
raise ClientFatalError("Aborting as change_list (%s) mentions resource at a location it does not have authority over (%s), override with --noauth" % (change_list,resource.uri))
### 3. Apply changes
num_updated = 0
num_deleted = 0
num_created = 0
for resource in src_changelist:
for resource in src_change_list:
uri = resource.uri
file = self.mapper.src_to_dst(uri)
if (resource.change == 'updated'):
@ -258,12 +258,12 @@ class Client(object):
(status,num_updated,num_deleted,num_created))
# 5. Store next link if available
if ((num_updated+num_deleted+num_created)>0):
links = self.extract_links(src_changelist)
links = self.extract_links(src_change_list)
if ('next' in links):
self.write_incremental_status(self.sitemap,links['next'])
self.logger.info("Written config with next incremental at %s" % (links['next']))
else:
self.logger.warning("Failed to extract next changelist location from changelist %s" % (changelist))
self.logger.warning("Failed to extract next change_list location from change_list %s" % (change_list))
# 6. Done
self.logger.debug("Completed incremental sync")
@ -275,7 +275,7 @@ class Client(object):
2. set mtime in local time to be equal to timestamp in UTC (should perhaps
or at least warn if different from LastModified from the GET response instead
but maybe warn if different (or just earlier than) the lastmod we expected
from the resourcelist
from the resource_list
"""
path = os.path.dirname(file)
distutils.dir_util.mkpath(path)
@ -345,29 +345,29 @@ class Client(object):
break
def explore_links(self):
"""Explore links from sitemap and between changelists"""
"""Explore links from sitemap and between change_lists"""
seen = dict()
is_changelist,links = self.explore_links_get(self.sitemap, seen=seen)
starting_changelist = self.sitemap
if (not is_changelist):
is_change_list,links = self.explore_links_get(self.sitemap, seen=seen)
starting_change_list = self.sitemap
if (not is_change_list):
if ('current' in links):
starting_changelist = links['current']
is_changelist,links = self.explore_links_get(links['current'], seen=seen)
starting_change_list = links['current']
is_change_list,links = self.explore_links_get(links['current'], seen=seen)
# Can we go backward?
if ('prev' in links and not links['prev'] in seen):
self.logger.warning("Will follow links backwards...")
while ('prev' in links and not links['prev'] in seen):
self.logger.warning("Following \"prev\" link")
is_changelist,links = self.explore_links_get(links['prev'], seen=seen)
is_change_list,links = self.explore_links_get(links['prev'], seen=seen)
else:
self.logger.warning("No links backwards")
# Can we go forward?
links = seen[starting_changelist]
links = seen[starting_change_list]
if ('next' in links and not links['next'] in seen):
self.logger.warning("Will follow links forwards...")
while ('next' in links and not links['next'] in seen):
self.logger.warning("Following \"next\" link")
is_changelist,links = self.explore_links_get(links['next'], seen=seen)
is_change_list,links = self.explore_links_get(links['next'], seen=seen)
else:
self.logger.warning("No links forwards")
@ -383,11 +383,11 @@ class Client(object):
if ('next' in links and links['next']==uri):
self.logger.warning("- self reference \"next\" link")
seen[uri]=links
return(s.changelist_read,links)
return(s.change_list_read,links)
def write_sitemap(self,outfile=None,capabilities=None,dump=None):
# Set up base_path->base_uri mappings, get resourcelist from disk
i = self.resourcelist
# Set up base_path->base_uri mappings, get resource_list from disk
i = self.resource_list
i.capabilities = capabilities
s=Sitemap(pretty_xml=True, allow_multifile=self.allow_multifile, mapper=self.mapper)
if (self.max_sitemap_entries is not None):
@ -398,44 +398,44 @@ class Client(object):
s.write(i,basename=outfile)
self.write_dump_if_requested(i,dump)
def changelist_sitemap(self,outfile=None,ref_sitemap=None,newref_sitemap=None,
def change_list_sitemap(self,outfile=None,ref_sitemap=None,newref_sitemap=None,
empty=None,capabilities=None,dump=None):
changelist = ChangeList()
changelist.capabilities = capabilities
change_list = ChangeList()
change_list.capabilities = capabilities
if (not empty):
# 1. Get and parse reference sitemap
old_inv = self.read_reference_sitemap(ref_sitemap)
# 2. Depending on whether a newref_sitemap was specified, either read that
# or build resourcelist from files on disk
# or build resource_list from files on disk
if (newref_sitemap is None):
# Get resourcelist from disk
new_inv = self.resourcelist
# Get resource_list from disk
new_inv = self.resource_list
else:
new_inv = self.read_reference_sitemap(newref_sitemap,name='new reference')
# 3. Calculate changelist
# 3. Calculate change_list
(same,updated,deleted,created)=old_inv.compare(new_inv)
changelist.add_changed_resources( updated, change='updated' )
changelist.add_changed_resources( deleted, change='deleted' )
changelist.add_changed_resources( created, change='created' )
# 4. Write out changelist
change_list.add_changed_resources( updated, change='updated' )
change_list.add_changed_resources( deleted, change='deleted' )
change_list.add_changed_resources( created, change='created' )
# 4. Write out change_list
s = Sitemap(pretty_xml=True, allow_multifile=self.allow_multifile, mapper=self.mapper)
if (self.max_sitemap_entries is not None):
s.max_sitemap_entries = self.max_sitemap_entries
if (outfile is None):
print s.resources_as_xml(changelist,changelist=True)
print s.resources_as_xml(change_list,change_list=True)
else:
s.write(changelist,basename=outfile,changelist=True)
self.write_dump_if_requested(changelist,dump)
s.write(change_list,basename=outfile,change_list=True)
self.write_dump_if_requested(change_list,dump)
def write_dump_if_requested(self,resourcelist,dump):
def write_dump_if_requested(self,resource_list,dump):
if (dump is None):
return
self.logger.info("Writing dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(resourcelist=resourcelist,dumpfile=dump)
d.write(resource_list=resource_list,dumpfile=dump)
def read_reference_sitemap(self,ref_sitemap,name='reference'):
"""Read reference sitemap and return the resourcelist
"""Read reference sitemap and return the resource_list
name parameter just uses in output messages to say what type
of sitemap is being read.
@ -462,7 +462,7 @@ class Client(object):
return(i)
def extract_links(self, rc, verbose=False):
"""Extract links from capabilities resourcelist or changelist
"""Extract links from capabilities resource_list or change_list
FIXME - when we finalize the form of links this should probably
go along with other capabilities functions somewhere general.
@ -472,8 +472,8 @@ class Client(object):
atts = rc.capabilities[href].get('attributes')
self.logger.debug("Capability: %s" % (str(rc.capabilities[href])))
if (atts is not None):
# split on spaces, check is changelist rel and diraction
if ('http://www.openarchives.org/rs/changelist' in atts):
# split on spaces, check is change_list rel and diraction
if ('http://www.openarchives.org/rs/change_list' in atts):
for linktype in ['next','prev','current']:
if (linktype in atts):
if (linktype in links):

38
resync/dump.py
View File

@ -8,9 +8,9 @@ class DumpError(Exception):
pass
class Dump(object):
"""Dump of resource content associated with an resourcelist or change set
"""Dump of resource content associated with an resource_list or change set
The resourcelist must be comprised of Resource objects
The resource_list must be comprised of Resource objects
which have the path attributes set to indicate the local
location of the copies of the resources.
"""
@ -21,31 +21,31 @@ class Dump(object):
self.max_size = 100*1024*1024 #100MB
self.max_files = 50000
def write(self, resourcelist=None, dumpfile=None):
def write(self, resource_list=None, dumpfile=None):
"""Write a dump file"""
self.check_files(resourcelist)
self.check_files(resource_list)
if (self.format == 'zip'):
self.write_zip(resourcelist,dumpfile)
self.write_zip(resource_list,dumpfile)
elif (self.format == 'warc'):
self.write_warc(resourcelist,dumpfile)
self.write_warc(resource_list,dumpfile)
else:
raise DumpError("Unknown dump format '%s'" % (self.format))
def write_zip(self, resourcelist=None, dumpfile=None):
def write_zip(self, resource_list=None, dumpfile=None):
"""Write a ZIP dump file"""
compression = ( ZIP_DEFLATED if self.compress else ZIP_STORED )
zf = ZipFile(dumpfile, mode="w", compression=compression, allowZip64=True)
# Write resourcelist first
# Write resource_list first
s = Sitemap(pretty_xml=True, allow_multifile=False)
zf.writestr('manifest.xml',s.resources_as_xml(resourcelist))
# Add all files in the resourcelist
for resource in resourcelist:
zf.writestr('manifest.xml',s.resources_as_xml(resource_list))
# Add all files in the resource_list
for resource in resource_list:
zf.write(resource.uri)
zf.close()
zipsize = os.path.getsize(dumpfile)
print "Wrote ZIP file dump %s with size %d bytes" % (dumpfile,zipsize)
def write_warc(self, resourcelist=None, dumpfile=None):
def write_warc(self, resource_list=None, dumpfile=None):
"""Write a WARC dump file"""
# Load library late as we want to be able to run rest of code
# without this installed
@ -54,8 +54,8 @@ class Dump(object):
except:
raise DumpError("Failed to load WARC library")
wf = WARCFile(dumpfile, mode="w", compress=self.compress)
# Add all files in the resourcelist
for resource in resourcelist:
# Add all files in the resource_list
for resource in resource_list:
wh = WARCHeader({})
wh.url = resource.uri
wh.ip_address = None
@ -69,12 +69,12 @@ class Dump(object):
warcsize = os.path.getsize(dumpfile)
print "Wrote WARC file dump %s with size %d bytes" % (dumpfile,warcsize)
def check_files(self,resourcelist):
"""Go though and check all files in resourcelist, add up size"""
if (len(resourcelist) > self.max_files):
raise DumpError("Number of files to dump (%d) exceeds maximum (%d)" % (len(resourcelist),self.max_files))
def check_files(self,resource_list):
"""Go though and check all files in resource_list, add up size"""
if (len(resource_list) > self.max_files):
raise DumpError("Number of files to dump (%d) exceeds maximum (%d)" % (len(resource_list),self.max_files))
total_size = 0 #total size of all files in bytes
for resource in resourcelist:
for resource in resource_list:
if (resource.path is None):
#explicit test because exception raised by getsize otherwise confusing
raise DumpError("No file path defined for resource %s" % resource.uri)

2
resync/resource_container.py
View File

@ -27,7 +27,7 @@ class ResourceContainer(object):
self.capabilities=(capabilities if (capabilities is not None) else {})
def __iter__(self):
"""Iterator over all the resources in this resourcelist
"""Iterator over all the resources in this resource_list
Baseline implementation use iterator given by resources property
"""

24
resync/resourcelist.py → resync/resource_list.py
View File

@ -1,12 +1,12 @@
"""ResourceSync resourcelist object
"""ResourceSync resource_list object
An resourcelist is a set of resources with some metadata for each
An resource_list is a set of resources with some metadata for each
resource. Comparison of inventories from a source and a
destination allows understanding of whether the two are in
sync or whether some resources need to be updated at the
destination.
The resourcelist object may also contain metadata regarding
The resource_list object may also contain metadata regarding
capabilities and discovery information.
"""
@ -28,7 +28,7 @@ class ResourceListDict(dict):
"""
def __iter__(self):
"""Iterator over all the resources in this resourcelist"""
"""Iterator over all the resources in this resource_list"""
self._iter_next_list = sorted(self.keys())
self._iter_next_list.reverse()
return(iter(self._iter_next, None))
@ -43,21 +43,21 @@ class ResourceListDict(dict):
"""Add just a single resource"""
uri = resource.uri
if (uri in self and not replace):
raise ResourceListDupeError("Attempt to add resource already in resourcelist")
raise ResourceListDupeError("Attempt to add resource already in resource_list")
self[uri]=resource
class ResourceListDupeError(Exception):
pass
class ResourceList(ResourceContainer):
"""Class representing an resourcelist of resources
"""Class representing an resource_list of resources
This same class is used for both the source and the destination
and is the central point of comparison the decide whether they
are in sync or what needs to be copied to bring the destinaton
into sync.
An resourcelist will admit only one resource with any given URI.
An resource_list will admit only one resource with any given URI.
Storage is unordered but the iterator imposes a canonical order
which is currently alphabetical by URI.
@ -68,18 +68,18 @@ class ResourceList(ResourceContainer):
self.capabilities=(capabilities if (capabilities is not None) else {})
def __iter__(self):
"""Iterator over all the resources in this resourcelist"""
"""Iterator over all the resources in this resource_list"""
return(iter(self.resources))
def __len__(self):
"""Return number of resources in this resourcelist"""
"""Return number of resources in this resource_list"""
return(len(self.resources))
def add(self, resource, replace=False):
"""Add a resource or an iterable collection of resources
Will throw a ValueError if the resource (ie. same uri) already
exists in the resourcelist, unless replace=True.
exists in the resource_list, unless replace=True.
"""
if isinstance(resource, collections.Iterable):
for r in resource:
@ -88,9 +88,9 @@ class ResourceList(ResourceContainer):
self.resources.add(resource,replace)
def compare(self,src):
"""Compare the current resourcelist object with the specified resourcelist
"""Compare the current resource_list object with the specified resource_list
The parameter src must also be an resourcelist object, it is assumed
The parameter src must also be an resource_list object, it is assumed
to be the source, and the current object is the destination. This
written to work for any objects in self and sc, provided that the
== operator can be used to compare them.

36
resync/resourcelist_builder.py → resync/resource_list_builder.py
View File

@ -2,7 +2,7 @@
Attributes:
- do_md5 set true to calculate MD5 sums for all files
- do_size set true to include file size in resourcelist
- do_size set true to include file size in resource_list
- exclude_dirs is a list of directory names to exclude
(defaults to ['CVS','.git'))
"""
@ -16,7 +16,7 @@ from urllib import URLopener
from xml.etree.ElementTree import parse
from resource import Resource
from resourcelist import ResourceList
from resource_list import ResourceList
from sitemap import Sitemap
from utils import compute_md5_for_file
@ -36,11 +36,11 @@ class ResourceListBuilder():
self.exclude_dirs = ['CVS','.git']
self.include_symlinks = False
# Used internally only:
self.logger = logging.getLogger('resourcelist_builder')
self.logger = logging.getLogger('resource_list_builder')
self.compiled_exclude_files = []
def add_exclude_files(self, exclude_patterns):
"""Add more patterns of files to exclude while building resourcelist"""
"""Add more patterns of files to exclude while building resource_list"""
for pattern in exclude_patterns:
self.exclude_files.append(pattern)
@ -56,36 +56,36 @@ class ResourceListBuilder():
return(True)
return(False)
def from_disk(self,resourcelist=None):
"""Create or extend resourcelist with resources from disk scan
def from_disk(self,resource_list=None):
"""Create or extend resource_list with resources from disk scan
Assumes very simple disk path to URL mapping: chop path and
replace with url_path. Returns the new or extended ResourceList
object.
If a resourcelist is specified then items are added to that rather
If a resource_list is specified then items are added to that rather
than creating a new one.
mapper=Mapper('http://example.org/path','/path/to/files')
mb = ResourceListBuilder(mapper=mapper)
m = resourcelist_from_disk()
m = resource_list_from_disk()
"""
num=0
# Either use resourcelist passed in or make a new one
if (resourcelist is None):
resourcelist = ResourceList()
# Either use resource_list passed in or make a new one
if (resource_list is None):
resource_list = ResourceList()
# Compile exclude pattern matches
self.compile_excludes()
# Run for each map in the mappings
for map in self.mapper.mappings:
self.logger.info("Scanning disk for %s" % (str(map)))
self.from_disk_add_map(resourcelist=resourcelist, map=map)
return(resourcelist)
self.from_disk_add_map(resource_list=resource_list, map=map)
return(resource_list)
def from_disk_add_map(self, resourcelist=None, map=None):
def from_disk_add_map(self, resource_list=None, map=None):
# sanity
if (resourcelist is None or map is None):
raise ValueError("Must specify resourcelist and map")
if (resource_list is None or map is None):
raise ValueError("Must specify resource_list and map")
path=map.dst_path
#print "walking: %s" % (path)
# for each file: create Resource object, add, increment counter
@ -118,10 +118,10 @@ class ResourceListBuilder():
if (self.do_size):
# add size
r.size=file_stat.st_size
resourcelist.add(r)
resource_list.add(r)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
self.logger.debug("Excluding dir %s" % (exclude))
dirs.remove(exclude)
return(resourcelist)
return(resource_list)

106
resync/sitemap.py
View File

@ -10,8 +10,8 @@ from datetime import datetime
import StringIO
from resource import Resource
from resourcelist import ResourceList, ResourceListDupeError
from changelist import ChangeList
from resource_list import ResourceList, ResourceListDupeError
from change_list import ChangeList
from mapper import Mapper, MapperError
from url_authority import UrlAuthority
@ -29,7 +29,7 @@ class SitemapIndexError(Exception):
return(self.message)
class SitemapIndex(ResourceList):
"""Reuse an resourcelist to hold the set of sitemaps"""
"""Reuse an resource_list to hold the set of sitemaps"""
pass
class SitemapError(Exception):
@ -50,21 +50,21 @@ class Sitemap(object):
self.mapper=mapper
self.max_sitemap_entries=50000
# Classes used when parsing
self.resourcelist_class=ResourceList
self.resource_list_class=ResourceList
self.resource_class=Resource
self.changelist_class=ChangeList
self.change_list_class=ChangeList
self.resourcechange_class=Resource
# Information recorded for logging
self.resources_created=None # Set during parsing sitemap
self.sitemaps_created=None # Set during parsing sitemapindex
self.content_length=None # Size of last sitemap read
self.bytes_read=0 # Aggregate of content_length values
self.changelist_read=None # Set true if changelist read
self.read_type=None # Either sitemap/sitemapindex/changelist/changelistindex
self.change_list_read=None # Set true if change_list read
self.read_type=None # Either sitemap/sitemapindex/change_list/change_listindex
##### General sitemap methods that also handle sitemapindexes #####
def write(self, resources=None, basename='/tmp/sitemap.xml', changelist=False):
def write(self, resources=None, basename='/tmp/sitemap.xml', change_list=False):
"""Write one or a set of sitemap files to disk
resources is a ResourceContainer that may be an ResourceList or
@ -74,10 +74,10 @@ class Sitemap(object):
basename is used as the name of the single sitemap file or the
sitemapindex for a set of sitemap files.
if changelist is set true then type information is added to indicate
that this sitemap file is a changelist and not an resourcelist.
if change_list is set true then type information is added to indicate
that this sitemap file is a change_list and not an resource_list.
Uses self.max_sitemap_entries to determine whether the resourcelist can
Uses self.max_sitemap_entries to determine whether the resource_list can
be written as one sitemap. If there are more entries and
self.allow_multifile is set true then a set of sitemap files,
with an sitemapindex, will be written.
@ -102,7 +102,7 @@ class Sitemap(object):
file = sitemap_prefix + ( "%05d" % (len(sitemaps)) ) + sitemap_suffix
self.logger.info("Writing sitemap %s..." % (file))
f = open(file, 'w')
f.write(self.resources_as_xml(chunk,changelist=changelist))
f.write(self.resources_as_xml(chunk,change_list=change_list))
f.close()
# Record timestamp
sitemaps[file] = os.stat(file).st_mtime
@ -111,13 +111,13 @@ class Sitemap(object):
self.logger.info("Wrote %d sitemaps" % (len(sitemaps)))
f = open(basename, 'w')
self.logger.info("Writing sitemapindex %s..." % (basename))
f.write(self.sitemapindex_as_xml(sitemaps=sitemaps,resourcelist=resources,capabilities=resources.capabilities,changelist=changelist))
f.write(self.sitemapindex_as_xml(sitemaps=sitemaps,resource_list=resources,capabilities=resources.capabilities,change_list=change_list))
f.close()
self.logger.info("Wrote sitemapindex %s" % (basename))
else:
f = open(basename, 'w')
self.logger.info("Writing sitemap %s..." % (basename))
f.write(self.resources_as_xml(chunk,capabilities=resources.capabilities,changelist=changelist))
f.write(self.resources_as_xml(chunk,capabilities=resources.capabilities,change_list=change_list))
f.close()
self.logger.info("Wrote sitemap %s" % (basename))
@ -144,21 +144,21 @@ class Sitemap(object):
next = chunk.pop()
return(chunk,next)
def read(self, uri=None, resources=None, changelist=None, index_only=False):
def read(self, uri=None, resources=None, change_list=None, index_only=False):
"""Read sitemap from a URI including handling sitemapindexes
Returns the resourcelist or changelist. If changelist is not specified (None)
Returns the resource_list or change_list. If change_list is not specified (None)
then it is assumed that an ResourceList is to be read, unless the XML
indicates a Changelist.
If changelist is True then a Changelist if expected; if changelist if False
If change_list is True then a Changelist if expected; if change_list if False
then an ResourceList is expected.
If index_only is True then individual sitemaps references in a sitemapindex
will not be read. This will result in no resources being returned and is
useful only to read the capabilities and metadata listed in the sitemapindex.
Will set self.read_type to a string value sitemap/sitemapindex/changelist/changelistindex
Will set self.read_type to a string value sitemap/sitemapindex/change_list/change_listindex
depleding on the type of the file expected/read.
Includes the subtlety that if the input URI is a local file and is a
@ -183,24 +183,24 @@ class Sitemap(object):
# check root element: urlset (for sitemap), sitemapindex or bad
self.sitemaps_created=0
root = etree.getroot()
# assume resourcelist but look to see whether this is a changelist
# as indicated with rs:type="changelist" on the root
resources_class = self.resourcelist_class
sitemap_xml_parser = self.resourcelist_parse_xml
self.changelist_read = False
# assume resource_list but look to see whether this is a change_list
# as indicated with rs:type="change_list" on the root
resources_class = self.resource_list_class
sitemap_xml_parser = self.resource_list_parse_xml
self.change_list_read = False
self.read_type = 'sitemap'
root_type = root.attrib.get('{'+RS_NS+'}type',None)
if (root_type is not None):
if (root_type == 'changelist'):
self.changelist_read = True
if (root_type == 'change_list'):
self.change_list_read = True
else:
self.logger.info("Bad value of rs:type on root element (%s), ignoring" % (root_type))
elif (changelist is True):
self.changelist_read = True
if (self.changelist_read):
self.read_type = 'changelist'
resources_class = self.changelist_class
sitemap_xml_parser = self.changelist_parse_xml
elif (change_list is True):
self.change_list_read = True
if (self.change_list_read):
self.read_type = 'change_list'
resources_class = self.change_list_class
sitemap_xml_parser = self.change_list_parse_xml
# now have make sure we have a place to put the data we read
if (resources is None):
resources=resources_class()
@ -364,7 +364,7 @@ class Sitemap(object):
##### ResourceContainer (ResourceList or Changelist) methods #####
def resources_as_xml(self, resources, num_resources=None, capabilities=None, changelist=False):
def resources_as_xml(self, resources, num_resources=None, capabilities=None, change_list=False):
"""Return XML for a set of resources in sitemap format
resources is either an iterable or iterator of Resource objects.
@ -375,8 +375,8 @@ class Sitemap(object):
# will include capabilities if allowed and if there are some
namespaces = { 'xmlns': SITEMAP_NS, 'xmlns:rs': RS_NS }
root = Element('urlset', namespaces)
if (changelist):
root.set('rs:type','changelist')
if (change_list):
root.set('rs:type','change_list')
if (self.pretty_xml):
root.text="\n"
if ( capabilities is not None and len(capabilities)>0 ):
@ -398,10 +398,10 @@ class Sitemap(object):
tree.write(xml_buf,encoding='UTF-8',xml_declaration=True,method='xml')
return(xml_buf.getvalue())
def resourcelist_parse_xml(self, fh=None, etree=None, resources=None):
def resource_list_parse_xml(self, fh=None, etree=None, resources=None):
"""Parse XML Sitemap from fh or etree and add resources to an ResourceList object
Returns the resourcelist.
Returns the resource_list.
Also sets self.resources_created to be the number of resources created.
We adopt a very lax approach here. The parsing is properly namespace
@ -412,9 +412,9 @@ class Sitemap(object):
indicates a sitemapindex then an SitemapIndexError() is thrown
and the etree passed along with it.
"""
resourcelist = resources #use resourcelist locally but want common argument name
if (resourcelist is None):
resourcelist=self.resourcelist_class()
resource_list = resources #use resource_list locally but want common argument name
if (resource_list is None):
resource_list=self.resource_list_class()
if (fh is not None):
etree=parse(fh)
elif (etree is None):
@ -425,19 +425,19 @@ class Sitemap(object):
for url_element in etree.findall('{'+SITEMAP_NS+"}url"):
r = self.resource_from_etree(url_element, self.resource_class)
try:
resourcelist.add( r )
resource_list.add( r )
except ResourceListDupeError:
self.logger.warning("dupe: %s (%s =? %s)" %
(r.uri,r.lastmod,resourcelist.resources[r.uri].lastmod))
(r.uri,r.lastmod,resource_list.resources[r.uri].lastmod))
self.resources_created+=1
resourcelist.capabilities = self.capabilities_from_etree(etree)
return(resourcelist)
resource_list.capabilities = self.capabilities_from_etree(etree)
return(resource_list)
elif (etree.getroot().tag == '{'+SITEMAP_NS+"}sitemapindex"):
raise SitemapIndexError("Got sitemapindex when expecting sitemap",etree)
else:
raise ValueError("XML is not sitemap or sitemapindex")
def changelist_parse_xml(self, fh=None, etree=None, resources=None):
def change_list_parse_xml(self, fh=None, etree=None, resources=None):
"""Parse XML Sitemap from fh or etree and add resources to an Changelist object
Returns the Changelist.
@ -451,9 +451,9 @@ class Sitemap(object):
indicates a sitemapindex then an SitemapIndexError() is thrown
and the etree passed along with it.
"""
changelist = resources #use resourcelist locally but want common argument name
if (changelist is None):
changelist=self.changelist_class()
change_list = resources #use resource_list locally but want common argument name
if (change_list is None):
change_list=self.change_list_class()
if (fh is not None):
etree=parse(fh)
elif (etree is None):
@ -463,10 +463,10 @@ class Sitemap(object):
self.resources_created=0
for url_element in etree.findall('{'+SITEMAP_NS+"}url"):
r = self.resource_from_etree(url_element, self.resourcechange_class)
changelist.add( r )
change_list.add( r )
self.resources_created+=1
changelist.capabilities = self.capabilities_from_etree(etree)
return(changelist)
change_list.capabilities = self.capabilities_from_etree(etree)
return(change_list)
elif (etree.getroot().tag == '{'+SITEMAP_NS+"}sitemapindex"):
raise SitemapIndexError("Got sitemapindex when expecting sitemap",etree)
else:
@ -474,7 +474,7 @@ class Sitemap(object):
##### Sitemap Index #####
def sitemapindex_as_xml(self, file=None, sitemaps={}, resourcelist=None, capabilities=None, changelist=False ):
def sitemapindex_as_xml(self, file=None, sitemaps={}, resource_list=None, capabilities=None, change_list=False ):
"""Return a sitemapindex as an XML string
Format:
@ -489,8 +489,8 @@ class Sitemap(object):
include_capabilities = capabilities and (len(capabilities)>0)
namespaces = { 'xmlns': SITEMAP_NS }
root = Element('sitemapindex', namespaces)
if (changelist):
root.set('rs:type','changelist')
if (change_list):
root.set('rs:type','change_list')
if (self.pretty_xml):
root.text="\n"
if (include_capabilities):

14
resync/test/test_changelist.py → resync/test/test_change_list.py
View File

@ -1,7 +1,7 @@
import unittest
from resync.resource import Resource
from resync.changelist import ChangeList
from resync.resourcelist import ResourceList
from resync.change_list import ChangeList
from resync.resource_list import ResourceList
class TestChangeList(unittest.TestCase):
@ -12,7 +12,7 @@ class TestChangeList(unittest.TestCase):
src.add( Resource('c',timestamp=1) )
src.add( Resource('a',timestamp=2) )
src.add( Resource('b',timestamp=2) )
self.assertEqual(len(src), 5, "5 changes in changelist")
self.assertEqual(len(src), 5, "5 changes in change_list")
def test2_with_repeats_again(self):
r1 = Resource(uri='a',size=1)
@ -26,7 +26,7 @@ class TestChangeList(unittest.TestCase):
i.add(r1d)
self.assertEqual( len(i), 3 )
def test3_changelist(self):
def test3_change_list(self):
src = ChangeList()
src.add( Resource('a',timestamp=1) )
src.add( Resource('b',timestamp=2) )
@ -52,7 +52,7 @@ class TestChangeList(unittest.TestCase):
added = ResourceList()
added.add( Resource('a',timestamp=1) )
added.add( Resource('d',timestamp=4))
self.assertEqual(len(added), 2, "2 things in added resourcelist")
self.assertEqual(len(added), 2, "2 things in added resource_list")
changes = ChangeList()
changes.add_changed_resources( added, change='created' )
self.assertEqual(len(changes), 2, "2 things added")
@ -68,10 +68,10 @@ class TestChangeList(unittest.TestCase):
updated = ResourceList()
updated.add( Resource('a',timestamp=5) )
updated.add( Resource('b',timestamp=6))
self.assertEqual(len(updated), 2, "2 things in updated resourcelist")
self.assertEqual(len(updated), 2, "2 things in updated resource_list")
changes.add_changed_resources( updated, change='updated' )
self.assertEqual(len(changes), 4, "4 = 2 old + 2 things updated")
# Make new resourcelist from the changes which should not have dupes
# Make new resource_list from the changes which should not have dupes
dst = ResourceList()
dst.add( changes, replace=True )
self.assertEqual(len(dst), 3, "3 unique resources")

10
resync/test/test_client.py
View File

@ -4,14 +4,14 @@ from resync.client import Client, ClientFatalError
class TestResource(unittest.TestCase):
def test1_make_resourcelist_empty(self):
def test1_make_resource_list_empty(self):
c = Client()
# No mapping is error
#
def wrap_resourcelist_property_call(c):
# do this because assertRaises( ClientFatalError, c.resourcelist ) doesn't work
return(c.resourcelist)
self.assertRaises( ClientFatalError, wrap_resourcelist_property_call, c )
def wrap_resource_list_property_call(c):
# do this because assertRaises( ClientFatalError, c.resource_list ) doesn't work
return(c.resource_list)
self.assertRaises( ClientFatalError, wrap_resource_list_property_call, c )
def test2_bad_source_uri(self):
c = Client()

4
resync/test/test_dump.py
View File

@ -1,6 +1,6 @@
import unittest
from resync.dump import Dump, DumpError
from resync.resourcelist import ResourceList
from resync.resource_list import ResourceList
from resync.resource import Resource
class TestDump(unittest.TestCase):
@ -10,7 +10,7 @@ class TestDump(unittest.TestCase):
i.add( Resource('http://ex.org/a', size=1, path='resync/test/testdata/a') )
i.add( Resource('http://ex.org/b', size=2, path='resync/test/testdata/b') )
d=Dump()
d.check_files(resourcelist=i)
d.check_files(resource_list=i)
self.assertEqual(d.total_size, 28)
#FIXME -- need some code to actually write and read dump

14
resync/test/test_examples_from_spec.py
View File

@ -2,26 +2,26 @@ import sys
import unittest
import StringIO
from resync.resource import Resource
from resync.resourcelist import ResourceList
from resync.resource_list import ResourceList
from resync.sitemap import Sitemap
class TestSitemap(unittest.TestCase):
def test_ex2_1(self):
"""ex2_1 is a simple resourcelist with 2 resources, no metadata"""
"""ex2_1 is a simple resource_list with 2 resources, no metadata"""
s=Sitemap()
fh=open('resync/test/testdata/examples_from_spec/ex2_1.xml')
si = s.resourcelist_parse_xml( fh=fh )
si = s.resource_list_parse_xml( fh=fh )
self.assertEqual( len(si.resources), 2, '2 resources')
sms = sorted(si.resources.keys())
self.assertEqual( sms, ['http://example.com/res1','http://example.com/res2'] )
self.assertEqual( si.resources['http://example.com/res1'].lastmod, None )
def test_ex2_2(self):
"""ex2_2 is a simple resourcelist with 2 resources, some metadata"""
"""ex2_2 is a simple resource_list with 2 resources, some metadata"""
s=Sitemap()
fh=open('resync/test/testdata/examples_from_spec/ex2_2.xml')
si = s.resourcelist_parse_xml( fh=fh )
si = s.resource_list_parse_xml( fh=fh )
self.assertEqual( len(si.resources), 2, '2 resources')
sms = sorted(si.resources.keys())
self.assertEqual( sms, ['http://example.com/res1','http://example.com/res2'] )
@ -31,10 +31,10 @@ class TestSitemap(unittest.TestCase):
self.assertEqual( si.resources['http://example.com/res2'].md5, '1e0d5cb8ef6ba40c99b14c0237be735e' )
def test_ex2_3(self):
"""ex2_3 is a simple changelist with 2 resources"""
"""ex2_3 is a simple change_list with 2 resources"""
s=Sitemap()
fh=open('resync/test/testdata/examples_from_spec/ex2_3.xml')
si = s.resourcelist_parse_xml( fh=fh )
si = s.resource_list_parse_xml( fh=fh )
self.assertEqual( len(si.resources), 2, '2 resources')
sms = sorted(si.resources.keys())
self.assertEqual( sms, ['http://example.com/res2.pdf','http://example.com/res3.tiff'] )

2
resync/test/test_resourcelist.py → resync/test/test_resource_list.py
View File

@ -1,6 +1,6 @@
import unittest
from resync.resource import Resource
from resync.resourcelist import ResourceList, ResourceListDupeError
from resync.resource_list import ResourceList, ResourceListDupeError
class TestResourceList(unittest.TestCase):

2
resync/test/test_resourcelist_builder.py → resync/test/test_resource_list_builder.py
View File

@ -2,7 +2,7 @@ import unittest
import re
import os
import time
from resync.resourcelist_builder import ResourceListBuilder
from resync.resource_list_builder import ResourceListBuilder
from resync.sitemap import Sitemap
from resync.mapper import Mapper

20
resync/test/test_sitemap.py
View File

@ -2,7 +2,7 @@ import sys
import unittest
import StringIO
from resync.resource import Resource
from resync.resourcelist import ResourceList
from resync.resource_list import ResourceList
from resync.sitemap import Sitemap, SitemapIndexError
# etree gives ParseError in 2.7, ExpatError in 2.6
@ -67,7 +67,7 @@ class TestSitemap(unittest.TestCase):
<url><loc>http://e.com/a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md hash="md5:Q2hlY2sgSW50ZWdyaXR5IQ==" size=\"12\" /></url>\
</urlset>'
s=Sitemap()
i=s.resourcelist_parse_xml(fh=StringIO.StringIO(xml))
i=s.resource_list_parse_xml(fh=StringIO.StringIO(xml))
self.assertEqual( s.resources_created, 1, 'got 1 resources')
r=i.resources['http://e.com/a']
self.assertTrue( r is not None, 'got the uri expected')
@ -83,23 +83,23 @@ class TestSitemap(unittest.TestCase):
<url><loc>/tmp/rs_test/src/file_b</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md size=\"32\" /></url>\
</urlset>'
s=Sitemap()
i=s.resourcelist_parse_xml(fh=StringIO.StringIO(xml))
i=s.resource_list_parse_xml(fh=StringIO.StringIO(xml))
self.assertEqual( s.resources_created, 2, 'got 2 resources')
def test_13_parse_illformed(self):
s=Sitemap()
# ExpatError in python2.6, ParserError in 2.7
self.assertRaises( etree_error_class, s.resourcelist_parse_xml, StringIO.StringIO('not xml') )
self.assertRaises( etree_error_class, s.resourcelist_parse_xml, StringIO.StringIO('<urlset><url>something</urlset>') )
self.assertRaises( etree_error_class, s.resource_list_parse_xml, StringIO.StringIO('not xml') )
self.assertRaises( etree_error_class, s.resource_list_parse_xml, StringIO.StringIO('<urlset><url>something</urlset>') )
def test_13_parse_valid_xml_but_other(self):
s=Sitemap()
self.assertRaises( ValueError, s.resourcelist_parse_xml, StringIO.StringIO('<urlset xmlns="http://example.org/other_namespace"> </urlset>') )
self.assertRaises( ValueError, s.resourcelist_parse_xml, StringIO.StringIO('<other xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </other>') )
self.assertRaises( ValueError, s.resource_list_parse_xml, StringIO.StringIO('<urlset xmlns="http://example.org/other_namespace"> </urlset>') )
self.assertRaises( ValueError, s.resource_list_parse_xml, StringIO.StringIO('<other xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </other>') )
def test_14_parse_sitemapindex_as_sitemap(self):
s=Sitemap()
self.assertRaises( SitemapIndexError, s.resourcelist_parse_xml, StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>') )
self.assertRaises( SitemapIndexError, s.resource_list_parse_xml, StringIO.StringIO('<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> </sitemapindex>') )
def test_20_parse_sitemapindex_empty(self):
s=Sitemap()
@ -141,7 +141,7 @@ class TestSitemap(unittest.TestCase):
self.assertEqual( sr[3], 'http://localhost:8888/resources/1000' )
self.assertEqual( sr[16], 'http://localhost:8888/resources/826' )
def test_30_parse_changelist(self):
def test_30_parse_change_list(self):
xml='<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n\
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">\
<url><loc>/tmp/rs_test/src/file_a</loc><lastmod>2012-03-14T18:37:36Z</lastmod><rs:md change="updated" size="12" /></url>\
@ -149,7 +149,7 @@ class TestSitemap(unittest.TestCase):
</urlset>'
s=Sitemap()
s.resource_class=Resource
c=s.changelist_parse_xml(fh=StringIO.StringIO(xml))
c=s.change_list_parse_xml(fh=StringIO.StringIO(xml))
self.assertEqual( s.resources_created, 2, 'got 2 resources')
i = iter(c)
r1 = i.next()