gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pep257

Browse Source
This commit is contained in:
Simeon Warner 2016-03-28 21:31:50 -04:00
parent 7aba374824
commit e8f102e073
26 changed files with 486 additions and 367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
resync/__init__.py
View File

@ -1,8 +1,9 @@
"""Module config for resync."""
from resync._version import __version__
"""Enable easy import for core classes, e.g.
from import Resource
"""
# Enable easy import for core classes, e.g.
# from resync import Resource
from resync.source_description import SourceDescription
from resync.capability_list import CapabilityList
from resync.resource_list import ResourceList

2
resync/_version.py
View File

@ -1,4 +1,4 @@
# This is the one place the version number for resync is stored
"""This is the one place the version number for resync is stored."""
#
# Format: x.y.z where
# x.y is spec version, see http://www.openarchives.org/rs/x.y/

18
resync/archives.py
View File

@ -1,4 +1,4 @@
"""ResourceSync Archive objects
"""ResourceSync Archive objects.
The ResourceSync Archives specification
http://www.openarchives.org/rs/archives specifies capabilities
@ -19,10 +19,12 @@ from .resource import Resource
from .sitemap import Sitemap
class ResourceListArchive(ListBaseWithIndex):
"""Class representing an Change List Archive"""
"""Class representing an Resource List Archive."""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
"""Initialize ResourceListArchive."""
self.resources_class = list if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
@ -30,27 +32,33 @@ class ResourceListArchive(ListBaseWithIndex):
capability_name='resourcelist-archive')
class ChangeListArchive(ListBaseWithIndex):
"""Class representing an Change List Archive"""
"""Class representing an Change List Archive."""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
"""Initialize ChangeListArchive."""
super(ChangeListArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changelist-archive')
class ResourceDumpArchive(ListBaseWithIndex):
"""Class representing an Resource Dump Archive"""
"""Class representing an Resource Dump Archive."""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
"""Initialize ResourceDumpArchive."""
super(ResourceDumpArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='resourcedump-archive',
resources_class=resources_class)
class ChangeDumpArchive(ListBaseWithIndex):
"""Class representing an Change Dump Archive"""
"""Class representing an Change Dump Archive."""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
"""Initialize ChangeDumpArchive."""
super(ChangeDumpArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changedump-archive',
resources_class=resources_class)

22
resync/capability_list.py
View File

@ -1,4 +1,4 @@
"""ResourceSync Capability List object
"""ResourceSync Capability List object.
An Capability List is a set of capabilitys with some metadata for
each capability. The Capability List object may also contain metadata
@ -13,7 +13,8 @@ from .list_base import ListBase
from .sitemap import Sitemap
class CapabilitySet(ResourceSet):
"""Class for storage of resources in a Capability List
"""Class for storage of resources in a Capability List.
Extends the ResourceSet to add checks to ensure that there are
never two entries for the same resource, and that values are
@ -21,13 +22,14 @@ class CapabilitySet(ResourceSet):
"""
def __init__(self):
"""Initialize CpabilitySet."""
self.order = [ 'resourcelist', 'resourcedump',
'changelist', 'changedump',
'resourcelist-archive', 'resourcedump-archive',
'changelist-archive', 'changedump-archive' ]
def __iter__(self):
"""Iterator over all the resources in capability order
"""Iterator over all the resources in capability order.
Deals with the case of unknown capabilities or duplicate entries
by using uri order for duplicates and adding any unknown ones
@ -62,7 +64,8 @@ class CapabilitySet(ResourceSet):
class CapabilityList(ListBase):
"""Class representing a Capability List
"""Class representing a Capability List.
An Capability List will admit only one resource with any given
URI. The iterator over resources is expected to return them in
@ -71,13 +74,14 @@ class CapabilityList(ListBase):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None):
"""Initialize CapabilityList."""
if (resources is None):
resources = CapabilitySet()
super(CapabilityList, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='capabilitylist')
def add(self, resource, replace=False):
"""Add a resource or an iterable collection of resources
"""Add a resource or an iterable collection of resources.
Will throw a ValueError if the resource (ie. same uri) already
exists in the capability_list, unless replace=True.
@ -89,7 +93,7 @@ class CapabilityList(ListBase):
self.resources.add(resource,replace)
def add_capability(self,capability=None,uri=None,name=None):
"""Specific add function for capabilities
"""Specific add function for capabilities.
Takes either:
- a capability object (derived from ListBase) as the first argument
@ -105,13 +109,13 @@ class CapabilityList(ListBase):
self.add( Resource(uri=uri,capability=name) )
def has_capability(self,name=None):
"""True if the Capability List includes the named capability"""
"""True if the Capability List includes the named capability."""
return( self.capability_info(name) is not None )
def capability_info(self,name=None):
"""Return information about the requested capability from this list
"""Return information about the requested capability from this list.
Will return None if there is no information about the requested capability
Will return None if there is no information about the requested capability.
"""
for r in self.resources:
if (r.capability == name):

10
resync/change_dump.py
View File

@ -1,4 +1,4 @@
"""ResourceSync Change Dump object
"""ResourceSync Change Dump object.
A Change Dump is a set of content dump package resources
with some metadata for each resource.
@ -13,7 +13,8 @@ http://www.openarchives.org/rs/resourcesync#ChangeDump
from .resource_list import ResourceList
class ChangeDump(ResourceList):
"""Class representing an Change Dump
"""Class representing an Change Dump.
A ChangeDump comprises a set of content dump packages that
are the resources listed. Properties similar to a ResourceList
@ -21,6 +22,11 @@ class ChangeDump(ResourceList):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None):
"""Initialize ChangeDump.
Simply sets capability_name to 'changedump' when
subclassing ResourceList.
"""
super(ChangeDump, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
mapper=mapper)
self.capability_name='changedump'

11
resync/change_dump_manifest.py
View File

@ -1,4 +1,4 @@
"""ResourceSync ChangeDumpManifest object
"""ResourceSync ChangeDumpManifest object.
A ChangeDumpManifest lists the set of files/resources included
within a content package that is included in a ChangeDump.
@ -14,7 +14,8 @@ http://www.openarchives.org/rs/resourcesync#ChangeDumpManifest
from .change_list import ChangeList
class ChangeDumpManifest(ChangeList):
"""Class representing a Change Dump Manifest
"""Class representing a Change Dump Manifest.
A ChangeDumpManifest comprises a set of files/resources
in a content package. Properties much Like a ChangeList
@ -22,6 +23,10 @@ class ChangeDumpManifest(ChangeList):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None):
"""Initialize ChangeDumpManifest.
Simply sets capability_name to 'changedump-manifest' when
subclassing ChangeList.
"""
super(ChangeDumpManifest, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper)
self.capability_name = 'changedump-manifest'

13
resync/change_list.py
View File

@ -1,4 +1,4 @@
"""ResourceSync ChangeList object
"""ResourceSync ChangeList object.
A ChangeList is a list of resource descriptions which includes
both metadata associated with the resource at some point in
@ -26,24 +26,25 @@ from .resource import Resource,ChangeTypeError
from .sitemap import Sitemap
class ChangeList(ListBaseWithIndex):
"""Class representing an Change List"""
"""Class representing an Change List."""
def __init__(self, resources=None, md=None, ln=None, uri=None,
mapper=None, resources_class=list):
"""Initialize ChangeList."""
super(ChangeList, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changelist', mapper=mapper,
resources_class=resources_class)
def add_if_changed(self, resource):
"""Add resource if change is not None else ChangeTypeError"""
"""Add resource if change is not None else ChangeTypeError."""
if (resource.change is not None):
self.resources.append(resource)
else:
raise ChangeTypeError(resource.change)
def add(self, resource):
"""Add a resource change or an iterable collection of them to
this ChangeList
"""Add a resource change or an iterable collection of them.
Allows multiple resource_change objects for the same
resource (ie. URI) and preserves the order of addition.
@ -55,7 +56,7 @@ class ChangeList(ListBaseWithIndex):
self.add_if_changed(resource)
def add_changed_resources(self, resources, change=None):
"""Add items from a ResourceContainer resources to this ChangeList
"""Add items from a ResourceContainer resources.
If change is specified then the attribute is set in the Resource
objects created.

28
resync/client.py
View File

@ -30,18 +30,8 @@ from .client_state import ClientState
from .list_base_with_index import ListBaseIndexError
from .w3c_datetime import str_to_datetime,datetime_to_str
def url_or_file_open(uri):
"""Wrapper around urlopen() to prepend file: if no scheme provided."""
if (not re.match(r'''\w+:''',uri)):
uri = 'file:'+uri
return(urlopen(uri))
class ClientFatalError(Exception):
"""Non-recoverable error in client, should include message to user."""
pass
class Client(object):
"""Implementation of a ResourceSync client.
Logging is used for both console output and for detailed logs for
@ -610,3 +600,19 @@ class Client(object):
self.logger.warning("Status: %15s (%s%s=%d, %s=%d, %s=%d)" %\
(status, same, words['created'], created,
words['updated'], updated, words['deleted'], deleted))
class ClientFatalError(Exception):
"""Non-recoverable error in client, should include message to user."""
pass
##### Misc functions. #####
def url_or_file_open(uri):
"""Wrapper around urlopen() to prepend file: if no scheme provided."""
if (not re.match(r'''\w+:''',uri)):
uri = 'file:'+uri
return(urlopen(uri))

1
resync/client_state.py
View File

@ -19,6 +19,7 @@ except ImportError: #python2
class ClientState(object):
"""Read and store client state on disk."""
def __init__(self):

19
resync/client_utils.py
View File

@ -1,9 +1,9 @@
"""Client Utilities
"""ResourceSync Client Utilities.
Factor out code shared by both the resync and resync-explorer
clients.
Copyright 2012,2013 Simeon Warner
Copyright 2012-2016 Simeon Warner
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ from resync.utils import UTCFormatter
def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log',
human=True, verbose=False, eval_mode=False,
default_logger='client', extra_loggers=None):
"""Initialize logging
"""Initialize logging.
Use of log levels:
DEBUG - very verbose, for evaluation of output (-e)
@ -41,7 +41,6 @@ def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log',
a file. This will be logfile if set, else default_logfile (which may
also be overridden).
"""
fmt = '%(asctime)s | %(name)s | %(levelname)s | %(message)s'
formatter = UTCFormatter(fmt)
@ -75,7 +74,7 @@ def init_logging(to_file=False, logfile=None, default_logfile='/tmp/resync.log',
def count_true_args(*args):
"""Count number of list of arguments that evaluate True"""
"""Count number of list of arguments that evaluate True."""
count=0
for arg in args:
if (arg):
@ -83,6 +82,10 @@ def count_true_args(*args):
return(count)
def parse_links(args_link):
"""Parse --link options.
Uses parse_link() to parse each option.
"""
links=[]
if (args_link is not None):
for link_str in args_link:
@ -93,7 +96,7 @@ def parse_links(args_link):
return(links)
def parse_link(link_str):
"""Parse --link option to add to <rs:ln> links
"""Parse one --link option to add to <rs:ln> links.
Input string of the form: rel,href,att1=val1,att2=val2
"""
@ -120,7 +123,7 @@ def parse_link(link_str):
return(atts)
def parse_capabilities(caps_str):
"""Parse list of capabilities in --capabilitylist option
"""Parse list of capabilities in --capabilitylist option.
Input string of the form: cap_name=uri,cap_name=uri
"""
@ -135,7 +138,7 @@ def parse_capabilities(caps_str):
return(capabilities)
def parse_capability_lists(cls_str):
"""Parse list of capability lists in --capabilitylistindex option
"""Parse list of capability lists in --capabilitylistindex option.
Input string of the form: uri,uri
"""

13
resync/dump.py
View File

@ -5,12 +5,8 @@ import os.path
from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
from resync.resource_dump_manifest import ResourceDumpManifest
class DumpError(Exception):
"""Error class used by Dump() objects."""
pass
class Dump(object):
"""Dump of content for a Resource Dump or Change Dump.
The resources itearable must be comprised of Resource objects
@ -182,3 +178,10 @@ class Dump(object):
return(real_path)
else:
return( os.path.relpath(real_path,self.path_prefix) )
class DumpError(Exception):
"""Error class used by Dump() objects."""
pass

71
resync/explorer.py
View File

@ -26,39 +26,8 @@ from .client_state import ClientState
from .resource import Resource
from .w3c_datetime import str_to_datetime,datetime_to_str
class XResource(object):
"""Information about a resource for the explorer.
Must have a uri but may also store:
acceptable_capabilities - None for any acceptable, 'resource' if a
resource rather than a capability document is expected, else
a list of capability names
checks - a set of information to check when then XResource is inspected
base_uri - on creation interpret any relative URI specified in the
context of base_uri, store the resulting full URI
"""
def __init__(self, uri, acceptable_capabilities=None, checks=None, context=None):
"""Initialize XResource object."""
self.uri=urljoin(context,uri)
self.acceptable_capabilities=acceptable_capabilities
self.checks=checks
class HeadResponse(object):
"""Object to mock up requests.head(...) response."""
def __init__(self):
"""Initialize with no status_code and no headers."""
self.status_code=None
self.headers={}
class ExplorerQuit(Exception):
"""Exception raised when user quits normally, no error."""
pass
class Explorer(Client):
"""Extension of the client code to explore a ResourceSync source.
Designed to support a text-based command-line client that starts from
@ -353,3 +322,41 @@ class Explorer(Client):
print(" WARNING - expanded relative URI to %s" % (full_uri))
uri = full_uri
return(uri)
class XResource(object):
"""Information about a resource for the explorer.
Must have a uri but may also store:
acceptable_capabilities - None for any acceptable, 'resource' if a
resource rather than a capability document is expected, else
a list of capability names
checks - a set of information to check when then XResource is inspected
base_uri - on creation interpret any relative URI specified in the
context of base_uri, store the resulting full URI
"""
def __init__(self, uri, acceptable_capabilities=None, checks=None, context=None):
"""Initialize XResource object."""
self.uri=urljoin(context,uri)
self.acceptable_capabilities=acceptable_capabilities
self.checks=checks
class HeadResponse(object):
"""Object to mock up requests.head(...) response."""
def __init__(self):
"""Initialize with no status_code and no headers."""
self.status_code=None
self.headers={}
class ExplorerQuit(Exception):
"""Exception raised when user quits normally, no error."""
pass

20
resync/list_base.py
View File

@ -1,4 +1,4 @@
"""Base class for ResourceSync capabilities with list of resources
"""Base class for ResourceSync capabilities with list of resources.
Adds Sitemap based IO to the ResourceContainer class and is
intended as the base class for ResourceList, ChangeList,
@ -26,7 +26,8 @@ from .resource_container import ResourceContainer
from .sitemap import Sitemap
class ListBase(ResourceContainer):
"""Class that adds Sitemap based IO to ResourceContainer
"""Class that adds Sitemap based IO to ResourceContainer.
resources - an iterable of resources
@ -43,6 +44,7 @@ class ListBase(ResourceContainer):
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
capability_name='unknown'):
"""Initialize ListBase."""
super(ListBase, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name=capability_name)
self.count = count
@ -54,11 +56,11 @@ class ListBase(ResourceContainer):
self.parsed_index = None
def __iter__(self):
"""Default to iterator provided by resources object"""
"""Default to iterator provided by resources object."""
return(iter(self.resources))
def __len__(self):
"""Number of entries in this list
"""Number of entries in this list.
To handle the case where self.resources is a generator or an iterator and
thus cannot provide len(...) we first check to see whether an explicit
@ -75,7 +77,7 @@ class ListBase(ResourceContainer):
##### INPUT #####
def read(self,uri=None):
"""Default case is just to parse document at this URI
"""Default case is just to parse document at this URI.
Intention is that the read() method may be overridden to support reading
of compound documents in more then one sitemapindex/sitemap.
@ -83,7 +85,7 @@ class ListBase(ResourceContainer):
self.parse(uri=uri)
def parse(self,uri=None,fh=None,str_data=None,**kwargs):
"""Parse a single XML document for this list
"""Parse a single XML document for this list.
Accepts either a uri (uri or default if parameter not specified),
or a filehandle (fh) or a string (str_data). Note that this method
@ -114,7 +116,7 @@ class ListBase(ResourceContainer):
##### OUTPUT #####
def as_xml(self):
"""Return XML serialization of this list
"""Return XML serialization of this list.
This code does not support the case where the list is too big for
a single XML document.
@ -124,7 +126,7 @@ class ListBase(ResourceContainer):
return s.resources_as_xml(self,sitemapindex=self.sitemapindex)
def write(self,basename="/tmp/resynclist.xml"):
"""Write a single sitemap or sitemapindex XML document
"""Write a single sitemap or sitemapindex XML document.
Must be overridden to support multi-file lists.
"""
@ -137,5 +139,5 @@ class ListBase(ResourceContainer):
##### UTILITY #####
def new_sitemap(self):
"""Create new Sitemap object with default settings"""
"""Create new Sitemap object with default settings."""
return Sitemap(pretty_xml=self.pretty_xml)

48
resync/list_base_with_index.py
View File

@ -1,7 +1,7 @@
"""Base class for ResourceSync capabilities with lists of resources including
support for both sitemaps and sitemapindexes.
"""Base class for ResourceSync capabilities with large lists of resources.
Extends ListBase to add support for sitemapindexes.
Handles possibly large lists by including support for both sitemaps and
sitemapindexes. Extends ListBase to add the support for sitemapindexes.
"""
import collections
@ -23,12 +23,9 @@ from .mapper import Mapper, MapperError
from .url_authority import UrlAuthority
from .utils import compute_md5_for_file
class ListBaseIndexError(Exception):
"""Exception for problems with sitemapindexes"""
pass
class ListBaseWithIndex(ListBase):
"""Class that add handling of sitemapindexes to ListBase
"""Class that add handling of sitemapindexes to ListBase.
Splitting of a list into multiple sitemaps with a sitemapindex is currently
handled based solely on the number of entries in the list. The configurable
@ -56,6 +53,7 @@ class ListBaseWithIndex(ListBase):
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
capability_name='unknown', allow_multifile=None, mapper=None,
resources_class=None):
"""Initialize ListBaseWithIndex."""
self.resources_class = list if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
@ -73,7 +71,7 @@ class ListBaseWithIndex(ListBase):
##### INPUT #####
def read(self, uri=None, resources=None, index_only=False):
"""Read sitemap from a URI including handling sitemapindexes
"""Read sitemap from a URI including handling sitemapindexes.
If index_only is True then individual sitemaps references in a sitemapindex
will not be read. This will result in no resources being returned and is
@ -123,7 +121,7 @@ class ListBaseWithIndex(ListBase):
def read_component_sitemap(self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
"""Read a component sitemap of a Resource List with index
"""Read a component sitemap of a Resource List with index.
Each component must be a sitemap with the
"""
@ -162,7 +160,7 @@ class ListBaseWithIndex(ListBase):
##### OUTPUT #####
def requires_multifile(self):
"""Returns False or the number of component sitemaps required
"""Return False or the number of component sitemaps required.
In the case that no len() is available for self.resources then
then self.count must be set beforehand to avoid an exception.
@ -173,7 +171,7 @@ class ListBaseWithIndex(ListBase):
return( int( math.ceil( len(self) / float(self.max_sitemap_entries) ) ) )
def as_xml(self, allow_multifile=False, basename="/tmp/sitemap.xml"):
"""Return XML serialization of this list
"""Return XML serialization of this list.
If this list can be serialized as a single sitemap then the
superclass method is used.
@ -192,7 +190,7 @@ class ListBaseWithIndex(ListBase):
raise ListBaseIndexError("Attempt to write single XML string for list with %d entries when max_sitemap_entries is set to %d" % (len(self),self.max_sitemap_entries))
def as_xml_index(self, basename="/tmp/sitemap.xml"):
"""Return a string of the index for a large list that is split
"""Return a string of the index for a large list that is split.
All we need to do is determine the number of component sitemaps will
be is and generate their URIs based on a pattern.
@ -215,7 +213,10 @@ class ListBaseWithIndex(ListBase):
return( index.as_xml() )
def as_xml_part(self, basename="/tmp/sitemap.xml", part_number=0):
"""Return a string of component sitemap part_number for a large list that is split
"""Return a string of component sitemap number part_number.
Used in the case of a large list that is split into component
sitemaps.
basename is used to create "index" links to the sitemapindex
@ -237,7 +238,7 @@ class ListBaseWithIndex(ListBase):
return( s.resources_as_xml(part) )
def write(self, basename='/tmp/sitemap.xml'):
"""Write one or a set of sitemap files to disk
"""Write one or a set of sitemap files to disk.
resources is a ResourceContainer that may be an ResourceList or
a ChangeList. This may be a generator so data is read as needed
@ -307,9 +308,7 @@ class ListBaseWithIndex(ListBase):
self.logger.info("Wrote sitemap %s" % (basename))
def index_as_xml(self):
"""Return XML serialization of this list taken to be sitemapindex entries
"""
"""XML serialization of this list taken to be sitemapindex entries."""
self.default_capability()
s = self.new_sitemap()
return s.resources_as_xml(self,sitemapindex=True)
@ -317,7 +316,7 @@ class ListBaseWithIndex(ListBase):
##### Utility #####
def get_resources_chunk(self, resource_iter, first=None):
"""Return next chunk of resources from resource_iter, and next item
"""Return next chunk of resources from resource_iter, and next item.
If first parameter is specified then this will be prepended to
the list.
@ -344,7 +343,7 @@ class ListBaseWithIndex(ListBase):
return(chunk,nxt)
def part_name(self, basename='/tmp/sitemap.xml', part_number=0):
"""Name (file or URI) for one component sitemap
"""Name (file or URI) for one component sitemap.
Works for both filenames and URIs because manipulates only the end
of the string.
@ -362,9 +361,16 @@ class ListBaseWithIndex(ListBase):
return( sitemap_prefix + ( "%05d" % (part_number) ) + sitemap_suffix )
def is_file_uri(self, uri):
"""Return true if uri looks like a local file URI, false otherwise
"""Return true if uri looks like a local file URI, false otherwise.
Test is to see whether have either an explicit file: URI or whether
there is no scheme name.
"""
return(re.match('file:',uri) or not re.match('\w{3,4}:',uri))
class ListBaseIndexError(Exception):
"""Exception for problems with sitemapindexes in ListBaseIndex."""
pass

47
resync/mapper.py
View File

@ -1,4 +1,5 @@
"""Map between source URIs and destination paths"""
"""Map between source URIs and destination paths."""
import os
import os.path
import re
@ -8,23 +9,26 @@ except ImportError: #python2
from urlparse import urlparse
import logging
class MapperError(Exception):
pass
class Mapper():
"""Mapper object to map between source URIs and destination paths.
Implemented as a list of Map objects.
"""
def __init__(self, mappings=None, use_default_path=False):
"""Initialize Mapper."""
self.logger = logging.getLogger('resync.mapper')
self.mappings=[]
if (mappings):
self.parse(mappings, use_default_path)
def __len__(self):
"""Length is number of mappings"""
"""Length is number of mappings."""
return(len(self.mappings))
def parse(self, mappings, use_default_path=False):
"""Parse a list of map strings (mappings)
"""Parse a list of map strings (mappings).
Accepts two distinct formats:
1. If there are exactly two entries then these may be the source base URI
@ -63,7 +67,7 @@ class Mapper():
self.mappings.append(Map(src_uri, dst_path))
def default_src_uri(self):
"""Default src_uri from mapping
"""Default src_uri from mapping.
This is take just to be the src_uri of the first entry
"""
@ -72,7 +76,7 @@ class Mapper():
raise MapperError("Can't get default source URI from empty mapping")
def unsafe(self):
"""True is one or more mapping is unsafe
"""True is one or more mapping is unsafe.
See Map.unsafe() for logic. Provide this as a test rather than
building into object creation/parse because it is useful to
@ -84,7 +88,7 @@ class Mapper():
return(False)
def dst_to_src(self,dst_file):
"""Map destination path to source URI"""
"""Map destination path to source URI."""
for map in self.mappings:
src_uri = map.dst_to_src(dst_file)
if (src_uri is not None):
@ -93,7 +97,7 @@ class Mapper():
raise MapperError("Unable to translate destination path (%s) into a source URI." % (dst_file))
def src_to_dst(self,src_uri):
"""Map source URI to destination path"""
"""Map source URI to destination path."""
for map in self.mappings:
dst_path = map.src_to_dst(src_uri)
if (dst_path is not None):
@ -102,7 +106,7 @@ class Mapper():
raise MapperError("Unable to translate source URI (%s) into a destination path." % (src_uri))
def path_from_uri(self,uri):
"""Make a safe path name from uri
"""Make a safe path name from uri.
In the case that uri is already a local path then the same path
is returned.
@ -118,6 +122,7 @@ class Mapper():
return(path)
def __repr__(self):
"""Human readable dump of all Maps in the Mapper."""
s = 'Mapper with %d maps:\n' % (len(self.mappings))
for map in self.mappings:
s += str(map) + '\n'
@ -125,7 +130,8 @@ class Mapper():
class Map:
"""A single map from source URI to destination path
"""A single map from source URI to destination path.
Both URI and destination paths are assumed to use / as the path
separator. No account is take for other path separators used
@ -134,18 +140,19 @@ class Map:
"""
def __init__(self,src_uri=None,dst_path=None):
"""Initialize Map object, strip trailing slashes from src_uri and dst_path."""
self.src_uri = self.strip_trailing_slashes(src_uri)
self.dst_path = self.strip_trailing_slashes(dst_path)
def strip_trailing_slashes(self,path):
"""Return input path minus any trailing slashes"""
"""Return input path minus any trailing slashes."""
m=re.match(r"(.*)/+$",path)
if (m is None):
return(path)
return(m.group(1))
def dst_to_src(self,dst_file):
"""Return the src URI from the dst filepath
"""Return the src URI from the dst filepath.
This does not rely on the destination filepath actually existing on the local
filesystem, just on pattern matching. Return source URI on success, None on
@ -158,7 +165,7 @@ class Map:
return(self.src_uri+'/'+rel_path)
def src_to_dst(self,src_uri):
"""Return the dst filepath from the src URI
"""Return the dst filepath from the src URI.
Returns None on failure, destination path on success.
"""
@ -169,7 +176,7 @@ class Map:
return(self.dst_path+'/'+rel_path)
def unsafe(self):
"""True is the mapping is unsafe for an update
"""True is the mapping is unsafe for an update.
Applies only to local source. Returns True if the paths for source and
destination are the same, or if one is a component of the other path.
@ -183,4 +190,12 @@ class Map:
return(s == lcp or d == lcp)
def __repr__(self):
"""Human readable string for one map."""
return("Map( %s -> %s )" % (self.src_uri, self.dst_path))
class MapperError(Exception):
"""Exception for errors in Mapper class."""
pass

212
resync/resource.py
View File

@ -1,49 +1,6 @@
"""Information about a web resource
Each web resource is identified by a URI and may optionally
have other metadata such as timestamp, length, md5, etc..
The lastmod property provides ISO8601 format string access
to the timestamp. The timestamp is stored internally as a
unix timestamp value in UTC. This limits the range of
possible lastmod values but covers all web-era values for
a good way into the future.
This object is optimized for size in the case whether there
is not large a large amount of data in the attributes. This
is done using __slots__ for the core attributes so that there
is no __dict__ defined for a Resource object. Core attributes
are:
uri - Resource URI
timestamp - Last-Modification time, has lastmod accessor
length - size in bytes
mime_type - MIME type
md5, sha1, sha256 - digests, have hash accessor
change - change type
path - path in dump
If non-core attributes are needed then the '_extra' attribute
has a dict of values. The ones explicitly used here are:
capability - Capability name
ts_at - at time, has md_at accessor
ts_completed - completed time, has md_completed accessor
ts_from - from time, has md_from accessor
ts_until - until time, has md_until accessor
The accessor names mime_type, md_from etc. are used to avoid conflict
with Python built-in functions and keywords type(), from etc..
The 'ln' attribute is used when it is necessary to add links
or other information to the object. Use of non-core attributes
or links results in dicts being created which is convenient
but will significantly increase the size of each Resource
object that contains such information.
"""
"""ResourceSync Resources - information about a web resource and changes."""
import re
try: #python3
from urllib.parse import urlparse
except: #python2
@ -51,27 +8,52 @@ except: #python2
from posixpath import basename
from .w3c_datetime import str_to_datetime, datetime_to_str
class ChangeTypeError(Exception):
"""Exception class raised by Resource for bad change attribute
The change attribute of a Resource object may be either None
or one of the values in Resource.CHANGE_TYPES. Checking is
disabled by setting Resource.CHANGE_TYPES False.
"""
def __init__(self, val):
self.supplied = val
def __repr__(self):
return "<ChangeTypeError: got %s, expected one of %s>" % \
(self.supplied,str(Resource.CHANGE_TYPES))
def __str__(self):
return repr(self)
class Resource(object):
"""ResourceSync Resource object.
Each web resource is identified by a URI and may optionally
have other metadata such as timestamp, length, md5, etc..
The lastmod property provides ISO8601 format string access
to the timestamp. The timestamp is stored internally as a
unix timestamp value in UTC. This limits the range of
possible lastmod values but covers all web-era values for
a good way into the future.
This object is optimized for size in the case whether there
is not large a large amount of data in the attributes. This
is done using __slots__ for the core attributes so that there
is no __dict__ defined for a Resource object. Core attributes
are:
uri - Resource URI
timestamp - Last-Modification time, has lastmod accessor
length - size in bytes
mime_type - MIME type
md5, sha1, sha256 - digests, have hash accessor
change - change type
path - path in dump
If non-core attributes are needed then the '_extra' attribute
has a dict of values. The ones explicitly used here are:
capability - Capability name
ts_at - at time, has md_at accessor
ts_completed - completed time, has md_completed accessor
ts_from - from time, has md_from accessor
ts_until - until time, has md_until accessor
The accessor names mime_type, md_from etc. are used to avoid conflict
with Python built-in functions and keywords type(), from etc..
The 'ln' attribute is used when it is necessary to add links
or other information to the object. Use of non-core attributes
or links results in dicts being created which is convenient
but will significantly increase the size of each Resource
object that contains such information.
"""
__slots__=('uri', 'timestamp', 'length', 'mime_type',
'md5', 'sha1', 'sha256', 'change', 'path',
'_extra', 'ln' )
@ -87,10 +69,11 @@ class Resource(object):
ts_from = None, md_from = None,
ts_until = None, md_until = None,
resource = None, ln = None):
"""Initialize object either from parameters specified or
from an existing Resource object. If explicit parameters
are specified then they will override values copied from
a resource object supplied.
"""Initialize Resource object.
Initialize either from parameters specified or from an existing
Resource object. If explicit parameters are specified then they
will override values copied from a Resource object supplied.
"""
# Initialize core attributes
self.uri = None
@ -159,6 +142,12 @@ class Resource(object):
raise ValueError("Cannot create resource without a URI")
def __setattr__(self, prop, value):
"""Attribute setter with check and support for extra attributes.
Because this class is optimized to use __slots__ it cannot
ordinarily be dynamically extended. We implement extension with
the idea of extra properties.
"""
# Add validity check for self.change
if (prop == 'change' and Resource.CHANGE_TYPES and
value is not None and not value in Resource.CHANGE_TYPES):
@ -185,67 +174,67 @@ class Resource(object):
@property
def lastmod(self):
"""The Last-Modified data in W3C Datetime syntax, Z notation"""
"""The Last-Modified data in W3C Datetime syntax, Z notation."""
return datetime_to_str(self.timestamp)
@lastmod.setter
def lastmod(self, lastmod):
"""Set timestamp from a W3C Datetime Last-Modified value"""
"""Set timestamp from a W3C Datetime Last-Modified value."""
self.timestamp = str_to_datetime(lastmod, context='lastmod')
@property
def md_at(self):
"""md_at values in W3C Datetime syntax, Z notation"""
"""md_at values in W3C Datetime syntax, Z notation."""
return datetime_to_str(self._get_extra('ts_at'))
@md_at.setter
def md_at(self, md_at):
"""Set at value from a W3C Datetime value"""
"""Set at value from a W3C Datetime value."""
self._set_extra( 'ts_at', str_to_datetime(md_at, context='md_at datetime') )
@property
def md_completed(self):
"""md_completed value in W3C Datetime syntax, Z notation"""
"""md_completed value in W3C Datetime syntax, Z notation."""
return datetime_to_str(self._get_extra('ts_completed'))
@md_completed.setter
def md_completed(self, md_completed):
"""Set md_completed value from a W3C Datetime value"""
"""Set md_completed value from a W3C Datetime value."""
self._set_extra( 'ts_completed', str_to_datetime(md_completed, context='md_completed datetime') )
@property
def md_from(self):
"""md_from value in W3C Datetime syntax, Z notation"""
"""md_from value in W3C Datetime syntax, Z notation."""
return datetime_to_str(self._get_extra('ts_from'))
@md_from.setter
def md_from(self, md_from):
"""Set md_from value from a W3C Datetime value"""
"""Set md_from value from a W3C Datetime value."""
self._set_extra( 'ts_from', str_to_datetime(md_from, context='md_from datetime') )
@property
def md_until(self):
"""md_until value in W3C Datetime syntax, Z notation"""
"""md_until value in W3C Datetime syntax, Z notation."""
return datetime_to_str(self._get_extra('ts_until'))
@md_until.setter
def md_until(self, md_until):
"""Set md_until value from a W3C Datetime value"""
"""Set md_until value from a W3C Datetime value."""
self._set_extra( 'ts_until', str_to_datetime(md_until, context='md_until datetime') )
@property
def capability(self):
"""Get Capability name string"""
"""Get Capability name string."""
return self._get_extra('capability')
@capability.setter
def capability(self, capability):
"""Set Capability name string"""
"""Set Capability name string."""
self._set_extra('capability', capability)
@property
def hash(self):
"""Provide access to the complete hash string
"""Provide access to the complete hash string.
The hash string may have zero or more hash values with
appropriate prefixes. All hash values are assumed to be
@ -264,7 +253,7 @@ class Resource(object):
@hash.setter
def hash(self, hash):
"""Parse space separated set of values
"""Parse space separated set of values.
See specification at:
http://tools.ietf.org/html/draft-snell-atompub-link-extensions-09
@ -295,7 +284,7 @@ class Resource(object):
raise ValueError(". ".join(errors))
def link(self,rel):
"""Look for link with specified rel, return else None
"""Look for link with specified rel, return else None.
Searches through dicts in self.ln looking for one with the
specified rel value. If there are multiple links with the
@ -310,14 +299,14 @@ class Resource(object):
return(None)
def link_href(self,rel):
"""Look for link with specified rel, return href from it or None"""
"""Look for link with specified rel, return href from it or None."""
link = self.link(rel)
if (link is not None):
link = link['href']
return(link)
def link_set(self,rel,href,allow_duplicates=False,**atts):
"""Set/create link with specified rel, set href and any other attributes
"""Set/create link with specified rel, set href and any other attributes.
Any link element must have both rel and href values, the specification
also defines the type attributes and others are permitted also. See
@ -345,47 +334,47 @@ class Resource(object):
link[k] = atts[k]
def link_add(self,rel,href,**atts):
"""Create an link with specified rel even if one with that rel already exists"""
"""Create an link with specified rel even if one with that rel already exists."""
self.link_set(rel,href,allow_duplicates=True,**atts)
@property
def describedby(self):
"""Convenient access to <rs:ln rel="describedby" href="uri">"""
"""Convenient access to <rs:ln rel="describedby" href="uri">."""
return(self.link_href('describedby'))
@describedby.setter
def describedby(self,uri):
"""Set ResourceSync Description link to given URI"""
"""Set ResourceSync Description link to given URI."""
self.link_set('describedby',uri)
@property
def up(self):
"""Get the URI of any ResourceSync rel="up" link"""
"""Get the URI of any ResourceSync rel="up" link."""
return(self.link_href('up'))
@up.setter
def up(self,uri):
"""Set rel="up" link to given URI"""
"""Set rel="up" link to given URI."""
self.link_set('up',uri)
@property
def index(self):
"""Get the URI of and ResourceSync rel="index" link"""
"""Get the URI of and ResourceSync rel="index" link."""
return(self.link_href('index'))
@index.setter
def index(self,uri):
"""Set rel="index" link to given URI"""
"""Set rel="index" link to given URI."""
self.link_set('index',uri)
@property
def contents(self):
"""Get the URI of and ResourceSync rel="contents" link"""
"""Get the URI of and ResourceSync rel="contents" link."""
return(self.link_href('index'))
@contents.setter
def contents(self,uri,type='application/xml'):
"""Set rel="contents" link to given URI
"""Set rel="contents" link to given URI.
Will also set the type="application/xml" unless overridden
"""
@ -393,19 +382,22 @@ class Resource(object):
@property
def basename(self):
"""The resource basename (http://example.com/resource/1 -> 1)"""
"""The resource basename.
For example from http://example.com/resource/1 returns 1
"""
parse_object = urlparse(self.uri)
return basename(parse_object.path)
def __eq__(self,other):
"""Equality test for resources allowing <1s difference in timestamp
"""Equality test for resources allowing <1s difference in timestamp.
See equal(...) for more details of equality test
"""
return( self.equal(other,delta=1.0) )
def equal(self,other,delta=0.0):
"""Equality or near equality test for resources
"""Equality or near equality test for resources.
Equality means:
1. same uri, AND
@ -432,7 +424,7 @@ class Resource(object):
return(True)
def __str__(self):
"""Return a human readable string for this resource
"""Return a human readable string for this resource.
Includes only the parts necessary for synchronizaion and
designed to support logging.
@ -446,7 +438,7 @@ class Resource(object):
return "[ " + " | ".join(s) + " ]"
def __repr__(self):
"""Return an unambigous representation of this resource
"""Return an unambigous representation of this resource.
Uses format like Python's representation of a dict() for
attributes. Includes only those attributes with values that
@ -458,3 +450,25 @@ class Resource(object):
if (val is not None):
s.append( repr(attr) + ': ' + repr(val) )
return "{" + ", ".join(s) + "}"
class ChangeTypeError(Exception):
"""Exception class raised by Resource for bad change attribute.
The change attribute of a Resource object may be either None
or one of the values in Resource.CHANGE_TYPES. Checking is
disabled by setting Resource.CHANGE_TYPES False.
"""
def __init__(self, val):
"""Initialize ChangeTypeError, store val."""
self.supplied = val
def __repr__(self):
"""Helpful error message."""
return "<ChangeTypeError: got %s, expected one of %s>" % \
(self.supplied,str(Resource.CHANGE_TYPES))
def __str__(self):
"""Helpful error message."""
return repr(self)

56
resync/resource_container.py
View File

@ -1,4 +1,4 @@
"""ResourceSync Resource Container object
"""ResourceSync Resource Container object.
All documents in ResourceSync have the same types of information:
they have some top-level metadata and links, and then a list of
@ -14,7 +14,8 @@ import collections
from .w3c_datetime import datetime_to_str
class ResourceContainer(object):
"""Class containing resource-like objects
"""Class containing resource-like objects.
Core functionality::
- resources property that is the set/list of resources
@ -31,6 +32,7 @@ class ResourceContainer(object):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, capability_name=None):
"""Initialize ResourceContainer."""
self.resources=(resources if (resources is not None) else [])
self.md=(md if (md is not None) else {})
self.ln=(ln if (ln is not None) else [])
@ -38,19 +40,19 @@ class ResourceContainer(object):
self.capability_name=capability_name
def __iter__(self):
"""Iterator over all the resources in this resource list
"""Iterator over all the resources in this resource list.
Baseline implementation use iterator given by resources property
Baseline implementation use iterator given by resources property.
"""
return(iter(self.resources))
def __getitem__(self,index):
"""Feed through for __getitem__ of resources property"""
"""Feed through for __getitem__ of resources property."""
return(self.resources[index])
@property
def capability(self):
"""Get/set the <rs:md capability="" .../> attribute"""
"""Get/set the <rs:md capability="" .../> attribute."""
if ('capability' in self.md):
return(self.md['capability'])
else:
@ -62,7 +64,7 @@ class ResourceContainer(object):
@property
def md_from(self):
"""Get/set the <rs:md from="" .../> attribute"""
"""Get/set the <rs:md from="" .../> attribute."""
if ('md_from' in self.md):
return(self.md['md_from'])
else:
@ -74,7 +76,7 @@ class ResourceContainer(object):
@property
def md_until(self):
"""Get/set the <rs:md until="" .../> attribute"""
"""Get/set the <rs:md until="" .../> attribute."""
if ('md_until' in self.md):
return(self.md['md_until'])
else:
@ -86,7 +88,7 @@ class ResourceContainer(object):
@property
def md_at(self):
"""Get/set the <rs:md at="" attribute"""
"""Get/set the <rs:md at="" attribute."""
if ('md_at' in self.md):
return(self.md['md_at'])
else:
@ -98,7 +100,7 @@ class ResourceContainer(object):
@property
def md_completed(self):
"""Get/set the <rs:md completed="" .../> attribute"""
"""Get/set the <rs:md completed="" .../> attribute."""
if ('md_completed' in self.md):
return(self.md['md_completed'])
else:
@ -109,7 +111,7 @@ class ResourceContainer(object):
self.md['md_completed']=self._str_datetime_now(md_completed)
def link(self,rel):
"""Look for link with specified rel, return else None"""
"""Look for link with specified rel, return else None."""
for link in self.ln:
if ('rel' in link and
link['rel']==rel):
@ -117,14 +119,14 @@ class ResourceContainer(object):
return(None)
def link_href(self,rel):
"""Look for link with specified rel, return href from it or None"""
"""Look for link with specified rel, return href from it or None."""
link = self.link(rel)
if (link is not None):
link = link['href']
return(link)
def link_set(self,rel,href,**atts):
"""Set/create link with specified rel, set href and any other attributes
"""Set/create link with specified rel, set href and any other attributes.
Any link element must have both rel and href values, the specification
also defines the type attributes and others are permitted also. See
@ -145,36 +147,36 @@ class ResourceContainer(object):
@property
def describedby(self):
"""Convenient access to <rs:ln rel="describedby" href="uri">"""
"""Convenient access to <rs:ln rel="describedby" href="uri">."""
return(self.link_href('describedby'))
@describedby.setter
def describedby(self,uri):
"""Set ResourceSync Description link to given URI"""
"""Set ResourceSync Description link to given URI."""
self.link_set('describedby',uri)
@property
def up(self):
"""Get the URI of any ResourceSync rel="up" link"""
"""Get the URI of any ResourceSync rel="up" link."""
return(self.link_href('up'))
@up.setter
def up(self,uri):
"""Set ResourceSync rel="up" link to given URI"""
"""Set ResourceSync rel="up" link to given URI."""
self.link_set('up',uri)
@property
def index(self):
"""Get the URI of and ResourceSync rel="index" link"""
"""Get the URI of and ResourceSync rel="index" link."""
return(self.link_href('index'))
@index.setter
def index(self,uri):
"""Set index link to given URI"""
"""Set index link to given URI."""
self.link_set('index',uri)
def default_capability(self):
"""Set capability name in md
"""Set capability name in md.
Every ResourceSync document should have the top-level
capability attributes.
@ -183,9 +185,9 @@ class ResourceContainer(object):
self.md['capability']=self.capability_name
def add(self, resource):
"""Add a resource or an iterable collection of resources to this container
"""Add a resource or an iterable collection of resources to this container.
Must be implemented in derived class
Must be implemented in derived class.
"""
if isinstance(resource, collections.Iterable):
for r in resource:
@ -194,14 +196,14 @@ class ResourceContainer(object):
self.resources.append(resource)
def uris(self):
"""Return list of all URIs, possibly including dupes"""
"""Return list of all URIs, possibly including dupes."""
uris = []
for r in self.resources:
uris.append(r.uri)
return(uris)
def prune_before(self, timestamp):
"""Remove all resources with timestamp earlier than that given
"""Remove all resources with timestamp earlier than that given.
Returns the number of entries removed. Will raise an excpetion
if there are any entries without a timestamp.
@ -219,7 +221,7 @@ class ResourceContainer(object):
return(n)
def prune_dupes(self):
"""Remove all but the last entry for a given resource URI
"""Remove all but the last entry for a given resource URI.
Returns the number of entries removed. Also removes all entries for a
given URI where the first entry is a create and the last entry is a
@ -250,14 +252,14 @@ class ResourceContainer(object):
return(n)
def __str__(self):
"""Return string of all resources in order given by interator"""
"""Return string of all resources in order given by interator."""
s = ''
for resource in self:
s += str(resource) + "\n"
return(s)
def _str_datetime_now(self, x=None):
"""Return datetime string for use with time attributes
"""Return datetime string for use with time attributes.
Handling depends on input:
'now' - returns datetime for now

15
resync/resource_dump.py
View File

@ -1,4 +1,4 @@
"""ResourceSync Resource Dump object
"""ResourceSync Resource Dump object.
A Resource Dump is a set of content package resources with
some metadata for each resource.
@ -12,7 +12,8 @@ http://www.openarchives.org/rs/resourcesync#ResourceDump
from .resource_list import ResourceList
class ResourceDump(ResourceList):
"""Class representing an Resource Dump
"""Class representing an Resource Dump.
A Resource Dump comprises a set of content packages that are
the resources listed. Properties similar to a ResourceList
@ -23,13 +24,15 @@ class ResourceDump(ResourceList):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None):
"""Initialize ResourceDump."""
super(ResourceDump, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
mapper=mapper)
self.capability_name='resourcedump'
def write(self, basename="/tmp/resource_dump.xml", dumpfile=None):
"""Write out a Resource Dump document and optionally also the actual dump files
"""Write out a ResourceDump document and optionally also the dump files.
Dump files will be written if the dumpfile argument is set to a base filename.
"""
super(ResourceDump,self).write(basename)
Dump files will be written if the dumpfile argument is set to a base
filename.
"""
super(ResourceDump,self).write(basename)

6
resync/resource_dump_manifest.py
View File

@ -1,4 +1,4 @@
"""ResourceSync ResourceDumpManifest object
"""ResourceSync ResourceDumpManifest object.
A ResourceDumpManifest lists the set of files/resources included
within a content package that is included in a ResourceDump.
@ -12,7 +12,8 @@ http://www.openarchives.org/rs/resourcesync#ResourceDumpManifest
from .resource_list import ResourceList
class ResourceDumpManifest(ResourceList):
"""Class representing a Resource Dump Manifest
"""Class representing a Resource Dump Manifest.
A ResourceDumpManifest comprises a set of files/resources
in a content package. Properties much Like a ResourceList
@ -20,5 +21,6 @@ class ResourceDumpManifest(ResourceList):
"""
def __init__(self, resources=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None):
"""Initialize ResourceDumpManifest."""
super(ResourceDumpManifest, self).__init__(resources=resources, md=md, ln=ln, uri=uri, mapper=mapper)
self.capability_name = 'resourcedump-manifest'

60
resync/resource_list.py
View File

@ -1,6 +1,6 @@
"""ResourceSync Resource List object
"""ResourceSync Resource List object.
An Resource List is a set of resources with some metadata for
A Resource List is a set of resources with some metadata for
each resource. Comparison of resource lists from a source and
a destination allows understanding of whether the two are in
sync or whether some resources need to be updated at the
@ -8,10 +8,10 @@ destination.
There may also be metadata about the Resource List, and links
to other ResourceSync documents. Metadata include the
timestamp of the ResourceList (md_at) and, optionally, the
timestamp of the Resource List (md_at) and, optionally, the
timestamp when creation of the Resource List was completed
(md_completed).at the top level. These include a creation timestamp (from)
and links to the Capability List.
(md_completed).at the top level. These include a creation timestamp
(from) and links to the Capability List.
Described in specification at:
http://www.openarchives.org/rs/resourcesync#DescResources
@ -34,7 +34,8 @@ from .url_authority import UrlAuthority
class ResourceListDict(dict):
"""Default implementation of class to store resources in ResourceList
"""Default implementation of class to store resources in ResourceList.
Key properties of this class are:
- has add(resource) method
@ -42,7 +43,7 @@ class ResourceListDict(dict):
"""
def __iter__(self):
"""Iterator over all the resources in this resource_list"""
"""Iterator over all the resources in this ResourceListDict."""
self._iter_next_list = sorted(self.keys())
self._iter_next_list.reverse()
return(iter(self._iter_next, None))
@ -54,17 +55,19 @@ class ResourceListDict(dict):
return(None)
def uris(self):
"""Extract sorted list of URIs for resources in this ResourceListDict."""
return(sorted(self.keys()))
def add(self, resource, replace=False):
"""Add just a single resource"""
"""Add just a single resource."""
uri = resource.uri
if (uri in self and not replace):
raise ResourceListDupeError("Attempt to add resource already in resource_list")
self[uri]=resource
class ResourceListOrdered(list):
"""Alternative implementation of class to store resources in ResourceList
"""Alternative implementation of class to store resources in ResourceList.
FIXME - This is a rather inefficient implementation which involves
scanning all resources to check for duplicates. Designed just to enable
@ -78,14 +81,14 @@ class ResourceListOrdered(list):
"""
def uris(self):
"""Extract list of all resource URIs (in the order added)"""
"""Extract list of all resource URIs (in the order added)."""
uris = []
for r in self:
uris.append(r.uri)
return(uris)
def add(self, resource, replace=False):
"""Add a single resource, check for dupes"""
"""Add a single resource, check for dupes."""
uri = resource.uri
for r in self:
if (uri == r.uri):
@ -98,55 +101,60 @@ class ResourceListOrdered(list):
self.append(resource)
class ResourceListDupeError(Exception):
"""Exception in case of duplicate resource."""
pass
class ResourceList(ListBaseWithIndex):
"""Class representing an resource_list of resources
"""Class representing a ResourceList.
This same class is used for both the source and the destination
and is the central point of comparison the decide whether they
are in sync or what needs to be copied to bring the destinaton
into sync.
A Resource List will admit only one resource with any given URI.
A ResourceList will admit only one resource with any given URI.
Typical usage for a small Resource List is:
Typical usage for a small ResourceList is:
rl = ResourceList()
rl.add( Resource(...) )
rl.add( Resource(...) )
print rl.as_xml()
rl = ResourceList()
rl.add( Resource(...) )
rl.add( Resource(...) )
print rl.as_xml()
The default storage is unordered but the iterator imposes a canonical
order which is alphabetical by URI. If it is desired to have
resources listed in the order they are added then the ResourceDictOrdered
class may be specified on creation:
rl = ResourceList( resources_class=ResourceDictOrdered )
rl = ResourceList( resources_class=ResourceDictOrdered )
In normal use it is expected that any Resource List Index will be
created automatically when writing out a large Resource List in
multiple sitemap files. However, should it be necessary to
explicitly create an index then this may be specified with:
rli = ResourceList( resources_class=ResourceDictOrdered )
rli.sitemapindex=True
rli = ResourceList( resources_class=ResourceDictOrdered )
rli.sitemapindex=True
See additional descriptions in ListBaseWithIndex and ListBase.
"""
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
allow_multifile=None, mapper=None, resources_class=ResourceListDict):
"""Initialize ResourceList."""
super(ResourceList, self).__init__(resources=resources, count=count, md=md, ln=ln, uri=uri,
capability_name = 'resourcelist',
allow_multifile=allow_multifile, mapper=mapper,
resources_class=resources_class)
def add(self, resource, replace=False):
"""Add a resource or an iterable collection of resources
"""Add a resource or an iterable collection of resources.
Will throw a ValueError if the resource (ie. same uri) already
exists in the resource_list, unless replace=True.
exists in the ResourceList, unless replace=True.
"""
if isinstance(resource, collections.Iterable):
for r in resource:
@ -155,9 +163,9 @@ class ResourceList(ListBaseWithIndex):
self.resources.add(resource,replace)
def compare(self,src):
"""Compare the current resource_list object with the specified resource_list
"""Compare this ResourceList object with that specified as src.
The parameter src must also be an resource_list object, it is assumed
The parameter src must also be a ResourceList object, it is assumed
to be the source, and the current object is the destination. This
written to work for any objects in self and sc, provided that the
== operator can be used to compare them.
@ -201,7 +209,7 @@ class ResourceList(ListBaseWithIndex):
return(same,updated,deleted,created)
def has_md5(self):
"""Return true if at least one contained resource-like object has md5 data"""
"""Return true if at least one contained resource-like object has md5 data."""
if (self.resources is None):
return(False)
for resource in self:

45
resync/resource_list_builder.py
View File

@ -1,17 +1,4 @@
"""ResourceListBuilder to create ResourceList objects
Currently implements build from files on disk only.
Attributes:
- set_path set true to add path attribute for each resource
- set_md5 set true to calculate MD5 sums for all files
- set_length set true to include file length in resource_list (defaults true)
- exclude_dirs is a list of directory names to exclude
(defaults to ['CVS','.git'))
FIXME - should add options to set sha1 and sha256 in addition or as
alternatives to md5.
"""
"""ResourceListBuilder to create ResourceList objects."""
import os
import os.path
@ -32,8 +19,23 @@ from .w3c_datetime import datetime_to_str
class ResourceListBuilder():
"""ResourceListBuilder to create ResourceList objects.
Currently implements build from files on disk only.
Attributes:
- set_path set true to add path attribute for each resource
- set_md5 set true to calculate MD5 sums for all files
- set_length set true to include file length in resource_list (defaults true)
- exclude_dirs is a list of directory names to exclude
(defaults to ['CVS','.git'))
FIXME - should add options to set sha1 and sha256 in addition or as
alternatives to md5.
"""
def __init__(self, mapper=None, set_md5=False, set_length=True, set_path=False):
"""Create ResourceListBuilder object, optionally set options
"""Create ResourceListBuilder object, optionally set options.
The mapper attribute must be set before a call to from_disk() in order to
map between local filenames and URIs.
@ -58,12 +60,12 @@ class ResourceListBuilder():
self.compiled_exclude_files = []
def add_exclude_files(self, exclude_patterns):
"""Add more patterns of files to exclude while building resource_list"""
"""Add more patterns of files to exclude while building resource_list."""
for pattern in exclude_patterns:
self.exclude_files.append(pattern)
def compile_excludes(self):
"""Compile a set of regexps for files to be exlcuded from scans"""
"""Compile a set of regexps for files to be exlcuded from scans."""
self.compiled_exclude_files = []
for pattern in self.exclude_files:
try:
@ -72,14 +74,14 @@ class ResourceListBuilder():
raise ValueError("Bad python regex in exclude '%s': %s" % (pattern,str(e)))
def exclude_file(self, file):
"""True if file should be exclude based on name pattern"""
"""True if file should be exclude based on name pattern."""
for pattern in self.compiled_exclude_files:
if (pattern.match(file)):
return(True)
return(False)
def from_disk(self, resource_list=None, paths=None):
"""Create or extend resource_list with resources from disk scan
"""Create or extend resource_list with resources from disk scan.
Assumes very simple disk path to URL mapping (in self.mapping): chop
path and replace with url_path. Returns the new or extended ResourceList
@ -126,8 +128,7 @@ class ResourceListBuilder():
return(resource_list)
def from_disk_add_path(self, path=None, resource_list=None):
"""Add to resource_list with resources from disk scan starting at path
"""
"""Add to resource_list with resources from disk scan starting at path."""
# sanity
if (path is None or resource_list is None or self.mapper is None):
raise ValueError("Must specify path, resource_list and mapper")
@ -151,7 +152,7 @@ class ResourceListBuilder():
self.add_file(resource_list=resource_list,file=path)
def add_file(self, resource_list=None, dir=None, file=None):
"""Add a single file to resource_list
"""Add a single file to resource_list.
Follows object settings of set_path, set_md5 and set_length.
"""

24
resync/resource_set.py
View File

@ -1,13 +1,14 @@
"""A set of Resource objects used for Capability List
Indexes and ResourceSync Description documents.
Ordinging is currently alphanumeric (using sorted(..)) on the
uri which is the key.
"""
"""Base class representing a set of Resource objects."""
class ResourceSet(dict):
"""Implementation of class to store resources in Capability List
Indexes and ResourceSync Description documents.
"""Base class representing a set of Resource objects.
The ResourceSet class is used for Capability List Indexes and
ResourceSync Description documents.
Ordering of Resources is currently alphanumeric (using sorted(..))
on the uri which is the key.
Key properties of this class are:
- has add(resource) method
@ -15,7 +16,7 @@ class ResourceSet(dict):
"""
def __iter__(self):
"""Iterator over all the resources in this resource_list"""
"""Iterator over all the resources in this ResourceSet."""
self._iter_next_list = sorted(self.keys())
self._iter_next_list.reverse()
return(iter(self._iter_next, None))
@ -27,11 +28,14 @@ class ResourceSet(dict):
return(None)
def add(self, resource, replace=False):
"""Add just a single resource"""
"""Add just a single resource."""
uri = resource.uri
if (uri in self and not replace):
raise ResourceSetDupeError("Attempt to add resource already in this set")
self[uri]=resource
class ResourceSetDupeError(Exception):
"""Exception for case of attempt to add duplicate resource."""
pass

43
resync/sitemap.py
View File

@ -1,4 +1,4 @@
"""Read and write ResourceSync documents as sitemaps"""
"""Read and write ResourceSync documents as sitemaps."""
import re
import os
@ -28,28 +28,37 @@ XML_ATT_NAME = {
}
class SitemapIndexError(Exception):
"""Exception on attempt to read a sitemapindex instead of sitemap
or vice-versa
"""Exception if sitemapindex read instead of sitemap or vice-versa.
Provides both a message and a place to store the etree so that
the parse tree may be reused as a sitemapingex.
the parse tree may be reused as a sitemapindex.
"""
def __init__(self, message=None, etree=None):
"""Initialize SitemapIndexError with supplied parameters."""
self.message = message
self.etree = etree
def __repr__(self):
"""Return just the message attribute."""
return(self.message)
class SitemapParseError(Exception):
"""Exception for sitemap parsing structural error."""
pass
class SitemapDupeError(Exception):
"""Exception for case of duplicate resources in sitemap."""
pass
class Sitemap(object):
"""Read and write sitemaps
"""Read and write sitemaps.
Implemented as a separate class that uses ResourceContainer
(ResourceList or ChangeList) and Resource classes as data objects.
@ -62,6 +71,7 @@ class Sitemap(object):
"""
def __init__(self, pretty_xml=False):
"""Initialize Sitemap object."""
self.logger = logging.getLogger('resync.sitemap')
self.pretty_xml=pretty_xml
# Classes used when parsing
@ -74,7 +84,7 @@ class Sitemap(object):
##### Write the XML for a sitemap or sitemapindex #####
def resources_as_xml(self, resources, sitemapindex=False, fh=None):
"""Write or return XML for a set of resources in sitemap format
"""Write or return XML for a set of resources in sitemap format.
Arguments:
- resources - either an iterable or iterator of Resource objects;
@ -121,9 +131,10 @@ class Sitemap(object):
##### Read/parse an XML sitemap or sitemapindex #####
def parse_xml(self, fh=None, etree=None, resources=None, capability=None, sitemapindex=None):
"""Parse XML Sitemap from fh or etree and add resources to a
resorces object (which must support the add method). Returns
the resources object.
"""Parse XML Sitemap and add to resources object.
Reads from fh or etree and adds resources to a resorces object
(which must support the add method). Returns the resources object.
Also sets self.resources_created to be the number of resources created.
We adopt a very lax approach here. The parsing is properly namespace
@ -215,7 +226,7 @@ class Sitemap(object):
##### Resource methods #####
def resource_etree_element(self, resource, element_name='url'):
"""Return xml.etree.ElementTree.Element representing the resource
"""Return xml.etree.ElementTree.Element representing the resource.
Returns and element for the specified resource, of the form <url>
with enclosed properties that are based on the sitemap with extensions
@ -249,7 +260,7 @@ class Sitemap(object):
return(e)
def resource_as_xml(self,resource):
"""Return string for the resource as part of an XML sitemap
"""Return string for the resource as part of an XML sitemap.
Returns a string with the XML snippet representing the resource,
without any XML declaration.
@ -267,7 +278,7 @@ class Sitemap(object):
return(s.replace("<?xml version='1.0' encoding='UTF-8'?>\n",''))
def resource_from_etree(self, etree, resource_class):
"""Construct a Resource from an etree
"""Construct a Resource from an etree.
Parameters:
etree - the etree to parse
@ -326,7 +337,7 @@ class Sitemap(object):
return(resource)
def md_from_etree(self, md_element, context=''):
"""Parse rs:md attributes returning a dict of the data
"""Parse rs:md attributes returning a dict of the data.
Parameters:
md_element - etree element <rs:md>
@ -359,7 +370,7 @@ class Sitemap(object):
def ln_from_etree(self,ln_element,context=''):
"""Parse rs:ln element from an etree, returning a dict of the data
"""Parse rs:ln element from an etree, returning a dict of the data.
Parameters:
md_element - etree element <rs:md>
@ -399,7 +410,7 @@ class Sitemap(object):
##### Metadata and link elements #####
def add_element_with_atts_to_etree(self, etree, name, atts):
"""Add element with name and atts to etree iff there are any atts
"""Add element with name and atts to etree iff there are any atts.
Parameters:
etree - an etree object
@ -418,7 +429,7 @@ class Sitemap(object):
etree.append(e)
def _xml_att_name(self,att):
"""Get XML attribute name corresponding to supplied Resource object attribute
"""Get XML attribute name corresponding to supplied Resource object attribute.
We cannot use the XML attribute names in Python because 'from' and others
conflict with Python reserved words. To be consistent all the extra timestamps

1
resync/source_description.py
View File

@ -28,6 +28,7 @@ from resync.resource_set import ResourceSet
from resync.list_base_with_index import ListBaseWithIndex
class SourceDescription(ListBaseWithIndex):
"""Class representing the set of Capability Lists supported.
Will admit only one resource with any given URI.

60
resync/url_authority.py
View File

@ -1,28 +1,4 @@
"""Determine whether one resource can speak authoritatively
about another based on DNS hierarchy of server names and
path hierarchy within URLs.
Two modes are supported:
strict=True: requires that a query URL has the same URI
scheme (e.g. http) as the master, is on the same server
or one in a sub-domain, and that the path component is
at the same level or below the master.
strict=False (default): requires only that a query URL
has the same URI scheme as the master, and is on the same
server or one in a sub-domain of the master.
Example use:
from resync.url_authority import UrlAuthority
auth = UrlAuthority("http://example.org/master")
if (auth.has_authority_over("http://example.com/res1")):
# will be true
if (auth.has_authority_over("http://other.com/res1")):
# will be false
"""
"""Determine URI authority based on DNS and paths."""
try: #python3
from urllib.parse import urlparse
@ -32,8 +8,36 @@ import os.path
class UrlAuthority(object):
"""Determine URI authority based on DNS and paths.
Determine whether one resource can speak authoritatively
about another based on DNS hierarchy of server names and
path hierarchy within URLs.
Two modes are supported:
strict=True: requires that a query URL has the same URI
scheme (e.g. http) as the master, is on the same server
or one in a sub-domain, and that the path component is
at the same level or below the master.
strict=False (default): requires only that a query URL
has the same URI scheme as the master, and is on the same
server or one in a sub-domain of the master.
Example use:
from resync.url_authority import UrlAuthority
auth = UrlAuthority("http://example.org/master")
if (auth.has_authority_over("http://example.com/res1")):
# will be true
if (auth.has_authority_over("http://other.com/res1")):
# will be false
"""
def __init__(self, url=None, strict=False):
"""Create object and optionally set master url and/or strict mode"""
"""Create object and optionally set master url and/or strict mode."""
self.url = url
self.strict = strict
if (self.url is not None):
@ -44,14 +48,14 @@ class UrlAuthority(object):
self.master_path='/not/very/likely'
def set_master(self, url):
"""Set the master url that this object works with"""
"""Set the master url that this object works with."""
m = urlparse(url)
self.master_scheme=m.scheme
self.master_netloc=m.netloc
self.master_path=os.path.dirname(m.path)
def has_authority_over(self, url):
"""Returns True of the current master has authority over url
"""Return True of the current master has authority over url.
In strict mode checks scheme, server and path. Otherwise checks
just that the server names match or the query url is a

1
resync/utils.py
View File

@ -4,6 +4,7 @@ from logging import Formatter
from datetime import datetime
class UTCFormatter(Formatter):
"""Format datetime values as ISO8601 UTC Z form.
Based on http://bit.ly/T2n3Xk