gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add archives

Browse Source
This commit is contained in:
Simeon Warner 2013-08-16 23:35:14 -04:00
parent eb46199db8
commit cf0a0b079b
17 changed files with 252 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
resync/__init__.py
View File

@ -3,11 +3,12 @@ from _version import __version__
"""Enable easy import for core classes, e.g.
from resync import Resource
"""
from resource import Resource
from source_description import SourceDescription
from capability_list import CapabilityList
from resource_list import ResourceList
from change_list import ChangeList
from resource_dump import ResourceDump
from change_dump import ChangeDump
from capability_list import CapabilityList
from source_description import SourceDescription
from archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive
from resource import Resource

53
resync/archives.py Normal file
View File

@ -0,0 +1,53 @@
"""ResourceSync Archive objects
The ResourceSync Archives specification
http://www.openarchives.org/rs/archives specifies capabilities
that provide archives of the 4 core capabilities. While some
optional attributes are different the basic structure is the
same for the Resource List Archive, Change List Archive,
Resource Dump Archive, and Change Dump Archive.
"""
import collections
from urllib import URLopener
from list_base_with_index import ListBaseWithIndex
from resource import Resource
from sitemap import Sitemap
class ResourceListArchive(ListBaseWithIndex):
"""Class representing an Change List Archive"""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
self.resources_class = list if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
super(ResourceListArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='resourcelist-archive')
class ChangeListArchive(ListBaseWithIndex):
"""Class representing an Change List Archive"""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
super(ChangeListArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changelist-archive')
class ResourceDumpArchive(ListBaseWithIndex):
"""Class representing an Resource Dump Archive"""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
super(ResourceDumpArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='resourcedump-archive',
resources_class=resources_class)
class ChangeDumpArchive(ListBaseWithIndex):
"""Class representing an Change Dump Archive"""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
super(ChangeDumpArchive, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changedump-archive',
resources_class=resources_class)

8
resync/change_list.py
View File

@ -26,12 +26,10 @@ class ChangeList(ListBaseWithIndex):
"""Class representing an Change List"""
def __init__(self, resources=None, md=None, ln=None, uri=None,
resources_class=None):
self.resources_class = list if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
resources_class=list):
super(ChangeList, self).__init__(resources=resources, md=md, ln=ln, uri=uri,
capability_name='changelist')
capability_name='changelist',
resources_class=resources_class)
# FIXME: Should probably be inline in add, rather than a class method
def maybeAdd(self, resource):

6
resync/list_base_with_index.py
View File

@ -51,7 +51,11 @@ class ListBaseWithIndex(ListBase):
"""
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
capability_name='unknown', allow_multifile=None, mapper=None):
capability_name='unknown', allow_multifile=None, mapper=None,
resources_class=None):
self.resources_class = list if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
super(ListBaseWithIndex, self).__init__(resources=resources, count=count, md=md, ln=ln,
uri=uri, capability_name=capability_name)
# specific to lists with indexes

12
resync/resource_list.py
View File

@ -129,14 +129,12 @@ class ResourceList(ListBaseWithIndex):
See additional descriptions in ListBaseWithIndex and ListBase.
"""
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None, allow_multifile=None, mapper=None,
resources_class=None):
self.resources_class = ResourceListDict if resources_class is None else resources_class
if (resources is None):
resources = self.resources_class()
def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
allow_multifile=None, mapper=None, resources_class=ResourceListDict):
super(ResourceList, self).__init__(resources=resources, count=count, md=md, ln=ln, uri=uri,
capability_name = 'resourcelist',
allow_multifile=allow_multifile, mapper=mapper)
capability_name = 'resourcelist',
allow_multifile=allow_multifile, mapper=mapper,
resources_class=resources_class)
def add(self, resource, replace=False):
"""Add a resource or an iterable collection of resources

9
resync/source_description.py
View File

@ -19,9 +19,9 @@ import collections
from resource import Resource
from resource_set import ResourceSet
from list_base import ListBase
from list_base_with_index import ListBaseWithIndex
class SourceDescription(ListBase):
class SourceDescription(ListBaseWithIndex):
"""Class representing the set of Capability Lists supported
Will admit only one resource with any given URI.
@ -31,10 +31,9 @@ class SourceDescription(ListBase):
"""
def __init__(self, resources=None, md=None, ln=None):
if (resources is None):
resources = ResourceSet()
super(SourceDescription, self).__init__(resources=resources, md=md, ln=ln,
capability_name='description')
capability_name='description',
resources_class=ResourceSet)
self.md['from']=None #usually don't want a from date
def add(self, resource, replace=False):

79
resync/test/test_examples_from_spec.py
View File

@ -5,13 +5,14 @@ import sys
import unittest
import StringIO
from resync.resource import Resource
from resync.source_description import SourceDescription
from resync.capability_list import CapabilityList
from resync.resource_list import ResourceList,ResourceListOrdered
from resync.resource_dump import ResourceDump
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.change_list import ChangeList
from resync.change_dump import ChangeDump
from resync.capability_list import CapabilityList
from resync.source_description import SourceDescription
from resync.archives import ResourceListArchive,ResourceDumpArchive,ChangeListArchive,ChangeDumpArchive
from resync.sitemap import Sitemap
class TestExamplesFromSpec(unittest.TestCase):
@ -19,10 +20,10 @@ class TestExamplesFromSpec(unittest.TestCase):
def test_all_simple_read(self):
"""Just try to read each one"""
for ex in ('archives_ex_2_1','archives_ex_2_2',
'archives_ex_3_1',
'archives_ex_3_1','archives_ex_3_2',
'archives_ex_4_1',
'archives_ex_5_1',
'archives_ex_6_1','archives_ex_6_2',
'archives_ex_6_1',
'resourcesync_ex_2_1','resourcesync_ex_2_2','resourcesync_ex_2_3',
'resourcesync_ex_2_4','resourcesync_ex_2_5','resourcesync_ex_2_6',
'resourcesync_ex_2_7','resourcesync_ex_2_8',
@ -406,6 +407,76 @@ class TestExamplesFromSpec(unittest.TestCase):
ex_xml = self._open_ex('resourcesync_ex_10_2').read()
self._assert_xml_equal( cl.as_xml(), ex_xml )
def test_build_archives_ex_3_1(self):
"""Resource List Archive listing 3 Resource Lists"""
rla = ResourceListArchive()
rla.up = 'http://example.com/dataset1/capabilitylist.xml'
rla.add( Resource( uri='http://example.com/resourcelist1.xml',
md_at='2012-11-03T09:00:00Z') )
rla.add( Resource( uri='http://example.com/resourcelist2.xml',
md_at='2012-12-03T09:00:00Z') )
rla.add( Resource( uri='http://example.com/resourcelist3.xml',
md_at='2013-01-03T09:00:00Z') )
ex_xml = self._open_ex('archives_ex_3_1').read()
self._assert_xml_equal( rla.as_xml(), ex_xml )
def test_build_archives_ex_3_2(self):
"""Resource List Archive Index listing 2 component Resource List Archives"""
rlai = ResourceListArchive()
rlai.sitemapindex = True
rlai.up = 'http://example.com/dataset1/capabilitylist.xml'
rlai.add( Resource( uri='http://example.com/resourcelistarchive00001.xml' ))
rlai.add( Resource( uri='http://example.com/resourcelistarchive00002.xml' ))
ex_xml = self._open_ex('archives_ex_3_2').read()
self._assert_xml_equal( rlai.as_xml(), ex_xml )
def test_build_archives_ex_4_1(self):
"""Resource Dump Archive listing 2 Resource Dumps"""
rda = ResourceDumpArchive()
rda.up = 'http://example.com/dataset1/capabilitylist.xml'
rda.add( Resource( uri='http://example.com/resourcedump1.xml',
lastmod='2012-11-03T09:05:42Z',
md_at="2012-11-03T09:00:00Z",
md_completed="2012-11-03T09:05:01Z" ) )
rda.add( Resource( uri='http://example.com/resourcedump2.xml',
lastmod='2012-12-03T09:06:12Z',
md_at="2012-12-03T09:00:00Z",
md_completed="2012-12-03T09:05:17Z" ) )
ex_xml = self._open_ex('archives_ex_4_1').read()
self._assert_xml_equal( rda.as_xml(), ex_xml )
def test_build_archives_ex_5_1(self):
"""Change List Archive listing 3 Change Lists"""
cla = ChangeListArchive()
cla.up = 'http://example.com/dataset1/capabilitylist.xml'
cla.add( Resource( uri='http://example.com/changelist1.xml',
md_from='2013-01-01T09:00:00Z',
md_until='2013-01-02T09:00:00Z') )
cla.add( Resource( uri='http://example.com/changelist2.xml',
md_from='2013-01-02T09:00:00Z',
md_until='2013-01-03T09:00:00Z') )
cla.add( Resource( uri='http://example.com/changelist3.xml',
md_from='2013-01-03T09:00:00Z',
md_until='2013-01-04T09:00:00Z') )
ex_xml = self._open_ex('archives_ex_5_1').read()
self._assert_xml_equal( cla.as_xml(), ex_xml )
def test_build_archives_ex_6_1(self):
"""Change Dump Archive listing 2 Change Dumps"""
cda = ChangeDumpArchive()
cda.up = 'http://example.com/dataset1/capabilitylist.xml'
cda.add( Resource( uri='http://example.com/changedump-w1.xml',
lastmod='2012-12-20T09:02:43Z',
md_from="2012-01-13T09:00:00Z",
md_until="2013-01-20T09:00:00Z" ) )
cda.add( Resource( uri='http://example.com/changedump-w2.xml',
lastmod='2012-12-27T09:01:57Z',
md_from="2012-01-20T09:00:00Z",
md_until="2013-01-27T09:00:00Z" ) )
ex_xml = self._open_ex('archives_ex_6_1').read()
self._assert_xml_equal( cda.as_xml(), ex_xml )
##### UTILITIES FOR (APPROX) COMPARISON OF XML IN EXAMPLES AND OUTPUT
def _assert_xml_equal(self,a,b):

39
resync/test/testdata/examples_from_spec/archives_ex_2_1.xml vendored
View File

@ -1,19 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="describedby"
href="http://example.com/info_about_set1_of_resources.xml"
type="application/xml"/>
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="resourcelist-archive"/>
href="http://example.com/resourcesync_description.xml"/>
<rs:md capability="capabilitylist"/>
<url>
<loc>http://example.com/resourcelist1.xml</loc>
<rs:md at="2012-11-03T09:00:00Z"/>
<loc>http://example.com/dataset1/resourcelist.xml</loc>
<rs:md capability="resourcelist"/>
</url>
<url>
<loc>http://example.com/resourcelist2.xml</loc>
<rs:md at="2012-12-03T09:00:00Z"/>
<loc>http://example.com/dataset1/resourcedump.xml</loc>
<rs:md capability="resourcedump"/>
</url>
<url>
<loc>http://example.com/resourcelist3.xml</loc>
<rs:md at="2013-01-03T09:00:00Z"/>
<loc>http://example.com/dataset1/changelist.xml</loc>
<rs:md capability="changelist"/>
</url>
<url>
<loc>http://example.com/dataset1/changedump.xml</loc>
<rs:md capability="changedump"/>
</url>
<url>
<loc>http://example.com/dataset1/resourcelist-archive.xml</loc>
<rs:md capability="resourcelist-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/resourcedump-archive.xml</loc>
<rs:md capability="resourcedump-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/changelist-archive.xml</loc>
<rs:md capability="changelist-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/changedump-archive.xml</loc>
<rs:md capability="changedump-archive"/>
</url>
</urlset>

26
resync/test/testdata/examples_from_spec/archives_ex_2_2.xml vendored
View File

@ -1,13 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="resourcelist-archive"/>
<sitemap>
<loc>http://example.com/resourcelistarchive00001.xml</loc>
</sitemap>
<sitemap>
<loc>http://example.com/resourcelistarchive00002.xml</loc>
</sitemap>
</sitemapindex>
<rs:ln rel="archives"
href="http://example.com/dataset1/changelistarchive.xml"/>
<rs:md capability="changelist"
from="2013-01-01T11:00:00Z"
until="2013-01-03T11:00:00Z"/>
<url>
<loc>http://example.com/res2.pdf</loc>
<lastmod>2013-01-02T13:00:00Z</lastmod>
<rs:md change="updated"/>
</url>
<url>
<loc>http://example.com/res3.tiff</loc>
<lastmod>2013-01-02T18:00:00Z</lastmod>
<rs:md change="deleted"/>
</url>
</urlset>

18
resync/test/testdata/examples_from_spec/archives_ex_3_1.xml vendored
View File

@ -3,17 +3,17 @@
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="resourcedump-archive"/>
<rs:md capability="resourcelist-archive"/>
<url>
<loc>http://example.com/resourcedump1.xml</loc>
<lastmod>2012-11-03T09:05:42Z</lastmod>
<rs:md at="2012-11-03T09:00:00Z"
completed="2012-11-03T09:05:01Z"/>
<loc>http://example.com/resourcelist1.xml</loc>
<rs:md at="2012-11-03T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/resourcedump2.xml</loc>
<lastmod>2012-12-03T09:06:12Z</lastmod>
<rs:md at="2012-12-03T09:00:00Z"
completed="2012-12-03T09:05:17Z"/>
<loc>http://example.com/resourcelist2.xml</loc>
<rs:md at="2012-12-03T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/resourcelist3.xml</loc>
<rs:md at="2013-01-03T09:00:00Z"/>
</url>
</urlset>

13
resync/test/testdata/examples_from_spec/archives_ex_3_2.xml vendored Normal file
View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="resourcelist-archive"/>
<sitemap>
<loc>http://example.com/resourcelistarchive00001.xml</loc>
</sitemap>
<sitemap>
<loc>http://example.com/resourcelistarchive00002.xml</loc>
</sitemap>
</sitemapindex>

21
resync/test/testdata/examples_from_spec/archives_ex_4_1.xml vendored
View File

@ -3,20 +3,17 @@
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="changelist-archive"/>
<rs:md capability="resourcedump-archive"/>
<url>
<loc>http://example.com/changelist1.xml</loc>
<rs:md from="2013-01-01T09:00:00Z"
until="2013-01-02T09:00:00Z"/>
<loc>http://example.com/resourcedump1.xml</loc>
<lastmod>2012-11-03T09:05:42Z</lastmod>
<rs:md at="2012-11-03T09:00:00Z"
completed="2012-11-03T09:05:01Z"/>
</url>
<url>
<loc>http://example.com/changelist2.xml</loc>
<rs:md from="2013-01-02T09:00:00Z"
until="2013-01-03T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/changelist3.xml</loc>
<rs:md from="2013-01-03T09:00:00Z"
until="2013-01-04T09:00:00Z"/>
<loc>http://example.com/resourcedump2.xml</loc>
<lastmod>2012-12-03T09:06:12Z</lastmod>
<rs:md at="2012-12-03T09:00:00Z"
completed="2012-12-03T09:05:17Z"/>
</url>
</urlset>

21
resync/test/testdata/examples_from_spec/archives_ex_5_1.xml vendored
View File

@ -3,17 +3,20 @@
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="changedump-archive"/>
<rs:md capability="changelist-archive"/>
<url>
<loc>http://example.com/changedump-w1.xml</loc>
<lastmod>2012-12-20T09:02:43Z</lastmod>
<rs:md from="2012-01-13T09:00:00Z"
until="2013-01-20T09:00:00Z"/>
<loc>http://example.com/changelist1.xml</loc>
<rs:md from="2013-01-01T09:00:00Z"
until="2013-01-02T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/changedump-w2.xml</loc>
<lastmod>2012-12-27T09:01:57Z</lastmod>
<rs:md from="2012-01-20T09:00:00Z"
until="2013-01-27T09:00:00Z"/>
<loc>http://example.com/changelist2.xml</loc>
<rs:md from="2013-01-02T09:00:00Z"
until="2013-01-03T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/changelist3.xml</loc>
<rs:md from="2013-01-03T09:00:00Z"
until="2013-01-04T09:00:00Z"/>
</url>
</urlset>

43
resync/test/testdata/examples_from_spec/archives_ex_6_1.xml vendored
View File

@ -1,42 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="describedby"
href="http://example.com/info_about_set1_of_resources.xml"
type="application/xml"/>
<rs:ln rel="up"
href="http://example.com/resourcesync_description.xml"/>
<rs:md capability="capabilitylist"/>
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="changedump-archive"/>
<url>
<loc>http://example.com/dataset1/resourcelist.xml</loc>
<rs:md capability="resourcelist"/>
<loc>http://example.com/changedump-w1.xml</loc>
<lastmod>2012-12-20T09:02:43Z</lastmod>
<rs:md from="2012-01-13T09:00:00Z"
until="2013-01-20T09:00:00Z"/>
</url>
<url>
<loc>http://example.com/dataset1/resourcedump.xml</loc>
<rs:md capability="resourcedump"/>
</url>
<url>
<loc>http://example.com/dataset1/changelist.xml</loc>
<rs:md capability="changelist"/>
</url>
<url>
<loc>http://example.com/dataset1/changedump.xml</loc>
<rs:md capability="changedump"/>
</url>
<url>
<loc>http://example.com/dataset1/resourcelist-archive.xml</loc>
<rs:md capability="resourcelist-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/resourcedump-archive.xml</loc>
<rs:md capability="resourcedump-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/changelist-archive.xml</loc>
<rs:md capability="changelist-archive"/>
</url>
<url>
<loc>http://example.com/dataset1/changedump-archive.xml</loc>
<rs:md capability="changedump-archive"/>
<loc>http://example.com/changedump-w2.xml</loc>
<lastmod>2012-12-27T09:01:57Z</lastmod>
<rs:md from="2012-01-20T09:00:00Z"
until="2013-01-27T09:00:00Z"/>
</url>
</urlset>

21
resync/test/testdata/examples_from_spec/archives_ex_6_2.xml vendored
View File

@ -1,21 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:rs="http://www.openarchives.org/rs/terms/">
<rs:ln rel="up"
href="http://example.com/dataset1/capabilitylist.xml"/>
<rs:ln rel="archives"
href="http://example.com/dataset1/changelistarchive.xml"/>
<rs:md capability="changelist"
from="2013-01-01T11:00:00Z"
until="2013-01-03T11:00:00Z"/>
<url>
<loc>http://example.com/res2.pdf</loc>
<lastmod>2013-01-02T13:00:00Z</lastmod>
<rs:md change="updated"/>
</url>
<url>
<loc>http://example.com/res3.tiff</loc>
<lastmod>2013-01-02T18:00:00Z</lastmod>
<rs:md change="deleted"/>
</url>
</urlset>

2
resync/test/testdata/examples_from_spec/resourcesync_ex_12_6.xml vendored
View File

@ -21,7 +21,7 @@
<rs:ln rel="timegate"
href="http://example.com/timegate/http://example.com/res1"/>
<rs:ln rel="timemap"
href="http://example.com/timemap/http://example.com/res1"/>
href="http://example.com/timemap/http://example.com/res1"
type="application/link-format"/>
</url>
</urlset>

2
resync/test/testdata/examples_from_spec/resourcesync_ex_12_9.xml vendored
View File

@ -4,7 +4,7 @@
<rs:ln rel="up"
href="http://aggregator1.example.com/dataset1/capabilitylist.xml"/>
<rs:md capability="changelist"
from="2013-01-03T11200:00Z"/>
from="2013-01-03T11:00:00Z"/>
<url>
<loc>http://aggregator1.example.com/res1.html</loc>
<lastmod>2013-01-03T20:00:00Z</lastmod>