gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add to and reorg client tests

Browse Source
This commit is contained in:
Simeon Warner 2016-03-26 16:42:16 -07:00
parent 20d50de0b9
commit f6c5ec88a9
8 changed files with 182 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/resync
View File

@ -42,7 +42,7 @@ def main():
rem = p.add_option_group('REMOTE MODES',
'These modes use a remote source that is specified in a set of uri=path mappings '
'and potentially also using an explicit --sitemap location. The default mode is '
'--baseline')
'--baseline. See also: resync-explorer for an interactive client.')
rem.add_option('--baseline', '-b', action='store_true',
help='baseline sync of resources from remote source (src) to local filesystem (dst)')
rem.add_option('--incremental', '--inc', '-i', action='store_true',

67
resync/client.py
View File

@ -1,4 +1,4 @@
"""ResourceSync client implementation"""
"""ResourceSync client implementation."""
import sys
try: #python3
@ -31,17 +31,18 @@ from .list_base_with_index import ListBaseIndexError
from .w3c_datetime import str_to_datetime,datetime_to_str
def url_or_file_open(uri):
"""Wrapper around urlopen() to prepend file: if no scheme provided"""
"""Wrapper around urlopen() to prepend file: if no scheme provided."""
if (not re.match(r'''\w+:''',uri)):
uri = 'file:'+uri
return(urlopen(uri))
class ClientFatalError(Exception):
"""Non-recoverable error in client, should include message to user"""
"""Non-recoverable error in client, should include message to user."""
pass
class Client(object):
"""Implementation of a ResourceSync client
"""Implementation of a ResourceSync client.
Logging is used for both console output and for detailed logs for
automated analysis. Levels used:
@ -51,7 +52,7 @@ class Client(object):
"""
def __init__(self, checksum=False, verbose=False, dryrun=False):
super(Client, self).__init__()
"""Initialize Client object with default parameters."""
self.checksum = checksum
self.verbose = verbose
self.dryrun = dryrun
@ -75,11 +76,11 @@ class Client(object):
def set_mappings(self,mappings):
"""Build and set Mapper object based on input mappings"""
"""Build and set Mapper object based on input mappings."""
self.mapper = Mapper(mappings, use_default_path=True)
def sitemap_uri(self,basename):
"""Get full URI (filepath) for sitemap based on basename"""
"""Get full URI (filepath) for sitemap based on basename."""
if (re.match(r"\w+:",basename)):
# looks like URI
return(basename)
@ -92,13 +93,13 @@ class Client(object):
@property
def sitemap(self):
"""Return the sitemap URI based on maps or explicit settings"""
"""Return the sitemap URI based on maps or explicit settings."""
if (self.sitemap_name is not None):
return(self.sitemap_name)
return(self.sitemap_uri(self.resource_list_name))
def build_resource_list(self, paths=None, set_path=False):
"""Return a resource list for files on local disk
"""Return a resource list for files on local disk.
The set of files is taken by disk scan from the paths specified or
else defaults to the paths specified in the current mappings
@ -134,16 +135,16 @@ class Client(object):
return(rl)
def log_event(self, change):
"""Log a Resource object as an event for automated analysis"""
"""Log a Resource object as an event for automated analysis."""
self.logger.debug( "Event: "+repr(change) )
def baseline_or_audit(self, allow_deletion=False, audit_only=False):
"""Baseline synchonization or audit
Both functions implemented in this routine because audit is a prerequisite
for a baseline sync. In the case of baseline sync the last timestamp seen
"""Baseline synchonization or audit.
Both functions implemented in this routine because audit is a prerequisite
for a baseline sync. In the case of baseline sync the last timestamp seen
is recorded as client state.
"""
"""
action = ( 'audit' if (audit_only) else 'baseline sync' )
self.logger.debug("Starting "+action)
### 0. Sanity checks
@ -216,7 +217,7 @@ class Client(object):
self.logger.debug("Completed %s" % (action))
def incremental(self, allow_deletion=False, change_list_uri=None, from_datetime=None):
"""Incremental synchronization
"""Incremental synchronization.
Use Change List to do incremental sync
"""
@ -339,7 +340,7 @@ class Client(object):
self.logger.debug("Completed incremental sync")
def update_resource(self, resource, file, change=None):
"""Update resource from uri to file on local system
"""Update resource from uri to file on local system.
Update means three things:
1. GET resources
@ -389,7 +390,7 @@ class Client(object):
return(num_updated)
def delete_resource(self, resource, file, allow_deletion=False):
"""Delete copy of resource in file on local system
"""Delete copy of resource in file on local system.
Will only actually do the deletion if allow_deletion is True. Regardless
of whether the deletion occurs, self.last_timestamp will be updated
@ -423,7 +424,7 @@ class Client(object):
return(num_deleted)
def parse_document(self):
"""Parse any ResourceSync document and show information
"""Parse any ResourceSync document and show information.
Will use sitemap URI taken either from explicit self.sitemap_name
or derived from the mappings supplied.
@ -455,7 +456,7 @@ class Client(object):
break
def explore(self):
"""Explore capabilities of a server interactvely
"""Explore capabilities of a server interactvely.
Will use sitemap URI taken either from explicit self.sitemap_name
or derived from the mappings supplied.
@ -488,7 +489,7 @@ class Client(object):
print("--explore done, bye...")
def explore_uri(self, uri, checks, caps, show_back=True):
"""Interactive exploration of document at uri
"""Interactive exploration of document at uri.
Will flag warnings if the document is not of type listed in caps
"""
@ -536,7 +537,7 @@ class Client(object):
return( options[inp].uri, checks, caps, inp )
def explore_show_summary(self,list,parsed_index,caps):
"""Show summary of one capability document
"""Show summary of one capability document.
Used as part of --explore.
FIXME - should look for <rs:ln rel="up"...> link and show that
@ -585,7 +586,7 @@ class Client(object):
return(options,capability)
def explore_show_head(self,uri,check_headers=None):
"""Do HEAD on uri and show infomation
"""Do HEAD on uri and show infomation.
Will also check headers against any values specified in
check_headers.
@ -609,7 +610,7 @@ class Client(object):
print(" %s: %s%s" % (header, response.headers[header], check_str))
def write_resource_list(self,paths=None,outfile=None,links=None,dump=None):
"""Write a Resource List or a Resource Dump for files on local disk
"""Write a Resource List or a Resource Dump for files on local disk.
Set of resources included is based on paths setting or else the mappings.
Optionally links can be added. Output will be to stdout unless outfile
@ -638,7 +639,7 @@ class Client(object):
def write_change_list(self,paths=None,outfile=None,ref_sitemap=None,newref_sitemap=None,
empty=None,links=None,dump=None):
"""Write a change list
"""Write a change list.
Unless the both ref_sitemap and newref_sitemap are specified then the Change
List is calculated between the reference an the current state of files on
@ -673,7 +674,7 @@ class Client(object):
self.write_dump_if_requested(cl,dump)
def write_capability_list(self,capabilities=None,outfile=None,links=None):
"""Write a Capability List to outfile or STDOUT"""
"""Write a Capability List to outfile or STDOUT."""
capl = CapabilityList(ln=links)
capl.pretty_xml = self.pretty_xml
if (capabilities is not None):
@ -685,7 +686,7 @@ class Client(object):
capl.write(basename=outfile)
def write_source_description(self,capability_lists=None,outfile=None,links=None):
"""Write a ResourceSync Description document to outfile or STDOUT"""
"""Write a ResourceSync Description document to outfile or STDOUT."""
rsd = SourceDescription(ln=links)
rsd.pretty_xml = self.pretty_xml
if (capability_lists is not None):
@ -697,18 +698,20 @@ class Client(object):
rsd.write(basename=outfile)
def write_dump_if_requested(self,resource_list,dump):
"""Write a dump to the file dump"""
"""Write a dump to the file dump."""
if (dump is None):
return
print("OOPS - FIXME - Wrinting dump to %s not yet implemented" % (dump))
return(1)
def read_reference_resource_list(self,ref_sitemap,name='reference'):
"""Read reference resource list and return the ResourceList object
"""Read reference resource list and return the ResourceList object.
name parameter just uses in output messages to say what type
The name parameter is used just in output messages to say what type
of resource list is being read.
"""
rl = ResourceList()
self.logger.info("Reading reference %s resource list from %s ..." % (name,ref_sitemap))
self.logger.info("Reading %s resource list from %s ..." % (name,ref_sitemap))
rl.mapper=self.mapper
rl.read(uri=ref_sitemap,index_only=(not self.allow_multifile))
num_entries = len(rl.resources)
@ -731,7 +734,7 @@ class Client(object):
def log_status(self, in_sync=True, incremental=False, audit=False,
same=None, created=0, updated=0, deleted=0, to_delete=0):
"""Write log message regarding status in standard form
"""Write log message regarding status in standard form.
Split this off so all messages from baseline/audit/incremental
are written in a consistent form.

5
setup.py
View File

@ -43,13 +43,16 @@ setup(
version=version,
packages=['resync'],
scripts=['bin/resync','bin/resync-explorer'],
classifiers=["Development Status :: 3 - Alpha",
classifiers=["Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent", #is this true? know Linux & OS X ok
"Programming Language :: Python",
"Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Software Development :: Libraries :: Python Modules",
"Environment :: Web Environment"],

23
tests/capture_stdout.py Normal file
View File

@ -0,0 +1,23 @@
"""Provide capture_stdout as contect manager."""
import sys, contextlib
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python
class Data(object):
pass
@contextlib.contextmanager
def capture_stdout():
old = sys.stdout
capturer = io.StringIO()
sys.stdout = capturer
data = Data()
yield data
sys.stdout = old
data.result = capturer.getvalue()

95
tests/test_client.py
View File

@ -1,36 +1,20 @@
from tests.testcase_with_tmpdir import TestCase
from tests.capture_stdout import capture_stdout
import unittest
import re
import logging
from testfixtures import LogCapture
import sys, contextlib
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import sys
import os.path
from resync.client import Client, ClientFatalError
from resync.change_list import ChangeList
# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python
class Data(object):
pass
@contextlib.contextmanager
def capture_stdout():
old = sys.stdout
capturer = io.StringIO()
sys.stdout = capturer
data = Data()
yield data
sys.stdout = old
data.result = capturer.getvalue()
class TestClient(unittest.TestCase):
class TestClient(TestCase):
@classmethod
def setUpClass(cls):
def extraSetUpClass(cls):
logging.basicConfig(level=logging.INFO)
def test01_make_resource_list_empty(self):
@ -109,7 +93,6 @@ class TestClient(unittest.TestCase):
# with an explicit paths setting only the specified paths will be included
with capture_stdout() as capturer:
c.write_resource_list(paths='tests/testdata/dir1')
sys.stderr.write(capturer.result)
self.assertTrue( re.search(r'<rs:md at="\S+" capability="resourcelist"', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc>', capturer.result ) )
self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc>', capturer.result ) )
@ -119,6 +102,59 @@ class TestClient(unittest.TestCase):
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
def test46_write_capability_list(self):
c = Client()
caps = {'resourcelist': 'http://a.b/rl',
'changelist': 'http://a.b/cl'}
# to STDOUT
with capture_stdout() as capturer:
c.write_capability_list(capabilities=caps)
self.assertTrue( re.search(r'http://a.b/rl', capturer.result ) )
# to file (just check that something is written)
outfile = os.path.join(self.tmpdir,'cl_out.xml')
c.write_capability_list(capabilities=caps, outfile=outfile)
self.assertTrue( os.path.getsize(outfile)>100 )
def test47_write_source_description(self):
c = Client()
# to STDOUT
with capture_stdout() as capturer:
c.write_source_description(capability_lists=['http://a.b/'], links=[{'rel':'c','href':'d'}])
self.assertTrue( re.search(r'http://a.b/', capturer.result ) )
# to file (just check that something is written)
outfile = os.path.join(self.tmpdir,'sd_out.xml')
c.write_source_description(capability_lists=['http://a.b/'], outfile=outfile, links=[{'rel':'c','href':'d'}])
self.assertTrue( os.path.getsize(outfile)>100 )
def test48_write_dump_if_requested(self):
c = Client()
# no dump file
self.assertFalse( c.write_dump_if_requested( ChangeList(), None ) )
# with dump file
with capture_stdout() as capturer:
c.write_dump_if_requested(ChangeList(),'/tmp/a_file')
self.assertTrue( re.search(r'FIXME', capturer.result) )
def test49_read_reference_resource_list(self):
c = Client()
with capture_stdout() as capturer:
rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml')
self.assertEqual( len(rl), 2 )
self.assertEqual( '', capturer.result )
c.verbose = True
with capture_stdout() as capturer:
rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml')
self.assertEqual( len(rl), 2 )
self.assertTrue( re.search(r'http://example.com/res2', capturer.result) )
c.verbose = True
c.max_sitemap_entries = 1
with capture_stdout() as capturer:
rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml')
self.assertEqual( len(rl), 2 )
self.assertTrue( re.search(r'http://example.com/res1', capturer.result) )
self.assertTrue( re.search(r'Showing first 1 entries', capturer.result) )
self.assertFalse( re.search(r'http://example.com/res2', capturer.result) )
def test50_log_status(self):
c = Client()
with LogCapture() as lc:
@ -135,6 +171,15 @@ class TestClient(unittest.TestCase):
c.log_status(audit=True)
self.assertEqual( lc.records[-1].msg,
'Status: IN SYNC (to create=0, to update=0, to delete=0)' )
c.log_status(in_sync=False, audit=True)
self.assertEqual( lc.records[-1].msg,
'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)' )
c.log_status(in_sync=False, audit=False, to_delete=1)
self.assertEqual( lc.records[-1].msg,
'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)' )
c.log_status(in_sync=False, audit=False, to_delete=1, incremental=1)
self.assertEqual( lc.records[-1].msg,
'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)' )
c.log_status(in_sync=False)
self.assertEqual( lc.records[-1].msg,
'Status: SYNCED (created=0, updated=0, deleted=0)' )

33
tests/test_dump.py
View File

@ -1,7 +1,7 @@
from tests.testcase_with_tmpdir import TestCase
import os.path
import unittest
import tempfile
import shutil
import sys
import zipfile
@ -12,34 +12,7 @@ from resync.resource import Resource
from resync.resource_dump_manifest import ResourceDumpManifest
from resync.change_dump_manifest import ChangeDumpManifest
class TestDump(unittest.TestCase):
_tmpdir=None
@classmethod
def setUpClass(cls):
# Create tmp dir to write to and check
cls._tmpdir=tempfile.mkdtemp()
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Failed to create tempdir to use for dump tests")
@classmethod
def tearDownClass(cls):
# Cleanup
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir))
shutil.rmtree(cls._tmpdir)
@property
def tmpdir(self):
# read-only access to _tmpdir, just in case... The rmtree scares me
#
# FIXME - Hack to work on python2.6 where setUpClass is not called, will
# FIXME - not have proper tidy as tearDownClass will not be called.
# FIXME - Remove when 2.6 no longer supported
if (not self._tmpdir and sys.version_info < (2,7)):
self.setUpClass()
return(self._tmpdir)
class TestDump(TestCase):
def test00_dump_zip_resource_list(self):
rl=ResourceDumpManifest()

25
tests/test_explorer.py
View File

@ -1,34 +1,15 @@
from tests.capture_stdout import capture_stdout
import unittest
import re
import logging
import sys, contextlib
try: #python2
# Must try this first as io also exists in python2
# but in the wrong one!
import StringIO as io
except ImportError: #python3
import io
import sys
from resync.client import Client, ClientFatalError
from resync.capability_list import CapabilityList
from resync.explorer import Explorer
from resync.resource import Resource
# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python
class Data(object):
pass
@contextlib.contextmanager
def capture_stdout():
old = sys.stdout
capturer = io.StringIO()
sys.stdout = capturer
data = Data()
yield data
sys.stdout = old
data.result = capturer.getvalue()
class TestExplorer(unittest.TestCase):
@classmethod

43
tests/testcase_with_tmpdir.py Normal file
View File

@ -0,0 +1,43 @@
"""Extension of unittest.TestCase to create a temp directory tmpdir."""
import os.path
import unittest
import tempfile
import shutil
class TestCase(unittest.TestCase):
"""Adds setUpClass an tearDownClass that create and destroy tmpdir."""
_tmpdir=None
@classmethod
def setUpClass(cls):
# Create tmp dir to write to and check
cls._tmpdir=tempfile.mkdtemp()
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Failed to create tempdir to use for dump tests")
try:
cls.extraSetUpClass()
except:
pass
@classmethod
def tearDownClass(cls):
# Cleanup
if (not os.path.isdir(cls._tmpdir)):
raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir))
shutil.rmtree(cls._tmpdir)
try:
cls.extraTearUpClass()
except:
pass
@property
def tmpdir(self):
# read-only access to _tmpdir, just in case... The rmtree scares me
#
# FIXME - Hack to work on python2.6 where setUpClass is not called, will
# FIXME - not have proper tidy as tearDownClass will not be called.
# FIXME - Remove when 2.6 no longer supported
if (not self._tmpdir and sys.version_info < (2,7)):
self.setUpClass()
return(self._tmpdir)