diff --git a/bin/resync b/bin/resync index c6e434d..5c595a8 100755 --- a/bin/resync +++ b/bin/resync @@ -42,7 +42,7 @@ def main(): rem = p.add_option_group('REMOTE MODES', 'These modes use a remote source that is specified in a set of uri=path mappings ' 'and potentially also using an explicit --sitemap location. The default mode is ' - '--baseline') + '--baseline. See also: resync-explorer for an interactive client.') rem.add_option('--baseline', '-b', action='store_true', help='baseline sync of resources from remote source (src) to local filesystem (dst)') rem.add_option('--incremental', '--inc', '-i', action='store_true', diff --git a/resync/client.py b/resync/client.py index 0207f95..2936bab 100644 --- a/resync/client.py +++ b/resync/client.py @@ -1,4 +1,4 @@ -"""ResourceSync client implementation""" +"""ResourceSync client implementation.""" import sys try: #python3 @@ -31,17 +31,18 @@ from .list_base_with_index import ListBaseIndexError from .w3c_datetime import str_to_datetime,datetime_to_str def url_or_file_open(uri): - """Wrapper around urlopen() to prepend file: if no scheme provided""" + """Wrapper around urlopen() to prepend file: if no scheme provided.""" if (not re.match(r'''\w+:''',uri)): uri = 'file:'+uri return(urlopen(uri)) class ClientFatalError(Exception): - """Non-recoverable error in client, should include message to user""" + """Non-recoverable error in client, should include message to user.""" + pass class Client(object): - """Implementation of a ResourceSync client + """Implementation of a ResourceSync client. Logging is used for both console output and for detailed logs for automated analysis. Levels used: @@ -51,7 +52,7 @@ class Client(object): """ def __init__(self, checksum=False, verbose=False, dryrun=False): - super(Client, self).__init__() + """Initialize Client object with default parameters.""" self.checksum = checksum self.verbose = verbose self.dryrun = dryrun @@ -75,11 +76,11 @@ class Client(object): def set_mappings(self,mappings): - """Build and set Mapper object based on input mappings""" + """Build and set Mapper object based on input mappings.""" self.mapper = Mapper(mappings, use_default_path=True) def sitemap_uri(self,basename): - """Get full URI (filepath) for sitemap based on basename""" + """Get full URI (filepath) for sitemap based on basename.""" if (re.match(r"\w+:",basename)): # looks like URI return(basename) @@ -92,13 +93,13 @@ class Client(object): @property def sitemap(self): - """Return the sitemap URI based on maps or explicit settings""" + """Return the sitemap URI based on maps or explicit settings.""" if (self.sitemap_name is not None): return(self.sitemap_name) return(self.sitemap_uri(self.resource_list_name)) def build_resource_list(self, paths=None, set_path=False): - """Return a resource list for files on local disk + """Return a resource list for files on local disk. The set of files is taken by disk scan from the paths specified or else defaults to the paths specified in the current mappings @@ -134,16 +135,16 @@ class Client(object): return(rl) def log_event(self, change): - """Log a Resource object as an event for automated analysis""" + """Log a Resource object as an event for automated analysis.""" self.logger.debug( "Event: "+repr(change) ) def baseline_or_audit(self, allow_deletion=False, audit_only=False): - """Baseline synchonization or audit - - Both functions implemented in this routine because audit is a prerequisite - for a baseline sync. In the case of baseline sync the last timestamp seen + """Baseline synchonization or audit. + + Both functions implemented in this routine because audit is a prerequisite + for a baseline sync. In the case of baseline sync the last timestamp seen is recorded as client state. - """ + """ action = ( 'audit' if (audit_only) else 'baseline sync' ) self.logger.debug("Starting "+action) ### 0. Sanity checks @@ -216,7 +217,7 @@ class Client(object): self.logger.debug("Completed %s" % (action)) def incremental(self, allow_deletion=False, change_list_uri=None, from_datetime=None): - """Incremental synchronization + """Incremental synchronization. Use Change List to do incremental sync """ @@ -339,7 +340,7 @@ class Client(object): self.logger.debug("Completed incremental sync") def update_resource(self, resource, file, change=None): - """Update resource from uri to file on local system + """Update resource from uri to file on local system. Update means three things: 1. GET resources @@ -389,7 +390,7 @@ class Client(object): return(num_updated) def delete_resource(self, resource, file, allow_deletion=False): - """Delete copy of resource in file on local system + """Delete copy of resource in file on local system. Will only actually do the deletion if allow_deletion is True. Regardless of whether the deletion occurs, self.last_timestamp will be updated @@ -423,7 +424,7 @@ class Client(object): return(num_deleted) def parse_document(self): - """Parse any ResourceSync document and show information + """Parse any ResourceSync document and show information. Will use sitemap URI taken either from explicit self.sitemap_name or derived from the mappings supplied. @@ -455,7 +456,7 @@ class Client(object): break def explore(self): - """Explore capabilities of a server interactvely + """Explore capabilities of a server interactvely. Will use sitemap URI taken either from explicit self.sitemap_name or derived from the mappings supplied. @@ -488,7 +489,7 @@ class Client(object): print("--explore done, bye...") def explore_uri(self, uri, checks, caps, show_back=True): - """Interactive exploration of document at uri + """Interactive exploration of document at uri. Will flag warnings if the document is not of type listed in caps """ @@ -536,7 +537,7 @@ class Client(object): return( options[inp].uri, checks, caps, inp ) def explore_show_summary(self,list,parsed_index,caps): - """Show summary of one capability document + """Show summary of one capability document. Used as part of --explore. FIXME - should look for link and show that @@ -585,7 +586,7 @@ class Client(object): return(options,capability) def explore_show_head(self,uri,check_headers=None): - """Do HEAD on uri and show infomation + """Do HEAD on uri and show infomation. Will also check headers against any values specified in check_headers. @@ -609,7 +610,7 @@ class Client(object): print(" %s: %s%s" % (header, response.headers[header], check_str)) def write_resource_list(self,paths=None,outfile=None,links=None,dump=None): - """Write a Resource List or a Resource Dump for files on local disk + """Write a Resource List or a Resource Dump for files on local disk. Set of resources included is based on paths setting or else the mappings. Optionally links can be added. Output will be to stdout unless outfile @@ -638,7 +639,7 @@ class Client(object): def write_change_list(self,paths=None,outfile=None,ref_sitemap=None,newref_sitemap=None, empty=None,links=None,dump=None): - """Write a change list + """Write a change list. Unless the both ref_sitemap and newref_sitemap are specified then the Change List is calculated between the reference an the current state of files on @@ -673,7 +674,7 @@ class Client(object): self.write_dump_if_requested(cl,dump) def write_capability_list(self,capabilities=None,outfile=None,links=None): - """Write a Capability List to outfile or STDOUT""" + """Write a Capability List to outfile or STDOUT.""" capl = CapabilityList(ln=links) capl.pretty_xml = self.pretty_xml if (capabilities is not None): @@ -685,7 +686,7 @@ class Client(object): capl.write(basename=outfile) def write_source_description(self,capability_lists=None,outfile=None,links=None): - """Write a ResourceSync Description document to outfile or STDOUT""" + """Write a ResourceSync Description document to outfile or STDOUT.""" rsd = SourceDescription(ln=links) rsd.pretty_xml = self.pretty_xml if (capability_lists is not None): @@ -697,18 +698,20 @@ class Client(object): rsd.write(basename=outfile) def write_dump_if_requested(self,resource_list,dump): - """Write a dump to the file dump""" + """Write a dump to the file dump.""" if (dump is None): return + print("OOPS - FIXME - Wrinting dump to %s not yet implemented" % (dump)) + return(1) def read_reference_resource_list(self,ref_sitemap,name='reference'): - """Read reference resource list and return the ResourceList object + """Read reference resource list and return the ResourceList object. - name parameter just uses in output messages to say what type + The name parameter is used just in output messages to say what type of resource list is being read. """ rl = ResourceList() - self.logger.info("Reading reference %s resource list from %s ..." % (name,ref_sitemap)) + self.logger.info("Reading %s resource list from %s ..." % (name,ref_sitemap)) rl.mapper=self.mapper rl.read(uri=ref_sitemap,index_only=(not self.allow_multifile)) num_entries = len(rl.resources) @@ -731,7 +734,7 @@ class Client(object): def log_status(self, in_sync=True, incremental=False, audit=False, same=None, created=0, updated=0, deleted=0, to_delete=0): - """Write log message regarding status in standard form + """Write log message regarding status in standard form. Split this off so all messages from baseline/audit/incremental are written in a consistent form. diff --git a/setup.py b/setup.py index babd71a..bb3fde3 100644 --- a/setup.py +++ b/setup.py @@ -43,13 +43,16 @@ setup( version=version, packages=['resync'], scripts=['bin/resync','bin/resync-explorer'], - classifiers=["Development Status :: 3 - Alpha", + classifiers=["Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", #is this true? know Linux & OS X ok "Programming Language :: Python", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", "Environment :: Web Environment"], diff --git a/tests/capture_stdout.py b/tests/capture_stdout.py new file mode 100644 index 0000000..a589b40 --- /dev/null +++ b/tests/capture_stdout.py @@ -0,0 +1,23 @@ +"""Provide capture_stdout as contect manager.""" + +import sys, contextlib +try: #python2 + # Must try this first as io also exists in python2 + # but in the wrong one! + import StringIO as io +except ImportError: #python3 + import io + +# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python +class Data(object): + pass + +@contextlib.contextmanager +def capture_stdout(): + old = sys.stdout + capturer = io.StringIO() + sys.stdout = capturer + data = Data() + yield data + sys.stdout = old + data.result = capturer.getvalue() diff --git a/tests/test_client.py b/tests/test_client.py index 67da627..781a84b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,36 +1,20 @@ +from tests.testcase_with_tmpdir import TestCase +from tests.capture_stdout import capture_stdout + import unittest import re import logging from testfixtures import LogCapture -import sys, contextlib -try: #python2 - # Must try this first as io also exists in python2 - # but in the wrong one! - import StringIO as io -except ImportError: #python3 - import io +import sys +import os.path from resync.client import Client, ClientFatalError +from resync.change_list import ChangeList -# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python -class Data(object): - pass - -@contextlib.contextmanager -def capture_stdout(): - old = sys.stdout - capturer = io.StringIO() - sys.stdout = capturer - data = Data() - yield data - sys.stdout = old - data.result = capturer.getvalue() - - -class TestClient(unittest.TestCase): +class TestClient(TestCase): @classmethod - def setUpClass(cls): + def extraSetUpClass(cls): logging.basicConfig(level=logging.INFO) def test01_make_resource_list_empty(self): @@ -109,7 +93,6 @@ class TestClient(unittest.TestCase): # with an explicit paths setting only the specified paths will be included with capture_stdout() as capturer: c.write_resource_list(paths='tests/testdata/dir1') - sys.stderr.write(capturer.result) self.assertTrue( re.search(r'http://example.org/dir1/file_a', capturer.result ) ) self.assertTrue( re.search(r'http://example.org/dir1/file_b', capturer.result ) ) @@ -119,6 +102,59 @@ class TestClient(unittest.TestCase): #self.assertTrue( re.search(r'http://example.org/dir1/file_a[\w\-:]+', capturer.result ) ) #self.assertTrue( re.search(r'http://example.org/dir1/file_b[\w\-:]+', capturer.result ) ) + def test46_write_capability_list(self): + c = Client() + caps = {'resourcelist': 'http://a.b/rl', + 'changelist': 'http://a.b/cl'} + # to STDOUT + with capture_stdout() as capturer: + c.write_capability_list(capabilities=caps) + self.assertTrue( re.search(r'http://a.b/rl', capturer.result ) ) + # to file (just check that something is written) + outfile = os.path.join(self.tmpdir,'cl_out.xml') + c.write_capability_list(capabilities=caps, outfile=outfile) + self.assertTrue( os.path.getsize(outfile)>100 ) + + def test47_write_source_description(self): + c = Client() + # to STDOUT + with capture_stdout() as capturer: + c.write_source_description(capability_lists=['http://a.b/'], links=[{'rel':'c','href':'d'}]) + self.assertTrue( re.search(r'http://a.b/', capturer.result ) ) + # to file (just check that something is written) + outfile = os.path.join(self.tmpdir,'sd_out.xml') + c.write_source_description(capability_lists=['http://a.b/'], outfile=outfile, links=[{'rel':'c','href':'d'}]) + self.assertTrue( os.path.getsize(outfile)>100 ) + + def test48_write_dump_if_requested(self): + c = Client() + # no dump file + self.assertFalse( c.write_dump_if_requested( ChangeList(), None ) ) + # with dump file + with capture_stdout() as capturer: + c.write_dump_if_requested(ChangeList(),'/tmp/a_file') + self.assertTrue( re.search(r'FIXME', capturer.result) ) + + def test49_read_reference_resource_list(self): + c = Client() + with capture_stdout() as capturer: + rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual( len(rl), 2 ) + self.assertEqual( '', capturer.result ) + c.verbose = True + with capture_stdout() as capturer: + rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual( len(rl), 2 ) + self.assertTrue( re.search(r'http://example.com/res2', capturer.result) ) + c.verbose = True + c.max_sitemap_entries = 1 + with capture_stdout() as capturer: + rl = c.read_reference_resource_list('tests/testdata/examples_from_spec/resourcesync_ex_1.xml') + self.assertEqual( len(rl), 2 ) + self.assertTrue( re.search(r'http://example.com/res1', capturer.result) ) + self.assertTrue( re.search(r'Showing first 1 entries', capturer.result) ) + self.assertFalse( re.search(r'http://example.com/res2', capturer.result) ) + def test50_log_status(self): c = Client() with LogCapture() as lc: @@ -135,6 +171,15 @@ class TestClient(unittest.TestCase): c.log_status(audit=True) self.assertEqual( lc.records[-1].msg, 'Status: IN SYNC (to create=0, to update=0, to delete=0)' ) + c.log_status(in_sync=False, audit=True) + self.assertEqual( lc.records[-1].msg, + 'Status: NOT IN SYNC (to create=0, to update=0, to delete=0)' ) + c.log_status(in_sync=False, audit=False, to_delete=1) + self.assertEqual( lc.records[-1].msg, + 'Status: PART SYNCED (created=0, updated=0, to delete (--delete)=1)' ) + c.log_status(in_sync=False, audit=False, to_delete=1, incremental=1) + self.assertEqual( lc.records[-1].msg, + 'Status: PART APPLIED (created=0, updated=0, to delete (--delete)=1)' ) c.log_status(in_sync=False) self.assertEqual( lc.records[-1].msg, 'Status: SYNCED (created=0, updated=0, deleted=0)' ) diff --git a/tests/test_dump.py b/tests/test_dump.py index bc2df39..0a21ac1 100644 --- a/tests/test_dump.py +++ b/tests/test_dump.py @@ -1,7 +1,7 @@ +from tests.testcase_with_tmpdir import TestCase + import os.path import unittest -import tempfile -import shutil import sys import zipfile @@ -12,34 +12,7 @@ from resync.resource import Resource from resync.resource_dump_manifest import ResourceDumpManifest from resync.change_dump_manifest import ChangeDumpManifest -class TestDump(unittest.TestCase): - - _tmpdir=None - - @classmethod - def setUpClass(cls): - # Create tmp dir to write to and check - cls._tmpdir=tempfile.mkdtemp() - if (not os.path.isdir(cls._tmpdir)): - raise Exception("Failed to create tempdir to use for dump tests") - - @classmethod - def tearDownClass(cls): - # Cleanup - if (not os.path.isdir(cls._tmpdir)): - raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir)) - shutil.rmtree(cls._tmpdir) - - @property - def tmpdir(self): - # read-only access to _tmpdir, just in case... The rmtree scares me - # - # FIXME - Hack to work on python2.6 where setUpClass is not called, will - # FIXME - not have proper tidy as tearDownClass will not be called. - # FIXME - Remove when 2.6 no longer supported - if (not self._tmpdir and sys.version_info < (2,7)): - self.setUpClass() - return(self._tmpdir) +class TestDump(TestCase): def test00_dump_zip_resource_list(self): rl=ResourceDumpManifest() diff --git a/tests/test_explorer.py b/tests/test_explorer.py index 53cd06b..6bc48fc 100644 --- a/tests/test_explorer.py +++ b/tests/test_explorer.py @@ -1,34 +1,15 @@ +from tests.capture_stdout import capture_stdout + import unittest import re import logging -import sys, contextlib -try: #python2 - # Must try this first as io also exists in python2 - # but in the wrong one! - import StringIO as io -except ImportError: #python3 - import io +import sys from resync.client import Client, ClientFatalError from resync.capability_list import CapabilityList from resync.explorer import Explorer from resync.resource import Resource -# From http://stackoverflow.com/questions/2654834/capturing-stdout-within-the-same-process-in-python -class Data(object): - pass - -@contextlib.contextmanager -def capture_stdout(): - old = sys.stdout - capturer = io.StringIO() - sys.stdout = capturer - data = Data() - yield data - sys.stdout = old - data.result = capturer.getvalue() - - class TestExplorer(unittest.TestCase): @classmethod diff --git a/tests/testcase_with_tmpdir.py b/tests/testcase_with_tmpdir.py new file mode 100644 index 0000000..2e05eeb --- /dev/null +++ b/tests/testcase_with_tmpdir.py @@ -0,0 +1,43 @@ +"""Extension of unittest.TestCase to create a temp directory tmpdir.""" +import os.path +import unittest +import tempfile +import shutil + +class TestCase(unittest.TestCase): + """Adds setUpClass an tearDownClass that create and destroy tmpdir.""" + + _tmpdir=None + + @classmethod + def setUpClass(cls): + # Create tmp dir to write to and check + cls._tmpdir=tempfile.mkdtemp() + if (not os.path.isdir(cls._tmpdir)): + raise Exception("Failed to create tempdir to use for dump tests") + try: + cls.extraSetUpClass() + except: + pass + + @classmethod + def tearDownClass(cls): + # Cleanup + if (not os.path.isdir(cls._tmpdir)): + raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir)) + shutil.rmtree(cls._tmpdir) + try: + cls.extraTearUpClass() + except: + pass + + @property + def tmpdir(self): + # read-only access to _tmpdir, just in case... The rmtree scares me + # + # FIXME - Hack to work on python2.6 where setUpClass is not called, will + # FIXME - not have proper tidy as tearDownClass will not be called. + # FIXME - Remove when 2.6 no longer supported + if (not self._tmpdir and sys.version_info < (2,7)): + self.setUpClass() + return(self._tmpdir)