diff --git a/resync/client.py b/resync/client.py index 670d10a..d796d21 100644 --- a/resync/client.py +++ b/resync/client.py @@ -97,9 +97,8 @@ class Client(object): mapper=self.mapper) resource_list.read(uri=uri) except Exception as e: - raise ClientFatalError( - "Can't read source resource list from %s (%s)" % - (uri, str(e))) + raise ClientError("Can't read source resource list from %s (%s)" % + (uri, str(e))) self.logger.debug("Finished reading resource list") return(resource_list) @@ -110,7 +109,7 @@ class Client(object): source description in another location, but a ClientFatalError if a source description is found but there is some problem using it. """ - self.logger.info("Reading capability list %s" % (uri)) + self.logger.info("Reading source description %s" % (uri)) try: sd = SourceDescription() sd.read(uri=uri) @@ -125,11 +124,25 @@ class Client(object): raise ClientFatalError( "Source description %s has multiple sources" % (uri)) self.logger.info("Finished reading source description") - raise ClientFatalError('debug') - return(cl) + # Now read capability list + cluri = sd.resources.first().uri + uri = urljoin(uri, cluri) # FIXME - Should relative URI handling be elsewhere? + self.logger.info("Reading capability list %s" % (uri)) + try: + cl = CapabilityList() + cl.read(uri=uri) + except Exception as e: + raise ClientError( + "Can't read capability list from %s (%s)" % + (uri, str(e))) + if (not cl.has_capability('resourcelist')): + raise ClientFatalError( + "Capability list %s does not describe a resource list" % (uri)) + rluri = cl.capability_info('resourcelist').uri + return(urljoin(uri, rluri)) def find_resource_list(self): - """Look for resource list by hueristics. + """Finf resource list by hueristics, returns ResourceList object. 1. Use explicitly specified self.sitemap_name (and fail if that doesn't work) @@ -146,23 +159,28 @@ class Client(object): return(self.read_resource_list(self.sitemap_name)) # 2 & 3 parts = urlsplit(self.sitemap) - uri_host = urlunsplit([parts[0],parts[1],'','','']) + uri_host = urlunsplit([parts[0], parts[1], '', '', '']) + errors = [] for uri in [urljoin(self.sitemap, '.well-known/resourcesync'), urljoin(uri_host, '.well-known/resourcesync')]: + uri = uri.lstrip('file:///') # urljoin adds this for local files try: - return(self.find_resource_list_from_source_description(uri)) + rluri = self.find_resource_list_from_source_description(uri) + return(self.read_resource_list(rluri)) except ClientError as e: - pass + errors.append(str(e)) # 4, 5 & 6 for uri in [urljoin(self.sitemap, 'resourcelist.xml'), urljoin(self.sitemap, 'sitemap.xml'), urljoin(uri_host, 'sitemap.xml')]: + uri = uri.lstrip('file:///') # urljoin adds this for local files try: return(self.read_resource_list(uri)) except ClientError as e: - pass + errors.append(str(e)) raise ClientFatalError( - "Failed to find source resource list from common patterns") + "Failed to find source resource list from common patterns (%s)" % + ". ".join(errors)) def build_resource_list(self, paths=None, set_path=False): """Return a resource list for files on local disk. diff --git a/resync/resource_set.py b/resync/resource_set.py index b03e818..ccd77bd 100644 --- a/resync/resource_set.py +++ b/resync/resource_set.py @@ -35,6 +35,10 @@ class ResourceSet(dict): "Attempt to add resource already in this set") self[uri] = resource + def first(self): + """First item obtained from iterator.""" + return(next(iter(self))) + class ResourceSetDupeError(Exception): """Exception for case of attempt to add duplicate resource.""" diff --git a/tests/test_client.py b/tests/test_client.py index 473640d..ce8dc40 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,5 +1,6 @@ from tests.testcase_with_tmpdir import TestCase from tests.capture_stdout import capture_stdout +from tests.webserver_context import webserver import unittest import re @@ -8,7 +9,7 @@ from testfixtures import LogCapture import sys import os.path -from resync.client import Client, ClientFatalError +from resync.client import Client, ClientError, ClientFatalError from resync.resource import Resource from resync.resource_list import ResourceList from resync.change_list import ChangeList @@ -36,6 +37,41 @@ class TestClient(TestCase): self.assertEqual(c.sitemap_uri('/abcd2'), '/abcd2') self.assertEqual(c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3') + def test04_read_resource_list(self): + c = Client() + rl = c.read_resource_list('tests/testdata/client/dir1/resourcelist.xml') + self.assertEqual(len(rl), 3) + rl = c.read_resource_list('file:tests/testdata/client/dir1/resourcelist.xml') + self.assertEqual(len(rl), 3) + self.assertEqual(len(rl), 3) + self.assertRaises(ClientError, c.read_resource_list, 'file://tests/testdata/client/dir1/resourcelist.xml') + self.assertRaises(ClientError, c.read_resource_list, 'file:///tests/testdata/client/dir1/resourcelist.xml') + self.assertRaises(ClientError, c.read_resource_list, 'DOES_NOT_EXIST') + + def test05_find_resource_list(self): + c = Client() + # Filesystem tests + c.set_mappings(['tests/testdata/find/find1', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find1') + c.set_mappings(['tests/testdata/find/find2', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find2') + c.set_mappings(['tests/testdata/find/find3', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find3') + # Tests requiring a server + with webserver('tests/testdata/find', 'localhost', 9999): + c.set_mappings(['http://localhost:9999/find1', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find1') + c.set_mappings(['http://localhost:9999/find2', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find2') + c.set_mappings(['http://localhost:9999/find3', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find3') + with webserver('tests/testdata/find/find1', 'localhost', 9999): + c.set_mappings(['http://localhost:9999/data', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find1') + with webserver('tests/testdata/find/find3', 'localhost', 9999): + c.set_mappings(['http://localhost:9999/data/data1', 'xxx']) + self.assertEqual(c.find_resource_list().up, 'find3') + def test10_baseline_or_audit(self): # FIXME - this is the guts of the client, tough to test, need to work # through more cases... diff --git a/tests/testcase_with_tmpdir.py b/tests/testcase_with_tmpdir.py index 8ae7125..80cdb47 100644 --- a/tests/testcase_with_tmpdir.py +++ b/tests/testcase_with_tmpdir.py @@ -26,7 +26,7 @@ class TestCase(unittest.TestCase): def tearDownClass(cls): # Cleanup if (not os.path.isdir(cls._tmpdir)): - raise Exception("Ooops, no tempdir (%s) to clean up?" % (tmpdir)) + raise Exception("Ooops, no tempdir (%s) to clean up?" % (cls._tmpdir)) shutil.rmtree(cls._tmpdir) try: cls.extraTearUpClass() diff --git a/tests/testdata/client/dir1/resourcelist.xml b/tests/testdata/client/dir1/resourcelist.xml index 7dc3982..434b1fc 100644 --- a/tests/testdata/client/dir1/resourcelist.xml +++ b/tests/testdata/client/dir1/resourcelist.xml @@ -1,7 +1,7 @@ - diff --git a/tests/testdata/find/find1/.well-known/resourcelist.xml b/tests/testdata/find/find1/.well-known/resourcelist.xml new file mode 100644 index 0000000..d589aa0 --- /dev/null +++ b/tests/testdata/find/find1/.well-known/resourcelist.xml @@ -0,0 +1,8 @@ + + + + + diff --git a/tests/testdata/find/find1/.well-known/resourcesync b/tests/testdata/find/find1/.well-known/resourcesync new file mode 100644 index 0000000..eac2edb --- /dev/null +++ b/tests/testdata/find/find1/.well-known/resourcesync @@ -0,0 +1,11 @@ + + + + + + ../data/capabilitylist.xml + + + diff --git a/tests/testdata/find/find1/data/capabilitylist.xml b/tests/testdata/find/find1/data/capabilitylist.xml new file mode 100644 index 0000000..2fe2171 --- /dev/null +++ b/tests/testdata/find/find1/data/capabilitylist.xml @@ -0,0 +1,9 @@ + + + + + resourcelist.xml + + + diff --git a/tests/testdata/find/find1/data/resourcelist.xml b/tests/testdata/find/find1/data/resourcelist.xml new file mode 100644 index 0000000..f218374 --- /dev/null +++ b/tests/testdata/find/find1/data/resourcelist.xml @@ -0,0 +1,11 @@ + + + + + + a_resource + 2017-05-28T00:00:00Z + + diff --git a/tests/testdata/find/find2/resourcelist.xml b/tests/testdata/find/find2/resourcelist.xml new file mode 100644 index 0000000..6076475 --- /dev/null +++ b/tests/testdata/find/find2/resourcelist.xml @@ -0,0 +1,11 @@ + + + + + + a_resource + 2017-05-28T00:00:00Z + + diff --git a/tests/testdata/find/find3/data/data1/res.txt b/tests/testdata/find/find3/data/data1/res.txt new file mode 100644 index 0000000..3de705a --- /dev/null +++ b/tests/testdata/find/find3/data/data1/res.txt @@ -0,0 +1 @@ +Text diff --git a/tests/testdata/find/find3/sitemap.xml b/tests/testdata/find/find3/sitemap.xml new file mode 100644 index 0000000..3a0b158 --- /dev/null +++ b/tests/testdata/find/find3/sitemap.xml @@ -0,0 +1,11 @@ + + + + + + a_resource + 2017-05-28T00:00:00Z + + diff --git a/tests/webserver_context.py b/tests/webserver_context.py new file mode 100644 index 0000000..74e2bf7 --- /dev/null +++ b/tests/webserver_context.py @@ -0,0 +1,88 @@ +"""Provides content manager that runs local webserver.""" +import contextlib +import os +import posixpath +import requests +import signal +import time +from multiprocessing import Process +from http.server import HTTPServer, SimpleHTTPRequestHandler +try: # python3 + from urllib.parse import unquote +except ImportError: # python2 + from urlparse import unquote + + +class MyHTTPRequestHandler(SimpleHTTPRequestHandler): + """Copy of SimpleHTTPRequestHandler with cls._base_path setting.""" + + _base_path = '/' + + def translate_path(self, path): + """Translate a /-separated PATH to the local filename syntax. + + Components that mean special things to the local file system + (e.g. drive or directory names) are ignored. (XXX They should + probably be diagnosed.) + + **Copied from http.server.SimpleHTTPRequestHandler with modification + of path** + """ + # abandon query parameters + path = path.split('?', 1)[0] + path = path.split('#', 1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = path.rstrip().endswith('/') + try: + path = unquote(path, errors='surrogatepass') + except UnicodeDecodeError: + path = unquote(path) + path = posixpath.normpath(path) + words = path.split('/') + words = filter(None, words) + path = self._base_path + for word in words: + if os.path.dirname(word) or word in (os.curdir, os.pardir): + # Ignore components that are not a simple file/directory name + continue + path = os.path.join(path, word) + if trailing_slash: + path += '/' + return path + + +def run_webserver(host='', port=9999): + """Run webserver at given host & port.""" + server_address = (host, port) + httpd = HTTPServer(server_address, MyHTTPRequestHandler) + httpd.serve_forever() + + +@contextlib.contextmanager +def webserver(dir='/tmp', host='', port=9999): + """Context Manager that provides a webserver serving files from dir.""" + MyHTTPRequestHandler._base_path = dir + p = Process(target=run_webserver, args=(host, port)) + p.start() + + # Wait for the server to be launched + for j in range(0, 10): + try: + requests.get("http://localhost:9999/", timeout=0.1) + break + except requests.exceptions.ConnectionError: + pass + time.sleep(0.1) + + try: + yield + finally: + # Closing the server + p.terminate() + + +if __name__ == '__main__': + with webserver(): + print('Started server...') + # Things with server go in here + print('Exited server')