Remove duplicate explore code, improve tests
This commit is contained in:
Simeon Warner
parent
ae8325e2d2
commit
4b6dc626ac
11
bin/resync
11
bin/resync
@ -51,8 +51,6 @@ def main():
|
||||
help="audit sync state of destination wrt source")
|
||||
rem.add_option('--parse', '-p', action='store_true',
|
||||
help="parse a remote sitemap/sitemapindex (from mapping or explicit --sitemap) and show summary information including document type and number of entries")
|
||||
rem.add_option('--explore','-x', action='store_true',
|
||||
help="read discovery information and show capabilities. Will use either an explicit --sitemap or the first mapping URI specified to determine which server to look for .well-known discovery information or which URI to try")
|
||||
# b. modes based solely on files on local disk
|
||||
loc = p.add_option_group('LOCAL MODES',
|
||||
'These modes act on files on the local disk')
|
||||
@ -173,8 +171,7 @@ def main():
|
||||
# Implement exclusive arguments and default --baseline (support for exclusive
|
||||
# groups in argparse is incomplete is python2.6)
|
||||
if (not args.baseline and not args.incremental and not args.audit and
|
||||
not args.parse and not args.explore and
|
||||
not args.resourcelist and not args.changelist and
|
||||
not args.parse and not args.resourcelist and not args.changelist and
|
||||
not args.capabilitylist and not args.sourcedescription and
|
||||
not args.resourcedump and not args.changedump):
|
||||
if (len(map)==0):
|
||||
@ -183,10 +180,10 @@ def main():
|
||||
return
|
||||
else:
|
||||
args.baseline=True
|
||||
elif (count_true_args(args.baseline,args.incremental,args.audit,args.parse,args.explore,
|
||||
elif (count_true_args(args.baseline,args.incremental,args.audit,args.parse,
|
||||
args.resourcelist,args.changelist,args.capabilitylist,
|
||||
args.sourcedescription,args.resourcedump,args.changedump)>1):
|
||||
p.error("Only one of --baseline, --incremental, --audit, --parse, --explore, --resourcelist, --changelist, --capabilitylist, --sourcedescription, --resourcedump, --changedump modes allowed")
|
||||
p.error("Only one of --baseline, --incremental, --audit, --parse, --resourcelist, --changelist, --capabilitylist, --sourcedescription, --resourcedump, --changedump modes allowed")
|
||||
|
||||
# Configure logging module and create logger instance
|
||||
init_logging( to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE,
|
||||
@ -243,8 +240,6 @@ def main():
|
||||
from_datetime=args.from_datetime)
|
||||
elif (args.parse):
|
||||
c.parse_document()
|
||||
elif (args.explore):
|
||||
c.explore()
|
||||
elif (args.resourcelist or args.resourcedump):
|
||||
c.write_resource_list(paths=args.paths,
|
||||
outfile=args.outfile,
|
||||
|
||||
158
resync/client.py
158
resync/client.py
@ -455,160 +455,6 @@ class Client(object):
|
||||
if ( n >= to_show ):
|
||||
break
|
||||
|
||||
def explore(self):
|
||||
"""Explore capabilities of a server interactvely.
|
||||
|
||||
Will use sitemap URI taken either from explicit self.sitemap_name
|
||||
or derived from the mappings supplied.
|
||||
"""
|
||||
uri = None
|
||||
if (self.sitemap_name is not None):
|
||||
uri = self.sitemap
|
||||
print("Taking location from --sitemap option")
|
||||
acceptable_capabilities = None #ie. any
|
||||
elif (len(self.mapper)>0):
|
||||
pu = urlparse(self.mapper.default_src_uri())
|
||||
uri = urlunparse( [ pu[0], pu[1], '/.well-known/resourcesync', '', '', '' ] )
|
||||
print("Will look for discovery information based on mappings")
|
||||
acceptable_capabilities = [ 'capabilitylist', 'capabilitylistindex' ]
|
||||
else:
|
||||
raise ClientFatalError("Neither explicit sitemap nor mapping specified")
|
||||
history = []
|
||||
inp = None
|
||||
checks = None
|
||||
while (inp!='q'):
|
||||
print()
|
||||
if (inp=='b'):
|
||||
if (len(history)<2):
|
||||
break #can't do this, exit
|
||||
history.pop() #throw away current
|
||||
uri=history.pop()
|
||||
acceptable_capabilities=None
|
||||
history.append(uri)
|
||||
(uri,checks,acceptable_capabilities,inp) = self.explore_uri(uri,checks,acceptable_capabilities,len(history)>1)
|
||||
print("--explore done, bye...")
|
||||
|
||||
def explore_uri(self, uri, checks, caps, show_back=True):
|
||||
"""Interactive exploration of document at uri.
|
||||
|
||||
Will flag warnings if the document is not of type listed in caps
|
||||
"""
|
||||
s=Sitemap()
|
||||
print("Reading %s" % (uri))
|
||||
options={}
|
||||
capability=None
|
||||
try:
|
||||
if (caps=='resource'):
|
||||
self.explore_show_head(uri,check_headers=checks)
|
||||
else:
|
||||
list = s.parse_xml(url_or_file_open(uri))
|
||||
(options,capability)=self.explore_show_summary(list,s.parsed_index,caps)
|
||||
except IOError as e:
|
||||
print("Cannot read %s (%s)\nGoing back" % (uri,str(e)))
|
||||
return('','','','b')
|
||||
except Exception as e:
|
||||
print("Cannot parse %s (%s)\nGoing back" % (uri,str(e)))
|
||||
return('','','','b')
|
||||
while (True):
|
||||
# don't offer number option for no resources/capabilities
|
||||
num_prompt = '' if (len(options)==0) else 'number, '
|
||||
up_prompt = 'b(ack), ' if (show_back) else ''
|
||||
inp = raw_input( "Follow [%s%sq(uit)]?" % (num_prompt,up_prompt) )
|
||||
if (inp in options.keys()):
|
||||
break
|
||||
if (inp == 'q' or inp == 'b'):
|
||||
return('','','',inp)
|
||||
checks = {}
|
||||
if ( options[inp].capability is None ):
|
||||
if (capability == 'capabilitylistindex'):
|
||||
# all links should be to capabilitylist documents
|
||||
caps = ['capabilitylist']
|
||||
elif (capability in ['resourcelist','changelist',
|
||||
'resourcedump','changedump']):
|
||||
caps = 'resource'
|
||||
else:
|
||||
r = options[inp]
|
||||
caps = [r.capability]
|
||||
if (r.length is not None):
|
||||
checks['content-length']=r.length
|
||||
if (r.lastmod is not None):
|
||||
checks['last-modified']=r.lastmod
|
||||
# FIXME - could do sanity check here and issue warnings if odd
|
||||
return( options[inp].uri, checks, caps, inp )
|
||||
|
||||
def explore_show_summary(self,list,parsed_index,caps):
|
||||
"""Show summary of one capability document.
|
||||
|
||||
Used as part of --explore.
|
||||
FIXME - should look for <rs:ln rel="up"...> link and show that
|
||||
"""
|
||||
num_entries = len(list.resources)
|
||||
capability = '(unknown capability)'
|
||||
if ('capability' in list.md):
|
||||
capability = list.md['capability']
|
||||
if (parsed_index):
|
||||
capability += 'index'
|
||||
print("Parsed %s document with %d entries:" % (capability,num_entries))
|
||||
if (caps is not None and capability not in caps):
|
||||
print("WARNING - expected a %s document" % (','.join(caps)))
|
||||
to_show = num_entries
|
||||
if (num_entries>21):
|
||||
to_show = 20
|
||||
# What entries are allowed?
|
||||
# FIXME - not complete
|
||||
entry_caps = []
|
||||
if (capability == 'capabilitylistindex'):
|
||||
entry_caps = ['capabilitylist']
|
||||
elif (capability == 'capabilitylist'):
|
||||
entry_caps = ['resourcelist','changelist','resourcedump','changedump','changelistindex']
|
||||
elif (capability == 'changelistindex'):
|
||||
entry_caps = ['changelist']
|
||||
options = {}
|
||||
n=0
|
||||
if ('up' in list.ln):
|
||||
options['up']=list.ln['up']
|
||||
print("[%s] %s" % ('up',list.ln['up'].uri))
|
||||
for r in list.resources:
|
||||
if (n>=to_show):
|
||||
print("(not showing remaining %d entries)" % (num_entries-n))
|
||||
break
|
||||
n+=1
|
||||
options[str(n)]=r
|
||||
print("[%d] %s" % (n,r.uri))
|
||||
if (r.capability is not None):
|
||||
warning = ''
|
||||
if (r.capability not in entry_caps):
|
||||
warning = " (EXPECTED %s)" % (' or '.join(entry_caps))
|
||||
print(" %s%s" % (r.capability,warning))
|
||||
elif (len(entry_caps)==1):
|
||||
r.capability=entry_caps[0]
|
||||
print(" capability not specified, should be %s" % (r.capability))
|
||||
return(options,capability)
|
||||
|
||||
def explore_show_head(self,uri,check_headers=None):
|
||||
"""Do HEAD on uri and show infomation.
|
||||
|
||||
Will also check headers against any values specified in
|
||||
check_headers.
|
||||
"""
|
||||
print("HEAD %s" % (uri))
|
||||
response = requests.head(uri)
|
||||
print(" status: %s" % (response.status_code))
|
||||
# generate normalized lastmod
|
||||
# if ('last-modified' in response.headers):
|
||||
# response.headers.add['lastmod'] = datetime_to_str(str_to_datetime(response.headers['last-modified']))
|
||||
# print some of the headers
|
||||
for header in ['content-length','last-modified','lastmod','content-type','etag']:
|
||||
if header in response.headers:
|
||||
check_str=''
|
||||
if (check_headers is not None and
|
||||
header in check_headers):
|
||||
if (response.headers[header] == check_headers[header]):
|
||||
check_str=' MATCHES EXPECTED VALUE'
|
||||
else:
|
||||
check_STR=' EXPECTED %s' % (check_headers[header])
|
||||
print(" %s: %s%s" % (header, response.headers[header], check_str))
|
||||
|
||||
def write_resource_list(self,paths=None,outfile=None,links=None,dump=None):
|
||||
"""Write a Resource List or a Resource Dump for files on local disk.
|
||||
|
||||
@ -626,8 +472,8 @@ class Client(object):
|
||||
if (outfile is None):
|
||||
outfile = self.default_resource_dump
|
||||
self.logger.info("Writing resource dump to %s..." % (dump))
|
||||
d = Dump(format=self.dump_format)
|
||||
d.write(resource_list=rl,dumpfile=outfile)
|
||||
d = Dump(resources=rl, format=self.dump_format)
|
||||
d.write(basename=outfile)
|
||||
else:
|
||||
if (outfile is None):
|
||||
try:
|
||||
|
||||
@ -9,13 +9,12 @@ import sys
|
||||
import os.path
|
||||
|
||||
from resync.client import Client, ClientFatalError
|
||||
from resync.resource_list import ResourceList
|
||||
from resync.change_list import ChangeList
|
||||
|
||||
class TestClient(TestCase):
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
@classmethod
|
||||
def extraSetUpClass(cls):
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
class TestClient(TestCase):
|
||||
|
||||
def test01_make_resource_list_empty(self):
|
||||
c = Client()
|
||||
@ -35,25 +34,6 @@ class TestClient(TestCase):
|
||||
self.assertEqual( c.sitemap_uri('/abcd2'), '/abcd2' )
|
||||
self.assertEqual( c.sitemap_uri('scheme:/abcd3'), 'scheme:/abcd3' )
|
||||
|
||||
def test06_write_capability_list(self):
|
||||
c = Client()
|
||||
with capture_stdout() as capturer:
|
||||
c.write_capability_list( { 'a':'uri_a', 'b':'uri_b' } )
|
||||
self.assertTrue( re.search(r'<urlset ',capturer.result) )
|
||||
self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"',capturer.result) )
|
||||
|
||||
def test07_write_source_description(self):
|
||||
c = Client()
|
||||
with capture_stdout() as capturer:
|
||||
c.write_source_description( [ 'a','b','c' ] )
|
||||
#print capturer.result
|
||||
self.assertTrue( re.search(r'<urlset ',capturer.result) )
|
||||
self.assertTrue( re.search(r'<rs:md capability="description" />',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>',capturer.result) )
|
||||
|
||||
def test20_parse_document(self):
|
||||
# Key property of the parse_document() method is that it parses the
|
||||
# document and identifies its type
|
||||
@ -101,23 +81,65 @@ class TestClient(TestCase):
|
||||
# size but not the datestamp
|
||||
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_a</loc><lastmod>[\w\-:]+</lastmod><rs:md length="20" /></url>', capturer.result ) )
|
||||
#self.assertTrue( re.search(r'<url><loc>http://example.org/dir1/file_b</loc><lastmod>[\w\-:]+</lastmod><rs:md length="45" /></url>', capturer.result ) )
|
||||
# to file
|
||||
outfile = os.path.join(self.tmpdir,'rl_out.xml')
|
||||
c.write_resource_list(paths='tests/testdata/dir1', outfile=outfile)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
# dump instead (default file)
|
||||
c.default_resource_dump = os.path.join(self.tmpdir,'rl_out_dump_def')
|
||||
outfile = c.default_resource_dump+'00000.zip'
|
||||
self.assertFalse( os.path.exists(outfile) )
|
||||
c.write_resource_list(paths='tests/testdata/dir1', dump=True)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
# (specific fuile)
|
||||
outbase = os.path.join(self.tmpdir,'rl_out_dump')
|
||||
outfile = outbase+'00000.zip'
|
||||
self.assertFalse( os.path.exists(outfile) )
|
||||
c.write_resource_list(paths='tests/testdata/dir1', dump=True, outfile=outbase)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
|
||||
def test45_write_change_list(self):
|
||||
c = Client()
|
||||
ex1 = 'tests/testdata/examples_from_spec/resourcesync_ex_1.xml'
|
||||
with capture_stdout() as capturer:
|
||||
c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1)
|
||||
self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
|
||||
# compare ex1 with testdata on disk
|
||||
c.set_mappings( ['http://example.org/','tests/testdata/'] )
|
||||
with capture_stdout() as capturer:
|
||||
c.write_change_list(ref_sitemap=ex1, paths='tests/testdata/dir1')
|
||||
self.assertTrue( re.search(r'<rs:md capability="changelist"', capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>http://example.com/res1</loc><rs:md change="deleted" /></url>', capturer.result) )
|
||||
# to file
|
||||
outfile = os.path.join(self.tmpdir,'cl_out.xml')
|
||||
c.write_change_list(ref_sitemap=ex1, newref_sitemap=ex1, outfile=outfile)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
|
||||
def test46_write_capability_list(self):
|
||||
c = Client()
|
||||
caps = {'resourcelist': 'http://a.b/rl',
|
||||
'changelist': 'http://a.b/cl'}
|
||||
# to STDOUT
|
||||
caps = { 'a':'uri_a', 'b':'uri_b' }
|
||||
# simple case to STDOUT
|
||||
with capture_stdout() as capturer:
|
||||
c.write_capability_list(capabilities=caps)
|
||||
self.assertTrue( re.search(r'http://a.b/rl', capturer.result ) )
|
||||
c.write_capability_list( caps )
|
||||
self.assertTrue( re.search(r'<urlset ',capturer.result) )
|
||||
self.assertTrue( re.search(r'<rs:md capability="capabilitylist" />',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>uri_a</loc><rs:md capability="a"',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>uri_b</loc><rs:md capability="b"',capturer.result) )
|
||||
# to file (just check that something is written)
|
||||
outfile = os.path.join(self.tmpdir,'cl_out.xml')
|
||||
outfile = os.path.join(self.tmpdir,'caps_out.xml')
|
||||
c.write_capability_list(capabilities=caps, outfile=outfile)
|
||||
self.assertTrue( os.path.getsize(outfile)>100 )
|
||||
|
||||
def test47_write_source_description(self):
|
||||
c = Client()
|
||||
# to STDOUT
|
||||
# simple case to STDOUT
|
||||
with capture_stdout() as capturer:
|
||||
c.write_source_description( [ 'a','b','c' ] )
|
||||
self.assertTrue( re.search(r'<urlset ',capturer.result) )
|
||||
self.assertTrue( re.search(r'<rs:md capability="description" />',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>a</loc><rs:md capability="capabilitylist" /></url>',capturer.result) )
|
||||
self.assertTrue( re.search(r'<url><loc>b</loc><rs:md capability="capabilitylist" /></url>',capturer.result) )
|
||||
# more complex case to STDOUT
|
||||
with capture_stdout() as capturer:
|
||||
c.write_source_description(capability_lists=['http://a.b/'], links=[{'rel':'c','href':'d'}])
|
||||
self.assertTrue( re.search(r'http://a.b/', capturer.result ) )
|
||||
|
||||
Loading…
Reference in New Issue
Block a user
