gian/resync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tidy examples to match split scripts

Browse Source
This commit is contained in:
Simeon Warner 2020-12-16 11:19:48 -05:00
parent ce9a15be74
commit 9f26daaf74
7 changed files with 52 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
README.md
View File

@ -3,7 +3,8 @@
[![Build Status](https://travis-ci.org/resync/resync.svg?branch=master)](https://travis-ci.org/resync/resync)
[![Test Coverage](https://coveralls.io/repos/github/resync/resync/badge.svg?branch=master)](https://coveralls.io/github/resync/resync)
**resync** is a ResourceSync client and library in python.
**resync** is a ResourceSync library with supporting client scipts,
written in python.
[ResourceSync](http://www.openarchives.org/rs/) is a synchronization
framework for the web consisting of various capabilities that allow
third party systems to remain synchronized with a server's evolving
@ -16,20 +17,20 @@ Typical client usage to synchronize from a source at
`http://source.example.com/` to a set of local files would be
```
resync.py http://source.example.com/
resync-sync http://source.example.com/
```
which will create or update a local directory `./source.example.com`.
Alternatively, the destination directory may be specified explicitly::
```
resync.py http://source.example.com/ /tmp/my_copy
resync-sync http://source.example.com/ /tmp/my_copy
```
Option details and a number of different modes are described with::
```
resync.py -h
resync-sync -h
```
## Python library usage
@ -37,7 +38,7 @@ resync.py -h
Typical library use in a source (create and output a Resource List)::
```
from resync import Resource,ResourceList
from resync import Resource, ResourceList
rl = ResourceList()
rl.add( Resource('http://example.com/res1', lastmod='2013-01-01') )
@ -83,9 +84,9 @@ python setup.py install
```
This will install the library code in the appropriate place within
a user-space python setup, including the clients `resync.py` and
`resync-explorer.py`. Use of `sudo` to install in system spaces is
generally discouraged.
a user-space python setup, including the clients `resync-sync`,
`resync-build` and `resync-explorer`. Use of `sudo` to install in
system spaces is generally discouraged.
The source code is maintained on [Github](https://github.com/resync/resync)
and there may be branches/versions available there that are not

15
examples/README_client_library_in_destination.py
View File

@ -1,11 +1,8 @@
#!/usr/bin/env python
if (True): #keep indentation of README
from resync import CapabilityList
# Read Capability List and show supported capabilities
cl = CapabilityList()
cl.read("https://raw.github.com/resync/resync/0.6/resync/test/testdata/examples_from_spec/resourcesync_ex_2_6.xml")
for resource in cl:
print "supports %s (at %s)" % (resource.capability,resource.uri)
from resync import CapabilityList
# Read Capability List and show supported capabilities
cl = CapabilityList()
cl.read("https://raw.github.com/resync/resync/0.6/resync/test/testdata/examples_from_spec/resourcesync_ex_2_6.xml")
for resource in cl:
print "supports %s (at %s)" % (resource.capability,resource.uri)

12
examples/README_client_library_in_source.py
View File

@ -1,9 +1,7 @@
#!/usr/bin/env python
if (True): #keep indentation of README
from resync import Resource, ResourceList
from resync import Resource,ResourceList
rl = ResourceList()
rl.add( Resource('http://example.com/res1', lastmod='2013-01-01') )
rl.add( Resource('http://example.com/res2', lastmod='2013-01-02') )
print rl.as_xml()
rl = ResourceList()
rl.add( Resource('http://example.com/res1', lastmod='2013-01-01') )
rl.add( Resource('http://example.com/res2', lastmod='2013-01-02') )
print rl.as_xml()

32
resync-build
View File

@ -37,37 +37,35 @@ def main():
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser._optionals = parser.add_argument_group(
'MODES OF OPERATION (must specify one only). The REMOTE modes use a '
'remote source that is specified in a set of uri=path mappings and '
'potentially also using an explicit --sitemap location. The LOCAL '
'modes operate only to create ResourceSync descriptions on the local '
'filesystem based on local content')
'MODES OF OPERATION (must specify one only). This script operates only '
'to create ResourceSync descriptions on the local filesystem based on '
'local content')
rem = parser.add_mutually_exclusive_group(required=True)
rem.add_argument('--write-resourcelist', '--write-resource-list', action='store_true',
help="LOCAL: write a resource list based on files on disk using uri=path mappings "
help="write a resource list based on files on disk using uri=path mappings "
"in reverse to calculate URIs from the local paths. Scans local disk "
"based either on explicit --paths setting, else starting from all local "
"paths specified in the mappings. Writes to STDOUT by default, override "
"with --outfile")
rem.add_argument('--write-changelist', '--write-change-list', action='store_true',
help="LOCAL: write a change list based on comparison of a reference sitemap "
help="write a change list based on comparison of a reference sitemap "
"(specify file with --reference) and either files on disk (using "
"the mapping provided) or a second sitemap (specify file with "
"--newreference). Otherwise follows --write-resourcelist options. Also accepts "
"the --empty option (with no mapping) to write and empty changelist.")
rem.add_argument('--write-capabilitylist', '--write-capability-list', type=str, action='store',
help="LOCAL: write a capability list based on the set of capabilities and "
help="write a capability list based on the set of capabilities and "
"URIs supplied in cap_name=URI,cap_name=URI format. Otherwise "
"follows --write-resourcelist options.")
rem.add_argument('--write-sourcedescription', '--write-source-description', type=str, action='store',
help="LOCAL: write a Source Description document based on the set of capability "
help="write a Source Description document based on the set of capability "
"list URIs supplied as a comma separated list. Otherwise "
"follows --write-resourcelist options.")
rem.add_argument('--write-resourcedump', '--write-resource-dump', '-d', action='store_true',
help="LOCAL: write a Resource Dump. Specify output file with --outfile and use other "
help="write a Resource Dump. Specify output file with --outfile and use other "
"options as for --write-resourcelist")
rem.add_argument('--write-changedump', '--write-change-dump', action='store_true',
help="LOCAL: write a Resource Dump. Specify output file with --outfile and use other "
help="write a Resource Dump. Specify output file with --outfile and use other "
"options as for --write-changelist")
# Positional arguments
@ -82,12 +80,6 @@ def main():
nam.add_argument('--paths', type=str, action='store',
help="explicit set of paths for disk scan --resourceslist or --changelist "
"generation")
nam.add_argument('--sitemap', type=str, action='store',
help="explicitly set sitemap name, overriding default sitemap.xml "
"appended to first source URI specified in the mappings")
nam.add_argument('--capabilitylist', '--capability-list', type=str, action='store',
help="explicitly set capability list URI to search for instead of "
"looking for the source description")
nam.add_argument('--reference', type=str, action='store',
help="reference sitemap name for --write-changelist calculation")
nam.add_argument('--newreference', type=str, action='store',
@ -143,18 +135,12 @@ def main():
if (args.map):
# Mappings apply to (almost) everything
c.set_mappings(args.map)
if (args.sitemap):
c.sitemap_name = args.sitemap
if (args.capabilitylist):
c.capability_list_uri = args.capabilitylist
if (args.warc):
c.dump_format = 'warc'
if (args.exclude):
c.exclude_patterns = args.exclude
if (args.multifile):
c.allow_multifile = not args.multifile
if (args.noauth):
c.noauth = args.noauth
if (args.max_sitemap_entries):
c.max_sitemap_entries = args.max_sitemap_entries

2
resync-explorer
View File

@ -62,7 +62,7 @@ def main():
opt = parser.add_argument_group('MISCELANEOUS OPTIONS')
opt.add_argument('--max-sitemap-entries', type=int, action='store',
help="override default size limits")
add_shared_misc_options(opt, default_logfile=DEFAULT_LOGFILE)
add_shared_misc_options(opt, default_logfile=DEFAULT_LOGFILE, include_remote=True)
args = parser.parse_args()

4
resync-sync
View File

@ -75,7 +75,7 @@ def main():
# Options that apply to multiple modes
opt = parser.add_argument_group('MISCELANEOUS OPTIONS')
add_shared_misc_options(opt, default_logfile=DEFAULT_LOGFILE)
add_shared_misc_options(opt, default_logfile=DEFAULT_LOGFILE, include_remote=True)
opt.add_argument('--delete', action='store_true',
help="allow files on destination to be deleted")
opt.add_argument('--empty', action='store_true',
@ -109,7 +109,7 @@ def main():
init_logging(to_file=args.logger, logfile=args.logfile, default_logfile=DEFAULT_LOGFILE,
verbose=args.verbose, eval_mode=args.eval)
process_shared_misc_options(args)
process_shared_misc_options(args, include_remote=True)
c = Client(hashes=args.hash,
verbose=args.verbose,

38
resync/client_utils.py
View File

@ -1,6 +1,6 @@
"""ResourceSync Client Utilities.
Code shared by both the resync and resync-explorer clients.
Code shared by client scripts.
"""
import argparse
@ -165,10 +165,10 @@ def parse_capability_lists(cls_str):
return(cls_str.split(','))
def add_shared_misc_options(opt, default_logfile):
def add_shared_misc_options(opt, default_logfile, include_remote=False):
"""Add shared miscellaneous options to the argument_group opt.
Options that both resync and resync-explorer use.
Options that the resync-sync, resync-build and resync-explorer scripts use.
"""
opt.add_argument('--hash', type=str, action='append',
help="use specified hash types in addition to last modification time "
@ -185,13 +185,14 @@ def add_shared_misc_options(opt, default_logfile):
"information, repeat option for multiple excludes)")
opt.add_argument('--multifile', '-m', action='store_true',
help="disable reading and output of sitemapindex for multifile sitemap")
opt.add_argument('--noauth', action='store_true',
help="disable checking of URL paths to ensure that the sitemaps refer "
"only to resources on the same server/sub-path etc. Use with care.")
opt.add_argument('--access-token', type=str, default=None,
help="include this access token (a bearer token) in web requests")
opt.add_argument('--delay', type=float, default=None,
help="add a delay between web requests (default is None)")
if include_remote:
opt.add_argument('--noauth', action='store_true',
help="disable checking of URL paths to ensure that the sitemaps refer "
"only to resources on the same server/sub-path etc. Use with care.")
opt.add_argument('--access-token', type=str, default=None,
help="include this access token (a bearer token) in web requests")
opt.add_argument('--delay', type=float, default=None,
help="add a delay between web requests (default is None)")
# Want these to show at the end
opt.add_argument('--logger', '-l', action='store_true',
help="create detailed log of client actions (will write "
@ -202,16 +203,17 @@ def add_shared_misc_options(opt, default_logfile):
help="verbose, show additional informational messages")
def process_shared_misc_options(args):
def process_shared_misc_options(args, include_remote=False):
"""Process shared miscellaneous options in args.
Options that both resync.py and resync-explorer.py use.
Parse options that the resync-sync, resync-build and resync-explorer scripts use.
"""
if args.checksum:
args.hash.append('md5')
if args.access_token:
set_url_or_file_open_config('bearer_token', args.access_token)
if args.delay:
if args.delay < 0.0:
raise argparse.ArgumentTypeError("--delay must be non-negative!")
set_url_or_file_open_config('delay', args.delay)
if include_remote:
if args.access_token:
set_url_or_file_open_config('bearer_token', args.access_token)
if args.delay:
if args.delay < 0.0:
raise argparse.ArgumentTypeError("--delay must be non-negative!")
set_url_or_file_open_config('delay', args.delay)