Add --delay option

CHANGES.md

@@ -9,6 +9,8 @@ ResourceSync specification which was standardized as ANSI/NISO Z39.99-2014
 v2.0.0 ???
   * Drop support Python 2.7
   * Rename command line clients: bin/resync -> resync.pl and bin/resync-explorer -> resync-explorer.py
+  * Add --access_token option to pass bearer token with web requests
+  * Add --delay option to pause between successive web requests
   * Drop Python 2.7, 3.3 & 3.4 from tests, add 3.7 & 3.8. Fix various new warnings and errors for 3.7 & 3.8
   * Switch from pep8 to pycodestyle in tests
   * Move libraries to support tests into test/testlib

resync/client_utils.py

@@ -3,16 +3,14 @@
 Code shared by both the resync and resync-explorer clients.
 """
 
-try:  # python3
-    from urllib.request import urlopen
-except ImportError:  # pragma: no cover  python2
-    from urllib import urlopen  # pragma: no cover
+import argparse
+from datetime import datetime
 import logging
 import logging.config
-from datetime import datetime
 import re
+from urllib.request import urlopen
 
-from .url_or_file_open import set_bearer_token
+from .url_or_file_open import set_url_or_file_open_config
 
 
 class ClientFatalError(Exception):
@@ -192,6 +190,8 @@ def add_shared_misc_options(opt, default_logfile):
                         "only to resources on the same server/sub-path etc. Use with care.")
     opt.add_option('--access-token', type=str, default=None,
                    help="include this access token (a bearer token) in web requests")
+    opt.add_option('--delay', type=float, default=None,
+                   help="add a delay between web requests (default is None)")
     # Want these to show at the end
     opt.add_option('--verbose', '-v', action='store_true',
                    help="verbose, show additional informational messages")
@@ -211,4 +211,8 @@ def process_shared_misc_options(args):
     if args.checksum:
         args.hash.append('md5')
     if args.access_token:
-        set_bearer_token(args.access_token)
+        set_url_or_file_open_config('bearer_token', args.access_token)
+    if args.delay:
+        if args.delay < 0.0:
+            raise argparse.ArgumentTypeError("--delay must be non-negative!")
+        set_url_or_file_open_config('delay', args.delay)

resync/url_or_file_open.py

@@ -1,24 +1,24 @@
 """Local version of urlopen that supports local files & web URLs, plus adds auth."""
 
 import re
+import time
 from urllib.request import Request, urlopen
 
 from . import __version__
 
 
-BEARER_TOKEN = None
+# Global configuration settings
+FIRST_REQUEST = False
+CONFIG = {
+    'bearer_token': None,
+    'delay': None
+}
 
 
-def set_bearer_token(token):
-    """Set the global Authorization header Bearer token.
-
-    FIXME - This token will be added blindy to all requests. This is insecure
-    if the --noauth setting is used allowing requests across different domains.
-    It would be better to have some scheme where a token is tide to a particular
-    domain, or domain pattern.
-    """
-    global BEARER_TOKEN
-    BEARER_TOKEN = token
+def set_url_or_file_open_config(key, value):
+    """Set the global config."""
+    global CONFIG
+    CONFIG[key] = value
 
 
 def url_or_file_open(uri):
@@ -26,7 +26,16 @@ def url_or_file_open(uri):
     if (not re.match(r'''\w+:''', uri)):
         uri = 'file:' + uri
     # Do we need to send an Authorization header?
+    # FIXME - This token will be added blindy to all requests. This is insecure
+    # if the --noauth setting is used allowing requests across different domains.
+    # It would be better to have some scheme where a token is tied to a particular
+    # domain, or domain pattern.
     headers = {'User-Agent': 'resync/' + __version__}
-    if BEARER_TOKEN is not None:
-        headers['Authorization'] = 'Bearer ' + BEARER_TOKEN
+    if CONFIG['bearer_token'] is not None:
+        headers['Authorization'] = 'Bearer ' + CONFIG['bearer_token']
+    # Have we got a delay set? Apply only to web requests after first
+    global FIRST_REQUEST
+    if not FIRST_REQUEST and CONFIG['delay'] is not None and not uri.startswith('file:'):
+        FIRST_REQUEST = False
+        time.sleep(CONFIG['delay'])
     return(urlopen(Request(url=uri, headers=headers)))