Update date and remove outdated travis links

Show when Content-Length unknown

CHANGES.md

@@ -1,5 +1,9 @@
 # resync change log
 
+v2.0.2 2022-09-30
+  * Add --user-agent option to set web User-Agent string
+  * Avoid misleading "(0 bytes)" output when no Content-Length given (https://github.com/resync/resync/issues/52)
+
 v2.0.1 2021-03-23
   * Route all URI and file requests through `resync/url_or_file_open.py` so that settings such as authentication headers can be consistently applied
   * Do not exclude any directories from sync by default, specify with --exclude

README.md

@@ -1,8 +1,5 @@
 # resync
 
-[![Build Status](https://travis-ci.org/resync/resync.svg?branch=main)](https://travis-ci.org/resync/resync)
-[![Test Coverage](https://coveralls.io/repos/github/resync/resync/badge.svg?branch=main)](https://coveralls.io/github/resync/resync)
-
 **resync** is a ResourceSync library with supporting client scipts,
 written in python.
 [ResourceSync](http://www.openarchives.org/rs/) is a synchronization

resync/__init__.py

@@ -2,7 +2,7 @@
 
 This is the one place the version number for resync is stored.
 """
-__version__ = '2.0.1'
+__version__ = '2.0.2'
 
 # Enable easy import for core classes, e.g.
 # from resync import Resource

resync/client_utils.py

@@ -191,6 +191,8 @@ def add_shared_misc_options(opt, default_logfile, include_remote=False):
                          help="include this access token (a bearer token) in web requests")
         opt.add_argument('--delay', type=float, default=None,
                          help="add a delay between web requests (default is None)")
+        opt.add_argument('--user-agent', type=str, default=None,
+                         help="set User-Agent string sent with web requests (default is resync/version)")
     # Want these to show at the end
     opt.add_argument('--logger', '-l', action='store_true',
                      help="create detailed log of client actions (will write "
@@ -211,7 +213,7 @@ def process_shared_misc_options(args, include_remote=False):
 
     Parse options that the resync-sync, resync-build and resync-explorer scripts use.
     """
-    if args.checksum:
+    if args.checksum and 'md5' not in args.hash:
         args.hash.append('md5')
     if include_remote:
         if args.access_token:
@@ -220,3 +222,5 @@ def process_shared_misc_options(args, include_remote=False):
             if args.delay < 0.0:
                 raise argparse.ArgumentTypeError("--delay must be non-negative!")
             set_url_or_file_open_config('delay', args.delay)
+        if args.user_agent:
+            set_url_or_file_open_config('user_agent', args.user_agent)

resync/list_base_with_index.py

@@ -167,15 +167,16 @@ class ListBaseWithIndex(ListBase):
                 "Failed to load sitemap from %s listed in sitemap index %s (%s)" %
                 (sitemap_uri, sitemapindex_uri, str(e)))
         # Get the Content-Length if we can (works fine for local files)
+        length_str = "size not given"
         try:
             self.content_length = int(fh.info()['Content-Length'])
+            length_str = "%d bytes" % self.content_length
             self.bytes_read += self.content_length
         except (KeyError, TypeError):
             # If we don't get a length then c'est la vie
             pass
         self.logger.info(
-            "Reading sitemap from %s (%d bytes)" %
+            "Reading sitemap from %s (%s)" % (sitemap_uri, length_str))
-            (sitemap_uri, self.content_length))
         component = sitemap.parse_xml(fh=fh, sitemapindex=False)
         # Copy resources into self, check any metadata
         for r in component:

resync/url_authority.py

@@ -14,19 +14,19 @@ class UrlAuthority(object):
     Two modes are supported:
 
     strict=True: requires that a query URL has the same URI
-    scheme (e.g. http) as the master, is on the same server
+    scheme (e.g. http) as the primary, is on the same server
     or one in a sub-domain, and that the path component is
-    at the same level or below the master.
+    at the same level or below the primary.
 
     strict=False (default): requires only that a query URL
-    has the same URI scheme as the master, and is on the same
+    has the same URI scheme as the primary, and is on the same
-    server or one in a sub-domain of the master.
+    server or one in a sub-domain of the primary.
 
     Example use:
 
         from resync.url_authority import UrlAuthority
 
-        auth = UrlAuthority("http://example.org/master")
+        auth = UrlAuthority("http://example.org/primary")
         if (auth.has_authority_over("http://example.com/res1")):
             # will be true
         if (auth.has_authority_over("http://other.com/res1")):
@@ -34,40 +34,40 @@ class UrlAuthority(object):
     """
 
     def __init__(self, url=None, strict=False):
-        """Create object and optionally set master url and/or strict mode."""
+        """Create object and optionally set primary url and/or strict mode."""
         self.url = url
         self.strict = strict
         if (self.url is not None):
-            self.set_master(self.url)
+            self.set_primary(self.url)
         else:
-            self.master_scheme = 'none'
+            self.primary_scheme = 'none'
-            self.master_netloc = 'none.none.none'
+            self.primary_netloc = 'none.none.none'
-            self.master_path = '/not/very/likely'
+            self.primary_path = '/not/very/likely'
 
-    def set_master(self, url):
+    def set_primary(self, url):
-        """Set the master url that this object works with."""
+        """Set the primary url that this object works with."""
         m = urlparse(url)
-        self.master_scheme = m.scheme
+        self.primary_scheme = m.scheme
-        self.master_netloc = m.netloc
+        self.primary_netloc = m.netloc
-        self.master_path = os.path.dirname(m.path)
+        self.primary_path = os.path.dirname(m.path)
 
     def has_authority_over(self, url):
-        """Return True of the current master has authority over url.
+        """Return True of the current primary has authority over url.
 
         In strict mode checks scheme, server and path. Otherwise checks
         just that the server names match or the query url is a
-        sub-domain of the master.
+        sub-domain of the primary.
         """
         s = urlparse(url)
-        if (s.scheme != self.master_scheme):
+        if (s.scheme != self.primary_scheme):
             return(False)
-        if (s.netloc != self.master_netloc):
+        if (s.netloc != self.primary_netloc):
-            if (not s.netloc.endswith('.' + self.master_netloc)):
+            if (not s.netloc.endswith('.' + self.primary_netloc)):
                 return(False)
             # Maybe should allow parallel for 3+ components, eg. a.example.org,
             # b.example.org
         path = os.path.dirname(s.path)
-        if (self.strict and path != self.master_path
+        if (self.strict and path != self.primary_path
-                and not path.startswith(self.master_path)):
+                and not path.startswith(self.primary_path)):
             return(False)
         return(True)

resync/url_or_file_open.py

@@ -11,7 +11,8 @@ from . import __version__
 NUM_REQUESTS = 0
 CONFIG = {
     'bearer_token': None,
-    'delay': None
+    'delay': None,
+    'user_agent': 'resync/' + __version__
 }
 
 
@@ -32,7 +33,7 @@ def url_or_file_open(uri, method=None, timeout=None):
     """
     if (not re.match(r'''\w+:''', uri)):
         uri = 'file:' + uri
-    headers = {'User-Agent': 'resync/' + __version__}
+    headers = {'User-Agent': CONFIG['user_agent']}
     # Do we need to send an Authorization header?
     # FIXME - This token will be added blindy to all requests. This is insecure
     # if the --noauth setting is used allowing requests across different domains.

tests/test_client_utils.py

@@ -1,11 +1,13 @@
 from .testlib import TestCase
 
+import argparse
 import logging
 import os.path
 import re
 import unittest
-from resync.client_utils import init_logging, count_true_args, parse_links, parse_link, parse_capabilities, parse_capability_lists
+from resync.client_utils import init_logging, count_true_args, parse_links, parse_link, parse_capabilities, parse_capability_lists, add_shared_misc_options, process_shared_misc_options
 from resync.client import ClientFatalError
+from resync.url_or_file_open import CONFIG
 
 
 class TestClientUtils(TestCase):
@@ -75,3 +77,63 @@ class TestClientUtils(TestCase):
     def test06_parse_capability_lists(self):
         # Input string of the form: uri,uri
         self.assertEqual(parse_capability_lists('a,b'), ['a', 'b'])
+
+    def test07_add_shared_misc_options(self):
+        """Test add_shared_misc_options method."""
+        parser = argparse.ArgumentParser()
+        add_shared_misc_options(parser, default_logfile='/tmp/abc.log')
+        args = parser.parse_args(['--hash', 'md5', '--hash', 'sha-1',
+                                  '--checksum',
+                                  '--from', '2020-01-01T01:01:01Z',
+                                  '--exclude', 'ex1', '--exclude', 'ex2',
+                                  '--multifile',
+                                  '--logger', '--logfile', 'log.out',
+                                  '--spec-version', '1.0',
+                                  '-v'])
+        self.assertEqual(args.hash, ['md5', 'sha-1'])
+        self.assertTrue(args.checksum)
+        self.assertEqual(args.from_datetime, '2020-01-01T01:01:01Z')
+        self.assertEqual(args.exclude, ['ex1', 'ex2'])
+        self.assertTrue(args.multifile)
+        self.assertTrue(args.logger)
+        self.assertEqual(args.logfile, 'log.out')
+        self.assertEqual(args.spec_version, '1.0')
+        self.assertTrue(args.verbose)
+        # Remote options
+        parser = argparse.ArgumentParser()
+        add_shared_misc_options(parser, default_logfile='/tmp/abc.log', include_remote=True)
+        args = parser.parse_args(['--noauth',
+                                  '--access-token', 'VerySecretToken',
+                                  '--delay', '1.23',
+                                  '--user-agent', 'rc/2.1.1'])
+        self.assertTrue(args.noauth)
+        self.assertEqual(args.access_token, 'VerySecretToken')
+        self.assertEqual(args.delay, 1.23)
+        self.assertEqual(args.user_agent, 'rc/2.1.1')
+        # Remote options note selected
+        parser = argparse.ArgumentParser()
+        add_shared_misc_options(parser, default_logfile='/tmp/abc.log', include_remote=False)
+        self.assertRaises(SystemExit, parser.parse_args, ['--access-token', 'VerySecretToken'])
+
+    def test08_process_shared_misc_options(self):
+        """Test process_shared_misc_options method."""
+        global CONFIG
+        config_copy = CONFIG.copy()
+        args = argparse.Namespace(hash=['sha-1'], checksum='md5')
+        process_shared_misc_options(args)
+        self.assertEqual(args.hash, ['sha-1', 'md5'])
+        # Remote options
+        args = argparse.Namespace(access_token='ExtraSecretToken',
+                                  delay=2.5,
+                                  user_agent='me',
+                                  checksum=None)
+        process_shared_misc_options(args, include_remote=True)
+        self.assertEqual(CONFIG['bearer_token'], 'ExtraSecretToken')
+        self.assertEqual(CONFIG['delay'], 2.5)
+        self.assertEqual(CONFIG['user_agent'], 'me')
+        # Negative delay is bad...
+        args = argparse.Namespace(access_token=None, delay=-1.0, user_agent=None, checksum=None)
+        self.assertRaises(argparse.ArgumentTypeError, process_shared_misc_options, args, include_remote=True)
+        # Config is a global so reset back to old version
+        for (k, v) in config_copy.items():
+            CONFIG[k] = v

tests/test_url_authority.py

@@ -91,7 +91,7 @@ class TestUrlAuthority(unittest.TestCase):
 
     def test07_no_init_data(self):
         uauth = UrlAuthority()
-        self.assertEqual(uauth.master_scheme, 'none')
+        self.assertEqual(uauth.primary_scheme, 'none')
         self.assertFalse(uauth.has_authority_over(
             'http://a.example.org/sitemap.xml'))
         self.assertFalse(uauth.has_authority_over(