qa/rgw: Adding GSSAPI testing task for Bucket Notification Testing

This adds a Kerberos setup task, tells the Kafka task to emit GSSAPI listeners and broker principals, and updates notification tests to pass Kerberos credentials through to the bucket-notification suite. It enables the RGW Kafka notification coverage to run against GSSAPI-secured brokers.

Signed-off-by: sujay-d07 <sujaydongre07@gmail.com>

qa/suites/rgw/notifications/overrides.yaml

@@ -11,6 +11,7 @@ overrides:
         rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
         rgw crypt require ssl: false
         rgw allow notification secrets in cleartext: true
+        rgw kafka sasl kerberos service name: kafka
   rgw:
     realm: MyRealm
     zonegroup: MyZoneGroup

qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml

@@ -1,4 +1,6 @@
 tasks:
+- kerberos:
+    client.0:
 - kafka:
     client.0:
       kafka_version: 3.9.2

qa/tasks/kafka.py

@@ -42,6 +42,8 @@ def install_kafka(ctx, config):
     assert isinstance(config, dict)
     log.info('Installing Kafka...')
 
+    kerberos = getattr(ctx, 'kerberos', None)
+
     # programmatically find a nearby mirror so as not to hammer archive.apache.org
     apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \
         "grep -o '<strong>[^<]*</strong>' | sed 's/<[^>]*>//g' | head -n 1"
@@ -101,7 +103,8 @@ def install_kafka(ctx, config):
             args=['sudo', 'chmod', 'o+rx', '/home/ubuntu'],
         )
 
-        broker_conf(ctx, client, kafka_dir)
+        client_kerberos = kerberos.get(client) if kerberos else None
+        broker_conf(ctx, client, kafka_dir, client_kerberos)
 
     try:
         yield
@@ -123,7 +126,7 @@ def install_kafka(ctx, config):
             )
 
 
-def broker_conf(ctx, client, kafka_dir):
+def broker_conf(ctx, client, kafka_dir, kerberos):
     """writing a custom server.properties config file"""
     (remote,) = ctx.cluster.only(client).remotes.keys()
     ip = remote.ip_address
@@ -170,8 +173,9 @@ def broker_conf(ctx, client, kafka_dir):
         "listener.name.mtls.ssl.truststore.location={tdir}/server.truststore.jks\n"
         "listener.name.mtls.ssl.truststore.password=mypassword\n"
         # SASL mechanisms
-        "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n"
+        "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI\n"
         "sasl.mechanism.inter.broker.protocol=PLAIN\n"
+        "sasl.kerberos.service.name={service_name}\n"
         'listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
         '  username="admin" \\\n'
         '  password="admin-secret" \\\n'
@@ -182,6 +186,11 @@ def broker_conf(ctx, client, kafka_dir):
         'listener.name.sasl_ssl.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
         '  username="admin" \\\n'
         '  password="admin-secret";\n'
+        'listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+        '  useKeyTab=true \\\n'
+        '  storeKey=true \\\n'
+        '  keyTab="{keytab}" \\\n'
+        '  principal="{principal}";\n'
         'listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
         '  username="admin" \\\n'
         '  password="admin-secret" \\\n'
@@ -192,6 +201,11 @@ def broker_conf(ctx, client, kafka_dir):
         'listener.name.sasl_plaintext.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
         '  username="admin" \\\n'
         '  password="admin-secret";\n'
+        'listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+        '  useKeyTab=true \\\n'
+        '  storeKey=true \\\n'
+        '  keyTab="{keytab}" \\\n'
+        '  principal="{principal}";\n'
     ).format(
         tdir=kafka_dir,
         ip=ip,
@@ -200,6 +214,9 @@ def broker_conf(ctx, client, kafka_dir):
         sasl_ssl=KAFKA_PORTS['SASL_SSL'],
         sasl_plaintext=KAFKA_PORTS['SASL_PLAINTEXT'],
         mtls=KAFKA_PORTS['MTLS'],
+        service_name=kerberos['service_name'] if kerberos else '',
+        keytab=kerberos['kafka_keytab'] if kerberos else '',
+        principal=kerberos['kafka_principal'] if kerberos else '',
     )
     file_name = 'server.properties'
     log.info("kafka conf file: %s", file_name)

qa/tasks/kerberos.py

@@ -0,0 +1,258 @@
+"""
+Deploy and configure a Kerberos KDC for Teuthology.
+
+This task stands up a single-node MIT Kerberos KDC and creates the service
+principals / keytabs used by the GSSAPI Kafka bucket-notification tests:
+
+  - kafka/<remote-ip>@CEPH.TEST  (used by the Kafka broker)
+  - rgw/<remote-ip>@CEPH.TEST    (used by RGW and the test consumer)
+
+"""
+import contextlib
+import logging
+
+from teuthology import contextutil
+from teuthology import misc as teuthology
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+REALM = 'CEPH.TEST'
+MASTER_PASSWORD = 'ceph-kdc-master'
+KEYTAB_DIR = '/etc/krb5-keytabs'
+KAFKA_KEYTAB = '{}/kafka.service.keytab'.format(KEYTAB_DIR)
+RGW_KEYTAB = '{}/rgw.service.keytab'.format(KEYTAB_DIR)
+SERVICE_NAME = 'kafka'
+
+
+def _distro_paths(remote):
+    """Return distro-specific KDC paths and service names."""
+    if remote.os.package_type == 'rpm':
+        return {
+            'kdc_conf_dir': '/var/kerberos/krb5kdc',
+            'kdc_service': 'krb5kdc',
+            'admin_service': 'kadmin',
+        }
+    return {
+        'kdc_conf_dir': '/etc/krb5kdc',
+        'kdc_service': 'krb5-kdc',
+        'admin_service': 'krb5-admin-server',
+    }
+
+
+@contextlib.contextmanager
+def install_kerberos(ctx, config):
+    """Install the Kerberos KDC, admin server and GSSAPI SASL plugin."""
+    assert isinstance(config, dict)
+    log.info('Installing Kerberos packages...')
+
+    deb_packages = [
+        'krb5-kdc', 'krb5-admin-server', 'krb5-user',
+        'libkrb5-dev', 'libsasl2-modules-gssapi-mit',
+    ]
+    rpm_packages = [
+        'krb5-devel', 'krb5-server', 'krb5-workstation',
+        'cyrus-sasl-gssapi',
+    ]
+
+    for (client, _) in config.items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        if remote.os.package_type == 'rpm':
+            remote.run(args=['sudo', 'dnf', 'install', '-y'] + rpm_packages)
+        else:
+            remote.run(args=['sudo', 'apt-get', 'update'])
+            remote.run(args=[
+                'sudo', 'DEBIAN_FRONTEND=noninteractive',
+                'apt-get', 'install', '-y',
+            ] + deb_packages)
+
+    try:
+        yield
+    finally:
+        log.info('Removing Kerberos packages...')
+        for (client, _) in config.items():
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            if remote.os.package_type == 'rpm':
+                remote.run(
+                    args=['sudo', 'dnf', 'remove', '-y'] + rpm_packages,
+                    check_status=False,
+                )
+            else:
+                remote.run(
+                    args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
+                          'apt-get', 'purge', '-y'] + deb_packages,
+                    check_status=False,
+                )
+
+
+@contextlib.contextmanager
+def setup_kdc(ctx, config):
+    """Create the realm, principals and keytabs and start the KDC."""
+    assert isinstance(config, dict)
+    log.info('Setting up Kerberos KDC...')
+
+    if not hasattr(ctx, 'kerberos'):
+        ctx.kerberos = {}
+
+    for (client, _) in config.items():
+        (remote,) = ctx.cluster.only(client).remotes.keys()
+        try:
+            ip = remote.ip_address
+            paths = _distro_paths(remote)
+            kafka_principal = 'kafka/{ip}@{realm}'.format(ip=ip, realm=REALM)
+            rgw_principal = 'rgw/{ip}@{realm}'.format(ip=ip, realm=REALM)
+
+            # /etc/krb5.conf: rdns=false keeps the SPN IP-based (no reverse
+            # lookup to a hostname); dns_lookup_kdc/realm=false keeps the KDC 
+            # and realm settings local and static, avoiding any DNS-related 
+            # issues in the test environment.
+            krb5_conf = (
+                "[libdefaults]\n"
+                "    default_realm = {realm}\n"
+                "    dns_lookup_kdc = false\n"
+                "    dns_lookup_realm = false\n"
+                "    rdns = false\n"
+                "    udp_preference_limit = 1\n"
+                "\n"
+                "[realms]\n"
+                "    {realm} = {{\n"
+                "        kdc = {ip}\n"
+                "        admin_server = {ip}\n"
+                "    }}\n"
+            ).format(realm=REALM, ip=ip)
+            remote.sudo_write_file('/etc/krb5.conf', krb5_conf)
+
+            # kdc.conf
+            kdc_conf = (
+                "[kdcdefaults]\n"
+                "    kdc_ports = 88\n"
+                "    kdc_tcp_ports = 88\n"
+                "\n"
+                "[realms]\n"
+                "    {realm} = {{\n"
+                "        max_life = 24h 0m 0s\n"
+                "        max_renewable_life = 7d 0h 0m 0s\n"
+                "    }}\n"
+            ).format(realm=REALM)
+            remote.run(args=['sudo', 'mkdir', '-p', paths['kdc_conf_dir']])
+            remote.sudo_write_file(
+                '{}/kdc.conf'.format(paths['kdc_conf_dir']), kdc_conf)
+            remote.sudo_write_file(
+                '{}/kadm5.acl'.format(paths['kdc_conf_dir']),
+                '*/admin@{realm}    *\n'.format(realm=REALM))
+
+            # create the KDC database
+            remote.run(args=[
+                'sudo', 'kdb5_util', 'create', '-s', '-r', REALM,
+                '-P', MASTER_PASSWORD,
+            ])
+
+            # create principals and extract keytabs
+            remote.run(args=[
+                'sudo', 'kadmin.local', '-q',
+                'addprinc -randkey {}'.format(kafka_principal),
+            ])
+            remote.run(args=[
+                'sudo', 'kadmin.local', '-q',
+                'addprinc -randkey {}'.format(rgw_principal),
+            ])
+            remote.run(args=['sudo', 'mkdir', '-p', KEYTAB_DIR])
+            remote.run(args=[
+                'sudo', 'kadmin.local', '-q',
+                'ktadd -k {keytab} {principal}'.format(
+                    keytab=KAFKA_KEYTAB, principal=kafka_principal),
+            ])
+            remote.run(args=[
+                'sudo', 'kadmin.local', '-q',
+                'ktadd -k {keytab} {principal}'.format(
+                    keytab=RGW_KEYTAB, principal=rgw_principal),
+            ])
+            # make the keytabs world-readable to avoid "keytab contains 
+            # no suitable keys" permission failures.
+            remote.run(args=[
+                'sudo', 'chmod', '644', KAFKA_KEYTAB, RGW_KEYTAB,
+            ])
+
+            # restart the KDC and admin server
+            remote.run(args=[
+                'sudo', 'systemctl', 'restart',
+                paths['kdc_service'], paths['admin_service'],
+            ])
+
+            # kinit with the RGW principal and list the tickets
+            remote.run(args=[
+                'kinit', '-kt', RGW_KEYTAB, rgw_principal, run.Raw('&&'), 'klist',
+            ])
+
+            ctx.kerberos[client] = {
+                'realm': REALM,
+                'service_name': SERVICE_NAME,
+                'principal': rgw_principal,
+                'keytab': RGW_KEYTAB,
+                'kafka_principal': kafka_principal,
+                'kafka_keytab': KAFKA_KEYTAB,
+                'ip': ip,
+            }
+        except Exception as exc:
+            log.exception('Kerberos setup failed for %s', client)
+            raise RuntimeError('Kerberos setup is failing') from exc
+
+    try:
+        yield
+    finally:
+        log.info('Tearing down Kerberos KDC...')
+        for (client, _) in config.items():
+            (remote,) = ctx.cluster.only(client).remotes.keys()
+            paths = _distro_paths(remote)
+            remote.run(
+                args=['sudo', 'systemctl', 'stop',
+                      paths['kdc_service'], paths['admin_service']],
+                check_status=False,
+            )
+            remote.run(
+                args=['sudo', 'kdb5_util', 'destroy', '-f'],
+                check_status=False,
+            )
+            remote.run(
+                args=['sudo', 'rm', '-rf', KEYTAB_DIR, '/etc/krb5.conf',
+                      '/tmp/krb5cc_bntests'],
+                check_status=False,
+            )
+            ctx.kerberos.pop(client, None)
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+    """
+    Set up a Kerberos KDC for the GSSAPI Kafka bucket-notification tests.
+
+    Must run before the kafka and notification-tests tasks::
+
+        tasks:
+        - kerberos:
+            client.0:
+        - kafka:
+            client.0:
+              kafka_version: 3.9.2
+        - notification-tests:
+            client.0:
+              extra_attr: ["kafka_test", "kafka_security_test"]
+    """
+    assert config is None or isinstance(config, list) \
+        or isinstance(config, dict), \
+        "task kerberos only supports a list or dictionary for configuration"
+
+    all_clients = ['client.{id}'.format(id=id_)
+                   for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+    if config is None:
+        config = all_clients
+    if isinstance(config, list):
+        config = dict.fromkeys(config)
+
+    log.debug('Kerberos config is %s', config)
+
+    with contextutil.nested(
+        lambda: install_kerberos(ctx=ctx, config=config),
+        lambda: setup_kdc(ctx=ctx, config=config),
+        ):
+        yield

qa/tasks/notification_tests.py

@@ -233,6 +233,15 @@ def run_tests(ctx, config):
         if kafka_dir:
             args.append('KAFKA_DIR={kafka_dir}'.format(kafka_dir=kafka_dir))
 
+        # when a KDC has been set up, point the GSSAPI client (the kafka-python
+        # consumer used by the tests) at the rgw keytab so it can auto-acquire
+        # and auto-renew its kerberos credentials without a separate kinit.
+        kerberos = getattr(ctx, 'kerberos', None)
+        if kerberos and client in kerberos:
+            keytab = kerberos[client]['keytab']
+            args.append('KRB5_CLIENT_KTNAME={keytab}'.format(keytab=keytab))
+            args.append('KRB5CCNAME=FILE:/tmp/krb5cc_bntests')
+
         args.extend([
             '{tdir}/ceph/src/test/rgw/bucket_notification/virtualenv/bin/python'.format(tdir=testdir),
             '-m', 'pytest',
@@ -251,10 +260,13 @@ def run_tests(ctx, config):
 @contextlib.contextmanager
 def task(ctx,config):
     """
-    To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server. Also you need to pass the
-    'extra_attr' to the notification tests. Following is the way how to run kafka and finally bucket notification tests::
+    To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server and have kerberos setup.
+    Also you need to pass the 'extra_attr' to the notification tests. Following is the way how to run kerberos, kafka, and 
+    finally bucket notification tests::
 
     tasks:
+    - kerberos:
+        client.0:
     - kafka:
         client.0:
           kafka_version: 2.6.0
@@ -294,13 +306,25 @@ def task(ctx,config):
 
     bntests_conf = {}
 
+    kerberos = getattr(ctx, 'kerberos', None)
+
     for client in clients:
         endpoint = ctx.rgw.role_endpoints.get(client)
         assert endpoint, 'bntests: no rgw endpoint for {}'.format(client)
 
+        if kerberos and client in kerberos:
+            kerberos_conf = {
+                'service_name': kerberos[client]['service_name'],
+                'principal': kerberos[client]['principal'],
+                'keytab': kerberos[client]['keytab'],
+            }
+        else:
+            kerberos_conf = {'service_name': '', 'principal': '', 'keytab': ''}
+
         cluster_name, _, _ = teuthology.split_role(client)
         bntests_conf[client] = ConfigObj(
             indent_type='',
+            write_empty_values=True,
             infile={
                 'DEFAULT':
                     {
@@ -310,7 +334,8 @@ def task(ctx,config):
                     'cluster':cluster_name,
                     'version':'v2'
                     },
-                's3 main':{}
+                's3 main':{},
+                'kerberos':kerberos_conf
             }
         )
 

src/test/rgw/bucket_notification/bootstrap

@@ -2,7 +2,7 @@
 set -e
 
 if [ -f /etc/debian_version ]; then
-    for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev; do
+    for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev libkrb5-dev; do
         if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then
             # add a space after old values
             missing="${missing:+$missing }$package"
@@ -14,7 +14,7 @@ if [ -f /etc/debian_version ]; then
     fi
 fi
 if [ -f /etc/redhat-release ]; then
-    for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel; do
+    for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel krb5-devel; do
         if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then
             missing="${missing:+$missing }$package"
         fi