diff --git a/qa/suites/rgw/notifications/overrides.yaml b/qa/suites/rgw/notifications/overrides.yaml
index b5d6e94ba6c..733eb0c80e2 100644
--- a/qa/suites/rgw/notifications/overrides.yaml
+++ b/qa/suites/rgw/notifications/overrides.yaml
@@ -11,6 +11,7 @@ overrides:
rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
rgw crypt require ssl: false
rgw allow notification secrets in cleartext: true
+ rgw kafka sasl kerberos service name: kafka
rgw:
realm: MyRealm
zonegroup: MyZoneGroup
diff --git a/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml b/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml
index 1acc2c74ca8..915b93199b2 100644
--- a/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml
+++ b/qa/suites/rgw/notifications/tasks/kafka/test_kafka.yaml
@@ -1,4 +1,6 @@
tasks:
+- kerberos:
+ client.0:
- kafka:
client.0:
kafka_version: 3.9.2
diff --git a/qa/tasks/kafka.py b/qa/tasks/kafka.py
index 7d3b4ec4bdc..0d2f095826d 100644
--- a/qa/tasks/kafka.py
+++ b/qa/tasks/kafka.py
@@ -42,6 +42,8 @@ def install_kafka(ctx, config):
assert isinstance(config, dict)
log.info('Installing Kafka...')
+ kerberos = getattr(ctx, 'kerberos', None)
+
# programmatically find a nearby mirror so as not to hammer archive.apache.org
apache_mirror_cmd="curl 'https://www.apache.org/dyn/closer.cgi' 2>/dev/null | " \
"grep -o '[^<]*' | sed 's/<[^>]*>//g' | head -n 1"
@@ -101,7 +103,8 @@ def install_kafka(ctx, config):
args=['sudo', 'chmod', 'o+rx', '/home/ubuntu'],
)
- broker_conf(ctx, client, kafka_dir)
+ client_kerberos = kerberos.get(client) if kerberos else None
+ broker_conf(ctx, client, kafka_dir, client_kerberos)
try:
yield
@@ -123,7 +126,7 @@ def install_kafka(ctx, config):
)
-def broker_conf(ctx, client, kafka_dir):
+def broker_conf(ctx, client, kafka_dir, kerberos):
"""writing a custom server.properties config file"""
(remote,) = ctx.cluster.only(client).remotes.keys()
ip = remote.ip_address
@@ -170,8 +173,9 @@ def broker_conf(ctx, client, kafka_dir):
"listener.name.mtls.ssl.truststore.location={tdir}/server.truststore.jks\n"
"listener.name.mtls.ssl.truststore.password=mypassword\n"
# SASL mechanisms
- "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n"
+ "sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI\n"
"sasl.mechanism.inter.broker.protocol=PLAIN\n"
+ "sasl.kerberos.service.name={service_name}\n"
'listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret" \\\n'
@@ -182,6 +186,11 @@ def broker_conf(ctx, client, kafka_dir):
'listener.name.sasl_ssl.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret";\n'
+ 'listener.name.sasl_ssl.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+ ' useKeyTab=true \\\n'
+ ' storeKey=true \\\n'
+ ' keyTab="{keytab}" \\\n'
+ ' principal="{principal}";\n'
'listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret" \\\n'
@@ -192,6 +201,11 @@ def broker_conf(ctx, client, kafka_dir):
'listener.name.sasl_plaintext.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required \\\n'
' username="admin" \\\n'
' password="admin-secret";\n'
+ 'listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \\\n'
+ ' useKeyTab=true \\\n'
+ ' storeKey=true \\\n'
+ ' keyTab="{keytab}" \\\n'
+ ' principal="{principal}";\n'
).format(
tdir=kafka_dir,
ip=ip,
@@ -200,6 +214,9 @@ def broker_conf(ctx, client, kafka_dir):
sasl_ssl=KAFKA_PORTS['SASL_SSL'],
sasl_plaintext=KAFKA_PORTS['SASL_PLAINTEXT'],
mtls=KAFKA_PORTS['MTLS'],
+ service_name=kerberos['service_name'] if kerberos else '',
+ keytab=kerberos['kafka_keytab'] if kerberos else '',
+ principal=kerberos['kafka_principal'] if kerberos else '',
)
file_name = 'server.properties'
log.info("kafka conf file: %s", file_name)
diff --git a/qa/tasks/kerberos.py b/qa/tasks/kerberos.py
new file mode 100644
index 00000000000..5e68eaacea0
--- /dev/null
+++ b/qa/tasks/kerberos.py
@@ -0,0 +1,258 @@
+"""
+Deploy and configure a Kerberos KDC for Teuthology.
+
+This task stands up a single-node MIT Kerberos KDC and creates the service
+principals / keytabs used by the GSSAPI Kafka bucket-notification tests:
+
+ - kafka/@CEPH.TEST (used by the Kafka broker)
+ - rgw/@CEPH.TEST (used by RGW and the test consumer)
+
+"""
+import contextlib
+import logging
+
+from teuthology import contextutil
+from teuthology import misc as teuthology
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+REALM = 'CEPH.TEST'
+MASTER_PASSWORD = 'ceph-kdc-master'
+KEYTAB_DIR = '/etc/krb5-keytabs'
+KAFKA_KEYTAB = '{}/kafka.service.keytab'.format(KEYTAB_DIR)
+RGW_KEYTAB = '{}/rgw.service.keytab'.format(KEYTAB_DIR)
+SERVICE_NAME = 'kafka'
+
+
+def _distro_paths(remote):
+ """Return distro-specific KDC paths and service names."""
+ if remote.os.package_type == 'rpm':
+ return {
+ 'kdc_conf_dir': '/var/kerberos/krb5kdc',
+ 'kdc_service': 'krb5kdc',
+ 'admin_service': 'kadmin',
+ }
+ return {
+ 'kdc_conf_dir': '/etc/krb5kdc',
+ 'kdc_service': 'krb5-kdc',
+ 'admin_service': 'krb5-admin-server',
+ }
+
+
+@contextlib.contextmanager
+def install_kerberos(ctx, config):
+ """Install the Kerberos KDC, admin server and GSSAPI SASL plugin."""
+ assert isinstance(config, dict)
+ log.info('Installing Kerberos packages...')
+
+ deb_packages = [
+ 'krb5-kdc', 'krb5-admin-server', 'krb5-user',
+ 'libkrb5-dev', 'libsasl2-modules-gssapi-mit',
+ ]
+ rpm_packages = [
+ 'krb5-devel', 'krb5-server', 'krb5-workstation',
+ 'cyrus-sasl-gssapi',
+ ]
+
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ if remote.os.package_type == 'rpm':
+ remote.run(args=['sudo', 'dnf', 'install', '-y'] + rpm_packages)
+ else:
+ remote.run(args=['sudo', 'apt-get', 'update'])
+ remote.run(args=[
+ 'sudo', 'DEBIAN_FRONTEND=noninteractive',
+ 'apt-get', 'install', '-y',
+ ] + deb_packages)
+
+ try:
+ yield
+ finally:
+ log.info('Removing Kerberos packages...')
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ if remote.os.package_type == 'rpm':
+ remote.run(
+ args=['sudo', 'dnf', 'remove', '-y'] + rpm_packages,
+ check_status=False,
+ )
+ else:
+ remote.run(
+ args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
+ 'apt-get', 'purge', '-y'] + deb_packages,
+ check_status=False,
+ )
+
+
+@contextlib.contextmanager
+def setup_kdc(ctx, config):
+ """Create the realm, principals and keytabs and start the KDC."""
+ assert isinstance(config, dict)
+ log.info('Setting up Kerberos KDC...')
+
+ if not hasattr(ctx, 'kerberos'):
+ ctx.kerberos = {}
+
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ try:
+ ip = remote.ip_address
+ paths = _distro_paths(remote)
+ kafka_principal = 'kafka/{ip}@{realm}'.format(ip=ip, realm=REALM)
+ rgw_principal = 'rgw/{ip}@{realm}'.format(ip=ip, realm=REALM)
+
+ # /etc/krb5.conf: rdns=false keeps the SPN IP-based (no reverse
+ # lookup to a hostname); dns_lookup_kdc/realm=false keeps the KDC
+ # and realm settings local and static, avoiding any DNS-related
+ # issues in the test environment.
+ krb5_conf = (
+ "[libdefaults]\n"
+ " default_realm = {realm}\n"
+ " dns_lookup_kdc = false\n"
+ " dns_lookup_realm = false\n"
+ " rdns = false\n"
+ " udp_preference_limit = 1\n"
+ "\n"
+ "[realms]\n"
+ " {realm} = {{\n"
+ " kdc = {ip}\n"
+ " admin_server = {ip}\n"
+ " }}\n"
+ ).format(realm=REALM, ip=ip)
+ remote.sudo_write_file('/etc/krb5.conf', krb5_conf)
+
+ # kdc.conf
+ kdc_conf = (
+ "[kdcdefaults]\n"
+ " kdc_ports = 88\n"
+ " kdc_tcp_ports = 88\n"
+ "\n"
+ "[realms]\n"
+ " {realm} = {{\n"
+ " max_life = 24h 0m 0s\n"
+ " max_renewable_life = 7d 0h 0m 0s\n"
+ " }}\n"
+ ).format(realm=REALM)
+ remote.run(args=['sudo', 'mkdir', '-p', paths['kdc_conf_dir']])
+ remote.sudo_write_file(
+ '{}/kdc.conf'.format(paths['kdc_conf_dir']), kdc_conf)
+ remote.sudo_write_file(
+ '{}/kadm5.acl'.format(paths['kdc_conf_dir']),
+ '*/admin@{realm} *\n'.format(realm=REALM))
+
+ # create the KDC database
+ remote.run(args=[
+ 'sudo', 'kdb5_util', 'create', '-s', '-r', REALM,
+ '-P', MASTER_PASSWORD,
+ ])
+
+ # create principals and extract keytabs
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'addprinc -randkey {}'.format(kafka_principal),
+ ])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'addprinc -randkey {}'.format(rgw_principal),
+ ])
+ remote.run(args=['sudo', 'mkdir', '-p', KEYTAB_DIR])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'ktadd -k {keytab} {principal}'.format(
+ keytab=KAFKA_KEYTAB, principal=kafka_principal),
+ ])
+ remote.run(args=[
+ 'sudo', 'kadmin.local', '-q',
+ 'ktadd -k {keytab} {principal}'.format(
+ keytab=RGW_KEYTAB, principal=rgw_principal),
+ ])
+ # make the keytabs world-readable to avoid "keytab contains
+ # no suitable keys" permission failures.
+ remote.run(args=[
+ 'sudo', 'chmod', '644', KAFKA_KEYTAB, RGW_KEYTAB,
+ ])
+
+ # restart the KDC and admin server
+ remote.run(args=[
+ 'sudo', 'systemctl', 'restart',
+ paths['kdc_service'], paths['admin_service'],
+ ])
+
+ # kinit with the RGW principal and list the tickets
+ remote.run(args=[
+ 'kinit', '-kt', RGW_KEYTAB, rgw_principal, run.Raw('&&'), 'klist',
+ ])
+
+ ctx.kerberos[client] = {
+ 'realm': REALM,
+ 'service_name': SERVICE_NAME,
+ 'principal': rgw_principal,
+ 'keytab': RGW_KEYTAB,
+ 'kafka_principal': kafka_principal,
+ 'kafka_keytab': KAFKA_KEYTAB,
+ 'ip': ip,
+ }
+ except Exception as exc:
+ log.exception('Kerberos setup failed for %s', client)
+ raise RuntimeError('Kerberos setup is failing') from exc
+
+ try:
+ yield
+ finally:
+ log.info('Tearing down Kerberos KDC...')
+ for (client, _) in config.items():
+ (remote,) = ctx.cluster.only(client).remotes.keys()
+ paths = _distro_paths(remote)
+ remote.run(
+ args=['sudo', 'systemctl', 'stop',
+ paths['kdc_service'], paths['admin_service']],
+ check_status=False,
+ )
+ remote.run(
+ args=['sudo', 'kdb5_util', 'destroy', '-f'],
+ check_status=False,
+ )
+ remote.run(
+ args=['sudo', 'rm', '-rf', KEYTAB_DIR, '/etc/krb5.conf',
+ '/tmp/krb5cc_bntests'],
+ check_status=False,
+ )
+ ctx.kerberos.pop(client, None)
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+ """
+ Set up a Kerberos KDC for the GSSAPI Kafka bucket-notification tests.
+
+ Must run before the kafka and notification-tests tasks::
+
+ tasks:
+ - kerberos:
+ client.0:
+ - kafka:
+ client.0:
+ kafka_version: 3.9.2
+ - notification-tests:
+ client.0:
+ extra_attr: ["kafka_test", "kafka_security_test"]
+ """
+ assert config is None or isinstance(config, list) \
+ or isinstance(config, dict), \
+ "task kerberos only supports a list or dictionary for configuration"
+
+ all_clients = ['client.{id}'.format(id=id_)
+ for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
+ if config is None:
+ config = all_clients
+ if isinstance(config, list):
+ config = dict.fromkeys(config)
+
+ log.debug('Kerberos config is %s', config)
+
+ with contextutil.nested(
+ lambda: install_kerberos(ctx=ctx, config=config),
+ lambda: setup_kdc(ctx=ctx, config=config),
+ ):
+ yield
diff --git a/qa/tasks/notification_tests.py b/qa/tasks/notification_tests.py
index cd1b4752c3b..5f258e219c3 100644
--- a/qa/tasks/notification_tests.py
+++ b/qa/tasks/notification_tests.py
@@ -233,6 +233,15 @@ def run_tests(ctx, config):
if kafka_dir:
args.append('KAFKA_DIR={kafka_dir}'.format(kafka_dir=kafka_dir))
+ # when a KDC has been set up, point the GSSAPI client (the kafka-python
+ # consumer used by the tests) at the rgw keytab so it can auto-acquire
+ # and auto-renew its kerberos credentials without a separate kinit.
+ kerberos = getattr(ctx, 'kerberos', None)
+ if kerberos and client in kerberos:
+ keytab = kerberos[client]['keytab']
+ args.append('KRB5_CLIENT_KTNAME={keytab}'.format(keytab=keytab))
+ args.append('KRB5CCNAME=FILE:/tmp/krb5cc_bntests')
+
args.extend([
'{tdir}/ceph/src/test/rgw/bucket_notification/virtualenv/bin/python'.format(tdir=testdir),
'-m', 'pytest',
@@ -251,10 +260,13 @@ def run_tests(ctx, config):
@contextlib.contextmanager
def task(ctx,config):
"""
- To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server. Also you need to pass the
- 'extra_attr' to the notification tests. Following is the way how to run kafka and finally bucket notification tests::
+ To run bucket notification tests under Kafka endpoint the prerequisite is to run the kafka server and have kerberos setup.
+ Also you need to pass the 'extra_attr' to the notification tests. Following is the way how to run kerberos, kafka, and
+ finally bucket notification tests::
tasks:
+ - kerberos:
+ client.0:
- kafka:
client.0:
kafka_version: 2.6.0
@@ -294,13 +306,25 @@ def task(ctx,config):
bntests_conf = {}
+ kerberos = getattr(ctx, 'kerberos', None)
+
for client in clients:
endpoint = ctx.rgw.role_endpoints.get(client)
assert endpoint, 'bntests: no rgw endpoint for {}'.format(client)
+ if kerberos and client in kerberos:
+ kerberos_conf = {
+ 'service_name': kerberos[client]['service_name'],
+ 'principal': kerberos[client]['principal'],
+ 'keytab': kerberos[client]['keytab'],
+ }
+ else:
+ kerberos_conf = {'service_name': '', 'principal': '', 'keytab': ''}
+
cluster_name, _, _ = teuthology.split_role(client)
bntests_conf[client] = ConfigObj(
indent_type='',
+ write_empty_values=True,
infile={
'DEFAULT':
{
@@ -310,7 +334,8 @@ def task(ctx,config):
'cluster':cluster_name,
'version':'v2'
},
- 's3 main':{}
+ 's3 main':{},
+ 'kerberos':kerberos_conf
}
)
diff --git a/src/test/rgw/bucket_notification/bootstrap b/src/test/rgw/bucket_notification/bootstrap
index 4d680c86d2f..535935426b0 100755
--- a/src/test/rgw/bucket_notification/bootstrap
+++ b/src/test/rgw/bucket_notification/bootstrap
@@ -2,7 +2,7 @@
set -e
if [ -f /etc/debian_version ]; then
- for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev; do
+ for package in python3-pip python3-dev python3-xmltodict python3-pika libevent-dev libxml2-dev libxslt-dev zlib1g-dev libkrb5-dev; do
if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then
# add a space after old values
missing="${missing:+$missing }$package"
@@ -14,7 +14,7 @@ if [ -f /etc/debian_version ]; then
fi
fi
if [ -f /etc/redhat-release ]; then
- for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel; do
+ for package in python3-pip python3-devel python3-xmltodict python3-pika libevent-devel libxml2-devel libxslt-devel zlib-devel krb5-devel; do
if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then
missing="${missing:+$missing }$package"
fi