From 6fa7167732085d651285a11fd85c1bc33a2afcd7 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 11:11:09 -0400 Subject: [PATCH 1/9] qa/rgw/tempest: rearrange tasks and overrides Signed-off-by: Casey Bodley --- qa/suites/rgw/tempest/0-install.yaml | 15 +++++++++ qa/suites/rgw/tempest/overrides.yaml | 14 ++++++++ .../tasks/{rgw_tempest.yaml => tempest.yaml} | 33 ------------------- 3 files changed, 29 insertions(+), 33 deletions(-) create mode 100644 qa/suites/rgw/tempest/0-install.yaml rename qa/suites/rgw/tempest/tasks/{rgw_tempest.yaml => tempest.yaml} (67%) diff --git a/qa/suites/rgw/tempest/0-install.yaml b/qa/suites/rgw/tempest/0-install.yaml new file mode 100644 index 00000000000..fc2cfcc7b6d --- /dev/null +++ b/qa/suites/rgw/tempest/0-install.yaml @@ -0,0 +1,15 @@ +tasks: +- install: +- ceph: +- tox: [ client.0 ] +- keystone: + client.0: + force-branch: stable/2023.1 + services: + - name: swift + type: object-store + description: Swift Service +- rgw: + client.0: + frontend_prefix: /swift + use-keystone-role: client.0 diff --git a/qa/suites/rgw/tempest/overrides.yaml b/qa/suites/rgw/tempest/overrides.yaml index e7a292ffd1f..9fb0e11e949 100644 --- a/qa/suites/rgw/tempest/overrides.yaml +++ b/qa/suites/rgw/tempest/overrides.yaml @@ -1,7 +1,21 @@ overrides: ceph: conf: + global: + osd_min_pg_log_entries: 10 + osd_max_pg_log_entries: 10 client: setuser: ceph setgroup: ceph debug rgw: 20 + rgw keystone api version: 3 + rgw keystone accepted roles: admin,member + rgw keystone implicit tenants: true + rgw keystone accepted admin roles: admin + rgw swift enforce content length: true + rgw swift account in url: true + rgw swift versioning enabled: true + rgw keystone admin domain: Default + rgw keystone admin user: admin + rgw keystone admin password: ADMIN + rgw keystone admin project: admin diff --git a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml similarity index 67% rename from qa/suites/rgw/tempest/tasks/rgw_tempest.yaml rename to qa/suites/rgw/tempest/tasks/tempest.yaml index ad9dc9dd502..560e41de5bb 100644 --- a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml +++ b/qa/suites/rgw/tempest/tasks/tempest.yaml @@ -1,18 +1,4 @@ tasks: -- install: -- ceph: -- tox: [ client.0 ] -- keystone: - client.0: - force-branch: stable/2023.1 - services: - - name: swift - type: object-store - description: Swift Service -- rgw: - client.0: - frontend_prefix: /swift - use-keystone-role: client.0 - tempest: client.0: sha1: 34.1.0 @@ -51,22 +37,3 @@ tasks: - .*test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time - .*test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time - .*test_account_services.AccountTest.test_list_no_account_metadata - -overrides: - ceph: - conf: - global: - osd_min_pg_log_entries: 10 - osd_max_pg_log_entries: 10 - client: - rgw keystone api version: 3 - rgw keystone accepted roles: admin,member - rgw keystone implicit tenants: true - rgw keystone accepted admin roles: admin - rgw swift enforce content length: true - rgw swift account in url: true - rgw swift versioning enabled: true - rgw keystone admin domain: Default - rgw keystone admin user: admin - rgw keystone admin password: ADMIN - rgw keystone admin project: admin From f19d05dcab8093424d9cd50943972fa7f1942f92 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 17:03:40 -0400 Subject: [PATCH 2/9] qa/keystone: merge overrides into config Signed-off-by: Casey Bodley --- qa/tasks/keystone.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py index 7aa785055c2..f0f09842e60 100644 --- a/qa/tasks/keystone.py +++ b/qa/tasks/keystone.py @@ -463,6 +463,8 @@ def task(ctx, config): config = all_clients if isinstance(config, list): config = dict.fromkeys(config) + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('keystone', {})) log.debug('Keystone config is %s', config) From 86b8b4b0198045b733235315c167ff1cb249a16b Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:00:28 -0400 Subject: [PATCH 3/9] qa/keystone: extract os_auth_args to helper function Signed-off-by: Casey Bodley --- qa/tasks/keystone.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py index f0f09842e60..27e808f6883 100644 --- a/qa/tasks/keystone.py +++ b/qa/tasks/keystone.py @@ -326,25 +326,26 @@ def dict_to_args(specials, items): args.extend(arg for arg in special_vals.values() if arg) return args +def os_auth_args(host, port): + return [ + '--os-username', 'admin', + '--os-password', 'ADMIN', + '--os-user-domain-id', 'default', + '--os-project-name', 'admin', + '--os-project-domain-id', 'default', + '--os-identity-api-version', '3', + '--os-auth-url', 'http://{host}:{port}/v3'.format(host=host, port=port), + ] + def run_section_cmds(ctx, cclient, section_cmd, specials, section_config_list): public_host, public_port = ctx.keystone.public_endpoints[cclient] - - auth_section = [ - ( 'os-username', 'admin' ), - ( 'os-password', 'ADMIN' ), - ( 'os-user-domain-id', 'default' ), - ( 'os-project-name', 'admin' ), - ( 'os-project-domain-id', 'default' ), - ( 'os-identity-api-version', '3' ), - ( 'os-auth-url', 'http://{host}:{port}/v3'.format(host=public_host, - port=public_port) ), - ] + auth_args = os_auth_args(public_host, public_port) for section_item in section_config_list: run_in_keystone_venv(ctx, cclient, - [ 'openstack' ] + section_cmd.split() + - dict_to_args(specials, auth_section + list(section_item.items())) + + [ 'openstack' ] + section_cmd.split() + auth_args + + dict_to_args(specials, list(section_item.items())) + [ '--debug' ]) def create_endpoint(ctx, cclient, service, url, adminurl=None): From 1456d12a9723f509717d706c09b252f9b810b300 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:01:15 -0400 Subject: [PATCH 4/9] qa/keystone: config can create ec2 credentials Signed-off-by: Casey Bodley --- qa/tasks/keystone.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py index 27e808f6883..fc45b9c2938 100644 --- a/qa/tasks/keystone.py +++ b/qa/tasks/keystone.py @@ -387,6 +387,8 @@ def fill_keystone(ctx, config): cconfig.get('projects', [])) run_section_cmds(ctx, cclient, 'user create --or-show', 'name', cconfig.get('users', [])) + run_section_cmds(ctx, cclient, 'ec2 credentials create', '', + cconfig.get('ec2 credentials', [])) run_section_cmds(ctx, cclient, 'role create --or-show', 'name', cconfig.get('roles', [])) run_section_cmds(ctx, cclient, 'role add', 'name', @@ -441,6 +443,9 @@ def task(ctx, config): - name: custom password: SECRET project: custom + ec2 credentials: + - project: custom + user: custom roles: [ name: custom ] role-mappings: - name: custom From f9872e58a1e5c0e0d1ca5d56e72e2333f5d8e810 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:02:24 -0400 Subject: [PATCH 5/9] qa/keystone: expose function to read a user's ec2 creds Signed-off-by: Casey Bodley --- qa/tasks/keystone.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/qa/tasks/keystone.py b/qa/tasks/keystone.py index fc45b9c2938..bffeeeae181 100644 --- a/qa/tasks/keystone.py +++ b/qa/tasks/keystone.py @@ -3,6 +3,8 @@ Deploy and configure Keystone for Teuthology """ import argparse import contextlib +from io import StringIO +import json import logging # still need this for python3.6 @@ -35,12 +37,12 @@ def toxvenv_sh(ctx, remote, args, **kwargs): activate = get_toxvenv_dir(ctx) + '/bin/activate' return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs) -def run_in_keystone_venv(ctx, client, args): - run_in_keystone_dir(ctx, client, +def run_in_keystone_venv(ctx, client, args, **kwargs): + return run_in_keystone_dir(ctx, client, [ 'source', '.tox/venv/bin/activate', run.Raw('&&') - ] + args) + ] + args, **kwargs) def get_keystone_venved_cmd(ctx, cmd, args, env=[]): kbindir = get_keystone_dir(ctx) + '/.tox/venv/bin/' @@ -420,6 +422,29 @@ def assign_ports(ctx, config, initial_port): return role_endpoints +def read_ec2_credentials(ctx, client, user): + """ + Look up EC2 credentials for the given user. + + Returns a dictionary of the form: + { + "Access": "b2c9a792ff934b50b7e5c6d8f0fbbc96", + "Secret": "53b34a24a8e244ca89f1d754f089b63a", + "Project ID": "49208b6cc1864a0ea1cd7de3b456db11", + "User ID": "3276c0e0116a4a3ab1dd462ae4846416" + } + """ + public_host, public_port = ctx.keystone.public_endpoints[client] + procs = run_in_keystone_venv(ctx, client, + ['openstack', 'ec2', 'credentials', 'list', + '--user', user, '--format', 'json', '--debug'] + + os_auth_args(public_host, public_port), + stdout=StringIO()) + assert len(procs) == 1 + response = json.loads(procs[0].stdout.getvalue()) + assert len(response) + return response[0] + @contextlib.contextmanager def task(ctx, config): """ @@ -476,6 +501,7 @@ def task(ctx, config): ctx.keystone = argparse.Namespace() ctx.keystone.public_endpoints = assign_ports(ctx, config, 5000) + ctx.keystone.read_ec2_credentials = read_ec2_credentials with contextutil.nested( lambda: download(ctx=ctx, config=config), From ff590c5ae467d3fb3cff920a1c3a04cf060097ec Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:11:15 -0400 Subject: [PATCH 6/9] qa/s3tests: create_users() takes all yaml config Signed-off-by: Casey Bodley --- qa/tasks/s3tests.py | 60 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py index 70b49c086b6..ad7d1b0fb5a 100644 --- a/qa/tasks/s3tests.py +++ b/qa/tasks/s3tests.py @@ -90,22 +90,21 @@ def _config_user(s3tests_conf, section, user): @contextlib.contextmanager -def create_users(ctx, config): +def create_users(ctx, config, s3tests_conf): """ Create a main and an alternate s3 user. """ - assert isinstance(config, dict) log.info('Creating rgw users...') testdir = teuthology.get_testdir(ctx) users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'} - for client in config['clients']: - s3tests_conf = config['s3tests_conf'][client] - s3tests_conf.setdefault('fixtures', {}) - s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-') + for client, cconfig in config.items(): + conf = s3tests_conf[client] + conf.setdefault('fixtures', {}) + conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-') for section, user in users.items(): - _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client)) - log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client)) + _config_user(conf, section, '{user}.{client}'.format(user=user, client=client)) + log.debug('Creating user {user} on {host}'.format(user=conf[section]['user_id'], host=client)) cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id # create user @@ -117,12 +116,12 @@ def create_users(ctx, config): 'radosgw-admin', '-n', client_with_id, 'user', 'create', - '--uid', s3tests_conf[section]['user_id'], - '--display-name', s3tests_conf[section]['display_name'], - '--email', s3tests_conf[section]['email'], + '--uid', conf[section]['user_id'], + '--display-name', conf[section]['display_name'], + '--email', conf[section]['email'], '--caps', 'user-policy=*', - '--access-key', s3tests_conf[section]['access_key'], - '--secret', s3tests_conf[section]['secret_key'], + '--access-key', conf[section]['access_key'], + '--secret', conf[section]['secret_key'], '--cluster', cluster_name, ], ) @@ -136,10 +135,10 @@ def create_users(ctx, config): 'radosgw-admin', '-n', client_with_id, 'mfa', 'create', - '--uid', s3tests_conf[section]['user_id'], - '--totp-serial', s3tests_conf[section]['totp_serial'], - '--totp-seed', s3tests_conf[section]['totp_seed'], - '--totp-seconds', s3tests_conf[section]['totp_seconds'], + '--uid', conf[section]['user_id'], + '--totp-serial', conf[section]['totp_serial'], + '--totp-seed', conf[section]['totp_seed'], + '--totp-seconds', conf[section]['totp_seconds'], '--totp-window', '8', '--totp-seed-type', 'base32', '--cluster', cluster_name, @@ -156,7 +155,7 @@ def create_users(ctx, config): 'radosgw-admin', '-n', client_with_id, 'caps', 'add', - '--uid', s3tests_conf[section]['user_id'], + '--uid', conf[section]['user_id'], '--caps', 'roles=*', '--cluster', cluster_name, ], @@ -169,26 +168,26 @@ def create_users(ctx, config): 'radosgw-admin', '-n', client_with_id, 'caps', 'add', - '--uid', s3tests_conf[section]['user_id'], + '--uid', conf[section]['user_id'], '--caps', 'oidc-provider=*', '--cluster', cluster_name, ], ) if "TOKEN" in os.environ: - s3tests_conf.setdefault('webidentity', {}) - s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN']) - s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD']) - s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB']) - s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP']) - s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN']) - s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT']) - s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM']) + conf.setdefault('webidentity', {}) + conf['webidentity'].setdefault('token',os.environ['TOKEN']) + conf['webidentity'].setdefault('aud',os.environ['AUD']) + conf['webidentity'].setdefault('sub',os.environ['SUB']) + conf['webidentity'].setdefault('azp',os.environ['AZP']) + conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN']) + conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT']) + conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM']) try: yield finally: - for client in config['clients']: + for client in config.keys(): for user in users.values(): uid = '{user}.{client}'.format(user=user, client=client) cluster_name, daemon_type, client_id = teuthology.split_role(client) @@ -638,10 +637,7 @@ def task(ctx, config): with contextutil.nested( lambda: download(ctx=ctx, config=config), - lambda: create_users(ctx=ctx, config=dict( - clients=clients, - s3tests_conf=s3tests_conf, - )), + lambda: create_users(ctx=ctx, config=config, s3tests_conf=s3tests_conf), lambda: configure(ctx=ctx, config=dict( clients=config, s3tests_conf=s3tests_conf, From fa70c782882a541b5beb1cb8ebfdc2a5119f1d88 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:25:19 -0400 Subject: [PATCH 7/9] qa/s3tests: enable testing against keystone users with ec2 Signed-off-by: Casey Bodley --- qa/tasks/s3tests.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/qa/tasks/s3tests.py b/qa/tasks/s3tests.py index ad7d1b0fb5a..7006602f2be 100644 --- a/qa/tasks/s3tests.py +++ b/qa/tasks/s3tests.py @@ -102,8 +102,31 @@ def create_users(ctx, config, s3tests_conf): conf = s3tests_conf[client] conf.setdefault('fixtures', {}) conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-') + + keystone_users = cconfig.get('keystone users', {}) for section, user in users.items(): _config_user(conf, section, '{user}.{client}'.format(user=user, client=client)) + + # for keystone users, read ec2 credentials into s3tests.conf instead + # of creating a local user + keystone_user = keystone_users.get(section) + if keystone_user: + project_name = keystone_user.pop('project') + creds = ctx.keystone.read_ec2_credentials(ctx, **keystone_user) + access = creds['Access'] + secret = creds['Secret'] + project_id = creds['Project ID'] + + conf[section]['access_key'] = access + conf[section]['secret_key'] = secret + conf[section]['user_id'] = project_id + conf[section]['display_name'] = project_name + + log.debug('Using keystone user {kuser} credentials ({access} : {secret}) for {pname}:{pid} on {host}'.format( + kuser=keystone_user['user'], access=access, secret=secret, + pname=project_name, pid=project_id, host=client)) + continue + log.debug('Creating user {user} on {host}'.format(user=conf[section]['user_id'], host=client)) cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id @@ -189,6 +212,9 @@ def create_users(ctx, config, s3tests_conf): finally: for client in config.keys(): for user in users.values(): + # don't need to delete keystone users + if not user in keystone_users: + continue uid = '{user}.{client}'.format(user=user, client=client) cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id @@ -501,6 +527,31 @@ def task(ctx, config): cloudtier_tests: True rgw_server: client.0 + To test against Keystone users with EC2 credentials:: + + tasks: + - ceph: + - rgw: [client.0 client.1] + - keystone: + client.0: + projects: + - name: myproject + description: my project + users: + - name: myuser + password: SECRET + project: myproject + ec2 credentials: + - project: myproject + user: myuser + - s3tests: + client.0: + keystone users: + s3 main: + client: client.0 + project: myproject + user: myuser + """ assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task' assert hasattr(ctx, 'tox'), 's3tests must run after the tox task' From 7b27f9dea979f6845e71dcc8eeffac7d1485eeab Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 4 Aug 2023 12:29:47 -0400 Subject: [PATCH 8/9] qa/rgw/tempest: run s3tests against keystone ec2 Fixes: https://tracker.ceph.com/issues/59424 Signed-off-by: Casey Bodley --- qa/suites/rgw/tempest/s3tests-branch.yaml | 1 + qa/suites/rgw/tempest/tasks/+ | 0 qa/suites/rgw/tempest/tasks/s3tests.yaml | 35 +++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 120000 qa/suites/rgw/tempest/s3tests-branch.yaml create mode 100644 qa/suites/rgw/tempest/tasks/+ create mode 100644 qa/suites/rgw/tempest/tasks/s3tests.yaml diff --git a/qa/suites/rgw/tempest/s3tests-branch.yaml b/qa/suites/rgw/tempest/s3tests-branch.yaml new file mode 120000 index 00000000000..bdcaca48ae0 --- /dev/null +++ b/qa/suites/rgw/tempest/s3tests-branch.yaml @@ -0,0 +1 @@ +.qa/rgw/s3tests-branch.yaml \ No newline at end of file diff --git a/qa/suites/rgw/tempest/tasks/+ b/qa/suites/rgw/tempest/tasks/+ new file mode 100644 index 00000000000..e69de29bb2d diff --git a/qa/suites/rgw/tempest/tasks/s3tests.yaml b/qa/suites/rgw/tempest/tasks/s3tests.yaml new file mode 100644 index 00000000000..f1ee8a17c4c --- /dev/null +++ b/qa/suites/rgw/tempest/tasks/s3tests.yaml @@ -0,0 +1,35 @@ +overrides: + ceph: + conf: + client: + rgw s3 auth use keystone: true + rgw crypt s3 kms backend: testing + rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= + rgw crypt require ssl: false + keystone: + client.0: + projects: + - name: s3tests + description: s3tests project + users: + - name: s3tests-main + password: SECRET + project: s3tests + ec2 credentials: + - project: s3tests + user: s3tests-main + roles: [ name: member ] + role-mappings: + - name: member + user: s3tests-main + project: s3tests + +tasks: +- s3tests: + client.0: + rgw_server: client.0 + keystone users: + s3 main: + client: client.0 + project: s3tests + user: s3tests-main From c19aab4c31e89defca9409440463f4654b6cafd5 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 9 Aug 2023 14:58:49 -0400 Subject: [PATCH 9/9] qa/rgw/tempest: override frontend_prefix in tempest task tempest relies on `frontend_prefix: /swift` to serve the swift api from the root path s3tests relies on the same for s3. move the frontend_prefix override into the tempest task so it doesn't apply to s3tests Signed-off-by: Casey Bodley --- qa/suites/rgw/tempest/0-install.yaml | 1 - qa/suites/rgw/tempest/tasks/+ | 0 qa/suites/rgw/tempest/tasks/tempest.yaml | 6 ++++++ 3 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 qa/suites/rgw/tempest/tasks/+ diff --git a/qa/suites/rgw/tempest/0-install.yaml b/qa/suites/rgw/tempest/0-install.yaml index fc2cfcc7b6d..f968db20c2b 100644 --- a/qa/suites/rgw/tempest/0-install.yaml +++ b/qa/suites/rgw/tempest/0-install.yaml @@ -11,5 +11,4 @@ tasks: description: Swift Service - rgw: client.0: - frontend_prefix: /swift use-keystone-role: client.0 diff --git a/qa/suites/rgw/tempest/tasks/+ b/qa/suites/rgw/tempest/tasks/+ deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/qa/suites/rgw/tempest/tasks/tempest.yaml b/qa/suites/rgw/tempest/tasks/tempest.yaml index 560e41de5bb..988c7515b51 100644 --- a/qa/suites/rgw/tempest/tasks/tempest.yaml +++ b/qa/suites/rgw/tempest/tasks/tempest.yaml @@ -1,3 +1,9 @@ +overrides: + rgw: + client.0: + # tempest tests expect the swift api at the root + frontend_prefix: /swift + tasks: - tempest: client.0: