gian/ceph-mirror
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2026-08-02 07:03:18 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #52813 from cbodley/wip-59424

Browse Source 
qa/rgw: run s3tests against keystone ec2

Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
This commit is contained in:
Casey Bodley 2023-11-21 13:33:58 +00:00 committed by GitHub
commit b0b050d31c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 198 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
qa/suites/rgw/tempest/0-install.yaml Normal file
View File

@ -0,0 +1,14 @@
tasks:
- install:
- ceph:
- tox: [ client.0 ]
- keystone:
client.0:
force-branch: stable/2023.1
services:
- name: swift
type: object-store
description: Swift Service
- rgw:
client.0:
use-keystone-role: client.0

14
qa/suites/rgw/tempest/overrides.yaml
View File

@ -1,7 +1,21 @@
overrides:
ceph:
conf:
global:
osd_min_pg_log_entries: 10
osd_max_pg_log_entries: 10
client:
setuser: ceph
setgroup: ceph
debug rgw: 20
rgw keystone api version: 3
rgw keystone accepted roles: admin,member
rgw keystone implicit tenants: true
rgw keystone accepted admin roles: admin
rgw swift enforce content length: true
rgw swift account in url: true
rgw swift versioning enabled: true
rgw keystone admin domain: Default
rgw keystone admin user: admin
rgw keystone admin password: ADMIN
rgw keystone admin project: admin

1
qa/suites/rgw/tempest/s3tests-branch.yaml Symbolic link
View File

@ -0,0 +1 @@
.qa/rgw/s3tests-branch.yaml

35
qa/suites/rgw/tempest/tasks/s3tests.yaml Normal file
View File

@ -0,0 +1,35 @@
overrides:
ceph:
conf:
client:
rgw s3 auth use keystone: true
rgw crypt s3 kms backend: testing
rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
rgw crypt require ssl: false
keystone:
client.0:
projects:
- name: s3tests
description: s3tests project
users:
- name: s3tests-main
password: SECRET
project: s3tests
ec2 credentials:
- project: s3tests
user: s3tests-main
roles: [ name: member ]
role-mappings:
- name: member
user: s3tests-main
project: s3tests
tasks:
- s3tests:
client.0:
rgw_server: client.0
keystone users:
s3 main:
client: client.0
project: s3tests
user: s3tests-main

37
qa/suites/rgw/tempest/tasks/rgw_tempest.yaml → qa/suites/rgw/tempest/tasks/tempest.yaml
View File

@ -1,18 +1,10 @@
tasks:
- install:
- ceph:
- tox: [ client.0 ]
- keystone:
client.0:
force-branch: stable/2023.1
services:
- name: swift
type: object-store
description: Swift Service
- rgw:
overrides:
rgw:
client.0:
# tempest tests expect the swift api at the root
frontend_prefix: /swift
use-keystone-role: client.0
tasks:
- tempest:
client.0:
sha1: 34.1.0
@ -51,22 +43,3 @@ tasks:
- .*test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time
- .*test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time
- .*test_account_services.AccountTest.test_list_no_account_metadata
overrides:
ceph:
conf:
global:
osd_min_pg_log_entries: 10
osd_max_pg_log_entries: 10
client:
rgw keystone api version: 3
rgw keystone accepted roles: admin,member
rgw keystone implicit tenants: true
rgw keystone accepted admin roles: admin
rgw swift enforce content length: true
rgw swift account in url: true
rgw swift versioning enabled: true
rgw keystone admin domain: Default
rgw keystone admin user: admin
rgw keystone admin password: ADMIN
rgw keystone admin project: admin

66
qa/tasks/keystone.py
View File

@ -3,6 +3,8 @@ Deploy and configure Keystone for Teuthology
"""
import argparse
import contextlib
from io import StringIO
import json
import logging
# still need this for python3.6
@ -35,12 +37,12 @@ def toxvenv_sh(ctx, remote, args, **kwargs):
activate = get_toxvenv_dir(ctx) + '/bin/activate'
return remote.sh(['source', activate, run.Raw('&&')] + args, **kwargs)
def run_in_keystone_venv(ctx, client, args):
run_in_keystone_dir(ctx, client,
def run_in_keystone_venv(ctx, client, args, **kwargs):
return run_in_keystone_dir(ctx, client,
[ 'source',
'.tox/venv/bin/activate',
run.Raw('&&')
] + args)
] + args, **kwargs)
def get_keystone_venved_cmd(ctx, cmd, args, env=[]):
kbindir = get_keystone_dir(ctx) + '/.tox/venv/bin/'
@ -326,25 +328,26 @@ def dict_to_args(specials, items):
args.extend(arg for arg in special_vals.values() if arg)
return args
def os_auth_args(host, port):
return [
'--os-username', 'admin',
'--os-password', 'ADMIN',
'--os-user-domain-id', 'default',
'--os-project-name', 'admin',
'--os-project-domain-id', 'default',
'--os-identity-api-version', '3',
'--os-auth-url', 'http://{host}:{port}/v3'.format(host=host, port=port),
]
def run_section_cmds(ctx, cclient, section_cmd, specials,
section_config_list):
public_host, public_port = ctx.keystone.public_endpoints[cclient]
auth_section = [
( 'os-username', 'admin' ),
( 'os-password', 'ADMIN' ),
( 'os-user-domain-id', 'default' ),
( 'os-project-name', 'admin' ),
( 'os-project-domain-id', 'default' ),
( 'os-identity-api-version', '3' ),
( 'os-auth-url', 'http://{host}:{port}/v3'.format(host=public_host,
port=public_port) ),
]
auth_args = os_auth_args(public_host, public_port)
for section_item in section_config_list:
run_in_keystone_venv(ctx, cclient,
[ 'openstack' ] + section_cmd.split() +
dict_to_args(specials, auth_section + list(section_item.items())) +
[ 'openstack' ] + section_cmd.split() + auth_args +
dict_to_args(specials, list(section_item.items())) +
[ '--debug' ])
def create_endpoint(ctx, cclient, service, url, adminurl=None):
@ -386,6 +389,8 @@ def fill_keystone(ctx, config):
cconfig.get('projects', []))
run_section_cmds(ctx, cclient, 'user create --or-show', 'name',
cconfig.get('users', []))
run_section_cmds(ctx, cclient, 'ec2 credentials create', '',
cconfig.get('ec2 credentials', []))
run_section_cmds(ctx, cclient, 'role create --or-show', 'name',
cconfig.get('roles', []))
run_section_cmds(ctx, cclient, 'role add', 'name',
@ -417,6 +422,29 @@ def assign_ports(ctx, config, initial_port):
return role_endpoints
def read_ec2_credentials(ctx, client, user):
"""
Look up EC2 credentials for the given user.
Returns a dictionary of the form:
{
"Access": "b2c9a792ff934b50b7e5c6d8f0fbbc96",
"Secret": "53b34a24a8e244ca89f1d754f089b63a",
"Project ID": "49208b6cc1864a0ea1cd7de3b456db11",
"User ID": "3276c0e0116a4a3ab1dd462ae4846416"
}
"""
public_host, public_port = ctx.keystone.public_endpoints[client]
procs = run_in_keystone_venv(ctx, client,
['openstack', 'ec2', 'credentials', 'list',
'--user', user, '--format', 'json', '--debug'] +
os_auth_args(public_host, public_port),
stdout=StringIO())
assert len(procs) == 1
response = json.loads(procs[0].stdout.getvalue())
assert len(response)
return response[0]
@contextlib.contextmanager
def task(ctx, config):
"""
@ -440,6 +468,9 @@ def task(ctx, config):
- name: custom
password: SECRET
project: custom
ec2 credentials:
- project: custom
user: custom
roles: [ name: custom ]
role-mappings:
- name: custom
@ -463,11 +494,14 @@ def task(ctx, config):
config = all_clients
if isinstance(config, list):
config = dict.fromkeys(config)
overrides = ctx.config.get('overrides', {})
teuthology.deep_merge(config, overrides.get('keystone', {}))
log.debug('Keystone config is %s', config)
ctx.keystone = argparse.Namespace()
ctx.keystone.public_endpoints = assign_ports(ctx, config, 5000)
ctx.keystone.read_ec2_credentials = read_ec2_credentials
with contextutil.nested(
lambda: download(ctx=ctx, config=config),

111
qa/tasks/s3tests.py
View File

@ -90,22 +90,44 @@ def _config_user(s3tests_conf, section, user):
@contextlib.contextmanager
def create_users(ctx, config):
def create_users(ctx, config, s3tests_conf):
"""
Create a main and an alternate s3 user.
"""
assert isinstance(config, dict)
log.info('Creating rgw users...')
testdir = teuthology.get_testdir(ctx)
users = {'s3 main': 'foo', 's3 alt': 'bar', 's3 tenant': 'testx$tenanteduser', 'iam': 'foobar'}
for client in config['clients']:
s3tests_conf = config['s3tests_conf'][client]
s3tests_conf.setdefault('fixtures', {})
s3tests_conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
for client, cconfig in config.items():
conf = s3tests_conf[client]
conf.setdefault('fixtures', {})
conf['fixtures'].setdefault('bucket prefix', 'test-' + client + '-{random}-')
keystone_users = cconfig.get('keystone users', {})
for section, user in users.items():
_config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
log.debug('Creating user {user} on {host}'.format(user=s3tests_conf[section]['user_id'], host=client))
_config_user(conf, section, '{user}.{client}'.format(user=user, client=client))
# for keystone users, read ec2 credentials into s3tests.conf instead
# of creating a local user
keystone_user = keystone_users.get(section)
if keystone_user:
project_name = keystone_user.pop('project')
creds = ctx.keystone.read_ec2_credentials(ctx, **keystone_user)
access = creds['Access']
secret = creds['Secret']
project_id = creds['Project ID']
conf[section]['access_key'] = access
conf[section]['secret_key'] = secret
conf[section]['user_id'] = project_id
conf[section]['display_name'] = project_name
log.debug('Using keystone user {kuser} credentials ({access} : {secret}) for {pname}:{pid} on {host}'.format(
kuser=keystone_user['user'], access=access, secret=secret,
pname=project_name, pid=project_id, host=client))
continue
log.debug('Creating user {user} on {host}'.format(user=conf[section]['user_id'], host=client))
cluster_name, daemon_type, client_id = teuthology.split_role(client)
client_with_id = daemon_type + '.' + client_id
# create user
@ -117,12 +139,12 @@ def create_users(ctx, config):
'radosgw-admin',
'-n', client_with_id,
'user', 'create',
'--uid', s3tests_conf[section]['user_id'],
'--display-name', s3tests_conf[section]['display_name'],
'--email', s3tests_conf[section]['email'],
'--uid', conf[section]['user_id'],
'--display-name', conf[section]['display_name'],
'--email', conf[section]['email'],
'--caps', 'user-policy=*',
'--access-key', s3tests_conf[section]['access_key'],
'--secret', s3tests_conf[section]['secret_key'],
'--access-key', conf[section]['access_key'],
'--secret', conf[section]['secret_key'],
'--cluster', cluster_name,
],
)
@ -136,10 +158,10 @@ def create_users(ctx, config):
'radosgw-admin',
'-n', client_with_id,
'mfa', 'create',
'--uid', s3tests_conf[section]['user_id'],
'--totp-serial', s3tests_conf[section]['totp_serial'],
'--totp-seed', s3tests_conf[section]['totp_seed'],
'--totp-seconds', s3tests_conf[section]['totp_seconds'],
'--uid', conf[section]['user_id'],
'--totp-serial', conf[section]['totp_serial'],
'--totp-seed', conf[section]['totp_seed'],
'--totp-seconds', conf[section]['totp_seconds'],
'--totp-window', '8',
'--totp-seed-type', 'base32',
'--cluster', cluster_name,
@ -156,7 +178,7 @@ def create_users(ctx, config):
'radosgw-admin',
'-n', client_with_id,
'caps', 'add',
'--uid', s3tests_conf[section]['user_id'],
'--uid', conf[section]['user_id'],
'--caps', 'roles=*',
'--cluster', cluster_name,
],
@ -169,27 +191,30 @@ def create_users(ctx, config):
'radosgw-admin',
'-n', client_with_id,
'caps', 'add',
'--uid', s3tests_conf[section]['user_id'],
'--uid', conf[section]['user_id'],
'--caps', 'oidc-provider=*',
'--cluster', cluster_name,
],
)
if "TOKEN" in os.environ:
s3tests_conf.setdefault('webidentity', {})
s3tests_conf['webidentity'].setdefault('token',os.environ['TOKEN'])
s3tests_conf['webidentity'].setdefault('aud',os.environ['AUD'])
s3tests_conf['webidentity'].setdefault('sub',os.environ['SUB'])
s3tests_conf['webidentity'].setdefault('azp',os.environ['AZP'])
s3tests_conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
s3tests_conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
s3tests_conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
conf.setdefault('webidentity', {})
conf['webidentity'].setdefault('token',os.environ['TOKEN'])
conf['webidentity'].setdefault('aud',os.environ['AUD'])
conf['webidentity'].setdefault('sub',os.environ['SUB'])
conf['webidentity'].setdefault('azp',os.environ['AZP'])
conf['webidentity'].setdefault('user_token',os.environ['USER_TOKEN'])
conf['webidentity'].setdefault('thumbprint',os.environ['THUMBPRINT'])
conf['webidentity'].setdefault('KC_REALM',os.environ['KC_REALM'])
try:
yield
finally:
for client in config['clients']:
for client in config.keys():
for user in users.values():
# don't need to delete keystone users
if not user in keystone_users:
continue
uid = '{user}.{client}'.format(user=user, client=client)
cluster_name, daemon_type, client_id = teuthology.split_role(client)
client_with_id = daemon_type + '.' + client_id
@ -502,6 +527,31 @@ def task(ctx, config):
cloudtier_tests: True
rgw_server: client.0
To test against Keystone users with EC2 credentials::
tasks:
- ceph:
- rgw: [client.0 client.1]
- keystone:
client.0:
projects:
- name: myproject
description: my project
users:
- name: myuser
password: SECRET
project: myproject
ec2 credentials:
- project: myproject
user: myuser
- s3tests:
client.0:
keystone users:
s3 main:
client: client.0
project: myproject
user: myuser
"""
assert hasattr(ctx, 'rgw'), 's3tests must run after the rgw task'
assert hasattr(ctx, 'tox'), 's3tests must run after the tox task'
@ -638,10 +688,7 @@ def task(ctx, config):
with contextutil.nested(
lambda: download(ctx=ctx, config=config),
lambda: create_users(ctx=ctx, config=dict(
clients=clients,
s3tests_conf=s3tests_conf,
)),
lambda: create_users(ctx=ctx, config=config, s3tests_conf=s3tests_conf),
lambda: configure(ctx=ctx, config=dict(
clients=config,
s3tests_conf=s3tests_conf,