diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 41fc84c19..f792cddb0 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -37,7 +37,7 @@ def setup_creds(env: RelengEnvironment): key = keys.get_ephemeral_key(env) $AWS_SECRET_ACCESS_KEY = key.secret_key $AWS_ACCESS_KEY_ID = key.id - $AWS_DEFAULT_REGION = 'garage' + $AWS_DEFAULT_REGION = env.s3_region $AWS_ENDPOINT_URL = env.s3_endpoint diff --git a/releng/environment.py b/releng/environment.py index b3bcb03d3..97a2b15ca 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -5,11 +5,12 @@ import functools import subprocess import dataclasses -S3_HOST = 's3.lix.systems' +S3_HOST = 's3-admin.afnix.fr' +S3_USER = 'lix-releng' DEFAULT_STORE_URI_BITS = { - 'region': 'garage', - 'endpoint': 's3.lix.systems', + 'region': 'global', + 'endpoint': 's3.afnix.fr', 'want-mass-query': 'true', 'write-nar-listing': 'true', 'ls-compression': 'zstd', @@ -54,7 +55,9 @@ class RelengEnvironment: git_repo: Callable[[], str] git_repo_is_gerrit: bool s3_endpoint: str + s3_region: str s3_ssh_host: str | None + s3_ssh_user: str | None docker_targets: list[DockerTarget] @@ -86,7 +89,9 @@ LOCAL = RelengEnvironment( git_repo_is_gerrit=False, docker_targets=[], s3_endpoint = 'http://localhost:3900', + s3_region = 'garage', s3_ssh_host = None, + s3_ssh_user = None, ) @@ -107,7 +112,9 @@ STAGING = RelengEnvironment( tags=['{version}', '{major}']), ], s3_endpoint = 'https://s3.lix.systems', + s3_region = 'garage', s3_ssh_host = S3_HOST, + s3_ssh_user = S3_USER, ) GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$') @@ -127,13 +134,13 @@ def guess_gerrit_remote(): PROD = RelengEnvironment( name='production', colour=functools.partial(sgr, RED), - docs_bucket='s3://docs', - cache_bucket='s3://cache', + docs_bucket='s3://docs.lix.systems', + cache_bucket='s3://cache.lix.systems', # FIXME: we should decrypt this with age into a tempdir in the future, but # the issue is how to deal with the recipients file. For now, we should # just delete it after doing a release. cache_store_overlay={'secret-key': 'prod.key'}, - releases_bucket='s3://releases', + releases_bucket='s3://releases.lix.systems', git_repo=guess_gerrit_remote, git_repo_is_gerrit=True, docker_targets=[ @@ -142,8 +149,10 @@ PROD = RelengEnvironment( tags=['{version}', '{major}']), DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']), ], - s3_endpoint = 'https://s3.lix.systems', + s3_endpoint = 'https://s3.afnix.fr', + s3_region = 'global', s3_ssh_host = S3_HOST, + s3_ssh_user = S3_USER, ) ENVIRONMENTS = { diff --git a/releng/garage-ephemeral-key/garage-ephemeral-key.py b/releng/garage-ephemeral-key/garage-ephemeral-key.py index 6848e1f99..141c1f51c 100644 --- a/releng/garage-ephemeral-key/garage-ephemeral-key.py +++ b/releng/garage-ephemeral-key/garage-ephemeral-key.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: 2024 Jade Lovelace +# SPDX-FileCopyrightText: 2026 Yureka Lilian # SPDX-License-Identifier: MIT import argparse import json import sys import datetime -import dataclasses import re -from typing import Any, Literal, Optional +from typing import Any import requests import os import logging @@ -14,27 +14,34 @@ import logging log = logging.getLogger(__name__) log.setLevel(logging.INFO) -fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', - datefmt='%b %d %H:%M:%S', - style='{') +fmt = logging.Formatter( + "{asctime} {levelname} {name}: {message}", + datefmt="%b %d %H:%M:%S", + style="{", +) if not any(isinstance(h, logging.StreamHandler) for h in log.handlers): hand = logging.StreamHandler() hand.setFormatter(fmt) log.addHandler(hand) -API_BASE = os.environ.get('GARAGE_ADMIN_API_BASE', 'http://localhost:3903') -API_KEY = os.environ['GARAGE_ADMIN_TOKEN'] +API_BASE = os.environ.get("GARAGE_ADMIN_API_BASE", "http://localhost:3903") +API_KEY = os.environ["GARAGE_ADMIN_TOKEN"] + +BUCKET_REGEX_STR = os.environ.get("BUCKET_REGEX", ".*") +BUCKET_REGEX = re.compile(BUCKET_REGEX_STR) def api(method, endpoint: str, resp_json=True, **kwargs) -> Any: - log.info('http %s %s', method, endpoint) - if not endpoint.startswith('https'): + log.info("http %s %s", method, endpoint) + if not endpoint.startswith("https"): endpoint = API_BASE + endpoint - resp = requests.request(method, - endpoint, - headers={'Authorization': f'Bearer {API_KEY}'}, - **kwargs) + resp = requests.request( + method, + endpoint, + headers={"Authorization": f"Bearer {API_KEY}"}, + **kwargs, + ) resp.raise_for_status() if resp_json: return resp.json() @@ -42,97 +49,64 @@ def api(method, endpoint: str, resp_json=True, **kwargs) -> Any: return resp -@dataclasses.dataclass -class Key: - name: str - id: str - secret_key: Optional[str] = None +def get_bucket_id(bucket_name: str) -> str: + resp: dict = api( + "GET", "/v2/GetBucketInfo", params={"globalAlias": bucket_name} + ) + return resp["id"] -@dataclasses.dataclass -class Bucket: - id: str - - -def keys() -> list[Key]: - data: list[dict] = api('GET', '/v1/key?list') - return [Key(name=k['name'], id=k['id']) for k in data] - - -def delete_key(key: Key): - api('DELETE', '/v1/key', resp_json=False, params={'id': key.id}) - - -def create_key(name: str) -> Key: - resp: dict = api('POST', '/v1/key', json={'name': name}) - return Key(name=resp['name'], - id=resp['accessKeyId'], - secret_key=resp['secretAccessKey']) - - -AccessType = Literal['read'] | Literal['write'] | Literal['owner'] - - -def get_bucket(bucket_name: str) -> Bucket: - resp: dict = api('GET', '/v1/bucket', params={'globalAlias': bucket_name}) - return Bucket(resp['id']) - - -def grant(bucket: Bucket, access_types: list[AccessType], key: Key): - access_types_dict = {k: True for k in access_types} - api('POST', - '/v1/bucket/allow', - json={ - 'bucketId': bucket.id, - 'accessKeyId': key.id, - 'permissions': access_types_dict, - }) - - -KEY_RE = re.compile(r'^.*ephemeral-(\d{14})$') -DATEFMT = '%Y%m%d%H%M%S' - - -def expired_keys(older_than: datetime.datetime) -> list[Key]: - ret = [] - for key in keys(): - if m := KEY_RE.match(key.name): - date = datetime.datetime.strptime(m.group(1), DATEFMT) - date = date.astimezone(datetime.UTC) - print(date) - if date < older_than: - ret.append(key) - return ret +DATEFMT = "%Y%m%d%H%M%S" def do_new(args): - buckets = [get_bucket(b) for b in args.buckets] + for b in args.buckets: + if not BUCKET_REGEX.match(b): + print(f"Bucket {b} not in allowed buckeds '{BUCKET_REGEX_STR}'") + exit(1) + bucket_ids = [get_bucket_id(b) for b in args.buckets] - def optional(s: str, whether) -> list[str]: - if whether: - return [s] - else: - return [] + key_name = args.name + "-" if args.name else "" + expiration = datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta( + seconds=args.age_secs + ) + key_name += "ephemeral-" + expiration.strftime(DATEFMT) - access_types: list[AccessType] = optional('read', args.read) + optional( - 'write', args.write) + optional('owner', args.owner) # type: ignore + key_resp: dict = api( + "POST", + "/v2/CreateKey", + json={ + "name": key_name, + "expiration": expiration.isoformat(), + "neverExpires": False, + }, + ) - key_name = args.name + '-' if args.name else '' - key_name += "ephemeral-" + ( - datetime.datetime.now(tz=datetime.UTC) + - datetime.timedelta(seconds=args.age_secs)).strftime(DATEFMT) + for b in bucket_ids: + api( + "POST", + "/v2/AllowBucketKey", + json={ + "accessKeyId": key_resp["accessKeyId"], + "bucketId": b, + "permissions": { + "read": args.read, + "write": args.write, + "owner": args.owner, + }, + }, + ) - k = create_key(key_name) - for b in buckets: - grant(b, access_types, k) - - print(json.dumps(dataclasses.asdict(k), indent=2)) - - -def do_clean(args): - older_than = datetime.datetime.now(tz=datetime.UTC) - for key in expired_keys(older_than): - delete_key(key) + print( + json.dumps( + { + "name": key_resp["name"], + "id": key_resp["accessKeyId"], + "secret_key": key_resp["secretAccessKey"], + }, + indent=2, + ) + ) def main(): @@ -148,28 +122,27 @@ def main(): new = sps.add_parser("new", help="Make an ephemeral key") new.add_argument("--name", help="Name prefix for the key") - new.add_argument("--read", - action="store_true", - help="Grant read access to buckets") - new.add_argument("--write", - action="store_true", - help="Grant write access to buckets") - new.add_argument("--owner", - action="store_true", - help="Grant owner access to buckets") - new.add_argument("--age-secs", - type=int, - required=True, - help="Maximum key lifetime in seconds") - new.add_argument("buckets", nargs='*', help="Buckets to grant access to") + new.add_argument( + "--read", action="store_true", help="Grant read access to buckets" + ) + new.add_argument( + "--write", action="store_true", help="Grant write access to buckets" + ) + new.add_argument( + "--owner", action="store_true", help="Grant owner access to buckets" + ) + new.add_argument( + "--age-secs", + type=int, + required=True, + help="Maximum key lifetime in seconds", + ) + new.add_argument("buckets", nargs="*", help="Buckets to grant access to") new.set_defaults(cmd=do_new) - clean = sps.add_parser("clean", help="Clean up old keys") - clean.set_defaults(cmd=do_clean) - args = ap.parse_args() args.cmd(args) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/releng/keys.py b/releng/keys.py index 158aac750..6e285121c 100644 --- a/releng/keys.py +++ b/releng/keys.py @@ -14,7 +14,7 @@ def get_ephemeral_key( env.docs_bucket.removeprefix('s3://'), ] if env.s3_ssh_host is not None: - command = ['ssh', '-l', 'root', env.s3_ssh_host, *command] + command = ['ssh', '-l', env.s3_ssh_user, *command] output = subprocess.check_output(command) d = json.loads(output.decode()) return environment.S3Credentials(name=d['name'],