guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

releng: Adapt for AFNix S3

Browse Source 
Change-Id: I29dbd62dcc70595ba3f2ac2a466a5c26a28aea99
(cherry picked from commit 0c63036c7d)
This commit is contained in:
Yureka
2026-05-04 19:01:44 +02:00
committed by Raito Bezarius
parent 5d6134064e
commit 7e56afa0b1
4 changed files with 104 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
releng/create_release.xsh
+1 -1
View File
@@ -37,7 +37,7 @@ def setup_creds(env: RelengEnvironment):
key = keys.get_ephemeral_key(env)
$AWS_SECRET_ACCESS_KEY = key.secret_key
$AWS_ACCESS_KEY_ID = key.id
$AWS_DEFAULT_REGION = 'garage'
$AWS_DEFAULT_REGION = env.s3_region
$AWS_ENDPOINT_URL = env.s3_endpoint
releng/environment.py
+16 -7
View File
@@ -5,11 +5,12 @@ import functools
import subprocess
import dataclasses
S3_HOST = 's3.lix.systems'
S3_HOST = 's3-admin.afnix.fr'
S3_USER = 'lix-releng'
DEFAULT_STORE_URI_BITS = {
'region': 'garage',
'endpoint': 's3.lix.systems',
'region': 'global',
'endpoint': 's3.afnix.fr',
'want-mass-query': 'true',
'write-nar-listing': 'true',
'ls-compression': 'zstd',
@@ -54,7 +55,9 @@ class RelengEnvironment:
git_repo: Callable[[], str]
git_repo_is_gerrit: bool
s3_endpoint: str
s3_region: str
s3_ssh_host: str | None
s3_ssh_user: str | None
docker_targets: list[DockerTarget]
@@ -86,7 +89,9 @@ LOCAL = RelengEnvironment(
git_repo_is_gerrit=False,
docker_targets=[],
s3_endpoint = 'http://localhost:3900',
s3_region = 'garage',
s3_ssh_host = None,
s3_ssh_user = None,
)
@@ -107,7 +112,9 @@ STAGING = RelengEnvironment(
tags=['{version}', '{major}']),
],
s3_endpoint = 'https://s3.lix.systems',
s3_region = 'garage',
s3_ssh_host = S3_HOST,
s3_ssh_user = S3_USER,
)
GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$')
@@ -127,13 +134,13 @@ def guess_gerrit_remote():
PROD = RelengEnvironment(
name='production',
colour=functools.partial(sgr, RED),
docs_bucket='s3://docs',
cache_bucket='s3://cache',
docs_bucket='s3://docs.lix.systems',
cache_bucket='s3://cache.lix.systems',
# FIXME: we should decrypt this with age into a tempdir in the future, but
# the issue is how to deal with the recipients file. For now, we should
# just delete it after doing a release.
cache_store_overlay={'secret-key': 'prod.key'},
releases_bucket='s3://releases',
releases_bucket='s3://releases.lix.systems',
git_repo=guess_gerrit_remote,
git_repo_is_gerrit=True,
docker_targets=[
@@ -142,8 +149,10 @@ PROD = RelengEnvironment(
tags=['{version}', '{major}']),
DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']),
],
s3_endpoint = 'https://s3.lix.systems',
s3_endpoint = 'https://s3.afnix.fr',
s3_region = 'global',
s3_ssh_host = S3_HOST,
s3_ssh_user = S3_USER,
)
ENVIRONMENTS = {
releng/garage-ephemeral-key/garage-ephemeral-key.py
+83 -110
View File
@@ -1,12 +1,12 @@
# SPDX-FileCopyrightText: 2024 Jade Lovelace
# SPDX-FileCopyrightText: 2026 Yureka Lilian <yureka@cyberchaos.dev>
# SPDX-License-Identifier: MIT
import argparse
import json
import sys
import datetime
import dataclasses
import re
from typing import Any, Literal, Optional
from typing import Any
import requests
import os
import logging
@@ -14,27 +14,34 @@ import logging
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
fmt = logging.Formatter('{asctime} {levelname} {name}: {message}',
datefmt='%b %d %H:%M:%S',
style='{')
fmt = logging.Formatter(
"{asctime} {levelname} {name}: {message}",
datefmt="%b %d %H:%M:%S",
style="{",
)
if not any(isinstance(h, logging.StreamHandler) for h in log.handlers):
hand = logging.StreamHandler()
hand.setFormatter(fmt)
log.addHandler(hand)
API_BASE = os.environ.get('GARAGE_ADMIN_API_BASE', 'http://localhost:3903')
API_KEY = os.environ['GARAGE_ADMIN_TOKEN']
API_BASE = os.environ.get("GARAGE_ADMIN_API_BASE", "http://localhost:3903")
API_KEY = os.environ["GARAGE_ADMIN_TOKEN"]
BUCKET_REGEX_STR = os.environ.get("BUCKET_REGEX", ".*")
BUCKET_REGEX = re.compile(BUCKET_REGEX_STR)
def api(method, endpoint: str, resp_json=True, **kwargs) -> Any:
log.info('http %s %s', method, endpoint)
if not endpoint.startswith('https'):
log.info("http %s %s", method, endpoint)
if not endpoint.startswith("https"):
endpoint = API_BASE + endpoint
resp = requests.request(method,
resp = requests.request(
method,
endpoint,
headers={'Authorization': f'Bearer {API_KEY}'},
**kwargs)
headers={"Authorization": f"Bearer {API_KEY}"},
**kwargs,
)
resp.raise_for_status()
if resp_json:
return resp.json()
@@ -42,97 +49,64 @@ def api(method, endpoint: str, resp_json=True, **kwargs) -> Any:
return resp
@dataclasses.dataclass
class Key:
name: str
id: str
secret_key: Optional[str] = None
def get_bucket_id(bucket_name: str) -> str:
resp: dict = api(
"GET", "/v2/GetBucketInfo", params={"globalAlias": bucket_name}
)
return resp["id"]
@dataclasses.dataclass
class Bucket:
id: str
def keys() -> list[Key]:
data: list[dict] = api('GET', '/v1/key?list')
return [Key(name=k['name'], id=k['id']) for k in data]
def delete_key(key: Key):
api('DELETE', '/v1/key', resp_json=False, params={'id': key.id})
def create_key(name: str) -> Key:
resp: dict = api('POST', '/v1/key', json={'name': name})
return Key(name=resp['name'],
id=resp['accessKeyId'],
secret_key=resp['secretAccessKey'])
AccessType = Literal['read'] | Literal['write'] | Literal['owner']
def get_bucket(bucket_name: str) -> Bucket:
resp: dict = api('GET', '/v1/bucket', params={'globalAlias': bucket_name})
return Bucket(resp['id'])
def grant(bucket: Bucket, access_types: list[AccessType], key: Key):
access_types_dict = {k: True for k in access_types}
api('POST',
'/v1/bucket/allow',
json={
'bucketId': bucket.id,
'accessKeyId': key.id,
'permissions': access_types_dict,
})
KEY_RE = re.compile(r'^.*ephemeral-(\d{14})$')
DATEFMT = '%Y%m%d%H%M%S'
def expired_keys(older_than: datetime.datetime) -> list[Key]:
ret = []
for key in keys():
if m := KEY_RE.match(key.name):
date = datetime.datetime.strptime(m.group(1), DATEFMT)
date = date.astimezone(datetime.UTC)
print(date)
if date < older_than:
ret.append(key)
return ret
DATEFMT = "%Y%m%d%H%M%S"
def do_new(args):
buckets = [get_bucket(b) for b in args.buckets]
for b in args.buckets:
if not BUCKET_REGEX.match(b):
print(f"Bucket {b} not in allowed buckeds '{BUCKET_REGEX_STR}'")
exit(1)
bucket_ids = [get_bucket_id(b) for b in args.buckets]
def optional(s: str, whether) -> list[str]:
if whether:
return [s]
else:
return []
key_name = args.name + "-" if args.name else ""
expiration = datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(
seconds=args.age_secs
)
key_name += "ephemeral-" + expiration.strftime(DATEFMT)
access_types: list[AccessType] = optional('read', args.read) + optional(
'write', args.write) + optional('owner', args.owner) # type: ignore
key_resp: dict = api(
"POST",
"/v2/CreateKey",
json={
"name": key_name,
"expiration": expiration.isoformat(),
"neverExpires": False,
},
)
key_name = args.name + '-' if args.name else ''
key_name += "ephemeral-" + (
datetime.datetime.now(tz=datetime.UTC) +
datetime.timedelta(seconds=args.age_secs)).strftime(DATEFMT)
for b in bucket_ids:
api(
"POST",
"/v2/AllowBucketKey",
json={
"accessKeyId": key_resp["accessKeyId"],
"bucketId": b,
"permissions": {
"read": args.read,
"write": args.write,
"owner": args.owner,
},
},
)
k = create_key(key_name)
for b in buckets:
grant(b, access_types, k)
print(json.dumps(dataclasses.asdict(k), indent=2))
def do_clean(args):
older_than = datetime.datetime.now(tz=datetime.UTC)
for key in expired_keys(older_than):
delete_key(key)
print(
json.dumps(
{
"name": key_resp["name"],
"id": key_resp["accessKeyId"],
"secret_key": key_resp["secretAccessKey"],
},
indent=2,
)
)
def main():
@@ -148,28 +122,27 @@ def main():
new = sps.add_parser("new", help="Make an ephemeral key")
new.add_argument("--name", help="Name prefix for the key")
new.add_argument("--read",
action="store_true",
help="Grant read access to buckets")
new.add_argument("--write",
action="store_true",
help="Grant write access to buckets")
new.add_argument("--owner",
action="store_true",
help="Grant owner access to buckets")
new.add_argument("--age-secs",
new.add_argument(
"--read", action="store_true", help="Grant read access to buckets"
)
new.add_argument(
"--write", action="store_true", help="Grant write access to buckets"
)
new.add_argument(
"--owner", action="store_true", help="Grant owner access to buckets"
)
new.add_argument(
"--age-secs",
type=int,
required=True,
help="Maximum key lifetime in seconds")
new.add_argument("buckets", nargs='*', help="Buckets to grant access to")
help="Maximum key lifetime in seconds",
)
new.add_argument("buckets", nargs="*", help="Buckets to grant access to")
new.set_defaults(cmd=do_new)
clean = sps.add_parser("clean", help="Clean up old keys")
clean.set_defaults(cmd=do_clean)
args = ap.parse_args()
args.cmd(args)
if __name__ == '__main__':
if __name__ == "__main__":
main()
releng/keys.py
+1 -1
View File
@@ -14,7 +14,7 @@ def get_ephemeral_key(
env.docs_bucket.removeprefix('s3://'),
]
if env.s3_ssh_host is not None:
command = ['ssh', '-l', 'root', env.s3_ssh_host, *command]
command = ['ssh', '-l', env.s3_ssh_user, *command]
output = subprocess.check_output(command)
d = json.loads(output.decode())
return environment.S3Credentials(name=d['name'],