From 3f811a415b76a77ee2cf4240fe898e0f7955c38c Mon Sep 17 00:00:00 2001 From: KFearsoff Date: Fri, 11 Apr 2025 23:29:47 +0400 Subject: [PATCH] releng: add local environment This takes the first baby steps towards resolving #765. This first test/workflow isn't the most useful thing ever, because it doesn't test the code path for building the manual. Still, it does a decent job at testing the basic git workflow, and the upload functionality. Change-Id: I16dd3a39addd6308ad3eb37c2e3dc3466584a4e3 --- flake.nix | 165 +++++++++-------- releng/create_release.xsh | 10 +- releng/environment.py | 31 +++- releng/garage-ephemeral-key/default.nix | 12 ++ .../garage-ephemeral-key.py | 175 ++++++++++++++++++ releng/keys.py | 13 +- releng/local/default.nix | 101 ++++++++++ tests/nixos/default.nix | 5 +- 8 files changed, 419 insertions(+), 93 deletions(-) create mode 100644 releng/garage-ephemeral-key/default.nix create mode 100644 releng/garage-ephemeral-key/garage-ephemeral-key.py create mode 100644 releng/local/default.nix diff --git a/flake.nix b/flake.nix index 78642029a..6fb909fa7 100644 --- a/flake.nix +++ b/flake.nix @@ -299,89 +299,98 @@ }); # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs); + tests = + import ./tests/nixos { + inherit + self + lib + nixpkgs + nixpkgsFor + ; + } + // { + nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs); - # This is x86_64-linux only, just because we have significantly - # cheaper x86_64-linux compute in CI. - # It is clangStdenv because clang's sanitizers are nicer. - asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override { - # Improve caching of non-code changes by not changing the - # derivation name every single time, since this will never be seen - # by users anyway. - versionSuffix = ""; - sanitize = [ - "address" - "undefined" - ]; - # it is very hard to make *every* CI build use this option such - # that we don't wind up building Lix twice, so we do it here where - # we are already doing so. - werror = true; - }; - - # Although this might be nicer to do with pre-commit, that would - # require adding 12MB of nodejs to the dev shell, whereas building it - # in CI with Nix avoids that at a cost of slower feedback on rarely - # touched files. - jsSyntaxCheck = - let - nixpkgs = nixpkgsFor.x86_64-linux.native; - inherit (nixpkgs) pkgs; - docSources = lib.fileset.toSource { - root = ./doc; - fileset = lib.fileset.fileFilter (f: f.hasExt "js") ./doc; - }; - in - pkgs.runCommand "js-syntax-check" { } '' - find ${docSources} -type f -print -exec ${pkgs.nodejs-slim}/bin/node --check '{}' ';' - touch $out - ''; - - # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin - clang-tidy = - let - nixpkgs = nixpkgsFor.x86_64-linux.native; - inherit (nixpkgs) pkgs; - in - pkgs.callPackage ./package.nix { - # Required since we don't support gcc stdenv - stdenv = pkgs.clangStdenv; + # This is x86_64-linux only, just because we have significantly + # cheaper x86_64-linux compute in CI. + # It is clangStdenv because clang's sanitizers are nicer. + asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override { + # Improve caching of non-code changes by not changing the + # derivation name every single time, since this will never be seen + # by users anyway. versionSuffix = ""; - lintInsteadOfBuild = true; + sanitize = [ + "address" + "undefined" + ]; + # it is very hard to make *every* CI build use this option such + # that we don't wind up building Lix twice, so we do it here where + # we are already doing so. + werror = true; }; - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; - runCommand "eval-nixos" { buildInputs = [ nix ]; } '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = 402242fca90874112b34718b8199d844e8b03d12 ]] - mkdir $out - ''; + # Although this might be nicer to do with pre-commit, that would + # require adding 12MB of nodejs to the dev shell, whereas building it + # in CI with Nix avoids that at a cost of slower feedback on rarely + # touched files. + jsSyntaxCheck = + let + nixpkgs = nixpkgsFor.x86_64-linux.native; + inherit (nixpkgs) pkgs; + docSources = lib.fileset.toSource { + root = ./doc; + fileset = lib.fileset.fileFilter (f: f.hasExt "js") ./doc; + }; + in + pkgs.runCommand "js-syntax-check" { } '' + find ${docSources} -type f -print -exec ${pkgs.nodejs-slim}/bin/node --check '{}' ';' + touch $out + ''; - nixpkgsLibTests = forAllSystems ( - system: - let - inherit (self.packages.${system}) nix; - pkgs = nixpkgsFor.${system}.native; - testWithNix = import (nixpkgs + "/lib/tests/test-with-nix.nix") { inherit pkgs lib nix; }; - in - pkgs.symlinkJoin { - name = "nixpkgs-lib-tests"; - paths = - [ testWithNix ] - # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override - # the evaluator used for the test. it will break again in the future, don't worry. - ++ lib.optionals pkgs.stdenv.isLinux [ - (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset - ]; - } - ); - }; + # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin + clang-tidy = + let + nixpkgs = nixpkgsFor.x86_64-linux.native; + inherit (nixpkgs) pkgs; + in + pkgs.callPackage ./package.nix { + # Required since we don't support gcc stdenv + stdenv = pkgs.clangStdenv; + versionSuffix = ""; + lintInsteadOfBuild = true; + }; + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + with nixpkgsFor.x86_64-linux.native; + runCommand "eval-nixos" { buildInputs = [ nix ]; } '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = 402242fca90874112b34718b8199d844e8b03d12 ]] + mkdir $out + ''; + + nixpkgsLibTests = forAllSystems ( + system: + let + inherit (self.packages.${system}) nix; + pkgs = nixpkgsFor.${system}.native; + testWithNix = import (nixpkgs + "/lib/tests/test-with-nix.nix") { inherit pkgs lib nix; }; + in + pkgs.symlinkJoin { + name = "nixpkgs-lib-tests"; + paths = + [ testWithNix ] + # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override + # the evaluator used for the test. it will break again in the future, don't worry. + ++ lib.optionals pkgs.stdenv.isLinux [ + (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset + ]; + } + ); + }; pre-commit = forAvailableSystems ( system: diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 34c29e337..0091a6728 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -41,7 +41,7 @@ def setup_creds(env: RelengEnvironment): $AWS_SECRET_ACCESS_KEY = key.secret_key $AWS_ACCESS_KEY_ID = key.id $AWS_DEFAULT_REGION = 'garage' - $AWS_ENDPOINT_URL = environment.S3_ENDPOINT + $AWS_ENDPOINT_URL = env.s3_endpoint def official_release_commit_tag(force_tag=False): @@ -255,13 +255,14 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False print('[+] git push to the repo') # We have to push the ref to gerrit for review at least such that the # commit is known, before we can push it as a tag. + repo = env.git_repo() if env.git_repo_is_gerrit: - git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}' + git push @(repo) f'{prev_branch}:refs/for/{prev_branch}' else: - git push @(env.git_repo) f'{prev_branch}:{prev_branch}' + git push @(repo) f'{prev_branch}:{prev_branch}' print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' + git push @(['-f'] if force_push_tag else []) @(repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): @@ -276,7 +277,6 @@ def build_manual(eval_result): (drv, manual) = next((x['drvPath'], x['outputs']['doc']) for x in eval_result if x['attr'] == 'build.x86_64-linux') print('[+] Building manual') realise([drv]) - cp --no-preserve=mode -T -vr @(manual)/share/doc/nix/manual @(MANUAL) diff --git a/releng/environment.py b/releng/environment.py index 2b6554ede..b3bcb03d3 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -6,7 +6,6 @@ import subprocess import dataclasses S3_HOST = 's3.lix.systems' -S3_ENDPOINT = 'https://s3.lix.systems' DEFAULT_STORE_URI_BITS = { 'region': 'garage', @@ -51,8 +50,11 @@ class RelengEnvironment: cache_bucket: str releases_bucket: str docs_bucket: str - git_repo: str + # We don't want to call functions that require computation (such as guess_gerrit_remote()) before they are needed + git_repo: Callable[[], str] git_repo_is_gerrit: bool + s3_endpoint: str + s3_ssh_host: str | None docker_targets: list[DockerTarget] @@ -65,6 +67,7 @@ class RelengEnvironment: SGR = '\x1b[' RED = '31;1m' GREEN = '32;1m' +BLUE = '34;1m' RESET = '0m' @@ -72,6 +75,21 @@ def sgr(colour: str, text: str) -> str: return f'{SGR}{colour}{text}{SGR}{RESET}' +LOCAL = RelengEnvironment( + name='local', + colour=functools.partial(sgr, BLUE), + docs_bucket='s3://local-docs', + cache_bucket='s3://local-cache', + cache_store_overlay={'secret-key': 'local.key', 'endpoint': 'localhost:3900', 'scheme': 'http'}, + releases_bucket='s3://local-releases', + git_repo=lambda: '../releng-target', + git_repo_is_gerrit=False, + docker_targets=[], + s3_endpoint = 'http://localhost:3900', + s3_ssh_host = None, +) + + STAGING = RelengEnvironment( name='staging', colour=functools.partial(sgr, GREEN), @@ -79,7 +97,7 @@ STAGING = RelengEnvironment( cache_bucket='s3://staging-cache', cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', - git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + git_repo=lambda: 'ssh://git@git.lix.systems/lix-project/lix-releng-staging', git_repo_is_gerrit=False, docker_targets=[ # latest will be auto tagged if appropriate @@ -88,6 +106,8 @@ STAGING = RelengEnvironment( DockerTarget('ghcr.io/lix-project/lix-releng-staging', tags=['{version}', '{major}']), ], + s3_endpoint = 'https://s3.lix.systems', + s3_ssh_host = S3_HOST, ) GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$') @@ -114,7 +134,7 @@ PROD = RelengEnvironment( # just delete it after doing a release. cache_store_overlay={'secret-key': 'prod.key'}, releases_bucket='s3://releases', - git_repo=guess_gerrit_remote(), + git_repo=guess_gerrit_remote, git_repo_is_gerrit=True, docker_targets=[ # latest will be auto tagged if appropriate @@ -122,9 +142,12 @@ PROD = RelengEnvironment( tags=['{version}', '{major}']), DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']), ], + s3_endpoint = 'https://s3.lix.systems', + s3_ssh_host = S3_HOST, ) ENVIRONMENTS = { + 'local': LOCAL, 'staging': STAGING, 'production': PROD, } diff --git a/releng/garage-ephemeral-key/default.nix b/releng/garage-ephemeral-key/default.nix new file mode 100644 index 000000000..4b63823af --- /dev/null +++ b/releng/garage-ephemeral-key/default.nix @@ -0,0 +1,12 @@ +{ + lib, + writePython3Bin, + python3Packages, +}: + +let + package = writePython3Bin "garage-ephemeral-key" { libraries = [ python3Packages.requests ]; } ( + builtins.readFile ./garage-ephemeral-key.py + ); +in +package diff --git a/releng/garage-ephemeral-key/garage-ephemeral-key.py b/releng/garage-ephemeral-key/garage-ephemeral-key.py new file mode 100644 index 000000000..6848e1f99 --- /dev/null +++ b/releng/garage-ephemeral-key/garage-ephemeral-key.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: 2024 Jade Lovelace +# SPDX-License-Identifier: MIT +import argparse +import json +import sys +import datetime +import dataclasses +import re +from typing import Any, Literal, Optional +import requests +import os +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + +fmt = logging.Formatter('{asctime} {levelname} {name}: {message}', + datefmt='%b %d %H:%M:%S', + style='{') + +if not any(isinstance(h, logging.StreamHandler) for h in log.handlers): + hand = logging.StreamHandler() + hand.setFormatter(fmt) + log.addHandler(hand) + +API_BASE = os.environ.get('GARAGE_ADMIN_API_BASE', 'http://localhost:3903') +API_KEY = os.environ['GARAGE_ADMIN_TOKEN'] + + +def api(method, endpoint: str, resp_json=True, **kwargs) -> Any: + log.info('http %s %s', method, endpoint) + if not endpoint.startswith('https'): + endpoint = API_BASE + endpoint + resp = requests.request(method, + endpoint, + headers={'Authorization': f'Bearer {API_KEY}'}, + **kwargs) + resp.raise_for_status() + if resp_json: + return resp.json() + else: + return resp + + +@dataclasses.dataclass +class Key: + name: str + id: str + secret_key: Optional[str] = None + + +@dataclasses.dataclass +class Bucket: + id: str + + +def keys() -> list[Key]: + data: list[dict] = api('GET', '/v1/key?list') + return [Key(name=k['name'], id=k['id']) for k in data] + + +def delete_key(key: Key): + api('DELETE', '/v1/key', resp_json=False, params={'id': key.id}) + + +def create_key(name: str) -> Key: + resp: dict = api('POST', '/v1/key', json={'name': name}) + return Key(name=resp['name'], + id=resp['accessKeyId'], + secret_key=resp['secretAccessKey']) + + +AccessType = Literal['read'] | Literal['write'] | Literal['owner'] + + +def get_bucket(bucket_name: str) -> Bucket: + resp: dict = api('GET', '/v1/bucket', params={'globalAlias': bucket_name}) + return Bucket(resp['id']) + + +def grant(bucket: Bucket, access_types: list[AccessType], key: Key): + access_types_dict = {k: True for k in access_types} + api('POST', + '/v1/bucket/allow', + json={ + 'bucketId': bucket.id, + 'accessKeyId': key.id, + 'permissions': access_types_dict, + }) + + +KEY_RE = re.compile(r'^.*ephemeral-(\d{14})$') +DATEFMT = '%Y%m%d%H%M%S' + + +def expired_keys(older_than: datetime.datetime) -> list[Key]: + ret = [] + for key in keys(): + if m := KEY_RE.match(key.name): + date = datetime.datetime.strptime(m.group(1), DATEFMT) + date = date.astimezone(datetime.UTC) + print(date) + if date < older_than: + ret.append(key) + return ret + + +def do_new(args): + buckets = [get_bucket(b) for b in args.buckets] + + def optional(s: str, whether) -> list[str]: + if whether: + return [s] + else: + return [] + + access_types: list[AccessType] = optional('read', args.read) + optional( + 'write', args.write) + optional('owner', args.owner) # type: ignore + + key_name = args.name + '-' if args.name else '' + key_name += "ephemeral-" + ( + datetime.datetime.now(tz=datetime.UTC) + + datetime.timedelta(seconds=args.age_secs)).strftime(DATEFMT) + + k = create_key(key_name) + for b in buckets: + grant(b, access_types, k) + + print(json.dumps(dataclasses.asdict(k), indent=2)) + + +def do_clean(args): + older_than = datetime.datetime.now(tz=datetime.UTC) + for key in expired_keys(older_than): + delete_key(key) + + +def main(): + ap = argparse.ArgumentParser(description="Garage ephemeral API keys tool") + + def fail(*args): + ap.print_help() + sys.exit(1) + + ap.set_defaults(cmd=fail) + + sps = ap.add_subparsers() + + new = sps.add_parser("new", help="Make an ephemeral key") + new.add_argument("--name", help="Name prefix for the key") + new.add_argument("--read", + action="store_true", + help="Grant read access to buckets") + new.add_argument("--write", + action="store_true", + help="Grant write access to buckets") + new.add_argument("--owner", + action="store_true", + help="Grant owner access to buckets") + new.add_argument("--age-secs", + type=int, + required=True, + help="Maximum key lifetime in seconds") + new.add_argument("buckets", nargs='*', help="Buckets to grant access to") + new.set_defaults(cmd=do_new) + + clean = sps.add_parser("clean", help="Clean up old keys") + clean.set_defaults(cmd=do_clean) + + args = ap.parse_args() + args.cmd(args) + + +if __name__ == '__main__': + main() diff --git a/releng/keys.py b/releng/keys.py index 14cff44dd..158aac750 100644 --- a/releng/keys.py +++ b/releng/keys.py @@ -5,14 +5,17 @@ from . import environment def get_ephemeral_key( env: environment.RelengEnvironment) -> environment.S3Credentials: - output = subprocess.check_output([ - 'ssh', '-l', 'root', environment.S3_HOST, 'garage-ephemeral-key', - 'new', '--name', f'releng-{env.name}', '--read', '--write', - '--age-secs', '3600', + output = None + command = [ + 'garage-ephemeral-key', 'new', '--name', f'releng-{env.name}', '--read', + '--write', '--age-secs', '3600', env.releases_bucket.removeprefix('s3://'), env.cache_bucket.removeprefix('s3://'), env.docs_bucket.removeprefix('s3://'), - ]) + ] + if env.s3_ssh_host is not None: + command = ['ssh', '-l', 'root', env.s3_ssh_host, *command] + output = subprocess.check_output(command) d = json.loads(output.decode()) return environment.S3Credentials(name=d['name'], id=d['id'], diff --git a/releng/local/default.nix b/releng/local/default.nix new file mode 100644 index 000000000..64c617a57 --- /dev/null +++ b/releng/local/default.nix @@ -0,0 +1,101 @@ +{ + self, + system, + lib, + config, + pkgs, + ... +}: +let + releng = ./..; + this-garage = pkgs.garage_1_x; + garage-ephemeral-key = pkgs.callPackage ../garage-ephemeral-key { + inherit (pkgs.writers) writePython3Bin; + }; + + build = self.release-jobs.all.build.${system}.doc; + + pythonPackages = ( + p: [ + p.yapf + p.requests + p.xdg-base-dirs + p.packaging + p.xonsh + ] + ); + pythonEnv = pkgs.python3.pythonOnBuildForHost.withPackages pythonPackages; +in +{ + name = "local-releng-manual-upload"; + + nodes = { + machine = + { config, pkgs, ... }: + { + services.garage.enable = true; + services.garage.package = this-garage; + services.garage.settings = { + replication_factor = 1; + + rpc_bind_addr = "[::]:3901"; + rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"; + + s3_api.api_bind_addr = "[::]:3900"; + s3_api.s3_region = "garage"; + s3_api.root_domain = ".localhost"; + + admin = { + api_bind_addr = "[::]:3903"; + admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4="; + }; + }; + + environment.systemPackages = [ + this-garage + pkgs.git + pkgs.build-release-notes + pkgs.jq + pkgs.nix-eval-jobs + pkgs.awscli2 + pythonEnv + garage-ephemeral-key + ]; + + environment.sessionVariables = { + GARAGE_ADMIN_TOKEN = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4="; + }; + }; + }; + testScript = '' + machine.wait_for_unit("garage") + machine.wait_for_open_port(3900) + + nodeId = machine.succeed("garage node id") + machine.succeed(f"garage layout assign -z dc1 -c 10G {nodeId}") + machine.succeed("garage layout apply --version 1") + machine.succeed("garage bucket create local-docs") + machine.succeed("garage bucket create local-cache") + machine.succeed("garage bucket create local-releases") + + machine.succeed("git init --bare releng-target") + + machine.succeed("mkdir repo") + machine.succeed("cd repo; git init --initial-branch release-2.92 .") + machine.succeed("cd repo; git config user.email 'local@test.com'") + machine.succeed("cd repo; git config user.name 'Test Releng'") + + machine.copy_from_host("${../../version.json}", "repo/version.json") + machine.copy_from_host("${releng}", "repo/releng") + machine.copy_from_host("${../../doc}", "repo/doc") + machine.succeed("cd repo; git add .") + machine.succeed("cd repo; git commit -m 'initial commit'") + machine.succeed("cd repo; git switch -c releng/2.92.4") + + machine.succeed("cd repo; python3 -m releng prepare") + machine.succeed("cd repo; python3 -m releng tag") + machine.succeed("cd repo; mkdir -p release/manual") + machine.succeed("cd repo; cp --no-preserve=mode -T -vr ${build}/share/doc/nix/manual ./release/manual") + machine.succeed("cd repo; python3 -m releng upload --noconfirm --environment local --target manual") + ''; +} diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 2463ae5cc..8f8873a8b 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -1,4 +1,4 @@ -{ lib, nixpkgs, nixpkgsFor }: +{ self, lib, nixpkgs, nixpkgsFor }: let @@ -17,6 +17,7 @@ let }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; + _module.args.self = self; }) // { # allow running tests against older nix versions via `nix eval --apply` @@ -40,6 +41,8 @@ let in { + local-releng = runNixOSTestFor "x86_64-linux" ../../releng/local; + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix;