releng: add local environment

This takes the first baby steps towards resolving #765. This first test/workflow isn't the most useful thing ever, because it doesn't test the code path for building the manual. Still, it does a decent job at testing the basic git workflow, and the upload functionality. Change-Id: I16dd3a39addd6308ad3eb37c2e3dc3466584a4e3
2025-05-09 00:01:22 +00:00
parent 6b3162be9c
commit 3f811a415b
8 changed files with 419 additions and 93 deletions
@@ -299,89 +299,98 @@
          });

        # System tests.
-        tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
-          nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs);
+        tests =
+          import ./tests/nixos {
+            inherit
+              self
+              lib
+              nixpkgs
+              nixpkgsFor
+              ;
+          }
+          // {
+            nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs);

-          # This is x86_64-linux only, just because we have significantly
-          # cheaper x86_64-linux compute in CI.
-          # It is clangStdenv because clang's sanitizers are nicer.
-          asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override {
-            # Improve caching of non-code changes by not changing the
-            # derivation name every single time, since this will never be seen
-            # by users anyway.
-            versionSuffix = "";
-            sanitize = [
-              "address"
-              "undefined"
-            ];
-            # it is very hard to make *every* CI build use this option such
-            # that we don't wind up building Lix twice, so we do it here where
-            # we are already doing so.
-            werror = true;
-          };
-
-          # Although this might be nicer to do with pre-commit, that would
-          # require adding 12MB of nodejs to the dev shell, whereas building it
-          # in CI with Nix avoids that at a cost of slower feedback on rarely
-          # touched files.
-          jsSyntaxCheck =
-            let
-              nixpkgs = nixpkgsFor.x86_64-linux.native;
-              inherit (nixpkgs) pkgs;
-              docSources = lib.fileset.toSource {
-                root = ./doc;
-                fileset = lib.fileset.fileFilter (f: f.hasExt "js") ./doc;
-              };
-            in
-            pkgs.runCommand "js-syntax-check" { } ''
-              find ${docSources} -type f -print -exec ${pkgs.nodejs-slim}/bin/node --check '{}' ';'
-              touch $out
-            '';
-
-          # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin
-          clang-tidy =
-            let
-              nixpkgs = nixpkgsFor.x86_64-linux.native;
-              inherit (nixpkgs) pkgs;
-            in
-            pkgs.callPackage ./package.nix {
-              # Required since we don't support gcc stdenv
-              stdenv = pkgs.clangStdenv;
+            # This is x86_64-linux only, just because we have significantly
+            # cheaper x86_64-linux compute in CI.
+            # It is clangStdenv because clang's sanitizers are nicer.
+            asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override {
+              # Improve caching of non-code changes by not changing the
+              # derivation name every single time, since this will never be seen
+              # by users anyway.
              versionSuffix = "";
-              lintInsteadOfBuild = true;
+              sanitize = [
+                "address"
+                "undefined"
+              ];
+              # it is very hard to make *every* CI build use this option such
+              # that we don't wind up building Lix twice, so we do it here where
+              # we are already doing so.
+              werror = true;
            };

-          # Make sure that nix-env still produces the exact same result
-          # on a particular version of Nixpkgs.
-          evalNixpkgs =
-            with nixpkgsFor.x86_64-linux.native;
-            runCommand "eval-nixos" { buildInputs = [ nix ]; } ''
-              type -p nix-env
-              # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
-              time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
-              [[ $(sha1sum < packages | cut -c1-40) = 402242fca90874112b34718b8199d844e8b03d12 ]]
-              mkdir $out
-            '';
+            # Although this might be nicer to do with pre-commit, that would
+            # require adding 12MB of nodejs to the dev shell, whereas building it
+            # in CI with Nix avoids that at a cost of slower feedback on rarely
+            # touched files.
+            jsSyntaxCheck =
+              let
+                nixpkgs = nixpkgsFor.x86_64-linux.native;
+                inherit (nixpkgs) pkgs;
+                docSources = lib.fileset.toSource {
+                  root = ./doc;
+                  fileset = lib.fileset.fileFilter (f: f.hasExt "js") ./doc;
+                };
+              in
+              pkgs.runCommand "js-syntax-check" { } ''
+                find ${docSources} -type f -print -exec ${pkgs.nodejs-slim}/bin/node --check '{}' ';'
+                touch $out
+              '';

-          nixpkgsLibTests = forAllSystems (
-            system:
-            let
-              inherit (self.packages.${system}) nix;
-              pkgs = nixpkgsFor.${system}.native;
-              testWithNix = import (nixpkgs + "/lib/tests/test-with-nix.nix") { inherit pkgs lib nix; };
-            in
-            pkgs.symlinkJoin {
-              name = "nixpkgs-lib-tests";
-              paths =
-                [ testWithNix ]
-                # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override
-                # the evaluator used for the test. it will break again in the future, don't worry.
-                ++ lib.optionals pkgs.stdenv.isLinux [
-                  (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset
-                ];
-            }
-          );
-        };
+            # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin
+            clang-tidy =
+              let
+                nixpkgs = nixpkgsFor.x86_64-linux.native;
+                inherit (nixpkgs) pkgs;
+              in
+              pkgs.callPackage ./package.nix {
+                # Required since we don't support gcc stdenv
+                stdenv = pkgs.clangStdenv;
+                versionSuffix = "";
+                lintInsteadOfBuild = true;
+              };
+
+            # Make sure that nix-env still produces the exact same result
+            # on a particular version of Nixpkgs.
+            evalNixpkgs =
+              with nixpkgsFor.x86_64-linux.native;
+              runCommand "eval-nixos" { buildInputs = [ nix ]; } ''
+                type -p nix-env
+                # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+                time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+                [[ $(sha1sum < packages | cut -c1-40) = 402242fca90874112b34718b8199d844e8b03d12 ]]
+                mkdir $out
+              '';
+
+            nixpkgsLibTests = forAllSystems (
+              system:
+              let
+                inherit (self.packages.${system}) nix;
+                pkgs = nixpkgsFor.${system}.native;
+                testWithNix = import (nixpkgs + "/lib/tests/test-with-nix.nix") { inherit pkgs lib nix; };
+              in
+              pkgs.symlinkJoin {
+                name = "nixpkgs-lib-tests";
+                paths =
+                  [ testWithNix ]
+                  # NOTE: nixpkgs 24.11 is being ... *creative*, and requires this dance to override
+                  # the evaluator used for the test. it will break again in the future, don't worry.
+                  ++ lib.optionals pkgs.stdenv.isLinux [
+                    (pkgs.callPackage "${nixpkgs}/ci/eval" { nixVersions.nix_2_24 = nix; }).attrpathsSuperset
+                  ];
+              }
+            );
+          };

        pre-commit = forAvailableSystems (
          system:
@@ -41,7 +41,7 @@ def setup_creds(env: RelengEnvironment):
    $AWS_SECRET_ACCESS_KEY = key.secret_key
    $AWS_ACCESS_KEY_ID = key.id
    $AWS_DEFAULT_REGION = 'garage'
-    $AWS_ENDPOINT_URL = environment.S3_ENDPOINT
+    $AWS_ENDPOINT_URL = env.s3_endpoint


 def official_release_commit_tag(force_tag=False):
@@ -255,13 +255,14 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False
        print('[+] git push to the repo')
        # We have to push the ref to gerrit for review at least such that the
        # commit is known, before we can push it as a tag.
+        repo = env.git_repo()
        if env.git_repo_is_gerrit:
-            git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}'
+            git push @(repo) f'{prev_branch}:refs/for/{prev_branch}'
        else:
-            git push @(env.git_repo) f'{prev_branch}:{prev_branch}'
+            git push @(repo) f'{prev_branch}:{prev_branch}'

        print('[+] git push tag')
-        git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
+        git push @(['-f'] if force_push_tag else []) @(repo) f'{VERSION}:refs/tags/{VERSION}'


 def do_tag_merge(force_tag=False, no_check_git=False):
@@ -276,7 +277,6 @@ def build_manual(eval_result):
    (drv, manual) = next((x['drvPath'], x['outputs']['doc']) for x in eval_result if x['attr'] == 'build.x86_64-linux')
    print('[+] Building manual')
    realise([drv])
-
    cp --no-preserve=mode -T -vr @(manual)/share/doc/nix/manual @(MANUAL)


@@ -6,7 +6,6 @@ import subprocess
 import dataclasses

 S3_HOST = 's3.lix.systems'
-S3_ENDPOINT = 'https://s3.lix.systems'

 DEFAULT_STORE_URI_BITS = {
    'region': 'garage',
@@ -51,8 +50,11 @@ class RelengEnvironment:
    cache_bucket: str
    releases_bucket: str
    docs_bucket: str
-    git_repo: str
+    # We don't want to call functions that require computation (such as guess_gerrit_remote()) before they are needed
+    git_repo: Callable[[], str]
    git_repo_is_gerrit: bool
+    s3_endpoint: str
+    s3_ssh_host: str | None

    docker_targets: list[DockerTarget]

@@ -65,6 +67,7 @@ class RelengEnvironment:
 SGR = '\x1b['
 RED = '31;1m'
 GREEN = '32;1m'
+BLUE = '34;1m'
 RESET = '0m'


@@ -72,6 +75,21 @@ def sgr(colour: str, text: str) -> str:
    return f'{SGR}{colour}{text}{SGR}{RESET}'


+LOCAL = RelengEnvironment(
+    name='local',
+    colour=functools.partial(sgr, BLUE),
+    docs_bucket='s3://local-docs',
+    cache_bucket='s3://local-cache',
+    cache_store_overlay={'secret-key': 'local.key', 'endpoint': 'localhost:3900', 'scheme': 'http'},
+    releases_bucket='s3://local-releases',
+    git_repo=lambda: '../releng-target',
+    git_repo_is_gerrit=False,
+    docker_targets=[],
+    s3_endpoint = 'http://localhost:3900',
+    s3_ssh_host = None,
+)
+
+
 STAGING = RelengEnvironment(
    name='staging',
    colour=functools.partial(sgr, GREEN),
@@ -79,7 +97,7 @@ STAGING = RelengEnvironment(
    cache_bucket='s3://staging-cache',
    cache_store_overlay={'secret-key': 'staging.key'},
    releases_bucket='s3://staging-releases',
-    git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging',
+    git_repo=lambda: 'ssh://git@git.lix.systems/lix-project/lix-releng-staging',
    git_repo_is_gerrit=False,
    docker_targets=[
        # latest will be auto tagged if appropriate
@@ -88,6 +106,8 @@ STAGING = RelengEnvironment(
        DockerTarget('ghcr.io/lix-project/lix-releng-staging',
                     tags=['{version}', '{major}']),
    ],
+    s3_endpoint = 'https://s3.lix.systems',
+    s3_ssh_host = S3_HOST,
 )

 GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$')
@@ -114,7 +134,7 @@ PROD = RelengEnvironment(
    # just delete it after doing a release.
    cache_store_overlay={'secret-key': 'prod.key'},
    releases_bucket='s3://releases',
-    git_repo=guess_gerrit_remote(),
+    git_repo=guess_gerrit_remote,
    git_repo_is_gerrit=True,
    docker_targets=[
        # latest will be auto tagged if appropriate
@@ -122,9 +142,12 @@ PROD = RelengEnvironment(
                     tags=['{version}', '{major}']),
        DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']),
    ],
+    s3_endpoint = 'https://s3.lix.systems',
+    s3_ssh_host = S3_HOST,
 )

 ENVIRONMENTS = {
+    'local': LOCAL,
    'staging': STAGING,
    'production': PROD,
 }
@@ -0,0 +1,12 @@
+{
+  lib,
+  writePython3Bin,
+  python3Packages,
+}:
+
+let
+  package = writePython3Bin "garage-ephemeral-key" { libraries = [ python3Packages.requests ]; } (
+    builtins.readFile ./garage-ephemeral-key.py
+  );
+in
+package
@@ -0,0 +1,175 @@
+# SPDX-FileCopyrightText: 2024 Jade Lovelace
+# SPDX-License-Identifier: MIT
+import argparse
+import json
+import sys
+import datetime
+import dataclasses
+import re
+from typing import Any, Literal, Optional
+import requests
+import os
+import logging
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+fmt = logging.Formatter('{asctime} {levelname} {name}: {message}',
+                        datefmt='%b %d %H:%M:%S',
+                        style='{')
+
+if not any(isinstance(h, logging.StreamHandler) for h in log.handlers):
+    hand = logging.StreamHandler()
+    hand.setFormatter(fmt)
+    log.addHandler(hand)
+
+API_BASE = os.environ.get('GARAGE_ADMIN_API_BASE', 'http://localhost:3903')
+API_KEY = os.environ['GARAGE_ADMIN_TOKEN']
+
+
+def api(method, endpoint: str, resp_json=True, **kwargs) -> Any:
+    log.info('http %s %s', method, endpoint)
+    if not endpoint.startswith('https'):
+        endpoint = API_BASE + endpoint
+    resp = requests.request(method,
+                            endpoint,
+                            headers={'Authorization': f'Bearer {API_KEY}'},
+                            **kwargs)
+    resp.raise_for_status()
+    if resp_json:
+        return resp.json()
+    else:
+        return resp
+
+
+@dataclasses.dataclass
+class Key:
+    name: str
+    id: str
+    secret_key: Optional[str] = None
+
+
+@dataclasses.dataclass
+class Bucket:
+    id: str
+
+
+def keys() -> list[Key]:
+    data: list[dict] = api('GET', '/v1/key?list')
+    return [Key(name=k['name'], id=k['id']) for k in data]
+
+
+def delete_key(key: Key):
+    api('DELETE', '/v1/key', resp_json=False, params={'id': key.id})
+
+
+def create_key(name: str) -> Key:
+    resp: dict = api('POST', '/v1/key', json={'name': name})
+    return Key(name=resp['name'],
+               id=resp['accessKeyId'],
+               secret_key=resp['secretAccessKey'])
+
+
+AccessType = Literal['read'] | Literal['write'] | Literal['owner']
+
+
+def get_bucket(bucket_name: str) -> Bucket:
+    resp: dict = api('GET', '/v1/bucket', params={'globalAlias': bucket_name})
+    return Bucket(resp['id'])
+
+
+def grant(bucket: Bucket, access_types: list[AccessType], key: Key):
+    access_types_dict = {k: True for k in access_types}
+    api('POST',
+        '/v1/bucket/allow',
+        json={
+            'bucketId': bucket.id,
+            'accessKeyId': key.id,
+            'permissions': access_types_dict,
+        })
+
+
+KEY_RE = re.compile(r'^.*ephemeral-(\d{14})$')
+DATEFMT = '%Y%m%d%H%M%S'
+
+
+def expired_keys(older_than: datetime.datetime) -> list[Key]:
+    ret = []
+    for key in keys():
+        if m := KEY_RE.match(key.name):
+            date = datetime.datetime.strptime(m.group(1), DATEFMT)
+            date = date.astimezone(datetime.UTC)
+            print(date)
+            if date < older_than:
+                ret.append(key)
+    return ret
+
+
+def do_new(args):
+    buckets = [get_bucket(b) for b in args.buckets]
+
+    def optional(s: str, whether) -> list[str]:
+        if whether:
+            return [s]
+        else:
+            return []
+
+    access_types: list[AccessType] = optional('read', args.read) + optional(
+        'write', args.write) + optional('owner', args.owner)  # type: ignore
+
+    key_name = args.name + '-' if args.name else ''
+    key_name += "ephemeral-" + (
+        datetime.datetime.now(tz=datetime.UTC) +
+        datetime.timedelta(seconds=args.age_secs)).strftime(DATEFMT)
+
+    k = create_key(key_name)
+    for b in buckets:
+        grant(b, access_types, k)
+
+    print(json.dumps(dataclasses.asdict(k), indent=2))
+
+
+def do_clean(args):
+    older_than = datetime.datetime.now(tz=datetime.UTC)
+    for key in expired_keys(older_than):
+        delete_key(key)
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Garage ephemeral API keys tool")
+
+    def fail(*args):
+        ap.print_help()
+        sys.exit(1)
+
+    ap.set_defaults(cmd=fail)
+
+    sps = ap.add_subparsers()
+
+    new = sps.add_parser("new", help="Make an ephemeral key")
+    new.add_argument("--name", help="Name prefix for the key")
+    new.add_argument("--read",
+                     action="store_true",
+                     help="Grant read access to buckets")
+    new.add_argument("--write",
+                     action="store_true",
+                     help="Grant write access to buckets")
+    new.add_argument("--owner",
+                     action="store_true",
+                     help="Grant owner access to buckets")
+    new.add_argument("--age-secs",
+                     type=int,
+                     required=True,
+                     help="Maximum key lifetime in seconds")
+    new.add_argument("buckets", nargs='*', help="Buckets to grant access to")
+    new.set_defaults(cmd=do_new)
+
+    clean = sps.add_parser("clean", help="Clean up old keys")
+    clean.set_defaults(cmd=do_clean)
+
+    args = ap.parse_args()
+    args.cmd(args)
+
+
+if __name__ == '__main__':
+    main()
@@ -5,14 +5,17 @@ from . import environment

 def get_ephemeral_key(
        env: environment.RelengEnvironment) -> environment.S3Credentials:
-    output = subprocess.check_output([
-        'ssh', '-l', 'root', environment.S3_HOST, 'garage-ephemeral-key',
-        'new', '--name', f'releng-{env.name}', '--read', '--write',
-        '--age-secs', '3600',
+    output = None
+    command = [
+        'garage-ephemeral-key', 'new', '--name', f'releng-{env.name}', '--read',
+        '--write', '--age-secs', '3600',
        env.releases_bucket.removeprefix('s3://'),
        env.cache_bucket.removeprefix('s3://'),
        env.docs_bucket.removeprefix('s3://'),
-    ])
+    ]
+    if env.s3_ssh_host is not None:
+        command = ['ssh', '-l', 'root', env.s3_ssh_host, *command]
+    output = subprocess.check_output(command)
    d = json.loads(output.decode())
    return environment.S3Credentials(name=d['name'],
                                     id=d['id'],
@@ -0,0 +1,101 @@
+{
+  self,
+  system,
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  releng = ./..;
+  this-garage = pkgs.garage_1_x;
+  garage-ephemeral-key = pkgs.callPackage ../garage-ephemeral-key {
+    inherit (pkgs.writers) writePython3Bin;
+  };
+
+  build = self.release-jobs.all.build.${system}.doc;
+
+  pythonPackages = (
+    p: [
+      p.yapf
+      p.requests
+      p.xdg-base-dirs
+      p.packaging
+      p.xonsh
+    ]
+  );
+  pythonEnv = pkgs.python3.pythonOnBuildForHost.withPackages pythonPackages;
+in
+{
+  name = "local-releng-manual-upload";
+
+  nodes = {
+    machine =
+      { config, pkgs, ... }:
+      {
+        services.garage.enable = true;
+        services.garage.package = this-garage;
+        services.garage.settings = {
+          replication_factor = 1;
+
+          rpc_bind_addr = "[::]:3901";
+          rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6";
+
+          s3_api.api_bind_addr = "[::]:3900";
+          s3_api.s3_region = "garage";
+          s3_api.root_domain = ".localhost";
+
+          admin = {
+            api_bind_addr = "[::]:3903";
+            admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=";
+          };
+        };
+
+        environment.systemPackages = [
+          this-garage
+          pkgs.git
+          pkgs.build-release-notes
+          pkgs.jq
+          pkgs.nix-eval-jobs
+          pkgs.awscli2
+          pythonEnv
+          garage-ephemeral-key
+        ];
+
+        environment.sessionVariables = {
+          GARAGE_ADMIN_TOKEN = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4=";
+        };
+      };
+  };
+  testScript = ''
+    machine.wait_for_unit("garage")
+    machine.wait_for_open_port(3900)
+
+    nodeId = machine.succeed("garage node id")
+    machine.succeed(f"garage layout assign -z dc1 -c 10G {nodeId}")
+    machine.succeed("garage layout apply --version 1")
+    machine.succeed("garage bucket create local-docs")
+    machine.succeed("garage bucket create local-cache")
+    machine.succeed("garage bucket create local-releases")
+
+    machine.succeed("git init --bare releng-target")
+
+    machine.succeed("mkdir repo")
+    machine.succeed("cd repo; git init --initial-branch release-2.92 .")
+    machine.succeed("cd repo; git config user.email 'local@test.com'")
+    machine.succeed("cd repo; git config user.name 'Test Releng'")
+
+    machine.copy_from_host("${../../version.json}", "repo/version.json")
+    machine.copy_from_host("${releng}", "repo/releng")
+    machine.copy_from_host("${../../doc}", "repo/doc")
+    machine.succeed("cd repo; git add .")
+    machine.succeed("cd repo; git commit -m 'initial commit'")
+    machine.succeed("cd repo; git switch -c releng/2.92.4")
+
+    machine.succeed("cd repo; python3 -m releng prepare")
+    machine.succeed("cd repo; python3 -m releng tag")
+    machine.succeed("cd repo; mkdir -p release/manual")
+    machine.succeed("cd repo; cp --no-preserve=mode -T -vr ${build}/share/doc/nix/manual ./release/manual")
+    machine.succeed("cd repo; python3 -m releng upload --noconfirm --environment local --target manual")
+  '';
+}
@@ -1,4 +1,4 @@
-{ lib, nixpkgs, nixpkgsFor }:
+{ self, lib, nixpkgs, nixpkgsFor }:

 let

@@ -17,6 +17,7 @@ let
      };
      _module.args.nixpkgs = nixpkgs;
      _module.args.system = system;
+      _module.args.self = self;
    })
    // {
      # allow running tests against older nix versions via `nix eval --apply`
@@ -40,6 +41,8 @@ let
 in

 {
+  local-releng = runNixOSTestFor "x86_64-linux" ../../releng/local;
+
  authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix;

  remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix;