From 0a5650647cfe3a071d6e72d3124d25ff8d4f91a1 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 18 Feb 2025 14:14:58 +0100 Subject: [PATCH] hic sunt dracones --- calculate-buildkite-steps.py | 141 ++++++++++++++++++++++++ flake.lock | 204 +++++++++++++++++++++++++++++++++++ flake.nix | 41 +++++++ static-pipeline.yaml | 9 ++ 4 files changed, 395 insertions(+) create mode 100755 calculate-buildkite-steps.py create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 static-pipeline.yaml diff --git a/calculate-buildkite-steps.py b/calculate-buildkite-steps.py new file mode 100755 index 000000000..ec2fa9612 --- /dev/null +++ b/calculate-buildkite-steps.py @@ -0,0 +1,141 @@ +import subprocess +import itertools +import json +import typing +import graphlib +import textwrap +import sys +import logging +import json + +logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.INFO) +log = logging.getLogger(__name__) + +SYSTEMS = { + 'x86_64-linux', + 'aarch64-linux', + # 'aarch64-darwin', + + # disabled due to low utility + # 'i686-linux', + + # disabled due to low capacity + # 'x86_64-darwin', +} +EVAL_WORKERS = 4 +EVAL_MEMORY = 4096 + +class Job(typing.NamedTuple): + drv: str + name: str + system: str + path: list[str] + expensive: bool + + def __hash__(self): + return hash(self.drv) + def __eq__(self, other): + return self.drv == other.drv + + def group(self): + return self.path[0] + def drv_hash(self): + return self.drv.rpartition("/")[2].partition("-")[0] + +class EvalFailure(Exception): + errors: list[str] + + def __init__(self, errors): + self.errors = errors + +def get_jobs() -> list[Job]: + nej = subprocess.Popen( + ["nix-eval-jobs", + "--flake", ".#hydraJobs", + "--option", "eval-cache", "false", + "--meta", + "--quiet", + "--workers", str(EVAL_WORKERS), + "--max-memory-size", str(EVAL_MEMORY), + "--show-trace", + "--force-recurse", + ], + stdout=subprocess.PIPE + ) + drvs = [] + errors = [] + while line := nej.stdout.readline(): + item = json.loads(line) + log.info("found %s", item['attr']) + if 'error' in item: + errors.append(item) + elif item['system'] in SYSTEMS: + drvs.append(item) + + if nej.wait(60) != 0: + raise subprocess.CalledProcessError(nej.returncode, nej.args) + elif errors: + raise EvalFailure(errors) + + return [ + # we use "has a mainProgram" as the marker for expensive builds. + # at time of writing this matches only lix builds, not n-e-j etc + Job(drv=j['drvPath'], name=j['attr'], system=j['system'], path=j['attrPath'], + expensive=j.get('meta', {}).get('mainProgram')) + for j in drvs + ] + +def get_dependencies(jobs: list[Job]) -> dict[Job, set[Job]]: + log.info("getting derivation infos") + by_drv = { j.drv: j for j in jobs } + drv_info = json.loads(subprocess.run( + ["nix", "derivation", "show", "-r", *by_drv.keys()], + capture_output=True, + check=True + ).stdout.decode()) + log.info("calculating dependencies") + deps = { drv: set(info["inputDrvs"].keys()) for drv, info in drv_info.items() } + for n in graphlib.TopologicalSorter(deps).static_order(): + for d in tuple(deps[n]): + deps[n].update(deps[d]) + return { j: set(( by_drv[d] for d in deps[j.drv] & by_drv.keys() )) for j in jobs } + +try: + # make unique based on drv hash to avoid errors when multiple attrs + # point to the same derivations. sort by attrpath for stable order. + jobs = [ + tuple(js[1])[0] + for all_jobs in [ get_jobs() ] + for sorted_jobs in [ sorted(all_jobs, key=lambda j: (j.drv, j.path)) ] + for js in itertools.groupby(sorted_jobs, lambda j: j.drv) + ] + deps = get_dependencies(jobs) + + commands = [ + { + 'group': group, + 'steps': [ + { + 'command': f'nix build --no-link -L ".#hydraJobs.{job.name}"', + 'label': ".".join(job.path), + 'key': job.drv_hash(), + 'depends_on': [ d.drv_hash() for d in deps[job] ], + 'agents': { + 'queue': job.system, + ('large' if job.expensive else 'small'): True, + }, + } + for job in items + ], + } + for group, items in itertools.groupby(sorted(jobs, key=Job.group), Job.group) + ] + + print(json.dumps({'steps': commands}, indent=2)) +except EvalFailure as e: + print("") + print("\x1b[1m\x1b[31mERRORS\x1b[0m encountered during evaluation:") + for e in e.errors: + print(f"\ton attribute hydraJobs.\x1b[1m{e['attr']}\x1b[0m:") + print(textwrap.indent(e['error'], "\t\t")) + sys.exit(1) diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..2f618b1e8 --- /dev/null +++ b/flake.lock @@ -0,0 +1,204 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flakey-profile": { + "locked": { + "lastModified": 1712898590, + "narHash": "sha256-FhGIEU93VHAChKEXx905TSiPZKga69bWl1VB37FK//I=", + "owner": "lf-", + "repo": "flakey-profile", + "rev": "243c903fd8eadc0f63d205665a92d4df91d42d9d", + "type": "github" + }, + "original": { + "owner": "lf-", + "repo": "flakey-profile", + "type": "github" + } + }, + "lix": { + "inputs": { + "flake-compat": "flake-compat", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs", + "nixpkgs-regression": "nixpkgs-regression", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1739846668, + "narHash": "sha256-B0+90JxZ7m+NVTb82Y8/R5OvKJrJhFDO2LtIxM7JNfw=", + "ref": "refs/heads/main", + "rev": "0069c59eda80e89ef4e42f7629cb15b051b6cadd", + "revCount": 17430, + "type": "git", + "url": "https://git.lix.systems/lix-project/lix" + }, + "original": { + "type": "git", + "url": "https://git.lix.systems/lix-project/lix" + } + }, + "lix-module": { + "inputs": { + "flake-utils": "flake-utils", + "flakey-profile": "flakey-profile", + "lix": [ + "lix" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1738176840, + "narHash": "sha256-NG3IRvRs3u3btVCN861FqHvgOwqcNT/Oy6PBG86F5/E=", + "ref": "refs/heads/main", + "rev": "621aae0f3cceaffa6d73a4fb0f89c08d338d729e", + "revCount": 133, + "type": "git", + "url": "https://git.lix.systems/lix-project/nixos-module" + }, + "original": { + "type": "git", + "url": "https://git.lix.systems/lix-project/nixos-module" + } + }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1724996935, + "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1733348545, + "narHash": "sha256-b4JrUmqT0vFNx42aEN9LTWOHomkTKL/ayLopflVf81U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9ecb50d2fae8680be74c08bb0a995c5383747f89", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1739758141, + "narHash": "sha256-uq6A2L7o1/tR6VfmYhZWoVAwb3gTy7j4Jx30MIrH0rE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c618e28f70257593de75a7044438efc1c1fc0791", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "flake": false, + "locked": { + "lastModified": 1733318908, + "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "6f4e2a2112050951a314d2733a994fbab94864c6", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "lix": "lix", + "lix-module": "lix-module", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..1208e0384 --- /dev/null +++ b/flake.nix @@ -0,0 +1,41 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + lix.url = "git+https://git.lix.systems/lix-project/lix"; + lix-module = { + url = "git+https://git.lix.systems/lix-project/nixos-module"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.lix.follows = "lix"; + }; + }; + + outputs = { nixpkgs, lix-module, ... }: + let + systems = [ "x86_64-linux" ]; + nixpkgsFor = builtins.listToAttrs + (map + (system: { + name = system; + value = import nixpkgs { + inherit system; + overlays = [ lix-module.overlays.default ]; + }; + }) + systems); + forEachSystem = f: builtins.mapAttrs f nixpkgsFor; + in + { + packages = forEachSystem (system: pkgs: { + default = pkgs.writeShellApplication { + name = "calculate-buildkite-steps"; + runtimeInputs = [ + pkgs.nix-eval-jobs + pkgs.python3 + ]; + text = '' + python ${./calculate-buildkite-steps.py} | buildkite-agent pipeline upload + ''; + }; + }); + }; +} diff --git a/static-pipeline.yaml b/static-pipeline.yaml new file mode 100644 index 000000000..021c06efb --- /dev/null +++ b/static-pipeline.yaml @@ -0,0 +1,9 @@ +# buildkite static setup file, in case the pipeline gets lost + +steps: + - label: ":clapper:" + agents: + queue: "x86_64-linux" + large: true + command: | + nix --tarball-ttl 0 run "git+$${BUILDKITE_REPO}?ref=refs/heads/ci-config"