hic sunt dracones
This commit is contained in:
eldritch horrors
Executable
+141
@@ -0,0 +1,141 @@
|
||||
import subprocess
|
||||
import itertools
|
||||
import json
|
||||
import typing
|
||||
import graphlib
|
||||
import textwrap
|
||||
import sys
|
||||
import logging
|
||||
import json
|
||||
|
||||
logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.INFO)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SYSTEMS = {
|
||||
'x86_64-linux',
|
||||
'aarch64-linux',
|
||||
# 'aarch64-darwin',
|
||||
|
||||
# disabled due to low utility
|
||||
# 'i686-linux',
|
||||
|
||||
# disabled due to low capacity
|
||||
# 'x86_64-darwin',
|
||||
}
|
||||
EVAL_WORKERS = 4
|
||||
EVAL_MEMORY = 4096
|
||||
|
||||
class Job(typing.NamedTuple):
|
||||
drv: str
|
||||
name: str
|
||||
system: str
|
||||
path: list[str]
|
||||
expensive: bool
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.drv)
|
||||
def __eq__(self, other):
|
||||
return self.drv == other.drv
|
||||
|
||||
def group(self):
|
||||
return self.path[0]
|
||||
def drv_hash(self):
|
||||
return self.drv.rpartition("/")[2].partition("-")[0]
|
||||
|
||||
class EvalFailure(Exception):
|
||||
errors: list[str]
|
||||
|
||||
def __init__(self, errors):
|
||||
self.errors = errors
|
||||
|
||||
def get_jobs() -> list[Job]:
|
||||
nej = subprocess.Popen(
|
||||
["nix-eval-jobs",
|
||||
"--flake", ".#hydraJobs",
|
||||
"--option", "eval-cache", "false",
|
||||
"--meta",
|
||||
"--quiet",
|
||||
"--workers", str(EVAL_WORKERS),
|
||||
"--max-memory-size", str(EVAL_MEMORY),
|
||||
"--show-trace",
|
||||
"--force-recurse",
|
||||
],
|
||||
stdout=subprocess.PIPE
|
||||
)
|
||||
drvs = []
|
||||
errors = []
|
||||
while line := nej.stdout.readline():
|
||||
item = json.loads(line)
|
||||
log.info("found %s", item['attr'])
|
||||
if 'error' in item:
|
||||
errors.append(item)
|
||||
elif item['system'] in SYSTEMS:
|
||||
drvs.append(item)
|
||||
|
||||
if nej.wait(60) != 0:
|
||||
raise subprocess.CalledProcessError(nej.returncode, nej.args)
|
||||
elif errors:
|
||||
raise EvalFailure(errors)
|
||||
|
||||
return [
|
||||
# we use "has a mainProgram" as the marker for expensive builds.
|
||||
# at time of writing this matches only lix builds, not n-e-j etc
|
||||
Job(drv=j['drvPath'], name=j['attr'], system=j['system'], path=j['attrPath'],
|
||||
expensive=j.get('meta', {}).get('mainProgram'))
|
||||
for j in drvs
|
||||
]
|
||||
|
||||
def get_dependencies(jobs: list[Job]) -> dict[Job, set[Job]]:
|
||||
log.info("getting derivation infos")
|
||||
by_drv = { j.drv: j for j in jobs }
|
||||
drv_info = json.loads(subprocess.run(
|
||||
["nix", "derivation", "show", "-r", *by_drv.keys()],
|
||||
capture_output=True,
|
||||
check=True
|
||||
).stdout.decode())
|
||||
log.info("calculating dependencies")
|
||||
deps = { drv: set(info["inputDrvs"].keys()) for drv, info in drv_info.items() }
|
||||
for n in graphlib.TopologicalSorter(deps).static_order():
|
||||
for d in tuple(deps[n]):
|
||||
deps[n].update(deps[d])
|
||||
return { j: set(( by_drv[d] for d in deps[j.drv] & by_drv.keys() )) for j in jobs }
|
||||
|
||||
try:
|
||||
# make unique based on drv hash to avoid errors when multiple attrs
|
||||
# point to the same derivations. sort by attrpath for stable order.
|
||||
jobs = [
|
||||
tuple(js[1])[0]
|
||||
for all_jobs in [ get_jobs() ]
|
||||
for sorted_jobs in [ sorted(all_jobs, key=lambda j: (j.drv, j.path)) ]
|
||||
for js in itertools.groupby(sorted_jobs, lambda j: j.drv)
|
||||
]
|
||||
deps = get_dependencies(jobs)
|
||||
|
||||
commands = [
|
||||
{
|
||||
'group': group,
|
||||
'steps': [
|
||||
{
|
||||
'command': f'nix build --no-link -L ".#hydraJobs.{job.name}"',
|
||||
'label': ".".join(job.path),
|
||||
'key': job.drv_hash(),
|
||||
'depends_on': [ d.drv_hash() for d in deps[job] ],
|
||||
'agents': {
|
||||
'queue': job.system,
|
||||
('large' if job.expensive else 'small'): True,
|
||||
},
|
||||
}
|
||||
for job in items
|
||||
],
|
||||
}
|
||||
for group, items in itertools.groupby(sorted(jobs, key=Job.group), Job.group)
|
||||
]
|
||||
|
||||
print(json.dumps({'steps': commands}, indent=2))
|
||||
except EvalFailure as e:
|
||||
print("")
|
||||
print("\x1b[1m\x1b[31mERRORS\x1b[0m encountered during evaluation:")
|
||||
for e in e.errors:
|
||||
print(f"\ton attribute hydraJobs.\x1b[1m{e['attr']}\x1b[0m:")
|
||||
print(textwrap.indent(e['error'], "\t\t"))
|
||||
sys.exit(1)
|
||||
Generated
+204
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1696426674,
|
||||
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flakey-profile": {
|
||||
"locked": {
|
||||
"lastModified": 1712898590,
|
||||
"narHash": "sha256-FhGIEU93VHAChKEXx905TSiPZKga69bWl1VB37FK//I=",
|
||||
"owner": "lf-",
|
||||
"repo": "flakey-profile",
|
||||
"rev": "243c903fd8eadc0f63d205665a92d4df91d42d9d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "lf-",
|
||||
"repo": "flakey-profile",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"lix": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"nix2container": "nix2container",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"nixpkgs-regression": "nixpkgs-regression",
|
||||
"pre-commit-hooks": "pre-commit-hooks"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1739846668,
|
||||
"narHash": "sha256-B0+90JxZ7m+NVTb82Y8/R5OvKJrJhFDO2LtIxM7JNfw=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "0069c59eda80e89ef4e42f7629cb15b051b6cadd",
|
||||
"revCount": 17430,
|
||||
"type": "git",
|
||||
"url": "https://git.lix.systems/lix-project/lix"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://git.lix.systems/lix-project/lix"
|
||||
}
|
||||
},
|
||||
"lix-module": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"flakey-profile": "flakey-profile",
|
||||
"lix": [
|
||||
"lix"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1738176840,
|
||||
"narHash": "sha256-NG3IRvRs3u3btVCN861FqHvgOwqcNT/Oy6PBG86F5/E=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "621aae0f3cceaffa6d73a4fb0f89c08d338d729e",
|
||||
"revCount": 133,
|
||||
"type": "git",
|
||||
"url": "https://git.lix.systems/lix-project/nixos-module"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://git.lix.systems/lix-project/nixos-module"
|
||||
}
|
||||
},
|
||||
"nix2container": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1724996935,
|
||||
"narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nlewo",
|
||||
"repo": "nix2container",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1733348545,
|
||||
"narHash": "sha256-b4JrUmqT0vFNx42aEN9LTWOHomkTKL/ayLopflVf81U=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "9ecb50d2fae8680be74c08bb0a995c5383747f89",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11-small",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-regression": {
|
||||
"locked": {
|
||||
"lastModified": 1643052045,
|
||||
"narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1739758141,
|
||||
"narHash": "sha256-uq6A2L7o1/tR6VfmYhZWoVAwb3gTy7j4Jx30MIrH0rE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "c618e28f70257593de75a7044438efc1c1fc0791",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-24.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"pre-commit-hooks": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1733318908,
|
||||
"narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=",
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"rev": "6f4e2a2112050951a314d2733a994fbab94864c6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"lix": "lix",
|
||||
"lix-module": "lix-module",
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
|
||||
lix.url = "git+https://git.lix.systems/lix-project/lix";
|
||||
lix-module = {
|
||||
url = "git+https://git.lix.systems/lix-project/nixos-module";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.lix.follows = "lix";
|
||||
};
|
||||
};
|
||||
|
||||
outputs = { nixpkgs, lix-module, ... }:
|
||||
let
|
||||
systems = [ "x86_64-linux" ];
|
||||
nixpkgsFor = builtins.listToAttrs
|
||||
(map
|
||||
(system: {
|
||||
name = system;
|
||||
value = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ lix-module.overlays.default ];
|
||||
};
|
||||
})
|
||||
systems);
|
||||
forEachSystem = f: builtins.mapAttrs f nixpkgsFor;
|
||||
in
|
||||
{
|
||||
packages = forEachSystem (system: pkgs: {
|
||||
default = pkgs.writeShellApplication {
|
||||
name = "calculate-buildkite-steps";
|
||||
runtimeInputs = [
|
||||
pkgs.nix-eval-jobs
|
||||
pkgs.python3
|
||||
];
|
||||
text = ''
|
||||
python ${./calculate-buildkite-steps.py} | buildkite-agent pipeline upload
|
||||
'';
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# buildkite static setup file, in case the pipeline gets lost
|
||||
|
||||
steps:
|
||||
- label: ":clapper:"
|
||||
agents:
|
||||
queue: "x86_64-linux"
|
||||
large: true
|
||||
command: |
|
||||
nix --tarball-ttl 0 run "git+$${BUILDKITE_REPO}?ref=refs/heads/ci-config"
|
||||
Reference in New Issue
Block a user