From 98a27fbfd248daffe7cc16e06948a87b069c1028 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Thu, 31 Jul 2025 16:11:55 +0200 Subject: [PATCH] bench: soup up the benchmark script add daemon support, fix the rebuild cases by using an installer config Change-Id: I47cbb0dd401abf5e041e9936077e502f5e0cabf9 --- bench/bench.py | 128 ++++++++++++---- bench/configuration.nix | 314 ---------------------------------------- 2 files changed, 98 insertions(+), 344 deletions(-) delete mode 100644 bench/configuration.nix diff --git a/bench/bench.py b/bench/bench.py index 8dfcf24a2..0248c1937 100755 --- a/bench/bench.py +++ b/bench/bench.py @@ -7,15 +7,43 @@ import os import json import tempfile import platform +import shlex +import textwrap -flake_args = ["--extra-experimental-features","'nix-command flakes'"] -# hyperfine has its own variable substitution, so we use that and pass build="{BUILD}" here. -# perf doesn't have variable substitution, so we call these with build being the actual build directory. +flake_args = ["--extra-experimental-features", "nix-command flakes"] cases = { - "search": lambda build: [f"{build}/bin/nix", *flake_args, "search", "--no-eval-cache", "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870", "hello"], - "rebuild": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import {}; system'"], - "rebuild_lh": lambda build: ["GC_INITIAL_HEAP_SIZE=10g", f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import {}; system'"], - "parse": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "-f", "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"], + "search": lambda build: [ + f"{build}/bin/nix", + *flake_args, + "search", + "--no-eval-cache", + "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870", + "hello", + ], + "rebuild": lambda build: [ + f"{build}/bin/nix", + *flake_args, + "eval", + "--raw", + "--impure", + "--expr", + textwrap.dedent(""" + (import { + configuration = ./bench/nixpkgs/nixos/modules/installer/cd-dvd/installation-cd-graphical-calamares-plasma6.nix; + }).config.system.build.toplevel + """).replace("\n", " "), + ], + "rebuild_lh": lambda build: [ + "GC_INITIAL_HEAP_SIZE=10g", + *cases['rebuild'](build), + ], + "parse": lambda build: [ + f"{build}/bin/nix", + *flake_args, + "eval", + "-f", + "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix", + ], } arg_parser = argparse.ArgumentParser() @@ -24,10 +52,26 @@ arg_parser = argparse.ArgumentParser() # mode, we would have to combine the JSON ourselves to support that, which # would probably be better done by writing a benchmarking script in # not-bash. -arg_parser.add_argument('builds', nargs='+', help="At least two build directories to compare, containing bin/nix") -arg_parser.add_argument('--cases', type=str, help="A comma-separated list of cases you want to run. Defaults to running all") -available_modes = [ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [] # perf doesn't run on Darwin -arg_parser.add_argument('--mode', choices=available_modes, default="walltime") +arg_parser.add_argument( + 'builds', + nargs='+', + help="At least two build directories to compare, containing bin/nix", +) +arg_parser.add_argument( + '--cases', + type=str, + help="A comma-separated list of cases you want to run. Defaults to running all", +) +arg_parser.add_argument( + '--mode', + choices=[ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [], # perf doesn't run on Darwin + default="walltime", +) +arg_parser.add_argument( + '--daemon', + action='store_true', + help='Run a temporary daemon for the benchmark instead of using a local store directly', +) args = arg_parser.parse_args() if len(args.builds) < 2: raise ValueError("need at least two build directories to compare") @@ -37,33 +81,51 @@ if args.cases is None: benchmarks = list(cases.keys()) else: for case in args.cases.split(","): - if case not in cases: raise ValueError(f"no such case: {case}") + if case not in cases: + raise ValueError(f"no such case: {case}") benchmarks.append(case) +def make_full_command(build, case): + cmd = " ".join(map(shlex.quote, cases[case](build))) + if args.daemon: + return " ".join([ + f"{build}/bin/nix --extra-experimental-features nix-command daemon &", + "trap 'kill %1' EXIT;", + f"NIX_REMOTE=daemon {cmd}", + ]) + else: + return cmd + def bench_walltime(env): - hyperfine_args = ["--parameter-list", "BUILD", ','.join(args.builds), "--warmup", "2", "--runs", "10"] for case in benchmarks: - case_command = cases[case]("{BUILD}") # see the comment on cases - subprocess.run([ - "taskset", "-c", "2,3", - "chrt", "-f","50", - "hyperfine", *hyperfine_args, "--export-json", f"bench/bench-{case}.json", "--export-markdown", f"bench/bench-{case}.md", "--", " ".join(case_command) - ], env=env, check=True) + for build in args.builds: + subprocess.run([ + "taskset", "-c", "2,3", + "chrt", "-f","50", + *[ + "hyperfine", "--warmup", "2", "--runs", "10", + "--export-json", f"bench/bench-{case}-{build}.json", + "--export-markdown", f"bench/bench-{case}-{build}.md", + "--", make_full_command(build, case), + ], + ], env=env, check=True) print("Benchmarks summary\n---\n") for case in benchmarks: - fd = open(f"bench/bench-{case}.json") - result_json = json.load(fd) - fd.close() - for result in result_json["results"]: + results = [] + for build in args.builds: + with open(f"bench/bench-{case}-{build}.json") as fd: + results.append(json.load(fd)["results"][0]) + for result in results: print(result["command"]) print("-" * min(80,len(result["command"]))) - attr_rounded = lambda attr: f"{result[attr]:.3f}" + def attr_rounded(attr): + return f"{result[attr]:.3f}" print(" mean: ", attr_rounded("mean"), "±", attr_rounded("stddev")) print(" user:", attr_rounded("user"), "| system", attr_rounded("system")) print(" median: ", attr_rounded("median")) print(" range: ", attr_rounded("min") + "s.." + attr_rounded("max")+"s") - print(" relative:", f"{result["mean"]/result_json["results"][0]["mean"]:.3f}") + print(" relative:", f"{result["mean"]/results[0]["mean"]:.3f}") print("\n") @@ -71,12 +133,14 @@ def bench_icount(env): perf_results_for: dict[str, list[tuple[str, float]]] = {} for case in benchmarks: for build in args.builds: - case_command = cases[case](build) # the perf stat -j output (incorrectly) localizes numbers, which will trip up the json parser. env["LC_ALL"]="C" + case_command = make_full_command(build, case) commandline = [ - "perf", "stat", "-o", f"bench/perf-{case}.json", "-j", "sh", "-c", " ".join(case_command) + "perf", "stat", "-o", f"bench/perf-{case}.json", "-j", + "sh", "-c", case_command, ] + print("running", case_command) subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL) # warmup run subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL) perf_fd = open(f"bench/perf-{case}.json") @@ -84,8 +148,9 @@ def bench_icount(env): perf_fd.close() instr = next(x for x in perf_data if x["event"] in ["instructions", "instructions:u"]) # an implementation of a find_first iterator - if case not in perf_results_for: perf_results_for[case] = [] - perf_results_for[case].append((" ".join(case_command), float(instr["counter-value"]))) + if case not in perf_results_for: + perf_results_for[case] = [] + perf_results_for[case].append((case_command, float(instr["counter-value"]))) print("Benchmarks summary\n---\n") for (case, entries) in perf_results_for.items(): @@ -108,7 +173,10 @@ with tempfile.TemporaryDirectory() as tmp_dir: subenv = os.environ.copy() subenv["NIX_CONF_DIR"] = "/var/empty" subenv["NIX_REMOTE"] = tmp_dir - subenv["NIX_PATH"] = "nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix" + subenv["NIX_PATH"] = ":".join([ + "nixpkgs=bench/nixpkgs", + ]) + subenv["NIX_DAEMON_SOCKET_PATH"] = f"{tmp_dir}/daemon" if args.mode == "walltime": bench_walltime(subenv) diff --git a/bench/configuration.nix b/bench/configuration.nix deleted file mode 100644 index c93f7b5d9..000000000 --- a/bench/configuration.nix +++ /dev/null @@ -1,314 +0,0 @@ -{ - config, - pkgs, - lib, - ... -}: - -{ - boot = { - initrd = { - availableKernelModules = [ - "xhci_pci" - "ahci" - ]; - kernelModules = [ "dm-snapshot" ]; - luks.devices = { - croot = { - device = "/dev/sdb"; - allowDiscards = true; - }; - }; - }; - kernelModules = [ "kvm-intel" ]; - kernelPackages = pkgs.linuxPackages_latest; - - loader = { - systemd-boot.enable = true; - efi.canTouchEfiVariables = true; - }; - }; - - hardware = { - enableRedistributableFirmware = true; - cpu.intel.updateMicrocode = true; - graphics.enable32Bit = true; - graphics.extraPackages = with pkgs; [ - vaapiIntel - intel-media-driver - intel-compute-runtime - ]; - }; - - fileSystems = { - "/" = { - device = "/dev/sda2"; - fsType = "xfs"; - options = [ "noatime" ]; - }; - - "/boot" = { - device = "/dev/sda1"; - fsType = "vfat"; - }; - - "/nas" = { - device = "nas:/"; - fsType = "nfs4"; - options = [ - "ro" - "x-systemd.automount" - ]; - }; - }; - swapDevices = [ { device = "/dev/swap"; } ]; - - networking = { - useDHCP = false; - hostName = "host"; - wireless = { - enable = true; - interfaces = [ "eth1" ]; - }; - interfaces = { - eth0.useDHCP = true; - eth1.useDHCP = true; - }; - wg-quick.interfaces = { - wg0 = { - address = [ "2001:db8::1" ]; - privateKeyFile = "/etc/secrets/wg0.key"; - peers = [ - { - publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; - endpoint = "[2001:db8::2]:61021"; - allowedIPs = [ "2001::db8:1::/64" ]; - } - ]; - }; - }; - - firewall.allowedUDPPorts = [ 4567 ]; - }; - - i18n = { - defaultLocale = "en_US.UTF-8"; - inputMethod.enable = true; - inputMethod.type = "ibus"; - }; - - services = { - libinput.enable = true; - xserver = { - enable = true; - xkb.layout = "us"; - xkb.variant = "altgr-intl"; - xkb.options = "ctrl:nocaps"; - wacom.enable = true; - videoDrivers = [ "modesetting" ]; - modules = [ pkgs.xf86_input_wacom ]; - - displayManager.sx.enable = true; - windowManager.i3.enable = true; - }; - - udev.extraHwdb = '' - # not like this mattered at all - # we're not running udev from here - ''; - - udev.extraRules = '' - # ACTION=="add", SUBSYSTEM=="input", ... - ''; - }; - - programs = { - light.enable = true; - wireshark = { - enable = true; - package = pkgs.wireshark-qt; - }; - gnupg.agent = { - enable = true; - }; - }; - - fonts.packages = with pkgs; [ - font-awesome - noto-fonts - noto-fonts-cjk-sans - noto-fonts-emoji - noto-fonts-extra - dejavu_fonts - powerline-fonts - source-code-pro - cantarell-fonts - ]; - - users = { - mutableUsers = false; - - users = { - user = { - isNormalUser = true; - group = "user"; - extraGroups = [ - "wheel" - "video" - "audio" - "dialout" - "users" - "kvm" - "wireshark" - ]; - password = "unimportant"; - }; - }; - - groups = { - user = { }; - }; - }; - - security = { - pam.loginLimits = [ - { - domain = "@audio"; - item = "memlock"; - type = "-"; - value = "unlimited"; - } - { - domain = "@audio"; - item = "rtprio"; - type = "-"; - value = "99"; - } - { - domain = "@audio"; - item = "nofile"; - type = "soft"; - value = "99999"; - } - { - domain = "@audio"; - item = "nofile"; - type = "hard"; - value = "99999"; - } - ]; - - sudo.extraRules = [ - { - users = [ "user" ]; - commands = [ - { - command = "${pkgs.linuxPackages.cpupower}/bin/cpupower"; - options = [ "NOPASSWD" ]; - } - ]; - } - ]; - }; - - environment.systemPackages = with pkgs; [ - a2jmidid - age - ardour - bemenu - blender - breeze-icons - breeze-qt5 - bubblewrap - calf - claws-mail - darktable - duperemove - emacs - feh - file - firefox - fluidsynth - adwaita-icon-theme - gnuplot - graphviz - helm - i3status-rust - inkscape - jack2 - jq - krita - ldns - libqalculate - libreoffice - man-pages - nix-diff - nix-index - nix-output-monitor - open-music-kontrollers.patchmatrix - pamixer - pavucontrol - pciutils - picom - pwgen - redshift - ripgrep - rlwrap - silver-searcher - soundfont-fluid - whois - wol - xclip - xdot - xdotool - xorg.xkbcomp - yt-dlp - zathura - borgbackup - linuxPackages.cpupower - mtr - kitty - xf86_input_wacom - ]; - - environment.pathsToLink = [ "/share/soundfonts" ]; - - systemd.user.services.run-python = { - after = [ "network-online.target" ]; - script = '' - exec ${pkgs.python3}/bin/python - ''; - serviceConfig = { - CapabilityBoundingSet = [ "" ]; - KeyringMode = "private"; - LockPersonality = true; - MemoryDenyWriteExecute = true; - NoNewPrivileges = true; - PrivateDevices = true; - PrivateTmp = true; - PrivateUsers = true; - ProcSubset = "pid"; - ProtectClock = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectKernelTunables = true; - ProtectProc = "invisible"; - ProtectSystem = "strict"; - RestrictAddressFamilies = "AF_INET AF_INET6"; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ - "@system-service" - "~ @resources @privileged" - ]; - UMask = "077"; - }; - }; - - system.stateVersion = "23.11"; -}