bench: soup up the benchmark script

add daemon support, fix the rebuild cases by using an installer config

Change-Id: I47cbb0dd401abf5e041e9936077e502f5e0cabf9

bench/bench.py

@@ -7,15 +7,43 @@ import os
 import json
 import tempfile
 import platform
+import shlex
+import textwrap
 
-flake_args = ["--extra-experimental-features","'nix-command flakes'"]
+flake_args = ["--extra-experimental-features", "nix-command flakes"]
-# hyperfine has its own variable substitution, so we use that and pass build="{BUILD}" here.
-# perf doesn't have variable substitution, so we call these with build being the actual build directory.
 cases = {
-    "search": lambda build: [f"{build}/bin/nix", *flake_args, "search", "--no-eval-cache", "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870", "hello"],
+    "search": lambda build: [
-    "rebuild": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
+        f"{build}/bin/nix",
-    "rebuild_lh": lambda build: ["GC_INITIAL_HEAP_SIZE=10g", f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
+        *flake_args,
-    "parse": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "-f", "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"],
+        "search",
+        "--no-eval-cache",
+        "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870",
+        "hello",
+    ],
+    "rebuild": lambda build: [
+        f"{build}/bin/nix",
+        *flake_args,
+        "eval",
+        "--raw",
+        "--impure",
+        "--expr",
+        textwrap.dedent("""
+            (import <nixpkgs/nixos> {
+                configuration = ./bench/nixpkgs/nixos/modules/installer/cd-dvd/installation-cd-graphical-calamares-plasma6.nix;
+            }).config.system.build.toplevel
+        """).replace("\n", " "),
+    ],
+    "rebuild_lh": lambda build: [
+        "GC_INITIAL_HEAP_SIZE=10g",
+        *cases['rebuild'](build),
+    ],
+    "parse": lambda build: [
+        f"{build}/bin/nix",
+        *flake_args,
+        "eval",
+        "-f",
+        "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix",
+    ],
 }
 
 arg_parser = argparse.ArgumentParser()
@@ -24,10 +52,26 @@ arg_parser = argparse.ArgumentParser()
 # mode, we would have to combine the JSON ourselves to support that, which
 # would probably be better done by writing a benchmarking script in
 # not-bash.
-arg_parser.add_argument('builds', nargs='+', help="At least two build directories to compare, containing bin/nix")
+arg_parser.add_argument(
-arg_parser.add_argument('--cases', type=str, help="A comma-separated list of cases you want to run. Defaults to running all")
+    'builds',
-available_modes = [ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [] # perf doesn't run on Darwin
+    nargs='+',
-arg_parser.add_argument('--mode', choices=available_modes, default="walltime")
+    help="At least two build directories to compare, containing bin/nix",
+)
+arg_parser.add_argument(
+    '--cases',
+    type=str,
+    help="A comma-separated list of cases you want to run. Defaults to running all",
+)
+arg_parser.add_argument(
+    '--mode',
+    choices=[ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [], # perf doesn't run on Darwin
+    default="walltime",
+)
+arg_parser.add_argument(
+    '--daemon',
+    action='store_true',
+    help='Run a temporary daemon for the benchmark instead of using a local store directly',
+)
 args = arg_parser.parse_args()
 if len(args.builds) < 2:
     raise ValueError("need at least two build directories to compare")
@@ -37,33 +81,51 @@ if args.cases is None:
     benchmarks = list(cases.keys())
 else:
     for case in args.cases.split(","):
-        if case not in cases: raise ValueError(f"no such case: {case}")
+        if case not in cases:
+            raise ValueError(f"no such case: {case}")
         benchmarks.append(case)
 
+def make_full_command(build, case):
+    cmd = " ".join(map(shlex.quote, cases[case](build)))
+    if args.daemon:
+        return " ".join([
+            f"{build}/bin/nix --extra-experimental-features nix-command daemon &",
+            "trap 'kill %1' EXIT;",
+            f"NIX_REMOTE=daemon {cmd}",
+        ])
+    else:
+        return cmd
+
 def bench_walltime(env):
-    hyperfine_args = ["--parameter-list", "BUILD", ','.join(args.builds), "--warmup", "2", "--runs", "10"]
     for case in benchmarks:
-        case_command = cases[case]("{BUILD}") # see the comment on cases
+        for build in args.builds:
             subprocess.run([
                 "taskset", "-c", "2,3",
                 "chrt", "-f","50",
-            "hyperfine", *hyperfine_args, "--export-json", f"bench/bench-{case}.json", "--export-markdown", f"bench/bench-{case}.md", "--", " ".join(case_command)
+                *[
+                    "hyperfine", "--warmup", "2", "--runs", "10",
+                    "--export-json", f"bench/bench-{case}-{build}.json",
+                    "--export-markdown", f"bench/bench-{case}-{build}.md",
+                    "--", make_full_command(build, case),
+                ],
             ], env=env, check=True)
 
     print("Benchmarks summary\n---\n")
     for case in benchmarks:
-        fd = open(f"bench/bench-{case}.json")
+        results = []
-        result_json = json.load(fd)
+        for build in args.builds:
-        fd.close()
+            with open(f"bench/bench-{case}-{build}.json") as fd:
-        for result in result_json["results"]:
+                results.append(json.load(fd)["results"][0])
+        for result in results:
             print(result["command"])
             print("-" * min(80,len(result["command"])))
-            attr_rounded = lambda attr: f"{result[attr]:.3f}"
+            def attr_rounded(attr):
+                return f"{result[attr]:.3f}"
             print("  mean:    ", attr_rounded("mean"), "±", attr_rounded("stddev"))
             print("            user:",  attr_rounded("user"), "| system", attr_rounded("system"))
             print("  median:  ", attr_rounded("median"))
             print("  range:   ", attr_rounded("min") + "s.." + attr_rounded("max")+"s")
-            print("  relative:", f"{result["mean"]/result_json["results"][0]["mean"]:.3f}")
+            print("  relative:", f"{result["mean"]/results[0]["mean"]:.3f}")
             print("\n")
 
 
@@ -71,12 +133,14 @@ def bench_icount(env):
     perf_results_for: dict[str, list[tuple[str, float]]] = {}
     for case in benchmarks:
         for build in args.builds:
-            case_command = cases[case](build)
             # the perf stat -j output (incorrectly) localizes numbers, which will trip up the json parser.
             env["LC_ALL"]="C"
+            case_command = make_full_command(build, case)
             commandline = [
-                "perf", "stat", "-o", f"bench/perf-{case}.json", "-j", "sh", "-c", " ".join(case_command)
+                "perf", "stat", "-o", f"bench/perf-{case}.json", "-j",
+                "sh", "-c", case_command,
             ]
+            print("running", case_command)
             subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL) # warmup run
             subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL)
             perf_fd = open(f"bench/perf-{case}.json")
@@ -84,8 +148,9 @@ def bench_icount(env):
             perf_fd.close()
 
             instr = next(x for x in perf_data if x["event"] in ["instructions", "instructions:u"]) # an implementation of a find_first iterator
-            if case not in perf_results_for: perf_results_for[case] = []
+            if case not in perf_results_for:
-            perf_results_for[case].append((" ".join(case_command), float(instr["counter-value"])))
+                perf_results_for[case] = []
+            perf_results_for[case].append((case_command, float(instr["counter-value"])))
 
     print("Benchmarks summary\n---\n")
     for (case, entries) in perf_results_for.items():
@@ -108,7 +173,10 @@ with tempfile.TemporaryDirectory() as tmp_dir:
     subenv = os.environ.copy()
     subenv["NIX_CONF_DIR"] = "/var/empty"
     subenv["NIX_REMOTE"] = tmp_dir
-    subenv["NIX_PATH"] = "nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
+    subenv["NIX_PATH"] = ":".join([
+        "nixpkgs=bench/nixpkgs",
+    ])
+    subenv["NIX_DAEMON_SOCKET_PATH"] = f"{tmp_dir}/daemon"
 
     if args.mode == "walltime":
         bench_walltime(subenv)

bench/configuration.nix

@@ -1,314 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}:
-
-{
-  boot = {
-    initrd = {
-      availableKernelModules = [
-        "xhci_pci"
-        "ahci"
-      ];
-      kernelModules = [ "dm-snapshot" ];
-      luks.devices = {
-        croot = {
-          device = "/dev/sdb";
-          allowDiscards = true;
-        };
-      };
-    };
-    kernelModules = [ "kvm-intel" ];
-    kernelPackages = pkgs.linuxPackages_latest;
-
-    loader = {
-      systemd-boot.enable = true;
-      efi.canTouchEfiVariables = true;
-    };
-  };
-
-  hardware = {
-    enableRedistributableFirmware = true;
-    cpu.intel.updateMicrocode = true;
-    graphics.enable32Bit = true;
-    graphics.extraPackages = with pkgs; [
-      vaapiIntel
-      intel-media-driver
-      intel-compute-runtime
-    ];
-  };
-
-  fileSystems = {
-    "/" = {
-      device = "/dev/sda2";
-      fsType = "xfs";
-      options = [ "noatime" ];
-    };
-
-    "/boot" = {
-      device = "/dev/sda1";
-      fsType = "vfat";
-    };
-
-    "/nas" = {
-      device = "nas:/";
-      fsType = "nfs4";
-      options = [
-        "ro"
-        "x-systemd.automount"
-      ];
-    };
-  };
-  swapDevices = [ { device = "/dev/swap"; } ];
-
-  networking = {
-    useDHCP = false;
-    hostName = "host";
-    wireless = {
-      enable = true;
-      interfaces = [ "eth1" ];
-    };
-    interfaces = {
-      eth0.useDHCP = true;
-      eth1.useDHCP = true;
-    };
-    wg-quick.interfaces = {
-      wg0 = {
-        address = [ "2001:db8::1" ];
-        privateKeyFile = "/etc/secrets/wg0.key";
-        peers = [
-          {
-            publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
-            endpoint = "[2001:db8::2]:61021";
-            allowedIPs = [ "2001::db8:1::/64" ];
-          }
-        ];
-      };
-    };
-
-    firewall.allowedUDPPorts = [ 4567 ];
-  };
-
-  i18n = {
-    defaultLocale = "en_US.UTF-8";
-    inputMethod.enable = true;
-    inputMethod.type = "ibus";
-  };
-
-  services = {
-    libinput.enable = true;
-    xserver = {
-      enable = true;
-      xkb.layout = "us";
-      xkb.variant = "altgr-intl";
-      xkb.options = "ctrl:nocaps";
-      wacom.enable = true;
-      videoDrivers = [ "modesetting" ];
-      modules = [ pkgs.xf86_input_wacom ];
-
-      displayManager.sx.enable = true;
-      windowManager.i3.enable = true;
-    };
-
-    udev.extraHwdb = ''
-      # not like this mattered at all
-      # we're not running udev from here
-    '';
-
-    udev.extraRules = ''
-      # ACTION=="add", SUBSYSTEM=="input", ...
-    '';
-  };
-
-  programs = {
-    light.enable = true;
-    wireshark = {
-      enable = true;
-      package = pkgs.wireshark-qt;
-    };
-    gnupg.agent = {
-      enable = true;
-    };
-  };
-
-  fonts.packages = with pkgs; [
-    font-awesome
-    noto-fonts
-    noto-fonts-cjk-sans
-    noto-fonts-emoji
-    noto-fonts-extra
-    dejavu_fonts
-    powerline-fonts
-    source-code-pro
-    cantarell-fonts
-  ];
-
-  users = {
-    mutableUsers = false;
-
-    users = {
-      user = {
-        isNormalUser = true;
-        group = "user";
-        extraGroups = [
-          "wheel"
-          "video"
-          "audio"
-          "dialout"
-          "users"
-          "kvm"
-          "wireshark"
-        ];
-        password = "unimportant";
-      };
-    };
-
-    groups = {
-      user = { };
-    };
-  };
-
-  security = {
-    pam.loginLimits = [
-      {
-        domain = "@audio";
-        item = "memlock";
-        type = "-";
-        value = "unlimited";
-      }
-      {
-        domain = "@audio";
-        item = "rtprio";
-        type = "-";
-        value = "99";
-      }
-      {
-        domain = "@audio";
-        item = "nofile";
-        type = "soft";
-        value = "99999";
-      }
-      {
-        domain = "@audio";
-        item = "nofile";
-        type = "hard";
-        value = "99999";
-      }
-    ];
-
-    sudo.extraRules = [
-      {
-        users = [ "user" ];
-        commands = [
-          {
-            command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
-            options = [ "NOPASSWD" ];
-          }
-        ];
-      }
-    ];
-  };
-
-  environment.systemPackages = with pkgs; [
-    a2jmidid
-    age
-    ardour
-    bemenu
-    blender
-    breeze-icons
-    breeze-qt5
-    bubblewrap
-    calf
-    claws-mail
-    darktable
-    duperemove
-    emacs
-    feh
-    file
-    firefox
-    fluidsynth
-    adwaita-icon-theme
-    gnuplot
-    graphviz
-    helm
-    i3status-rust
-    inkscape
-    jack2
-    jq
-    krita
-    ldns
-    libqalculate
-    libreoffice
-    man-pages
-    nix-diff
-    nix-index
-    nix-output-monitor
-    open-music-kontrollers.patchmatrix
-    pamixer
-    pavucontrol
-    pciutils
-    picom
-    pwgen
-    redshift
-    ripgrep
-    rlwrap
-    silver-searcher
-    soundfont-fluid
-    whois
-    wol
-    xclip
-    xdot
-    xdotool
-    xorg.xkbcomp
-    yt-dlp
-    zathura
-    borgbackup
-    linuxPackages.cpupower
-    mtr
-    kitty
-    xf86_input_wacom
-  ];
-
-  environment.pathsToLink = [ "/share/soundfonts" ];
-
-  systemd.user.services.run-python = {
-    after = [ "network-online.target" ];
-    script = ''
-      exec ${pkgs.python3}/bin/python
-    '';
-    serviceConfig = {
-      CapabilityBoundingSet = [ "" ];
-      KeyringMode = "private";
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      NoNewPrivileges = true;
-      PrivateDevices = true;
-      PrivateTmp = true;
-      PrivateUsers = true;
-      ProcSubset = "pid";
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = true;
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectKernelTunables = true;
-      ProtectProc = "invisible";
-      ProtectSystem = "strict";
-      RestrictAddressFamilies = "AF_INET AF_INET6";
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-        "@system-service"
-        "~ @resources @privileged"
-      ];
-      UMask = "077";
-    };
-  };
-
-  system.stateVersion = "23.11";
-}