bench: soup up the benchmark script
add daemon support, fix the rebuild cases by using an installer config Change-Id: I47cbb0dd401abf5e041e9936077e502f5e0cabf9
This commit is contained in:
eldritch horrors
+94
-26
@@ -7,15 +7,43 @@ import os
|
||||
import json
|
||||
import tempfile
|
||||
import platform
|
||||
import shlex
|
||||
import textwrap
|
||||
|
||||
flake_args = ["--extra-experimental-features","'nix-command flakes'"]
|
||||
# hyperfine has its own variable substitution, so we use that and pass build="{BUILD}" here.
|
||||
# perf doesn't have variable substitution, so we call these with build being the actual build directory.
|
||||
flake_args = ["--extra-experimental-features", "nix-command flakes"]
|
||||
cases = {
|
||||
"search": lambda build: [f"{build}/bin/nix", *flake_args, "search", "--no-eval-cache", "github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870", "hello"],
|
||||
"rebuild": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
|
||||
"rebuild_lh": lambda build: ["GC_INITIAL_HEAP_SIZE=10g", f"{build}/bin/nix", *flake_args, "eval", "--raw", "--impure", "--expr", "'with import <nixpkgs/nixos> {}; system'"],
|
||||
"parse": lambda build: [f"{build}/bin/nix", *flake_args, "eval", "-f", "bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"],
|
||||
"search": lambda build: [
|
||||
f"{build}/bin/nix",
|
||||
*flake_args,
|
||||
"search",
|
||||
"--no-eval-cache",
|
||||
"github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870",
|
||||
"hello",
|
||||
],
|
||||
"rebuild": lambda build: [
|
||||
f"{build}/bin/nix",
|
||||
*flake_args,
|
||||
"eval",
|
||||
"--raw",
|
||||
"--impure",
|
||||
"--expr",
|
||||
textwrap.dedent("""
|
||||
(import <nixpkgs/nixos> {
|
||||
configuration = ./bench/nixpkgs/nixos/modules/installer/cd-dvd/installation-cd-graphical-calamares-plasma6.nix;
|
||||
}).config.system.build.toplevel
|
||||
""").replace("\n", " "),
|
||||
],
|
||||
"rebuild_lh": lambda build: [
|
||||
"GC_INITIAL_HEAP_SIZE=10g",
|
||||
*cases['rebuild'](build),
|
||||
],
|
||||
"parse": lambda build: [
|
||||
f"{build}/bin/nix",
|
||||
*flake_args,
|
||||
"eval",
|
||||
"-f",
|
||||
"bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix",
|
||||
],
|
||||
}
|
||||
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
@@ -24,10 +52,26 @@ arg_parser = argparse.ArgumentParser()
|
||||
# mode, we would have to combine the JSON ourselves to support that, which
|
||||
# would probably be better done by writing a benchmarking script in
|
||||
# not-bash.
|
||||
arg_parser.add_argument('builds', nargs='+', help="At least two build directories to compare, containing bin/nix")
|
||||
arg_parser.add_argument('--cases', type=str, help="A comma-separated list of cases you want to run. Defaults to running all")
|
||||
available_modes = [ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [] # perf doesn't run on Darwin
|
||||
arg_parser.add_argument('--mode', choices=available_modes, default="walltime")
|
||||
arg_parser.add_argument(
|
||||
'builds',
|
||||
nargs='+',
|
||||
help="At least two build directories to compare, containing bin/nix",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
'--cases',
|
||||
type=str,
|
||||
help="A comma-separated list of cases you want to run. Defaults to running all",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
'--mode',
|
||||
choices=[ "walltime" ] + [ "icount" ] if platform.system() == 'Linux' else [], # perf doesn't run on Darwin
|
||||
default="walltime",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
'--daemon',
|
||||
action='store_true',
|
||||
help='Run a temporary daemon for the benchmark instead of using a local store directly',
|
||||
)
|
||||
args = arg_parser.parse_args()
|
||||
if len(args.builds) < 2:
|
||||
raise ValueError("need at least two build directories to compare")
|
||||
@@ -37,33 +81,51 @@ if args.cases is None:
|
||||
benchmarks = list(cases.keys())
|
||||
else:
|
||||
for case in args.cases.split(","):
|
||||
if case not in cases: raise ValueError(f"no such case: {case}")
|
||||
if case not in cases:
|
||||
raise ValueError(f"no such case: {case}")
|
||||
benchmarks.append(case)
|
||||
|
||||
def make_full_command(build, case):
|
||||
cmd = " ".join(map(shlex.quote, cases[case](build)))
|
||||
if args.daemon:
|
||||
return " ".join([
|
||||
f"{build}/bin/nix --extra-experimental-features nix-command daemon &",
|
||||
"trap 'kill %1' EXIT;",
|
||||
f"NIX_REMOTE=daemon {cmd}",
|
||||
])
|
||||
else:
|
||||
return cmd
|
||||
|
||||
def bench_walltime(env):
|
||||
hyperfine_args = ["--parameter-list", "BUILD", ','.join(args.builds), "--warmup", "2", "--runs", "10"]
|
||||
for case in benchmarks:
|
||||
case_command = cases[case]("{BUILD}") # see the comment on cases
|
||||
for build in args.builds:
|
||||
subprocess.run([
|
||||
"taskset", "-c", "2,3",
|
||||
"chrt", "-f","50",
|
||||
"hyperfine", *hyperfine_args, "--export-json", f"bench/bench-{case}.json", "--export-markdown", f"bench/bench-{case}.md", "--", " ".join(case_command)
|
||||
*[
|
||||
"hyperfine", "--warmup", "2", "--runs", "10",
|
||||
"--export-json", f"bench/bench-{case}-{build}.json",
|
||||
"--export-markdown", f"bench/bench-{case}-{build}.md",
|
||||
"--", make_full_command(build, case),
|
||||
],
|
||||
], env=env, check=True)
|
||||
|
||||
print("Benchmarks summary\n---\n")
|
||||
for case in benchmarks:
|
||||
fd = open(f"bench/bench-{case}.json")
|
||||
result_json = json.load(fd)
|
||||
fd.close()
|
||||
for result in result_json["results"]:
|
||||
results = []
|
||||
for build in args.builds:
|
||||
with open(f"bench/bench-{case}-{build}.json") as fd:
|
||||
results.append(json.load(fd)["results"][0])
|
||||
for result in results:
|
||||
print(result["command"])
|
||||
print("-" * min(80,len(result["command"])))
|
||||
attr_rounded = lambda attr: f"{result[attr]:.3f}"
|
||||
def attr_rounded(attr):
|
||||
return f"{result[attr]:.3f}"
|
||||
print(" mean: ", attr_rounded("mean"), "±", attr_rounded("stddev"))
|
||||
print(" user:", attr_rounded("user"), "| system", attr_rounded("system"))
|
||||
print(" median: ", attr_rounded("median"))
|
||||
print(" range: ", attr_rounded("min") + "s.." + attr_rounded("max")+"s")
|
||||
print(" relative:", f"{result["mean"]/result_json["results"][0]["mean"]:.3f}")
|
||||
print(" relative:", f"{result["mean"]/results[0]["mean"]:.3f}")
|
||||
print("\n")
|
||||
|
||||
|
||||
@@ -71,12 +133,14 @@ def bench_icount(env):
|
||||
perf_results_for: dict[str, list[tuple[str, float]]] = {}
|
||||
for case in benchmarks:
|
||||
for build in args.builds:
|
||||
case_command = cases[case](build)
|
||||
# the perf stat -j output (incorrectly) localizes numbers, which will trip up the json parser.
|
||||
env["LC_ALL"]="C"
|
||||
case_command = make_full_command(build, case)
|
||||
commandline = [
|
||||
"perf", "stat", "-o", f"bench/perf-{case}.json", "-j", "sh", "-c", " ".join(case_command)
|
||||
"perf", "stat", "-o", f"bench/perf-{case}.json", "-j",
|
||||
"sh", "-c", case_command,
|
||||
]
|
||||
print("running", case_command)
|
||||
subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL) # warmup run
|
||||
subprocess.run(commandline, env=env, check=True, stdout=subprocess.DEVNULL)
|
||||
perf_fd = open(f"bench/perf-{case}.json")
|
||||
@@ -84,8 +148,9 @@ def bench_icount(env):
|
||||
perf_fd.close()
|
||||
|
||||
instr = next(x for x in perf_data if x["event"] in ["instructions", "instructions:u"]) # an implementation of a find_first iterator
|
||||
if case not in perf_results_for: perf_results_for[case] = []
|
||||
perf_results_for[case].append((" ".join(case_command), float(instr["counter-value"])))
|
||||
if case not in perf_results_for:
|
||||
perf_results_for[case] = []
|
||||
perf_results_for[case].append((case_command, float(instr["counter-value"])))
|
||||
|
||||
print("Benchmarks summary\n---\n")
|
||||
for (case, entries) in perf_results_for.items():
|
||||
@@ -108,7 +173,10 @@ with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
subenv = os.environ.copy()
|
||||
subenv["NIX_CONF_DIR"] = "/var/empty"
|
||||
subenv["NIX_REMOTE"] = tmp_dir
|
||||
subenv["NIX_PATH"] = "nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
|
||||
subenv["NIX_PATH"] = ":".join([
|
||||
"nixpkgs=bench/nixpkgs",
|
||||
])
|
||||
subenv["NIX_DAEMON_SOCKET_PATH"] = f"{tmp_dir}/daemon"
|
||||
|
||||
if args.mode == "walltime":
|
||||
bench_walltime(subenv)
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
|
||||
{
|
||||
boot = {
|
||||
initrd = {
|
||||
availableKernelModules = [
|
||||
"xhci_pci"
|
||||
"ahci"
|
||||
];
|
||||
kernelModules = [ "dm-snapshot" ];
|
||||
luks.devices = {
|
||||
croot = {
|
||||
device = "/dev/sdb";
|
||||
allowDiscards = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
kernelModules = [ "kvm-intel" ];
|
||||
kernelPackages = pkgs.linuxPackages_latest;
|
||||
|
||||
loader = {
|
||||
systemd-boot.enable = true;
|
||||
efi.canTouchEfiVariables = true;
|
||||
};
|
||||
};
|
||||
|
||||
hardware = {
|
||||
enableRedistributableFirmware = true;
|
||||
cpu.intel.updateMicrocode = true;
|
||||
graphics.enable32Bit = true;
|
||||
graphics.extraPackages = with pkgs; [
|
||||
vaapiIntel
|
||||
intel-media-driver
|
||||
intel-compute-runtime
|
||||
];
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/" = {
|
||||
device = "/dev/sda2";
|
||||
fsType = "xfs";
|
||||
options = [ "noatime" ];
|
||||
};
|
||||
|
||||
"/boot" = {
|
||||
device = "/dev/sda1";
|
||||
fsType = "vfat";
|
||||
};
|
||||
|
||||
"/nas" = {
|
||||
device = "nas:/";
|
||||
fsType = "nfs4";
|
||||
options = [
|
||||
"ro"
|
||||
"x-systemd.automount"
|
||||
];
|
||||
};
|
||||
};
|
||||
swapDevices = [ { device = "/dev/swap"; } ];
|
||||
|
||||
networking = {
|
||||
useDHCP = false;
|
||||
hostName = "host";
|
||||
wireless = {
|
||||
enable = true;
|
||||
interfaces = [ "eth1" ];
|
||||
};
|
||||
interfaces = {
|
||||
eth0.useDHCP = true;
|
||||
eth1.useDHCP = true;
|
||||
};
|
||||
wg-quick.interfaces = {
|
||||
wg0 = {
|
||||
address = [ "2001:db8::1" ];
|
||||
privateKeyFile = "/etc/secrets/wg0.key";
|
||||
peers = [
|
||||
{
|
||||
publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||
endpoint = "[2001:db8::2]:61021";
|
||||
allowedIPs = [ "2001::db8:1::/64" ];
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
firewall.allowedUDPPorts = [ 4567 ];
|
||||
};
|
||||
|
||||
i18n = {
|
||||
defaultLocale = "en_US.UTF-8";
|
||||
inputMethod.enable = true;
|
||||
inputMethod.type = "ibus";
|
||||
};
|
||||
|
||||
services = {
|
||||
libinput.enable = true;
|
||||
xserver = {
|
||||
enable = true;
|
||||
xkb.layout = "us";
|
||||
xkb.variant = "altgr-intl";
|
||||
xkb.options = "ctrl:nocaps";
|
||||
wacom.enable = true;
|
||||
videoDrivers = [ "modesetting" ];
|
||||
modules = [ pkgs.xf86_input_wacom ];
|
||||
|
||||
displayManager.sx.enable = true;
|
||||
windowManager.i3.enable = true;
|
||||
};
|
||||
|
||||
udev.extraHwdb = ''
|
||||
# not like this mattered at all
|
||||
# we're not running udev from here
|
||||
'';
|
||||
|
||||
udev.extraRules = ''
|
||||
# ACTION=="add", SUBSYSTEM=="input", ...
|
||||
'';
|
||||
};
|
||||
|
||||
programs = {
|
||||
light.enable = true;
|
||||
wireshark = {
|
||||
enable = true;
|
||||
package = pkgs.wireshark-qt;
|
||||
};
|
||||
gnupg.agent = {
|
||||
enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
fonts.packages = with pkgs; [
|
||||
font-awesome
|
||||
noto-fonts
|
||||
noto-fonts-cjk-sans
|
||||
noto-fonts-emoji
|
||||
noto-fonts-extra
|
||||
dejavu_fonts
|
||||
powerline-fonts
|
||||
source-code-pro
|
||||
cantarell-fonts
|
||||
];
|
||||
|
||||
users = {
|
||||
mutableUsers = false;
|
||||
|
||||
users = {
|
||||
user = {
|
||||
isNormalUser = true;
|
||||
group = "user";
|
||||
extraGroups = [
|
||||
"wheel"
|
||||
"video"
|
||||
"audio"
|
||||
"dialout"
|
||||
"users"
|
||||
"kvm"
|
||||
"wireshark"
|
||||
];
|
||||
password = "unimportant";
|
||||
};
|
||||
};
|
||||
|
||||
groups = {
|
||||
user = { };
|
||||
};
|
||||
};
|
||||
|
||||
security = {
|
||||
pam.loginLimits = [
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "memlock";
|
||||
type = "-";
|
||||
value = "unlimited";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "rtprio";
|
||||
type = "-";
|
||||
value = "99";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "nofile";
|
||||
type = "soft";
|
||||
value = "99999";
|
||||
}
|
||||
{
|
||||
domain = "@audio";
|
||||
item = "nofile";
|
||||
type = "hard";
|
||||
value = "99999";
|
||||
}
|
||||
];
|
||||
|
||||
sudo.extraRules = [
|
||||
{
|
||||
users = [ "user" ];
|
||||
commands = [
|
||||
{
|
||||
command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
|
||||
options = [ "NOPASSWD" ];
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
a2jmidid
|
||||
age
|
||||
ardour
|
||||
bemenu
|
||||
blender
|
||||
breeze-icons
|
||||
breeze-qt5
|
||||
bubblewrap
|
||||
calf
|
||||
claws-mail
|
||||
darktable
|
||||
duperemove
|
||||
emacs
|
||||
feh
|
||||
file
|
||||
firefox
|
||||
fluidsynth
|
||||
adwaita-icon-theme
|
||||
gnuplot
|
||||
graphviz
|
||||
helm
|
||||
i3status-rust
|
||||
inkscape
|
||||
jack2
|
||||
jq
|
||||
krita
|
||||
ldns
|
||||
libqalculate
|
||||
libreoffice
|
||||
man-pages
|
||||
nix-diff
|
||||
nix-index
|
||||
nix-output-monitor
|
||||
open-music-kontrollers.patchmatrix
|
||||
pamixer
|
||||
pavucontrol
|
||||
pciutils
|
||||
picom
|
||||
pwgen
|
||||
redshift
|
||||
ripgrep
|
||||
rlwrap
|
||||
silver-searcher
|
||||
soundfont-fluid
|
||||
whois
|
||||
wol
|
||||
xclip
|
||||
xdot
|
||||
xdotool
|
||||
xorg.xkbcomp
|
||||
yt-dlp
|
||||
zathura
|
||||
borgbackup
|
||||
linuxPackages.cpupower
|
||||
mtr
|
||||
kitty
|
||||
xf86_input_wacom
|
||||
];
|
||||
|
||||
environment.pathsToLink = [ "/share/soundfonts" ];
|
||||
|
||||
systemd.user.services.run-python = {
|
||||
after = [ "network-online.target" ];
|
||||
script = ''
|
||||
exec ${pkgs.python3}/bin/python
|
||||
'';
|
||||
serviceConfig = {
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
KeyringMode = "private";
|
||||
LockPersonality = true;
|
||||
MemoryDenyWriteExecute = true;
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProcSubset = "pid";
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectProc = "invisible";
|
||||
ProtectSystem = "strict";
|
||||
RestrictAddressFamilies = "AF_INET AF_INET6";
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
SystemCallArchitectures = "native";
|
||||
SystemCallFilter = [
|
||||
"@system-service"
|
||||
"~ @resources @privileged"
|
||||
];
|
||||
UMask = "077";
|
||||
};
|
||||
};
|
||||
|
||||
system.stateVersion = "23.11";
|
||||
}
|
||||
Reference in New Issue
Block a user