diff --git a/doc/manual/rl-next/machines-toml.md b/doc/manual/rl-next/machines-toml.md new file mode 100644 index 000000000..478ce25ca --- /dev/null +++ b/doc/manual/rl-next/machines-toml.md @@ -0,0 +1,10 @@ +--- +synopsis: "Allow remote builders to be configured using TOML" +cls: [4533] +category: "Features" +credits: [commentator2.0, Qyriad] +--- +Lix now supports configuring remote builders using a TOML file instead of the old, very cursed and incomprehensible format. +This comes with not only a human-understandable file, but also with better messages and error reports on misconfiguration. + +A more detailed Documentation can be found on the [distributed-builds](@docroot@/advanced-topics/distributed-builds.md) Wiki-page diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 6781e672d..626b0bc0d 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -45,102 +45,13 @@ contains Nix. > > If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. + +## Configuration + The list of remote machines can be specified on the command line or in -the Lix configuration file. The former is convenient for testing. For -example, the following command allows you to build a derivation for -`x86_64-darwin` on a Linux machine: - -```console -$ uname -Linux - -$ nix build --impure \ - --expr '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ - --builders 'ssh://mac x86_64-darwin' -[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac - -$ cat ./result -Darwin -``` - -It is possible to specify multiple builders separated by a semicolon or -a newline, e.g. - -```console - --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' -``` - -Each machine specification consists of the following elements, separated -by spaces. Only the first element is required. To leave a field at its -default, set it to `-`. - -1. The URI of the remote store in the format - `ssh://[username@]hostname[?port=]`, e.g. `ssh://nix@mac` or `ssh://mac`. - If the ssh server is not listening on port 22 (e.g. port 1337 in this case) - the URI would be `ssh://nix@mac?port=1337` - For backward compatibility, `ssh://` may be omitted. The hostname - may be an alias defined in your `~/.ssh/config`. - -2. A comma-separated list of Nix platform type identifiers, such as - `x86_64-darwin`. It is possible for a machine to support multiple - platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this - defaults to the local platform type. - -3. The SSH identity file to be used to log in to the remote machine. If - omitted, SSH will use its regular identities. - -4. The maximum number of builds that Lix will execute in parallel on - the machine. Typically this should be equal to the number of CPU - cores. For instance, the machine `itchy` in the example will execute - up to 8 builds in parallel. - -5. The “speed factor”, indicating the relative speed of the machine. If - there are multiple machines of the right type, Lix will prefer the - fastest, taking load into account. - -6. A comma-separated list of *supported features*. If a derivation has - the `requiredSystemFeatures` attribute, then Lix will only perform - the derivation on a machine that has the specified features. For - instance, the attribute - - ```nix - requiredSystemFeatures = [ "kvm" ]; - ``` - - will cause the build to be performed on a machine that has the `kvm` - feature. - -7. A comma-separated list of *mandatory features*. A machine will only - be used to build a derivation if all of the machine’s mandatory - features appear in the derivation’s `requiredSystemFeatures` - attribute. - -8. The (base64-encoded) public host key of the remote machine. If omitted, SSH - will use its regular known-hosts file. Specifically, the field is calculated - via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. - -For example, the machine specification - - nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm - nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 - nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark - -specifies several machines that can perform `i686-linux` builds. -However, `poochie` will only do builds that have the attribute - -```nix -requiredSystemFeatures = [ "benchmark" ]; -``` - -or - -```nix -requiredSystemFeatures = [ "benchmark" "kvm" ]; -``` - -`itchy` cannot do builds that require `kvm`, but `scratchy` does support -such builds. For regular builds, `itchy` will be preferred over -`scratchy` because it has a higher speed factor. +the Lix configuration file. The former is convenient for testing. +Additionally, there are two supported formats to configure remote builders: +The legacy, "space"-separated format and starting with Lix 2.95.0, a TOML. Remote builders can also be configured in `nix.conf`, e.g. @@ -159,3 +70,169 @@ option `builders-use-substitutes` in your local `nix.conf`. To build only on remote builders and disable building on the local machine, you can use the option `--max-jobs 0`. + +--- + +Each machine specification consists of the following attributes. +How those are combined within the configuration file differs for the formats, and will be explained further down. + +1. `uri` (**required**) + The URI of the remote store in the format + `ssh[-ng]://[username@]hostname[?port=]`, e.g. `ssh://nix@mac` or `ssh://mac`. + If the ssh server is not listening on port 22 (e.g. port 1337 in this case) + the URI would be `ssh[-ng]://nix@mac?port=1337`. The hostname + may be an alias defined in your `~/.ssh/config`. + +2. `system-types` (**optional**) + A list of Nix platform type identifiers, such as + `x86_64-darwin`. It is possible for a machine to support multiple + platform types, e.g., `i686-linux` and `x86_64-linux`. + + Defaults to the local platform type + +3. `ssh-key` (**optional**) + The SSH identity file to be used to log in to the remote machine. + + Defaults to SSHs regular identities. + +4. `jobs` (**optional**) + The maximum number of builds that Lix will execute in parallel on + the machine. Typically, this should be equal to the number of CPU + cores divided by the cores within the target machines configuration, i.e. `jobs * cores ~= cpu cores` + + Defaults to 1; must be a positive integer. + +5. `speed-factor` + The “speed factor”, indicating the relative speed of the machine. If + there are multiple machines of the right type, Lix will prefer the + fastest, taking load into account. + + Defaults to 1; must be a positive float. + +6. `supported-features` (**optional**) + A list of *supported features*. If a derivation has + the `requiredSystemFeatures` attribute, then Lix will only schedule + the derivation on a machine that has the specified features. For + example, the attribute + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + will cause the build to be performed on a machine that has the `kvm` + feature. + + Defaults to an empty list. + +7. `mandatory-features` (**optional**) + A list of *mandatory features*. A machine will only + be used to build a derivation if all the machine’s mandatory + features appear in the derivation’s `requiredSystemFeatures` + attribute. + + Defaults to an empty list. + +8. `ssh-public-host-key` (**optional**) + The public host key of the remote machine. + + Defaults to basic ssh behavior (checking contests of the known-hosts file) + + +### Using a TOML configuration + +Each machine is configured as an attribute within the map called `machines`. +The attributes name is the machines name. +Attributes can be in any order. + +For example: + +```toml +version = 1 + +[machines.andesite] +uri = "ssh://lix@andesite.lix.systems" # toml also allows for comments +system-types = ["i686-linux"] +jobs = 8 +speed-factor = 1.0 +supported-features = ["kvm"] +ssh-key = "/home/deepslate/.ssh/id_ed25519" + +[machines.diorite] +uri = "ssh://lix@diorite.lix.systems" +system-types = ["i686-linux"] +jobs = 8 +speed-factor = 2.0 +ssh-key = "/home/deepslate/.ssh/id_ed25519" + +[machines.granite] +uri = "ssh://lix@granite.lix.systems" +system-types = ["i686-linux"] +jobs = 1 +speed-factor = 2.0 +supported-features = ["kvm", "benchmark"] +ssh-key = "/home/deepslate/.ssh/id_ed25519" + +``` + +> **Note** +> +> If the version tag is omitted (e.g. in the CLI), it defaults to the latest version. +> It is strongly recommended to always provide a version tag for configuration within files to avoid breakage. + +For testing purposes, one can also define a builder ad hoc on the CLI as follows: +`--builders 'machines.andesite = {uri = "ssh://lix@andesite.lix.systems", jobs = 8}'` + + +### Using the legacy format +> **Warning** +> +> This format is frozen and new features / configuration options will not be backported to this format. + +It is possible to specify multiple builders separated by a semicolon or +a newline, e.g. + +```console + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' +``` + +Every machine specification consists of the elements listed in the section above, seperated by any amount of spaces or tabs. +The Attributes need to be provided **in order** and without names. +To leave a field at its default, set it to `-`. +Lists are colon seperated, without additional spaces. + +``` +lix@andesite.lix.systems i686-linux /home/deepslate/.ssh/id_ed25519 8 1 kvm +lix@diorite.lix.systems i686-linux /home/deepslate/.ssh/id_ed25519 8 2 +lix@granite.lix.systems i686-linux /home/deepslate/.ssh/id_ed25519 1 2 kvm benchmark +``` + +#### Special handling of fields +- `uri`: Due to backward compatibility, the `ssh://` may be omitted for the store-uri. +- `ssh-public-host-key`: The key must be provided encoded in base64. Specifically calculated via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub` + + +### Format detection + +At first, the given configuration is being parsed syntactically as a toml. +If parsing fails and the given configuration contains a `"` the error is presented to the user, as those characters are necessary for TOML, but disallowed for the legacy format. +Otherwise, parsing is retried using the legacy format. +If non-syntactic errors are detected within the toml, the exception will always be shown to the user directly. + + +## Builder selection +The configuration(s) above specify several machines that can perform `i686-linux` builds. +However, `granite` will only do builds that have the attribute + +```nix +requiredSystemFeatures = [ "benchmark" ]; +``` + +or + +```nix +requiredSystemFeatures = [ "benchmark" "kvm" ]; +``` + +`diorite` cannot do builds that require `kvm`, but `andesite` does support +such builds. For regular builds, `diorite` will be preferred over +`andesite` because it has a higher speed factor. diff --git a/lix/libstore/machines.cc b/lix/libstore/machines.cc index 07a8267e2..b471d5803 100644 --- a/lix/libstore/machines.cc +++ b/lix/libstore/machines.cc @@ -1,11 +1,15 @@ #include "lix/libstore/machines.hh" + #include "lix/libstore/globals.hh" #include "lix/libstore/store-api.hh" #include "lix/libutil/async.hh" #include "lix/libutil/strings.hh" +#include #include +#include + namespace nix { bool Machine::systemSupported(const std::string & system) const @@ -181,12 +185,284 @@ static Machines parseBuilderLines(const std::vector & builders) return result; } +Machines getMachines() +{ + const auto builderLines = expandBuilderLines(settings.builders); + return parseBuilderLines(builderLines); +} + +} + +namespace machines_toml_parsing { + +static constexpr int MIN_VERSION = 1; +static constexpr int LATEST_VERSION = 1; +// Toml format: +// [[machines.andesite]] +// uri = "..." +// +// [[machines.diorite]] +// ... + +template +static toml::result parse(const toml::value & data, const std::string & key) +{ + try { + return toml::success(toml::get(data.at(key))); + } catch (toml::type_error & e) { // NOLINT(lix-foreign-exceptions) + return toml::failure({e.what()}); + } catch (std::out_of_range & _) { // NOLINT(lix-foreign-exceptions) + const auto ei = + toml::make_error_info(fmt("%s must be present", key), data, "but was not set"); + return toml::failure(toml::format_error(ei)); + } +} + +template +static toml::result +parse(const toml::value & data, const std::string & key, T defaultValue) +{ + if (!data.contains(key)) { + return toml::success(defaultValue); + } + try { + return toml::success(toml::get(data.at(key))); + } catch (toml::type_error & e) { // NOLINT(lix-foreign-exceptions) + // invalid value + return toml::failure({e.what()}); + } +} + +static const std::set EXPECTED_KEYS = { + "uri", + "system-types", + "ssh-key", + "jobs", + "speed-factor", + "supported-features", + "mandatory-features", + "ssh-public-host-key", +}; + +static toml::result getSpeedFactor(const toml::value & data) +{ + if (data.contains("speed-factor")) { + auto sf = data.at("speed-factor"); + if (sf.is_integer()) { + return toml::success(static_cast(sf.as_integer())); + } + if (sf.is_floating()) { + return toml::success(static_cast(sf.as_floating())); + } + return toml::failure(toml::format_error(toml::make_error_info( + "bad_cast to floating for `speed-factor`", sf, "Was neither an integer nor a float" + ))); + } + return toml::success(1.0f); +} + +static toml::result> parseMachine(const toml::value & data) +{ + std::vector errs; + + if (!data.is_table()) { + errs.push_back(toml::format_error(toml::make_error_info( + "Each machine must be a table", data, "This should be a table. Did you mean `.uri = `?" + ))); + return toml::failure(errs); + } + + // parsing + auto storeUri = parse(data, "uri"); + auto systemTypes = parse>( + data, "system-types", std::vector{settings.thisSystem} + ); + auto sshKey = parse(data, "ssh-key", ""); + auto maxJobs = parse(data, "jobs", 1U); + auto speedFactor = getSpeedFactor(data); + auto supportedFeatures = + parse>(data, "supported-features", std::vector{}); + auto mandatoryFeatures = + parse>(data, "mandatory-features", std::vector{}); + auto sshPublicHostKey = parse(data, "ssh-public-host-key", ""); + + // parsing validation + if (storeUri.is_err()) { + errs.push_back(storeUri.as_err()); + } + if (systemTypes.is_err()) { + errs.push_back(systemTypes.as_err()); + } + if (sshKey.is_err()) { + errs.push_back(sshKey.as_err()); + } + if (maxJobs.is_err()) { + errs.push_back(maxJobs.as_err()); + } + if (speedFactor.is_err()) { + errs.push_back(speedFactor.as_err()); + } + if (supportedFeatures.is_err()) { + errs.push_back(supportedFeatures.as_err()); + } + if (mandatoryFeatures.is_err()) { + errs.push_back(mandatoryFeatures.as_err()); + } + if (sshPublicHostKey.is_err()) { + errs.push_back(sshPublicHostKey.as_err()); + } + + // value validation + if (maxJobs.is_ok() && maxJobs.as_ok() < 0) { + auto ei = + toml::make_error_info("jobs must be >= 0", data.at("jobs"), "but got negative value"); + errs.push_back(toml::format_error(ei)); + } + + if (speedFactor.is_ok() && speedFactor.as_ok() < 0.0) { + auto ei = toml::make_error_info( + "speed factor must be >= 0", data.at("speed-factor"), "but got negative value" + ); + errs.push_back(toml::format_error(ei)); + } + + for (const auto & [key, _] : data.as_table()) { + if (!EXPECTED_KEYS.contains(key)) { + errs.push_back(toml::format_error(toml::make_error_info( + fmt("unexpected key `%s`", key), data.at(key), "should not be present" + ))); + } + } + + if (!errs.empty()) { + return toml::failure(errs); + } + return toml::success({ + storeUri.unwrap(), + std::set(systemTypes.unwrap().begin(), systemTypes.unwrap().end()), + sshKey.unwrap(), + static_cast(maxJobs.unwrap()), + speedFactor.unwrap(), + std::set(supportedFeatures.unwrap().begin(), supportedFeatures.unwrap().end()), + std::set(mandatoryFeatures.unwrap().begin(), mandatoryFeatures.unwrap().end()), + base64Encode(sshPublicHostKey.unwrap()), + }); +} + +static toml::result> parseToml(const toml::value & data) +{ + auto const array_name = "machines"; + std::vector parserErrors; + Machines machines; + // Empty config + if (data.size() == 0) { + return toml::success({}); + } + if (!data.is_table()) { + parserErrors.push_back( + "Top level must be a table. This should never throw as this is required by the toml " + "SPEC" + ); + return toml::failure(parserErrors); + } + + if (auto config_version = parse(data, "version", LATEST_VERSION); config_version.is_err()) + { + parserErrors.push_back(config_version.as_err()); + } else if (config_version.as_ok() < MIN_VERSION || config_version.as_ok() > LATEST_VERSION) { + parserErrors.push_back( + fmt("Unable to parse Machines of version %d, only versions between %d and %d are " + "supported.", + config_version.as_ok(), + MIN_VERSION, + LATEST_VERSION) + ); + } + if (!parserErrors.empty()) { + return toml::failure(parserErrors); + } + + auto & tbl = data.as_table(); + std::string unexpected_keys; + for (auto it = tbl.begin(); it != tbl.end(); ++it) { + if (it->first == array_name || it->first == "version") { + // expected keys + continue; + } + unexpected_keys += ", " + it->first; + } + if (unexpected_keys.size()) { + parserErrors.push_back(fmt("unexpected keys found: %s", unexpected_keys.erase(0, 2))); + } + + if (!data.at(array_name).is_table()) { + parserErrors.push_back( + fmt("Expected key `%s` to be a table of name -> machine configurations", array_name) + ); + return toml::failure(parserErrors); + } + + for (const auto & [name, machine] : data.at(array_name).as_table()) { + auto const res = parseMachine(machine); + if (res.is_err()) { + auto err = res.as_err(); + parserErrors.push_back(fmt("for machine %s:", name)); + parserErrors.insert(parserErrors.end(), err.begin(), err.end()); + } else { + machines.push_back(res.unwrap()); + } + } + + if (!parserErrors.empty()) { + return toml::failure(parserErrors); + } + return toml::success(machines); +} + +static std::optional getMachines() +{ + toml::value data; + auto buildersStr = settings.builders.get(); + try { + if (buildersStr.size() > 0 && buildersStr.at(0) == '@') { + data = toml::parse(buildersStr.substr(1)); + } else { + data = toml::parse_str(settings.builders); + } + } catch (toml::syntax_error const & e) { // NOLINT(lix-foreign-exceptions) + if (toLower(buildersStr).contains("toml") || buildersStr.contains("\"")) { + // Yes, we are sure this is a TOML and no this shitty legacy format + // so we can safely throw the syntax error here + throw UsageError(fmt("invalid Machines TOML syntax: \n%s", e.what())); + } + return {}; + } catch (toml::file_io_error const & _) { // NOLINT(lix-foreign-exceptions) + // sadly we have to do this otherwise we break the old format, + // which requires **silently ignoring** invalid files + return {}; + } + auto const fromToml = parseToml(data); + if (fromToml.is_ok()) { + return fromToml.unwrap(); + } + + auto const & errs = fromToml.as_err(); + std::string msg = "invalid Machines TOML:\n"; + msg += concatStringsSep("\n", errs); + + throw UsageError(msg); +} + } Machines getMachines() { - const auto builderLines = machines_legacy_parsing::expandBuilderLines(settings.builders); - return machines_legacy_parsing::parseBuilderLines(builderLines); + auto const toml_result = machines_toml_parsing::getMachines(); + if (toml_result.has_value()) { + return toml_result.value(); + } + debug("Trying again with legacy format"); + return machines_legacy_parsing::getMachines(); } } diff --git a/tests/unit/libstore/machines_toml.cc b/tests/unit/libstore/machines_toml.cc new file mode 100644 index 000000000..f374901b7 --- /dev/null +++ b/tests/unit/libstore/machines_toml.cc @@ -0,0 +1,285 @@ +#include "lix/libutil/file-system.hh" +#include "lix/libstore/machines.hh" +#include "lix/libstore/globals.hh" +#include "lix/libutil/logging.hh" +#include "lix/libutil/strings.hh" + +#include + +using testing::Contains; +using testing::ElementsAre; +using testing::EndsWith; +using testing::Eq; +using testing::Field; +using testing::HasSubstr; +using testing::SizeIs; + +using nix::absPath; +using nix::FormatError; +using nix::getMachines; +using nix::Machine; +using nix::Machines; +using nix::pathExists; +using nix::settings; +using nix::UsageError; + +TEST(machines, getMachinesTOMLWithEmptyBuilders) +{ + settings.builders.override(""); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesTOMLUriOnly) +{ + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"ssh://nix@scratchy.labs.cs.uu.nl\"" + ); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesTOMLMultipleMachines) +{ + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "[machines.itchy]\n" + "uri = \"nix@itchy.labs.cs.uu.nl\"\n" + ); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(2)); + EXPECT_THAT( + actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))) + ); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesTOMLWithCorrectCompleteSingleBuilder) +{ + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "system-types = [\"i686-linux\"]\n" + "ssh-key = \"/home/nix/.ssh/id_scratchy_auto\"\n" + "jobs = 8\n" + "speed-factor = 3.0\n" + "supported-features = [\"kvm\"]\n" + "mandatory-features = [\"benchmark\"]\n" + "ssh-public-host-key = \"ssh-ed25519 " + "AAAAC3NzaC1lZDI1NTE5AAAAIJYfqESaiQlOrL3Wm1Q9s9q8b4mjj2nIuyqCZub5aGPi nix@scratchy\"\n" + ); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT( + actual[0], + Field( + &Machine::sshPublicHostKey, + Eq("c3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSUpZZnFFU2FpUWxPckwzV20xUTlzOXE4YjR" + "tamoybkl1eXFDWnViNWFHUGkgbml4QHNjcmF0Y2h5") + ) + ); +} + +TEST(machines, getMachinesTOMLBothFloatFormats) +{ + settings.builders.override( + "[machines.andesite]\n" + "uri = \"ssh://lix@andesite.lix.systems\"\n" + "speed-factor = 3\n" + ); + auto actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + settings.builders.override( + "[machines.diorite]\n" + "uri = \"ssh://lix@diorite.lix.systems\"\n" + "speed-factor = 3.1\n" + ); + actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3.1f))); +} + +TEST(machines, getMachinesTOMLWithMultiOptions) +{ + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "system-types = [\"Arch1\", \"Arch2\"]\n" + "supported-features = [\"SupportedFeature1\", \"SupportedFeature2\"]\n" + "mandatory-features = [\"MandatoryFeature1\", \"MandatoryFeature2\"]\n" + ); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); + EXPECT_THAT( + actual[0], + Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2")) + ); + EXPECT_THAT( + actual[0], + Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2")) + ); +} + +#define EXPECT_MESSAGE_THROW(EXPR, EXC, MSG) \ + EXPECT_THROW( \ + { \ + try { \ + EXPR; \ + } catch (const EXC & e) { \ + EXPECT_THAT(e.what(), HasSubstr(MSG)); \ + throw; \ + } \ + }, \ + EXC \ + ); + +TEST(machines, getMachinesTOMLExtraKeys) +{ + settings.builders.override( + "[machines.andesite]\n" + "uri = \"ssh://lix@andesite.lix.systems\"\n" + "extra-key = 3\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "unexpected key `extra-key`"); + settings.builders.override( + "[machines.andesite]\n" + "uri = \"ssh://lix@andesite.lix.systems\"\n" + "another-key = 3\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "unexpected key `another-key`"); +} + +/* + This should throw a syntax error, but actually parses successfully and puts weird shit in the uri +field instead Other parsers (e.g. pythons tomllib) do successfully throw a syntax error here, but +toml11 doesn't AAAAAAAAAAAAAAAAAAAAAA An upstream issue was created for this on 2025-12-01 +(https://github.com/ToruNiina/toml11/issues/303) + +Note: this somehow worked once or twice, but now its broken again (and CI agrees that it's broken) +*/ +TEST(machines, getMachinesTOMLNoQuotationOnUri) +{ + GTEST_SKIP() << "See upstream issue https://github.com/ToruNiina/toml11/issues/303"; + settings.builders.override( + "[machines.invalid_syntax]\n" + "uri = ssh://lix@andesite.lix.systems\n" + "maxJobs = -3\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "bad format: unknown value appeared"); +} + +TEST(machines, getMachinesTOMLWithIncorrectTyping) +{ + settings.builders.override("[machines.a]"); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "uri must be present"); + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "jobs = -3\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "jobs must be >= 0"); + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "jobs = \"three\"\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "bad_cast to integer"); + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "jobs = 8\n" + "speed-factor = -3.0\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "speed factor must be >= 0"); + settings.builders.override( + "[machines.scratchy]\n" + "uri = \"nix@scratchy.labs.cs.uu.nl\"\n" + "jobs = 8\n" + "speed-factor = \"three\"\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "bad_cast to floating"); + + settings.builders.override( + "[[machines]]\n" + "uri = \"lix@andesite.lix.systems\"\n" + "[[machines]]\n" + "uri = \"lix@diorite.lix.systems\"\n" + ); + EXPECT_MESSAGE_THROW( + getMachines(), + UsageError, + "Expected key `machines` to be a table of name -> machine configurations" + ); + + settings.builders.override("machines.a = \"lix@andesite.lix.sytems\"\n"); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "Each machine must be a table"); +} + +TEST(machines, getMachinesTOMLBadVersion) +{ + settings.builders.override( + "version = \"hello\"\n" + "machines = {}\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "bad_cast to integer"); +} + +TEST(machines, getMachinesTOMLTooHighVersion) +{ + settings.builders.override( + "version = 42\n" + "machines = {}\n" + ); + EXPECT_MESSAGE_THROW( + getMachines(), + UsageError, + "Unable to parse Machines of version 42, only versions between 1 and 1 are supported." + ); +} + +TEST(machines, getMachinesTOMLTooLowVersion) +{ + settings.builders.override( + "version = -1\n" + "machines = {}\n" + ); + EXPECT_MESSAGE_THROW( + getMachines(), + UsageError, + "Unable to parse Machines of version -1, only versions between 1 and 1 are supported." + ); +} + +TEST(machines, getMachinesTOMLInvalidSyntaxButClearlyTOML) +{ + settings.builders.override( + "version = 1\n" + "[machines]\n" + "[machines.hello]\n" + "uri = \"ssh://hello\"\n" + " = 5\n" + ); + EXPECT_MESSAGE_THROW(getMachines(), UsageError, "invalid Machines TOML syntax:"); +} + +#undef EXPECT_MESSAGE_THROW diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 6e11081fa..2b5d9f469 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -141,6 +141,7 @@ libstore_tests_sources = files( 'libstore/derived-path.cc', 'libstore/filetransfer.cc', 'libstore/machines_legacy.cc', + 'libstore/machines_toml.cc', 'libstore/nar-info-disk-cache.cc', 'libstore/outputs-spec.cc', 'libstore/path.cc',