From 5f50741ce97fb3b2e4a1896bdf568fd11dafebce Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Sun, 27 Oct 2024 15:18:00 +0100 Subject: [PATCH] feat: add support for external `lix-` prefixed commands in the CLI This feature allows calling external binaries starting with `lix-` as subcommands directly within the `lix` CLI, similar to how Git handles external commands. For example, a binary named `lix-example` can now be invoked using `lix example`. This change opens up the possibility for broader community-driven extensions of the Lix CLI, offering flexibility that Flakes has somewhat restricted by centralizing certain features around its own model of dependency management. By enabling users to introduce custom subcommands, we encourage experimentation and diverse workflows, addressing the needs of those who seek more modular, feature-rich approaches to package management and system configuration beyond what Flakes currently offers [1] [2] [3]. This adds `lix` as a new binary to be able to use this feature. [1]: https://github.com/nmattia/niv [2]: https://github.com/andir/npins [3]: https://github.com/nikstur/lon Change-Id: Ic6344424a6a46fc9fb30432f00e21c3509659f8a Signed-off-by: Raito Bezarius --- doc/manual/rl-next/extend-cli.md | 37 +++++ lix/libutil/args.cc | 156 +++++++++++++++++- lix/libutil/args.hh | 74 +++++++-- .../lix-custom-sub-commands.md | 5 + lix/libutil/meson.build | 1 + lix/nix/main.cc | 11 +- lix/nix/meson.build | 1 + lix/nix/nix.md | 27 +++ tests/functional/external-commands.sh | 68 ++++++++ tests/functional/meson.build | 1 + 10 files changed, 360 insertions(+), 21 deletions(-) create mode 100644 doc/manual/rl-next/extend-cli.md create mode 100644 lix/libutil/experimental-features/lix-custom-sub-commands.md create mode 100644 tests/functional/external-commands.sh diff --git a/doc/manual/rl-next/extend-cli.md b/doc/manual/rl-next/extend-cli.md new file mode 100644 index 000000000..5c51f41fd --- /dev/null +++ b/doc/manual/rl-next/extend-cli.md @@ -0,0 +1,37 @@ +--- +synopsis: "`lix foo` now invokes `lix-foo` from PATH" +cls: [2119] +category: Features +credits: raito +--- + +Lix introduces the ability to extend the Nix command line by adding custom +binaries to the `PATH`, similar to how Git integrates with other tools. This +feature allows developers and end users to enhance their workflow by +integrating additional functionalities directly into the Nix CLI. + +#### Examples + +For example, a user can create a custom deployment tool, `lix-deploy-tool`, and +place it in their `PATH`. This allows them to execute `lix deploy-tool` +directly from the command line, streamlining the process of deploying +applications without needing to switch contexts or use separate commands. + +#### Limitations + +For now, autocompletion is supported to discover new custom commands, but the +documentation will not render them. Argument autocompletion of the custom +command is not supported either. + +This is also locked behind a new experimental feature called +`lix-custom-sub-commands` to enable developing all the required features. + +Only the top-level `lix` command can be extended, this is an artificial +limitation for the time being until we flesh out this feature. + +#### Outline + +In the future, this feature may pave the way for moving the Flake subcommand +line to its own standalone binary, allowing for a more focused approach to +managing Nix Flakes while letting the community explore alternatives to +dependency management. diff --git a/lix/libutil/args.cc b/lix/libutil/args.cc index fa063f28e..67d91c09c 100644 --- a/lix/libutil/args.cc +++ b/lix/libutil/args.cc @@ -381,28 +381,139 @@ void Args::completeDir(AddCompletions & completions, size_t, std::string_view pr _completePath(completions, prefix, true); } +static bool isAcceptableLixSubcommandExe(const std::filesystem::path & exe_path) +{ + namespace fs = std::filesystem; + + return fs::is_regular_file(exe_path) && + (fs::status(exe_path).permissions() & (fs::perms::owner_exec | fs::perms::group_exec | fs::perms::others_exec)) != fs::perms::none; +} + +std::shared_ptr searchForCustomSubcommand(AsyncIoRoot & aio, const std::string_view & command, const std::string_view & prefix, const Strings & searchPaths) +{ + namespace fs = std::filesystem; + for (auto searchPath : searchPaths) { + if (searchPath.empty()) continue; + + auto path = fs::path(searchPath) / fs::path(prefix).concat(command); + + try { + if (isAcceptableLixSubcommandExe(path)) { + debug("Found requested external subcommand '%s' in '%s'", command, path); + return std::make_shared(aio, path); + } + } catch (fs::filesystem_error & fs_exc) { + if (fs_exc.code() != std::errc::no_such_file_or_directory && fs_exc.code() != std::errc::not_a_directory && fs_exc.code() != std::errc::permission_denied) { + throw SysError("while searching for the subcommand '%1%' in search path '%2%': '%3%'", command, searchPath, fs_exc.what()); + } + } + } + + return nullptr; +} + +Strings searchForAllAvailableCustomSubcommands(const std::string_view & prefix, const Strings & searchPaths) +{ + namespace fs = std::filesystem; + Strings commandNames; + + for (auto searchPath : searchPaths) { + if (searchPath.empty()) continue; + + if (!fs::exists(searchPath) || !fs::is_directory(searchPath)) { + // TODO(Raito): this will break all the time our functional tests + // for people with garbage in their $PATH which is my personal case. + + // warn("The search path '%s' for custom subcommands does not exist or is not a directory, ignoring...", searchPath); + continue; + } + + // Browse per prefix. + for (const auto& entry : fs::directory_iterator(searchPath)) { + try { + if (isAcceptableLixSubcommandExe(entry.path())) { + auto filename = entry.path().filename().string(); + + if (filename.starts_with(prefix)) { + auto suffix = filename.substr(prefix.size()); + + debug("Found custom subcommand ('%s') '%s'", filename, suffix); + + commandNames.push_back(suffix); + } + } + } catch (fs::filesystem_error & fs_exc) { + if (fs_exc.code() != std::errc::no_such_file_or_directory && fs_exc.code() != std::errc::not_a_directory && fs_exc.code() != std::errc::permission_denied) { + throw SysError("while searching for all available commands in search path '%1%', while analyzing '%2%': %3%'", searchPath, entry.path(), fs_exc.what()); + } + } + } + } + + return commandNames; +} + std::optional Command::experimentalFeature () { return { Xp::NixCommand }; } -MultiCommand::MultiCommand(const Commands & commands_) - : commands(commands_) +MultiCommand::MultiCommand(const Commands & commands_, bool allowExternal) + : commands(commands_), + customCommandSearchPaths( + allowExternal ? tokenizeString( + getEnv("PATH").value_or(""), + ":" + ) : Strings() + ), + isExternalSubcommand(false) { expectArgs({ .label = "subcommand", .optional = true, .handler = {[=,this](std::string s) { assert(!command); - auto i = commands.find(s); - if (i == commands.end()) { + auto it = commands.find(s); + + // NOTE: this logic does not rely + // on `allowExternal`, indeed: + // if external subcommands are not allowed, the search paths should be empty which will short-circuit any search and will never ever return external subcommands. + + // If `i` is not found, look into custom subcommands. + if (it == commands.end()) { + debug("looking for %s", s); + auto possibleNewSubcommand = searchForCustomSubcommand(aio(), s, ExternalCommand::lixExternalPrefix, customCommandSearchPaths); + + if (possibleNewSubcommand) { + command = {s, ref(possibleNewSubcommand)}; + isExternalSubcommand = true; + } + } else { + command = {s, it->second(aio())}; + } + + // By this point, we tried everything: + // (a) built-in commands + // (b) filesystem view of external subcommands + // + // We are going to do the expensive thing of looking for all external subcommands now + // for error reporting purpose. + if (!command) { std::set commandNames; + auto customCommands = searchForAllAvailableCustomSubcommands("lix-", customCommandSearchPaths); + // As we are going to throw an error, there's no need to fill the hot cache of subcommands. + for (auto & name : customCommands) + commandNames.insert(name); for (auto & [name, _] : commands) commandNames.insert(name); auto suggestions = Suggestions::bestMatches(commandNames, s); throw UsageError(suggestions, "'%s' is not a recognised command", s); } - command = {s, i->second(aio())}; + + if (isExternalSubcommand) { + debug("Found external subcommand for %s", s); + } + command->second->parent = this; }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { @@ -413,11 +524,14 @@ MultiCommand::MultiCommand(const Commands & commands_) }); categories[Command::catDefault] = "Available commands"; + if (allowExternal) { + categories[Command::catCustom] = "External custom commands"; + } } bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { - if (Args::processFlag(pos, end)) return true; + if (!isExternalSubcommand && Args::processFlag(pos, end)) return true; if (command && command->second->processFlag(pos, end)) return true; return false; } @@ -452,4 +566,34 @@ nlohmann::json MultiCommand::toJSON() return res; } +ExternalCommand::ExternalCommand(AsyncIoRoot & aio, std::filesystem::path absoluteBinaryPath) : aio_(aio), absoluteBinaryPath(absoluteBinaryPath) { + // NOTE: on shell invocation, argv[0] is the basename of the binary invoked. + // We just reproduce this behavior. + externalArgv.push_back(this->absoluteBinaryPath.filename()); +} + +bool ExternalCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + externalArgv.push_back(*pos++); + + // All flags are recognised as we leave + // parsing to the external commands. + return true; +} + +bool ExternalCommand::processArgs(const Strings & args, bool finish) +{ + for (const auto & arg : args) { + externalArgv.push_back(arg); + } + + return true; +} + +void ExternalCommand::run() { + execv(absoluteBinaryPath.c_str(), stringsToCharPtrs(externalArgv).data()); + + throw SysError(errno, "failed to execute external command '%1%'", absoluteBinaryPath); +} + } diff --git a/lix/libutil/args.hh b/lix/libutil/args.hh index 7bca5cdc5..8a5026547 100644 --- a/lix/libutil/args.hh +++ b/lix/libutil/args.hh @@ -14,6 +14,7 @@ #include #include #include +#include namespace nix { @@ -189,6 +190,8 @@ protected: /** * Process a single flag and its arguments, pulling from an iterator * of raw CLI args as needed. + * + * @return false if the flag is not recognised. */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); @@ -229,17 +232,23 @@ protected: */ std::list processedArgs; - /** - * Process some positional arugments - * - * @param finish: We have parsed everything else, and these are the only - * arguments left. Used because we accumulate some "pending args" we might - * have left over. - */ - virtual bool processArgs(const Strings & args, bool finish); + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + * + * @return true if the passed arguments were fully consumed and no further processing is + * required, false if the passed arguments should be processed with more context. + * + */ + virtual bool processArgs(const Strings & args, bool finish); - virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) - { return pos; } + virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) + { + return pos; + } std::set hiddenCategories; @@ -334,6 +343,7 @@ struct Command : virtual public Args typedef int Category; static constexpr Category catDefault = 0; + static constexpr Category catCustom = 1000; virtual std::optional experimentalFeature (); @@ -350,6 +360,8 @@ class MultiCommand : public Command { public: Commands commands; + Strings customCommandSearchPaths; + bool isExternalSubcommand; std::map categories; @@ -358,7 +370,7 @@ public: */ std::optional>> command; - MultiCommand(const Commands & commands); + MultiCommand(const Commands & commands, bool allowExternal = false); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; @@ -367,6 +379,46 @@ public: nlohmann::json toJSON() override; }; +/** + * An external command wrapper which is represented by a external binary. + * i.e. `lix-flakes`. + */ +class ExternalCommand : virtual public Command +{ + Strings externalArgv; + AsyncIoRoot & aio_; +public: + std::filesystem::path absoluteBinaryPath; + static constexpr std::string_view lixExternalPrefix = "lix-"; + + ExternalCommand(AsyncIoRoot & aio, std::filesystem::path absoluteBinaryPath); + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + virtual bool processArgs(const Strings & args, bool finish) override; + virtual void run() override; + virtual std::optional experimentalFeature () override { + return Xp::LixCustomSubCommands; + } + + // Create a custom category section. + virtual Category category() override { return catCustom; } + + virtual AsyncIoRoot & aio() override { return aio_; } +}; + +/* This returns a Command handle + * if the command name exist in one of the search paths and points to an executable regular file. + * + * i.e. if $searchpath/$command exist for any $searchpath in `searchPaths` and $searchpath/$command links to an executable regular file. + */ +std::shared_ptr searchForCustomSubcommand(const std::string_view & command, const Strings & searchPaths); +/* This will read all directories in searchPaths one by one and look for all executable regular files which starts with `$prefix-`. + * Finally, it will return the list of commands stripped of their `$prefix` prefix. + * + * If you need to know about a specific command, prefer `searchForCustomSubcommand`. + */ +Strings searchForAllAvailableCustomSubcommands(const std::string_view & prefix, const Strings & searchPaths); + struct Completion { std::string completion; std::string description; diff --git a/lix/libutil/experimental-features/lix-custom-sub-commands.md b/lix/libutil/experimental-features/lix-custom-sub-commands.md new file mode 100644 index 000000000..8026ca55d --- /dev/null +++ b/lix/libutil/experimental-features/lix-custom-sub-commands.md @@ -0,0 +1,5 @@ +--- +name: lix-custom-sub-commands +internalName: LixCustomSubCommands +--- +Allows Lix to invoke a custom command via its main binary `lix`, i.e. `lix-foo` gets invoked when `lix foo` is executed. diff --git a/lix/libutil/meson.build b/lix/libutil/meson.build index d3e5daba1..62b4cf65a 100644 --- a/lix/libutil/meson.build +++ b/lix/libutil/meson.build @@ -142,6 +142,7 @@ experimental_feature_definitions = files( 'experimental-features/fetch-closure.md', 'experimental-features/flakes.md', 'experimental-features/impure-derivations.md', + 'experimental-features/lix-custom-sub-commands.md', 'experimental-features/nix-command.md', 'experimental-features/no-url-literals.md', 'experimental-features/parse-toml-timestamps.md', diff --git a/lix/nix/main.cc b/lix/nix/main.cc index 006e3dee7..6c7e48bad 100644 --- a/lix/nix/main.cc +++ b/lix/nix/main.cc @@ -114,9 +114,9 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs AsyncIoRoot & aio_; AsyncIoRoot & aio() override { return aio_; } - NixArgs(AsyncIoRoot & aio) - : MultiCommand(RegisterCommand::getCommandsFor({})) - , MixCommonArgs("nix") + NixArgs(const std::string & programName, AsyncIoRoot & aio) + : MultiCommand(RegisterCommand::getCommandsFor({}), true) + , MixCommonArgs(programName) , aio_(aio) { categories.clear(); @@ -406,7 +406,10 @@ void mainWrapped(AsyncIoRoot & aio, int argc, char * * argv) verbosity = lvlInfo; } - NixArgs args(aio); + // NOTE: out of over-cautiousness for backward compatibility, + // the program name had always been `nix` for a long time. + // Only when we invoke it as `lix`, we should propagate `lix`. + NixArgs args(programName == "lix" ? programName : "nix", aio); if (argc == 2 && std::string(argv[1]) == "__dump-cli") { logger->cout(args.dumpCli()); diff --git a/lix/nix/meson.build b/lix/nix/meson.build index f08c5851a..45fd6abb4 100644 --- a/lix/nix/meson.build +++ b/lix/nix/meson.build @@ -153,6 +153,7 @@ nix_symlinks = [ 'nix-prefetch-url', 'nix-shell', 'nix-store', + 'lix', ] foreach linkname : nix_symlinks diff --git a/lix/nix/nix.md b/lix/nix/nix.md index 192b1d1c1..06868d098 100644 --- a/lix/nix/nix.md +++ b/lix/nix/nix.md @@ -1,5 +1,32 @@ R""( +# Custom commands + +> **Warning** \ +> Custom commands are part of the unstable +> [lix-custom-sub-commands experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-lix-custom-sub-commands), +> and subject to change without notice. + +Lix allows users to define custom subcommands by placing executables in the system's `PATH` that follow the naming convention `lix-`. When a user runs `lix `, Lix will attempt to locate and execute `lix-` as a separate process. + +Auto-completion of custom commands is not supported yet. + +## Usage + +A custom Lix command must be an executable script or binary named `lix-` and be accessible from the `PATH`. When the user invokes `lix `, Lix will execute `lix-` with the given arguments. + +For example, if an executable named `lix-example` exists in the `PATH`, running: + +```console +$ lix example arg1 arg2 +``` + +will be equivalent to running: + +```console +$ lix-example arg1 arg2 +``` + # Examples * Create a new flake: diff --git a/tests/functional/external-commands.sh b/tests/functional/external-commands.sh new file mode 100644 index 000000000..5891457fd --- /dev/null +++ b/tests/functional/external-commands.sh @@ -0,0 +1,68 @@ +source common.sh + +cd "$TEST_ROOT" + +# To start off, we will produce some custom binaries that redirects to known functionality. +EXTRA_BINARIES_DIR=$(mktemp -d) +# trap "rm -rf $EXTRA_BINARIES_DIR" EXIT + +# Create an extra command as an alias of an existing one. +create_extra_command() { + local existing_command="$1" + local script_path="$EXTRA_BINARIES_DIR/lix-$existing_command" + + cat > "$script_path" <$TEST_ROOT/stdout 2>$TEST_ROOT/stderr + + # Test that an external subcommand can be run successfully. + expect 0 lix --extra-experimental-features 'lix-custom-sub-commands' copy-closure --version 1>/dev/null + expect 0 lix --extra-experimental-features 'lix-custom-sub-commands' collect-garbage --version 1>/dev/null + + # Test that the external subcommand can be run beyond `--help` processing successfully. + expect 0 lix --extra-experimental-features 'lix-custom-sub-commands' collect-garbage --dry-run 1>/dev/null + + # Test that an external subcommand without the experimental flag will fail + # without mentioning the experimental feature. + expect 1 lix copy-closure --help 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +} + +testSimpleExternalCommands +# Test that an external subcommand can be run successfully with a slightly modified PATH. +( + # Split the PATH into its first component and the rest + FIRST_PATH=${PATH%%:*} # The first directory + REST_PATH=${PATH#*:} # The rest of the PATH + + echo "modified PATH: $PATH" + # Rebuild PATH with the first directory moved to the second position + export PATH=$(echo $REST_PATH | cut -d: -f1):$FIRST_PATH:$(echo $REST_PATH | cut -d: -s -f2-) + + testSimpleExternalCommands +) + +# TODO: Test flags handling. + +# TODO: Short, long and multiple flags should be tested as well. +# TODO: `--` special flag? +# TODO: test positional arguments, but only `nix-copy-closure` implements some and it's pesky to test here. diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 2d905d79f..231177da3 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -199,6 +199,7 @@ functional_tests_scripts = [ 'substitute-truncated-nar.sh', 'regression-484.sh', 'regression-reference-checks.sh', + 'external-commands.sh' ] # Plugin tests require shared libraries support.