Merge changes from topic "profile-v3" into main

* changes:
  nix3-profile: remove check "name" attr in manifests
  Add profile migration test
  nix3-profile: make element names stable
  getNameFromURL(): Support uppercase characters in attribute names
  nix3-profile: remove indices
  nix3-profile: allow using human-readable names to select packages
  implement parsing human-readable names from URLs
This commit is contained in:
Qyriad
2024-05-02 20:15:48 +00:00
committed by Gerrit Code Review
16 changed files with 401 additions and 139 deletions
+63 -29
View File
@@ -1,8 +1,12 @@
#include <set>
#include "cmd-profiles.hh"
#include "built-path.hh"
#include "builtins/buildenv.hh"
#include "logging.hh"
#include "names.hh"
#include "store-api.hh"
#include "url-name.hh"
namespace nix
{
@@ -116,6 +120,8 @@ ProfileManifest::ProfileManifest(EvalState & state, const Path & profile)
sOriginalUrl = "originalUri";
break;
case 2:
[[fallthrough]];
case 3:
sUrl = "url";
sOriginalUrl = "originalUrl";
break;
@@ -123,7 +129,10 @@ ProfileManifest::ProfileManifest(EvalState & state, const Path & profile)
throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version);
}
for (auto & e : json["elements"]) {
auto elems = json["elements"];
for (auto & elem : elems.items()) {
auto & e = elem.value();
ProfileElement element;
for (auto & p : e["storePaths"]) {
element.storePaths.insert(state.store->parseStorePath((std::string) p));
@@ -139,7 +148,16 @@ ProfileManifest::ProfileManifest(EvalState & state, const Path & profile)
e["attrPath"],
e["outputs"].get<ExtendedOutputsSpec>()};
}
elements.emplace_back(std::move(element));
// TODO(Qyriad): holy crap this chain of ternaries needs cleanup.
std::string name =
elems.is_object()
? elem.key()
: element.source
? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier())
: element.identifier();
addElement(name, std::move(element));
}
} else if (pathExists(profile + "/manifest.nix")) {
// FIXME: needed because of pure mode; ugly.
@@ -151,15 +169,37 @@ ProfileManifest::ProfileManifest(EvalState & state, const Path & profile)
for (auto & drvInfo : drvInfos) {
ProfileElement element;
element.storePaths = {drvInfo.queryOutPath()};
elements.emplace_back(std::move(element));
addElement(std::move(element));
}
}
}
void ProfileManifest::addElement(std::string_view nameCandidate, ProfileElement element)
{
std::string finalName(nameCandidate);
for (unsigned i = 1; elements.contains(finalName); ++i) {
finalName = nameCandidate + "-" + std::to_string(i);
}
elements.insert_or_assign(finalName, std::move(element));
}
void ProfileManifest::addElement(ProfileElement element)
{
auto name =
element.source
? getNameFromURL(parseURL(element.source->to_string()))
: std::nullopt;
auto finalName = name.value_or(element.identifier());
addElement(finalName, std::move(element));
}
nlohmann::json ProfileManifest::toJSON(Store & store) const
{
auto array = nlohmann::json::array();
for (auto & element : elements) {
auto es = nlohmann::json::object();
for (auto & [name, element] : elements) {
auto paths = nlohmann::json::array();
for (auto & path : element.storePaths) {
paths.push_back(store.printStorePath(path));
@@ -174,11 +214,11 @@ nlohmann::json ProfileManifest::toJSON(Store & store) const
obj["attrPath"] = element.source->attrPath;
obj["outputs"] = element.source->outputs;
}
array.push_back(obj);
es[name] = obj;
}
nlohmann::json json;
json["version"] = 2;
json["elements"] = array;
json["version"] = 3;
json["elements"] = es;
return json;
}
@@ -189,7 +229,7 @@ StorePath ProfileManifest::build(ref<Store> store)
StorePathSet references;
Packages pkgs;
for (auto & element : elements) {
for (auto & [name, element] : elements) {
for (auto & path : element.storePaths) {
if (element.active) {
pkgs.emplace_back(store->printStorePath(path), true, element.priority);
@@ -235,35 +275,29 @@ void ProfileManifest::printDiff(
const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent
)
{
auto prevElems = prev.elements;
std::sort(prevElems.begin(), prevElems.end());
auto curElems = cur.elements;
std::sort(curElems.begin(), curElems.end());
auto i = prevElems.begin();
auto j = curElems.begin();
auto prevElemIt = prev.elements.begin();
auto curElemIt = cur.elements.begin();
bool changes = false;
while (i != prevElems.end() || j != curElems.end()) {
if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) {
logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions());
while (prevElemIt != prev.elements.end() || curElemIt != cur.elements.end()) {
if (curElemIt != cur.elements.end() && (prevElemIt == prev.elements.end() || prevElemIt->first > curElemIt->first)) {
logger->cout("%s%s: ∅ -> %s", indent, curElemIt->second.identifier(), curElemIt->second.versions());
changes = true;
++j;
} else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) {
logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions());
++curElemIt;
} else if (prevElemIt != prev.elements.end() && (curElemIt == cur.elements.end() || prevElemIt->first < curElemIt->first)) {
logger->cout("%s%s: %s -> ∅", indent, prevElemIt->second.identifier(), prevElemIt->second.versions());
changes = true;
++i;
++prevElemIt;
} else {
auto v1 = i->versions();
auto v2 = j->versions();
auto v1 = prevElemIt->second.versions();
auto v2 = curElemIt->second.versions();
if (v1 != v2) {
logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2);
logger->cout("%s%s: %s -> %s", indent, prevElemIt->second.identifier(), v1, v2);
changes = true;
}
++i;
++j;
++prevElemIt;
++curElemIt;
}
}
+6 -1
View File
@@ -6,6 +6,8 @@
#include "flake/flakeref.hh"
#include "get-drvs.hh"
#include "types.hh"
#include "url.hh"
#include "url-name.hh"
#include <string>
#include <set>
@@ -54,7 +56,7 @@ struct ProfileElement
struct ProfileManifest
{
std::vector<ProfileElement> elements;
std::map<std::string, ProfileElement> elements;
ProfileManifest() { }
@@ -64,6 +66,9 @@ struct ProfileManifest
StorePath build(ref<Store> store);
void addElement(std::string_view nameCandidate, ProfileElement element);
void addElement(ProfileElement element);
static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent);
};
+2
View File
@@ -31,6 +31,7 @@ libutil_sources = files(
'tarfile.cc',
'thread-pool.cc',
'url.cc',
'url-name.cc',
'util.cc',
'xml-writer.cc',
)
@@ -92,6 +93,7 @@ libutil_headers = files(
'topo-sort.hh',
'types.hh',
'url-parts.hh',
'url-name.hh',
'url.hh',
'util.hh',
'variant-wrapper.hh',
+59
View File
@@ -0,0 +1,59 @@
#include <iostream>
#include <regex>
#include "url-name.hh"
namespace nix {
static std::string const attributeNamePattern("[a-zA-Z0-9_-]+");
static std::regex const lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
static std::string const pathSegmentPattern("[a-zA-Z0-9_-]+");
static std::regex const lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
static std::regex const secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
static std::regex const gitProviderRegex("github|gitlab|sourcehut");
static std::regex const gitSchemeRegex("git($|\\+.*)");
static std::regex const defaultOutputRegex(".*\\.default($|\\^.*)");
std::optional<std::string> getNameFromURL(ParsedURL const & url)
{
std::smatch match;
/* If there is a dir= argument, use its value */
if (url.query.count("dir") > 0) {
return url.query.at("dir");
}
/* If the fragment isn't a "default" and contains two attribute elements, use the last one */
if (std::regex_match(url.fragment, match, lastAttributeRegex)) {
return match.str(1);
}
/* If this is a github/gitlab/sourcehut flake, use the repo name */
if (
std::regex_match(url.scheme, gitProviderRegex)
&& std::regex_match(url.path, match, secondPathSegmentRegex)
) {
return match.str(1);
}
/* If it is a regular git flake, use the directory name */
if (
std::regex_match(url.scheme, gitSchemeRegex)
&& std::regex_match(url.path, match, lastPathSegmentRegex)
) {
return match.str(1);
}
/* If everything failed but there is a non-default fragment, use it in full */
if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
return url.fragment;
/* If there is no fragment, take the last element of the path */
if (std::regex_match(url.path, match, lastPathSegmentRegex))
return match.str(1);
/* If even that didn't work, the URL does not contain enough info to determine a useful name */
return {};
}
}
+26
View File
@@ -0,0 +1,26 @@
#pragma once
///@file url-name.hh, for some hueristic-ish URL parsing.
#include <string>
#include <optional>
#include "url.hh"
#include "url-parts.hh"
#include "util.hh"
#include "split.hh"
namespace nix {
/**
* Try to extract a reasonably unique and meaningful, human-readable
* name of a flake output from a parsed URL.
* When nullopt is returned, the callsite should use information available
* to it outside of the URL to determine a useful name.
* This is a heuristic approach intended for user interfaces.
* @return nullopt if the extracted name is not useful to identify a
* flake output, for example because it is empty or "default".
* Otherwise returns the extracted name.
*/
std::optional<std::string> getNameFromURL(ParsedURL const & url);
}
+1
View File
@@ -19,6 +19,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*";
const static std::string segmentRegex = "(?:" + pcharRegex + "*)";
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
+1 -1
View File
@@ -16,7 +16,7 @@ ParsedURL parseURL(const std::string & url)
"((" + schemeRegex + "):"
+ "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
+ "(?:\\?(" + queryRegex + "))?"
+ "(?:#(" + queryRegex + "))?",
+ "(?:#(" + fragmentRegex + "))?",
std::regex::ECMAScript);
std::smatch match;
+4 -4
View File
@@ -6,13 +6,13 @@ R""(
```console
# nix profile list
Index: 0
Name: gdb
Flake attribute: legacyPackages.x86_64-linux.gdb
Original flake URL: flake:nixpkgs
Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca
Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1
Index: 1
Name: blender-bin
Flake attribute: packages.x86_64-linux.default
Original flake URL: flake:blender-bin
Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender
@@ -26,7 +26,7 @@ R""(
# nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default
```
will build the package with index 1 shown above.
will build the package with name blender-bin shown above.
# Description
@@ -34,7 +34,7 @@ This command shows what packages are currently installed in a
profile. For each installed package, it shows the following
information:
* `Index`: An integer that can be used to unambiguously identify the
* `Name`: A unique name used to unambiguously identify the
package in invocations of `nix profile remove` and `nix profile
upgrade`.
+2 -2
View File
@@ -2,10 +2,10 @@ R""(
# Examples
* Remove a package by position:
* Remove a package by name:
```console
# nix profile remove 3
# nix profile remove hello
```
* Remove a package by attribute path:
+5 -10
View File
@@ -9,21 +9,16 @@ R""(
# nix profile upgrade '.*'
```
* Upgrade a specific package:
* Upgrade a specific package by name:
```console
# nix profile upgrade hello
```
```console
# nix profile upgrade packages.x86_64-linux.hello
```
* Upgrade a specific profile element by number:
```console
# nix profile list
0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify …
# nix profile upgrade 0
```
# Description
This command upgrades a previously installed package in a Nix profile,
+118 -77
View File
@@ -105,7 +105,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
element.updateStorePaths(getEvalStore(), store, res);
manifest.elements.push_back(std::move(element));
manifest.addElement(std::move(element));
}
try {
@@ -115,7 +115,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
// See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102
auto findRefByFilePath = [&]<typename Iterator>(Iterator begin, Iterator end) {
for (auto it = begin; it != end; it++) {
auto profileElement = *it;
auto & profileElement = it->second;
for (auto & storePath : profileElement.storePaths) {
if (conflictError.fileA.starts_with(store->printStorePath(storePath))) {
return std::pair(conflictError.fileA, profileElement.toInstallables(*store));
@@ -183,35 +183,40 @@ public:
std::string pattern;
std::regex reg;
};
typedef std::variant<size_t, Path, RegexPattern> Matcher;
using Matcher = std::variant<Path, RegexPattern>;
std::vector<Matcher> getMatchers(ref<Store> store)
{
std::vector<Matcher> res;
for (auto & s : _matchers) {
if (auto n = string2Int<size_t>(s))
res.push_back(*n);
else if (store->isStorePath(s))
if (auto n = string2Int<size_t>(s)) {
throw Error("'nix profile' no longer supports indices ('%d')", *n);
} else if (store->isStorePath(s)) {
res.push_back(s);
else
} else {
res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)});
}
}
return res;
}
bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers)
bool matches(
Store const & store,
// regex_match doesn't take a string_view lol
std::string const & name,
ProfileElement const & element,
std::vector<Matcher> const & matchers
)
{
for (auto & matcher : matchers) {
if (auto n = std::get_if<size_t>(&matcher)) {
if (*n == pos) return true;
} else if (auto path = std::get_if<Path>(&matcher)) {
if (auto path = std::get_if<Path>(&matcher)) {
if (element.storePaths.count(store.parseStorePath(*path))) return true;
} else if (auto regex = std::get_if<RegexPattern>(&matcher)) {
if (element.source
&& std::regex_match(element.source->attrPath, regex->reg))
if (std::regex_match(name, regex->reg)) {
return true;
}
}
}
@@ -241,10 +246,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
ProfileManifest newManifest;
for (size_t i = 0; i < oldManifest.elements.size(); ++i) {
auto & element(oldManifest.elements[i]);
if (!matches(*store, element, i, matchers)) {
newManifest.elements.push_back(std::move(element));
for (auto & [name, element] : oldManifest.elements) {
if (!matches(*store, name, element, matchers)) {
newManifest.elements.insert_or_assign(name, std::move(element));
} else {
notice("removing '%s'", element.identifier());
}
@@ -257,9 +261,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
if (removedCount == 0) {
for (auto matcher: matchers) {
if (const size_t * index = std::get_if<size_t>(&matcher)){
warn("'%d' is not a valid index", *index);
} else if (const Path * path = std::get_if<Path>(&matcher)){
if (const Path * path = std::get_if<Path>(&matcher)) {
warn("'%s' does not match any paths", *path);
} else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
warn("'%s' does not match any packages", regex->pattern);
@@ -292,64 +294,99 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
auto matchers = getMatchers(store);
Installables installables;
std::vector<size_t> indices;
std::vector<ProfileElement *> elems;
auto matchedCount = 0;
auto upgradedCount = 0;
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
if (element.source
&& !element.source->originalRef.input.isLocked()
&& matches(*store, element, i, matchers))
{
upgradedCount++;
Activity act(*logger, lvlChatty, actUnknown,
fmt("checking '%s' for updates", element.source->attrPath));
auto installable = make_ref<InstallableFlake>(
this,
getEvalState(),
FlakeRef(element.source->originalRef),
"",
element.source->outputs,
Strings{element.source->attrPath},
Strings{},
lockFlags);
auto derivedPaths = installable->toDerivedPaths();
if (derivedPaths.empty()) continue;
auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info);
// `InstallableFlake` should use `ExtraPathInfoFlake`.
assert(infop);
auto & info = *infop;
if (element.source->lockedRef == info.flake.lockedRef) continue;
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->lockedRef, info.flake.lockedRef);
element.source = ProfileElementSource {
.originalRef = installable->flakeRef,
.lockedRef = info.flake.lockedRef,
.attrPath = info.value.attrPath,
.outputs = installable->extendedOutputsSpec,
};
installables.push_back(installable);
indices.push_back(i);
for (auto & [name, element] : manifest.elements) {
if (!matches(*store, name, element, matchers)) {
continue;
}
matchedCount += 1;
if (!element.source) {
warn(
"Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades",
element.identifier()
);
continue;
}
if (element.source->originalRef.input.isLocked()) {
warn(
"Found package '%s', but it was installed from a locked flake reference so it can't be upgraded",
element.identifier()
);
continue;
}
upgradedCount++;
Activity act(
*logger,
lvlChatty,
actUnknown,
fmt("checking '%s' for updates", element.source->attrPath),
Logger::Fields{element.source->attrPath}
);
auto installable = make_ref<InstallableFlake>(
this,
getEvalState(),
FlakeRef(element.source->originalRef),
"",
element.source->outputs,
Strings{element.source->attrPath},
Strings{},
lockFlags
);
auto derivedPaths = installable->toDerivedPaths();
if (derivedPaths.empty()) {
continue;
}
auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info);
// `InstallableFlake` should use `ExtraPathInfoFlake`.
assert(infop);
auto & info = *infop;
if (element.source->lockedRef == info.flake.lockedRef) {
continue;
}
printInfo(
"upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath,
element.source->lockedRef,
info.flake.lockedRef
);
element.source = ProfileElementSource {
.originalRef = installable->flakeRef,
.lockedRef = info.flake.lockedRef,
.attrPath = info.value.attrPath,
.outputs = installable->extendedOutputsSpec,
};
installables.push_back(installable);
elems.push_back(&element);
}
if (upgradedCount == 0) {
for (auto & matcher : matchers) {
if (const size_t * index = std::get_if<size_t>(&matcher)){
warn("'%d' is not a valid index", *index);
} else if (const Path * path = std::get_if<Path>(&matcher)){
warn("'%s' does not match any paths", *path);
} else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
warn("'%s' does not match any packages", regex->pattern);
if (matchedCount == 0) {
for (auto & matcher : matchers) {
if (const Path * path = std::get_if<Path>(&matcher)){
warn("'%s' does not match any paths", *path);
} else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)) {
warn("'%s' does not match any packages", regex->pattern);
}
}
} else {
warn("Found some packages but none of them could be upgraded");
}
warn ("Use 'nix profile list' to see the current profile.");
}
@@ -360,7 +397,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
for (size_t i = 0; i < installables.size(); ++i) {
auto & installable = installables.at(i);
auto & element = manifest.elements[indices.at(i)];
auto & element = *elems.at(i);
element.updateStorePaths(
getEvalStore(),
store,
@@ -392,12 +429,16 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
if (json) {
std::cout << manifest.toJSON(*store).dump() << "\n";
} else {
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
if (i) logger->cout("");
logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s",
i,
element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
for (auto const & [i, nameElemPair] : enumerate(manifest.elements)) {
auto & [name, element] = nameElemPair;
if (i) {
logger->cout("");
}
logger->cout(
"Name: " ANSI_BOLD "%s" ANSI_NORMAL "%s",
name,
element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL
);
if (element.source) {
logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string());
logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
+5 -3
View File
@@ -208,7 +208,8 @@ struct CmdUpgradeNix : MixDryRun, EvalCommand
// Find which profile element has Nix in it.
// It should be impossible to *not* have Nix, since we grabbed this
// store path by looking for things with bin/nix-env in them anyway.
auto findNix = [&](ProfileElement const & elem) -> bool {
auto findNix = [&](std::pair<std::string, ProfileElement> const & nameElemPair) -> bool {
auto const & [name, elem] = nameElemPair;
for (auto const & ePath : elem.storePaths) {
auto const nixEnv = store->printStorePath(ePath) + "/bin/nix-env";
if (pathExists(nixEnv)) {
@@ -226,14 +227,15 @@ struct CmdUpgradeNix : MixDryRun, EvalCommand
// *Should* be impossible...
assert(elemWithNix != std::end(manifest.elements));
auto const nixElemName = elemWithNix->first;
// Now create a new profile element for the new Nix version...
ProfileElement elemForNewNix = {
.storePaths = {newNix},
};
// ...and splork it into the manifest where the old profile element was.
// (Remember, elemWithNix is an iterator)
*elemWithNix = elemForNewNix;
manifest.elements.at(nixElemName) = elemForNewNix;
// Build the new profile, and switch to it.
StorePath const newProfile = manifest.build(store);