diff --git a/flake.lock b/flake.lock index cf2b2d724..854d695da 100644 --- a/flake.lock +++ b/flake.lock @@ -48,11 +48,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1721091462, - "narHash": "sha256-0cmEeoOiB91BviTJHzIyxkY+Gxv3O8ZnnExVAoXEFGI=", + "lastModified": 1723331518, + "narHash": "sha256-JVnQ3OLbXQAlkOluFc3gWhZMbhared1Rg5YvNEc92m0=", "ref": "refs/heads/main", - "rev": "6b4d46e9e0e1dd80e0977684ab20d14bcd1a6bc3", - "revCount": 15967, + "rev": "5137cea99044d54337e439510a647743110b2d7d", + "revCount": 16128, "type": "git", "url": "https://git.lix.systems/lix-project/lix" }, @@ -111,11 +111,11 @@ "nix2container": { "flake": false, "locked": { - "lastModified": 1712990762, - "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=", + "lastModified": 1720642556, + "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", "owner": "nlewo", "repo": "nix2container", - "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e", + "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", "type": "github" }, "original": { @@ -126,11 +126,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1720691131, - "narHash": "sha256-CWT+KN8aTPyMIx8P303gsVxUnkinIz0a/Cmasz1jyIM=", + "lastModified": 1723282977, + "narHash": "sha256-oTK91aOlA/4IsjNAZGMEBz7Sq1zBS0Ltu4/nIQdYDOg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a046c1202e11b62cbede5385ba64908feb7bfac4", + "rev": "a781ff33ae258bbcfd4ed6e673860c3e923bf2cc", "type": "github" }, "original": { @@ -159,11 +159,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1712055707, - "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 122b32ceb..fcab7b541 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -368,7 +368,7 @@ static std::map queryPathInfos( auto references = ServeProto::Serialise::read(localStore, conn); readLongLong(conn.from); // download size auto narSize = readLongLong(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashType::SHA256); auto ca = ContentAddress::parseOpt(readString(conn.from)); readStrings(conn.from); // sigs ValidPathInfo info(localStore.parseStorePath(storePathS), narHash); @@ -397,8 +397,7 @@ static void copyPathFromRemote( /* Receive the NAR from the remote and add it to the destination store. Meanwhile, extract all the info from the NAR that getBuildOutput() needs. */ - auto source2 = sinkToSource([&](Sink & sink) - { + auto coro = [&]() -> WireFormatGenerator { /* Note: we should only send the command to dump the store path to the remote if the NAR is actually going to get read by the destination store, which won't happen if this path @@ -409,11 +408,11 @@ static void copyPathFromRemote( conn.to << ServeProto::Command::DumpStorePath << localStore.printStorePath(info.path); conn.to.flush(); - TeeSource tee(conn.from, sink); - extractNarData(tee, localStore.printStorePath(info.path), narMembers); - }); + co_yield extractNarDataFilter(conn.from, localStore.printStorePath(info.path), narMembers); + }; + GeneratorSource source2{coro()}; - destStore.addToStore(info, *source2, NoRepair, NoCheckSigs); + destStore.addToStore(info, source2, NoRepair, NoCheckSigs); } static void copyPathsFromRemote( diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 938513693..d0b99939c 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -34,11 +34,8 @@ BuildOutput getBuildOutput( auto outputS = store->printStorePath(output); if (!narMembers.count(outputS)) { printInfo("fetching NAR contents of '%s'...", outputS); - auto source = sinkToSource([&](Sink & sink) - { - sink << store->narFromPath(output); - }); - extractNarData(*source, outputS, narMembers); + GeneratorSource source{store->narFromPath(output)}; + extractNarData(source, outputS, narMembers); } } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 5b679d187..7cd50877b 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -534,7 +534,7 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, product.type, product.subtype, product.fileSize ? std::make_optional(*product.fileSize) : std::nullopt, - product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base16, false)) : std::nullopt, + product.sha256hash ? std::make_optional(product.sha256hash->to_string(Base::Base16, false)) : std::nullopt, product.path, product.name, product.defaultPath); diff --git a/src/hydra-queue-runner/nar-extractor.cc b/src/hydra-queue-runner/nar-extractor.cc index 9f0eb4319..7a4dd9964 100644 --- a/src/hydra-queue-runner/nar-extractor.cc +++ b/src/hydra-queue-runner/nar-extractor.cc @@ -42,7 +42,7 @@ struct Extractor : ParseSink void preallocateContents(uint64_t size) override { expectedSize = size; - hashSink = std::make_unique(htSHA256); + hashSink = std::make_unique(HashType::SHA256); } void receiveContents(std::string_view data) override @@ -76,7 +76,19 @@ void extractNarData( const Path & prefix, NarMemberDatas & members) { - Extractor extractor(members, prefix); - parseDump(extractor, source); - // Note: this point may not be reached if we're in a coroutine. + auto parser = extractNarDataFilter(source, prefix, members); + while (parser.next()) { + // ignore raw data + } +} + +nix::WireFormatGenerator extractNarDataFilter( + Source & source, + const Path & prefix, + NarMemberDatas & members) +{ + return [](Source & source, const Path & prefix, NarMemberDatas & members) -> WireFormatGenerator { + Extractor extractor(members, prefix); + co_yield parseAndCopyDump(extractor, source); + }(source, prefix, members); } diff --git a/src/hydra-queue-runner/nar-extractor.hh b/src/hydra-queue-runner/nar-extractor.hh index 45b2706c2..e6ba91b07 100644 --- a/src/hydra-queue-runner/nar-extractor.hh +++ b/src/hydra-queue-runner/nar-extractor.hh @@ -21,3 +21,8 @@ void extractNarData( nix::Source & source, const nix::Path & prefix, NarMemberDatas & members); + +nix::WireFormatGenerator extractNarDataFilter( + nix::Source & source, + const nix::Path & prefix, + NarMemberDatas & members); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index bc58a8d00..9cc4357e7 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -727,7 +727,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref product.fileSize = row[2].as(); } if (!row[3].is_null()) - product.sha256hash = Hash::parseAny(row[3].as(), htSHA256); + product.sha256hash = Hash::parseAny(row[3].as(), HashType::SHA256); if (!row[4].is_null()) product.path = row[4].as(); product.name = row[5].as();