From f6c0aea824869b9d915dc4c95fd464592d70534a Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Sun, 20 Jul 2025 19:39:11 +0200 Subject: [PATCH] libstore/http: expose `HttpBinaryCacheStore` in the header Including the `.cc` is possible but is suspicious. Change-Id: Ie18fef1e30e517edff4ab96f4a9c339e2b3145b5 Signed-off-by: Raito Bezarius --- lix/libstore/http-binary-cache-store.cc | 288 +++++++++--------------- lix/libstore/http-binary-cache-store.hh | 107 +++++++++ 2 files changed, 217 insertions(+), 178 deletions(-) diff --git a/lix/libstore/http-binary-cache-store.cc b/lix/libstore/http-binary-cache-store.cc index 3bb5bcc03..eea66a463 100644 --- a/lix/libstore/http-binary-cache-store.cc +++ b/lix/libstore/http-binary-cache-store.cc @@ -1,6 +1,5 @@ #include "lix/libstore/http-binary-cache-store.hh" #include "lix/libstore/binary-cache-store.hh" -#include "lix/libstore/filetransfer.hh" #include "lix/libstore/globals.hh" #include "lix/libstore/nar-info-disk-cache.hh" #include "lix/libutil/async-io.hh" @@ -12,196 +11,129 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : BinaryCacheStoreConfig +std::string HttpBinaryCacheStoreConfig::doc() { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + return +#include "http-binary-cache-store.md" + ; +} - const std::string name() override { return "HTTP Binary Cache Store"; } - - std::string doc() override - { - return - #include "http-binary-cache-store.md" - ; - } -}; - -class HttpBinaryCacheStore : public BinaryCacheStore +HttpBinaryCacheStore::HttpBinaryCacheStore( + const std::string & scheme, const Path & _cacheUri, HttpBinaryCacheStoreConfig config +) + : Store(config) + , BinaryCacheStore(config) + , config_(std::move(config)) + , cacheUri(scheme + "://" + _cacheUri) { -private: - - HttpBinaryCacheStoreConfig config_; - Path cacheUri; - - struct State - { - bool enabled = true; - std::chrono::steady_clock::time_point disabledUntil; - }; - - Sync _state; - -public: - - HttpBinaryCacheStore( - const std::string & scheme, - const Path & _cacheUri, - HttpBinaryCacheStoreConfig config) - : Store(config) - , BinaryCacheStore(config) - , config_(std::move(config)) - , cacheUri(scheme + "://" + _cacheUri) - { - if (cacheUri.back() == '/') - cacheUri.pop_back(); - - diskCache = getNarInfoDiskCache(); + if (cacheUri.back() == '/') { + cacheUri.pop_back(); } - HttpBinaryCacheStoreConfig & config() override { return config_; } - const HttpBinaryCacheStoreConfig & config() const override { return config_; } + diskCache = getNarInfoDiskCache(); +} - std::string getUri() override - { - return cacheUri; +kj::Promise> HttpBinaryCacheStore::init() +try { + // FIXME: do this lazily? + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { + config_.wantMassQuery.setDefault(cacheInfo->wantMassQuery); + config_.priority.setDefault(cacheInfo->priority); + } else { + try { + TRY_AWAIT(BinaryCacheStore::init()); + } catch (UploadToHTTP &) { + throw Error("'%s' does not appear to be a binary cache", cacheUri); + } + diskCache->createCache(cacheUri, config_.storeDir, config_.wantMassQuery, config_.priority); } + co_return result::success(); +} catch (...) { + co_return result::current_exception(); +} + +FileTransferOptions HttpBinaryCacheStore::makeOptions(Headers && headers) +{ + return {.headers = std::move(headers)}; +} + +void HttpBinaryCacheStore::maybeDisable() +{ + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); + } +} + +void HttpBinaryCacheStore::checkEnabled() +{ + auto state(_state.lock()); + if (state->enabled) { + return; + } + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); +} + +kj::Promise> HttpBinaryCacheStore::fileExists(const std::string & path) +try { + checkEnabled(); - kj::Promise> init() override try { - // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - config_.wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config_.priority.setDefault(cacheInfo->priority); - } else { - try { - TRY_AWAIT(BinaryCacheStore::init()); - } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); - } - diskCache->createCache( - cacheUri, config_.storeDir, config_.wantMassQuery, config_.priority + co_return TRY_AWAIT(getFileTransfer()->exists(makeURI(path), makeOptions())); + } catch (FileTransferError & e) { + maybeDisable(); + throw; + } +} catch (...) { + co_return result::current_exception(); +} + +kj::Promise> HttpBinaryCacheStore::upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType +) +try { + auto data = StreamToSourceAdapter(istream).drain(); + try { + TRY_AWAIT(getFileTransfer()->upload( + makeURI(path), std::move(data), makeOptions({{"Content-Type", mimeType}}) + )); + } catch (FileTransferError & e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + } + co_return result::success(); +} catch (...) { + co_return result::current_exception(); +} + +kj::Promise>> +HttpBinaryCacheStore::getFile(const std::string & path) +try { + checkEnabled(); + try { + co_return TRY_AWAIT(getFileTransfer()->download(makeURI(path), makeOptions())).second; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) { + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, getUri() ); } - co_return result::success(); - } catch (...) { - co_return result::current_exception(); + maybeDisable(); + throw; } - /** Override this to configure additional curl options on the request. - * e.g. authentication method or key material. - * - * This is meant mostly for plugins who wish to perform their own - * custom initialization. - */ - virtual FileTransferOptions makeOptions(Headers && headers = {}) - { - return {.headers = std::move(headers)}; - } - - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - -protected: - - void maybeDisable() - { - auto state(_state.lock()); - if (state->enabled && settings.tryFallback) { - int t = 60; - printError("disabling binary cache '%s' for %s seconds", getUri(), t); - state->enabled = false; - state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); - } - } - - void checkEnabled() - { - auto state(_state.lock()); - if (state->enabled) return; - if (std::chrono::steady_clock::now() > state->disabledUntil) { - state->enabled = true; - debug("re-enabling binary cache '%s'", getUri()); - return; - } - throw SubstituterDisabled("substituter '%s' is disabled", getUri()); - } - - kj::Promise> fileExists(const std::string & path) override - try { - checkEnabled(); - - try { - co_return TRY_AWAIT(getFileTransfer()->exists(makeURI(path), makeOptions())); - } catch (FileTransferError & e) { - maybeDisable(); - throw; - } - } catch (...) { - co_return result::current_exception(); - } - - kj::Promise> upsertFile( - const std::string & path, - std::shared_ptr> istream, - const std::string & mimeType - ) override - try { - auto data = StreamToSourceAdapter(istream).drain(); - try { - TRY_AWAIT(getFileTransfer()->upload( - makeURI(path), std::move(data), makeOptions({{"Content-Type", mimeType}}) - )); - } catch (FileTransferError & e) { - throw UploadToHTTP( - "while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg() - ); - } - co_return result::success(); - } catch (...) { - co_return result::current_exception(); - } - - std::string makeURI(const std::string & path) - { - return path.starts_with("https://") || path.starts_with("http://") - || path.starts_with("file://") - ? path - : cacheUri + "/" + path; - } - - kj::Promise>> getFile(const std::string & path) override - try { - checkEnabled(); - try { - co_return TRY_AWAIT(getFileTransfer()->download(makeURI(path), makeOptions())).second; - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); - maybeDisable(); - throw; - } - } catch (...) { - co_return result::current_exception(); - } - - /** - * This isn't actually necessary read only. We support "upsert" now, so we - * have a notion of authentication via HTTP POST/PUT. - * - * For now, we conservatively say we don't know. - * - * \todo try to expose our HTTP authentication status. - */ - kj::Promise>> isTrustedClient() override - { - return {result::success(std::nullopt)}; - } -}; +} catch (...) { + co_return result::current_exception(); +} void registerHttpBinaryCacheStore() { StoreImplementations::add(); diff --git a/lix/libstore/http-binary-cache-store.hh b/lix/libstore/http-binary-cache-store.hh index 097c5480b..70d363dd0 100644 --- a/lix/libstore/http-binary-cache-store.hh +++ b/lix/libstore/http-binary-cache-store.hh @@ -1,8 +1,115 @@ #pragma once ///@file +#include "lix/libstore/binary-cache-store.hh" +#include "lix/libstore/filetransfer.hh" + +#include + namespace nix { +struct HttpBinaryCacheStoreConfig : BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + std::string doc() override; +}; + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + HttpBinaryCacheStoreConfig config_; + Path cacheUri; + + struct State + { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync _state; + +public: + + HttpBinaryCacheStore( + const std::string & scheme, const Path & _cacheUri, HttpBinaryCacheStoreConfig config + ); + + HttpBinaryCacheStoreConfig & config() override + { + return config_; + } + const HttpBinaryCacheStoreConfig & config() const override + { + return config_; + } + + std::string getUri() override + { + return cacheUri; + } + + kj::Promise> init() override; + + /** Override this to configure additional curl options on the request. + * e.g. authentication method or key material. + * + * This is meant mostly for plugins who wish to perform their own + * custom initialization. + */ + virtual FileTransferOptions makeOptions(Headers && headers = {}); + + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) { + ret.insert("file"); + } + return ret; + } + +protected: + + void maybeDisable(); + void checkEnabled(); + + kj::Promise> fileExists(const std::string & path) override; + kj::Promise> upsertFile( + const std::string & path, + std::shared_ptr> istream, + const std::string & mimeType + ) override; + kj::Promise>> getFile(const std::string & path) override; + + std::string makeURI(const std::string & path) + { + return path.starts_with("https://") || path.starts_with("http://") + || path.starts_with("file://") + ? path + : cacheUri + "/" + path; + } + + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + kj::Promise>> isTrustedClient() override + { + return {result::success(std::nullopt)}; + } +}; + void registerHttpBinaryCacheStore(); }