guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

libstore/http: expose HttpBinaryCacheStore in the header

Browse Source 
Including the `.cc` is possible but is suspicious.

Change-Id: Ie18fef1e30e517edff4ab96f4a9c339e2b3145b5
Signed-off-by: Raito Bezarius <raito@lix.systems>
This commit is contained in:
Raito Bezarius
2025-07-23 15:35:32 +00:00
parent cc73479558
commit f6c0aea824
2 changed files with 217 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lix/libstore/http-binary-cache-store.cc
+110 -178
View File
@@ -1,6 +1,5 @@
#include "lix/libstore/http-binary-cache-store.hh"
#include "lix/libstore/binary-cache-store.hh"
#include "lix/libstore/filetransfer.hh"
#include "lix/libstore/globals.hh"
#include "lix/libstore/nar-info-disk-cache.hh"
#include "lix/libutil/async-io.hh"
@@ -12,196 +11,129 @@ namespace nix {
MakeError(UploadToHTTP, Error);
struct HttpBinaryCacheStoreConfig : BinaryCacheStoreConfig
std::string HttpBinaryCacheStoreConfig::doc()
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
return
#include "http-binary-cache-store.md"
;
}
const std::string name() override { return "HTTP Binary Cache Store"; }
std::string doc() override
{
return
#include "http-binary-cache-store.md"
;
}
};
class HttpBinaryCacheStore : public BinaryCacheStore
HttpBinaryCacheStore::HttpBinaryCacheStore(
const std::string & scheme, const Path & _cacheUri, HttpBinaryCacheStoreConfig config
)
: Store(config)
, BinaryCacheStore(config)
, config_(std::move(config))
, cacheUri(scheme + "://" + _cacheUri)
{
private:
HttpBinaryCacheStoreConfig config_;
Path cacheUri;
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
HttpBinaryCacheStore(
const std::string & scheme,
const Path & _cacheUri,
HttpBinaryCacheStoreConfig config)
: Store(config)
, BinaryCacheStore(config)
, config_(std::move(config))
, cacheUri(scheme + "://" + _cacheUri)
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
diskCache = getNarInfoDiskCache();
if (cacheUri.back() == '/') {
cacheUri.pop_back();
}
HttpBinaryCacheStoreConfig & config() override { return config_; }
const HttpBinaryCacheStoreConfig & config() const override { return config_; }
diskCache = getNarInfoDiskCache();
}
std::string getUri() override
{
return cacheUri;
kj::Promise<Result<void>> HttpBinaryCacheStore::init()
try {
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
config_.wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config_.priority.setDefault(cacheInfo->priority);
} else {
try {
TRY_AWAIT(BinaryCacheStore::init());
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", cacheUri);
}
diskCache->createCache(cacheUri, config_.storeDir, config_.wantMassQuery, config_.priority);
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
FileTransferOptions HttpBinaryCacheStore::makeOptions(Headers && headers)
{
return {.headers = std::move(headers)};
}
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
void HttpBinaryCacheStore::checkEnabled()
{
auto state(_state.lock());
if (state->enabled) {
return;
}
if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true;
debug("re-enabling binary cache '%s'", getUri());
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
}
kj::Promise<Result<bool>> HttpBinaryCacheStore::fileExists(const std::string & path)
try {
checkEnabled();
kj::Promise<Result<void>> init() override
try {
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
config_.wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config_.priority.setDefault(cacheInfo->priority);
} else {
try {
TRY_AWAIT(BinaryCacheStore::init());
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", cacheUri);
}
diskCache->createCache(
cacheUri, config_.storeDir, config_.wantMassQuery, config_.priority
co_return TRY_AWAIT(getFileTransfer()->exists(makeURI(path), makeOptions()));
} catch (FileTransferError & e) {
maybeDisable();
throw;
}
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<void>> HttpBinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType
)
try {
auto data = StreamToSourceAdapter(istream).drain();
try {
TRY_AWAIT(getFileTransfer()->upload(
makeURI(path), std::move(data), makeOptions({{"Content-Type", mimeType}})
));
} catch (FileTransferError & e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<box_ptr<AsyncInputStream>>>
HttpBinaryCacheStore::getFile(const std::string & path)
try {
checkEnabled();
try {
co_return TRY_AWAIT(getFileTransfer()->download(makeURI(path), makeOptions())).second;
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) {
throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, getUri()
);
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
maybeDisable();
throw;
}
/** Override this to configure additional curl options on the request.
* e.g. authentication method or key material.
*
* This is meant mostly for plugins who wish to perform their own
* custom initialization.
*/
virtual FileTransferOptions makeOptions(Headers && headers = {})
{
return {.headers = std::move(headers)};
}
static std::set<std::string> uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp) ret.insert("file");
return ret;
}
protected:
void maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
void checkEnabled()
{
auto state(_state.lock());
if (state->enabled) return;
if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true;
debug("re-enabling binary cache '%s'", getUri());
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", getUri());
}
kj::Promise<Result<bool>> fileExists(const std::string & path) override
try {
checkEnabled();
try {
co_return TRY_AWAIT(getFileTransfer()->exists(makeURI(path), makeOptions()));
} catch (FileTransferError & e) {
maybeDisable();
throw;
}
} catch (...) {
co_return result::current_exception();
}
kj::Promise<Result<void>> upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType
) override
try {
auto data = StreamToSourceAdapter(istream).drain();
try {
TRY_AWAIT(getFileTransfer()->upload(
makeURI(path), std::move(data), makeOptions({{"Content-Type", mimeType}})
));
} catch (FileTransferError & e) {
throw UploadToHTTP(
"while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()
);
}
co_return result::success();
} catch (...) {
co_return result::current_exception();
}
std::string makeURI(const std::string & path)
{
return path.starts_with("https://") || path.starts_with("http://")
|| path.starts_with("file://")
? path
: cacheUri + "/" + path;
}
kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) override
try {
checkEnabled();
try {
co_return TRY_AWAIT(getFileTransfer()->download(makeURI(path), makeOptions())).second;
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
}
} catch (...) {
co_return result::current_exception();
}
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
* For now, we conservatively say we don't know.
*
* \todo try to expose our HTTP authentication status.
*/
kj::Promise<Result<std::optional<TrustedFlag>>> isTrustedClient() override
{
return {result::success(std::nullopt)};
}
};
} catch (...) {
co_return result::current_exception();
}
void registerHttpBinaryCacheStore() {
StoreImplementations::add<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig>();
lix/libstore/http-binary-cache-store.hh
+107
View File
@@ -1,8 +1,115 @@
#pragma once
///@file
#include "lix/libstore/binary-cache-store.hh"
#include "lix/libstore/filetransfer.hh"
#include <curl/curl.h>
namespace nix {
struct HttpBinaryCacheStoreConfig : BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
const std::string name() override
{
return "HTTP Binary Cache Store";
}
std::string doc() override;
};
class HttpBinaryCacheStore : public BinaryCacheStore
{
private:
HttpBinaryCacheStoreConfig config_;
Path cacheUri;
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
HttpBinaryCacheStore(
const std::string & scheme, const Path & _cacheUri, HttpBinaryCacheStoreConfig config
);
HttpBinaryCacheStoreConfig & config() override
{
return config_;
}
const HttpBinaryCacheStoreConfig & config() const override
{
return config_;
}
std::string getUri() override
{
return cacheUri;
}
kj::Promise<Result<void>> init() override;
/** Override this to configure additional curl options on the request.
* e.g. authentication method or key material.
*
* This is meant mostly for plugins who wish to perform their own
* custom initialization.
*/
virtual FileTransferOptions makeOptions(Headers && headers = {});
static std::set<std::string> uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp) {
ret.insert("file");
}
return ret;
}
protected:
void maybeDisable();
void checkEnabled();
kj::Promise<Result<bool>> fileExists(const std::string & path) override;
kj::Promise<Result<void>> upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType
) override;
kj::Promise<Result<box_ptr<AsyncInputStream>>> getFile(const std::string & path) override;
std::string makeURI(const std::string & path)
{
return path.starts_with("https://") || path.starts_with("http://")
|| path.starts_with("file://")
? path
: cacheUri + "/" + path;
}
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
* For now, we conservatively say we don't know.
*
* \todo try to expose our HTTP authentication status.
*/
kj::Promise<Result<std::optional<TrustedFlag>>> isTrustedClient() override
{
return {result::success(std::nullopt)};
}
};
void registerHttpBinaryCacheStore();
}