diff --git a/doc/manual/rl-next/crash-in-syslog.md b/doc/manual/rl-next/crash-in-syslog.md new file mode 100644 index 000000000..f9181d9d2 --- /dev/null +++ b/doc/manual/rl-next/crash-in-syslog.md @@ -0,0 +1,12 @@ +--- +synopsis: "Crashs land in syslog now" +cls: [2640] +category: Improvements +credits: [jade] +--- + +When Lix crashes with unexpected exceptions and in some other conditions, it prints bug reporting instructions. +Previously, these only landed in stderr and not in syslog. +However, on larger Lix installations, it may be the case that Lix crashes in the client without the logs landing in the system logs, which impeded diagnosis. + +Now, such crashes always land in syslog too. diff --git a/doc/manual/rl-next/parallelise-signs.md b/doc/manual/rl-next/parallelise-signs.md index 6be48b7d5..23f7bdf1e 100644 --- a/doc/manual/rl-next/parallelise-signs.md +++ b/doc/manual/rl-next/parallelise-signs.md @@ -1,6 +1,6 @@ --- synopsis: "Paralellise `nix store sign` using a thread pool" -issues: [399] +issues: [fj#399] cls: [2606] category: Fixes credits: [Lunaphied] diff --git a/lix/libmain/crash-handler.cc b/lix/libmain/crash-handler.cc index 70c50b509..9ad3fbc01 100644 --- a/lix/libmain/crash-handler.cc +++ b/lix/libmain/crash-handler.cc @@ -1,5 +1,6 @@ #include "lix/libmain/crash-handler.hh" #include "lix/libutil/fmt.hh" +#include "lix/libutil/logging.hh" #include #include @@ -7,24 +8,25 @@ namespace nix { namespace { + void onTerminate() { - std::cerr << "Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included:\n\n"; + logFatal("Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included:\n"); try { std::exception_ptr eptr = std::current_exception(); if (eptr) { std::rethrow_exception(eptr); } else { - std::cerr << "std::terminate() called without exception\n"; + logFatal("std::terminate() called without exception"); } } catch (const std::exception & ex) { - std::cerr << "Exception: " << boost::core::demangle(typeid(ex).name()) << ": " << ex.what() << "\n"; + logFatal(fmt("Exception: %s: %s", boost::core::demangle(typeid(ex).name()), ex.what())); } catch (...) { - std::cerr << "Unknown exception! Spooky.\n"; + logFatal("Unknown exception! Spooky."); } - std::cerr << "Stack trace:\n"; - nix::printStackTrace(); + logFatal("Stack trace:"); + logFatal(getStackTrace()); std::abort(); } diff --git a/lix/libstore/build/worker.cc b/lix/libstore/build/worker.cc index 8742c63de..2e23ab0ba 100644 --- a/lix/libstore/build/worker.cc +++ b/lix/libstore/build/worker.cc @@ -18,8 +18,8 @@ struct ErrorHandler : kj::TaskSet::ErrorHandler { void taskFailed(kj::Exception && e) override { - printError("unexpected async failure in Worker: %s", kj::str(e).cStr()); - abort(); + logFatal(fmt("unexpected async failure in Worker: %s", kj::str(e).cStr())); + std::terminate(); } } errorHandler; } diff --git a/lix/libutil/fmt.cc b/lix/libutil/fmt.cc index b45448e3b..a587f3294 100644 --- a/lix/libutil/fmt.cc +++ b/lix/libutil/fmt.cc @@ -1,4 +1,5 @@ #include "lix/libutil/fmt.hh" // IWYU pragma: keep +#include // Darwin and FreeBSD stdenv do not define _GNU_SOURCE but do have _Unwind_Backtrace. #if __APPLE__ || __FreeBSD__ #define BOOST_STACKTRACE_GNU_SOURCE_NOT_REQUIRED @@ -16,9 +17,11 @@ template HintFmt::HintFmt(const std::string &, const uint64_t &, const char * co HintFmt::HintFmt(const std::string & literal) : HintFmt("%s", Uncolored(literal)) {} -void printStackTrace() +std::string getStackTrace() { - std::cerr << boost::stacktrace::stacktrace() << std::endl; + std::stringstream ss; + ss << boost::stacktrace::stacktrace(); + return ss.str(); } } diff --git a/lix/libutil/fmt.hh b/lix/libutil/fmt.hh index 2c0a07c88..8a797c934 100644 --- a/lix/libutil/fmt.hh +++ b/lix/libutil/fmt.hh @@ -11,8 +11,8 @@ extern template class boost::basic_format; namespace nix { -/** Prints a C++ stack trace to stderr using boost stacktrace */ -void printStackTrace(); +/** Gets a C++ stack trace using boost stacktrace */ +std::string getStackTrace(); /** * Values wrapped in this struct are printed in magenta. diff --git a/lix/libutil/logging.cc b/lix/libutil/logging.cc index 50ed4b83c..3bad9e50a 100644 --- a/lix/libutil/logging.cc +++ b/lix/libutil/logging.cc @@ -10,6 +10,7 @@ #include #include #include +#include namespace nix { @@ -126,27 +127,6 @@ Verbosity verbosityFromIntClamped(int val) return static_cast(clamped); } -void writeLogsToStderr(std::string_view s) -{ - static std::mutex lock; - - // make sure only one thread uses this function at any given time. - // multiple concurrent threads can have deleterious effects on log - // output, especially when layering structured formats (like JSON) - // on top of a SimpleLogger which is itself not thread-safe. every - // Logger instance should be thread-safe in an ideal world, but we - // cannot really enforce that on a per-logger level at this point. - std::unique_lock _lock(lock); - try { - writeFull(STDERR_FILENO, s, false); - } catch (SysError & e) { - /* Ignore failing writes to stderr. We need to ignore write - errors to ensure that cleanup code that logs to stderr runs - to completion if the other side of stderr has been closed - unexpectedly. */ - } -} - Logger * makeSimpleLogger(bool printBuildLogs) { return new SimpleLogger(printBuildLogs); @@ -367,4 +347,32 @@ Activity::~Activity() } } +void writeLogsToStderr(std::string_view s) +{ + static std::mutex lock; + + // make sure only one thread uses this function at any given time. + // multiple concurrent threads can have deleterious effects on log + // output, especially when layering structured formats (like JSON) + // on top of a SimpleLogger which is itself not thread-safe. every + // Logger instance should be thread-safe in an ideal world, but we + // cannot really enforce that on a per-logger level at this point. + std::unique_lock _lock(lock); + try { + writeFull(STDERR_FILENO, s, false); + } catch (SysError & e) { + /* Ignore failing writes to stderr. We need to ignore write + errors to ensure that cleanup code that logs to stderr runs + to completion if the other side of stderr has been closed + unexpectedly. */ + } +} + +void logFatal(std::string const & s) +{ + writeLogsToStderr(s + "\n"); + // std::string for guaranteed null termination + syslog(LOG_CRIT, "%s", s.c_str()); +} + } diff --git a/lix/libutil/logging.hh b/lix/libutil/logging.hh index 1afa103e1..08c7c382d 100644 --- a/lix/libutil/logging.hh +++ b/lix/libutil/logging.hh @@ -280,4 +280,8 @@ inline void warn(const std::string & fs, const Args & ... args) void writeLogsToStderr(std::string_view s); +/** Logs a fatal message as loudly as possible. This will go into syslog as well as stderr. + * The purpose of this function is making failures with redirected stderr louder. */ +void logFatal(std::string const & s); + }