Delete a left-over trash directory before doing a GC

Doing this once makes subsequent operations like garbage collecting
more efficient since we don't have to call makeMutable() first.

configure.ac

@@ -354,7 +354,6 @@ fi
 AC_SUBST(tarFlags)
 
 
-
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    src/Makefile

doc/manual/conf-file.xml

@@ -33,18 +33,18 @@ env-keep-derivations = false
 <para>You can override settings using the <option>--option</option>
 flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
-<para>The following settings are currently available: 
+<para>The following settings are currently available:
 
 <variablelist>
 
-  
+
   <varlistentry xml:id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
     will keep the outputs of non-garbage derivations.  If
     <literal>false</literal> (default), outputs will be deleted unless
     they are GC roots themselves (or reachable from other roots).</para>
- 
+
     <para>In general, outputs must be registered as roots separately.
     However, even if the output of a derivation is registered as a
     root, the collector will still delete store paths that are used
@@ -53,7 +53,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     this option to <literal>true</literal>.</para></listitem>
 
   </varlistentry>
-  
+
 
   <varlistentry xml:id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
 
@@ -71,7 +71,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
   <varlistentry><term><literal>env-keep-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
@@ -95,7 +95,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
   <varlistentry xml:id="conf-build-max-jobs"><term><literal>build-max-jobs</literal></term>
 
     <listitem><para>This option defines the maximum number of jobs
@@ -234,7 +234,27 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
+  <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
+
+    <listitem><para>When builds are performed in a chroot environment,
+    Nix will mount some directories from the normal file system
+    hierarchy inside the chroot.  These are the Nix store, the
+    temporary build directory (usually
+    <filename>/tmp/nix-build-<replaceable>drvname</replaceable>-<replaceable>number</replaceable></filename>),
+    the <literal>/proc</literal> filesystem, and the directories
+    listed here.  The default is <literal>/dev /dev/pts</literal>,
+    since these contain files needed by many builds (such as
+    <filename>/dev/null</filename>).  You can use the syntax
+    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
+    to mount a path in a different location in the chroot; for
+    instance, <literal>/bin=/nix-bin</literal> will mount the
+    directory <literal>/nix-bin</literal> as <literal>/bin</literal>
+    inside the chroot.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>build-use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
@@ -243,7 +263,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-    
+
   <varlistentry><term><literal>build-fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
@@ -253,34 +273,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-    
-  <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
 
-    <listitem><para>When builds are performed in a chroot environment,
-    Nix will mount (using <command>mount --bind</command> on Linux)
-    some directories from the normal file system hierarchy inside the
-    chroot.  These are the Nix store, the temporary build directory
-    (usually
-    <filename>/tmp/nix-<replaceable>pid</replaceable>-<replaceable>number</replaceable></filename>)
-    and the directories listed here.  The default is <literal>dev
-    /proc</literal>.  Files in <filename>/dev</filename> (such as
-    <filename>/dev/null</filename>) are needed by many builds, and
-    some files in <filename>/proc</filename> may also be needed
-    occasionally.</para>
-
-    <para>The value used on NixOS is
-    
-<programlisting>
-build-use-chroot = /dev /proc /bin</programlisting>
-
-    to make the <filename>/bin/sh</filename> symlink available (which
-    is still needed by many builders).</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  
   <varlistentry><term><literal>build-cache-failures</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will
@@ -331,8 +324,8 @@ build-use-chroot = /dev /proc /bin</programlisting>
   <varlistentry><term><literal>binary-caches</literal></term>
 
     <listitem><para>A list of URLs of binary caches, separated by
-    whitespace.  The default is empty.<!-- The default is
-    <literal>http://nixos.org/binary-cache</literal>. --></para></listitem>
+    whitespace.  The default is
+    <literal>http://nixos.org/binary-cache</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -341,10 +334,12 @@ build-use-chroot = /dev /proc /bin</programlisting>
 
     <listitem><para>A list of names of files that will be read to
     obtain additional binary cache URLs.  The default is
-    <literal>/nix/var/nix/profiles/per-user/root/channels/binary-caches/*</literal>,
-    which ensures that Nix will use the binary caches corresponding to
-    the channels installed by root.  Do not set this option to read
-    files created by untrusted users!</para></listitem>
+    <literal>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels/binary-caches/*</literal>.
+    Note that when you’re using the Nix daemon,
+    <replaceable>username</replaceable> is always equal to
+    <literal>root</literal>, so Nix will only use the binary caches
+    provided by the channels installed by root.  Do not set this
+    option to read files created by untrusted users!</para></listitem>
 
   </varlistentry>
 
@@ -415,7 +410,7 @@ build-use-chroot = /dev /proc /bin</programlisting>
 
   </varlistentry>
 
-  
+
   <varlistentry><term><literal>auto-optimise-store</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (the default),

doc/manual/env-common.xml

@@ -283,7 +283,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   <listitem><para>This variable should be set to
   <literal>daemon</literal> if you want to use the Nix daemon to
-  executed Nix operations, which is necessary in <link
+  execute Nix operations. This is necessary in <link
   linkend="ssec-multi-user">multi-user Nix installations</link>.
   Otherwise, it should be left unset.</para></listitem>
 

doc/manual/nix-env.xml

@@ -93,11 +93,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     <listitem><para>Specifies the profile to be used by those
     operations that operate on a profile (designated below as the
-    <emphasis>active profile</emphasis>).  A profile is sequence of
+    <emphasis>active profile</emphasis>).  A profile is a sequence of
     user environments called <emphasis>generations</emphasis>, one of
-    which is the <emphasis>current generation</emphasis>.  The default
-    profile is the target of the symbolic link
-    <filename>~/.nix-profile</filename> (see below).</para></listitem>
+    which is the <emphasis>current
+    generation</emphasis>.</para></listitem>
 
   </varlistentry>
 
@@ -1256,6 +1255,16 @@ error: no generation older than the current (91) exists</screen>
 <refsection condition="manpage"><title>Environment variables</title>
 
 <variablelist>
+  
+  <varlistentry><term><envar>NIX_PROFILE</envar></term>
+
+    <listitem><para>Location of the Nix profile.  Defaults to the
+    target of the symlink <filename>~/.nix-profile</filename>, if it
+    exists, or <filename>/nix/var/nix/profiles/default</filename>
+    otherwise.</para></listitem>
+
+  </varlistentry>
+
   <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
 </variablelist>
 

doc/manual/package-management.xml

@@ -538,7 +538,7 @@ dependencies) to a file, and then unpack that file into another Nix
 store.  For example,
 
 <screen>
-$ nix-store --export $(type -p firefox) > firefox.closure</screen>
+$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen>
 
 writes the closure of Firefox to a file.  You can then copy this file
 to another machine and install the closure:
@@ -552,7 +552,7 @@ another command, e.g. to copy and install a closure directly to/on
 another machine:
 
 <screen>
-$ nix-store --export $(type -p firefox) | bzip2 | \
+$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \
     ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
 
 But note that <command>nix-copy-closure</command> is generally more
@@ -583,7 +583,7 @@ $ nix-env -i firefox</screen>
 and Nix has to build a path that it sees is already present in
 <filename>/mnt/nix</filename>, then it will just copy from there
 instead of building it from source.</para>
- 
+
 
 </section>
 

doc/manual/release-notes.xml

@@ -6,6 +6,23 @@
 
 
 
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.3"><title>Release 1.3 (January 4, 2013)</title>
+
+<para>This is primarily a bug fix release.  When this version is first
+run on Linux, it removes any immutable bits from the Nix store and
+increases the schema version of the Nix store.  (The previous release
+removed support for setting the immutable bit; this release clears any
+remaining immutable bits to make certain operations more
+efficient.)</para>
+
+<para>This release has contributions from Eelco Dolstra and Stuart
+Pernsteiner.</para>
+
+</section>
+
+
 <!--==================================================================-->
 
 <section xml:id="ssec-relnotes-1.2"><title>Release 1.2 (December 6, 2012)</title>

perl/lib/Nix/Utils.pm

@@ -23,7 +23,7 @@ sub uniq {
 
 sub writeFile {
     my ($fn, $s) = @_;
-    open TMP, ">$fn" or die;
+    open TMP, ">$fn" or die "cannot create file `$fn': $!";
     print TMP "$s" or die;
     close TMP or die;
 }
@@ -31,7 +31,7 @@ sub writeFile {
 sub readFile {
     local $/ = undef;
     my ($fn) = @_;
-    open TMP, "<$fn" or die;
+    open TMP, "<$fn" or die "cannot open file `$fn': $!";
     my $s = <TMP>;
     close TMP or die;
     return $s;

release.nix

@@ -89,10 +89,15 @@ let
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
           --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
           --enable-gc
+          --sysconfdir=/etc
         '';
 
         enableParallelBuilding = true;
 
+        makeFlags = "profiledir=$(out)/etc/profile.d";
+
+        installFlags = "sysconfdir=$(out)/etc";
+
         doInstallCheck = true;
       };
 
@@ -229,7 +234,7 @@ let
       memSize = 1024;
       meta.schedulingPriority = prio;
       configureFlags = "--sysconfdir=/etc";
-      debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" ];
+      debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" ];
       doInstallCheck = true;
     };
 

scripts/Makefile.am

@@ -7,9 +7,11 @@ noinst_SCRIPTS = nix-profile.sh \
   find-runtime-roots.pl build-remote.pl nix-reduce-build \
   copy-from-other-stores.pl nix-http-export.cgi
 
+profiledir = $(sysconfdir)/profile.d
+
 install-exec-local: download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl find-runtime-roots.pl
-	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
-	$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
+	$(INSTALL) -d $(DESTDIR)$(profiledir)
+	$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(profiledir)/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 

scripts/download-from-binary-cache.pl.in

@@ -36,6 +36,8 @@ my %requests;
 my %scheduled;
 my $caBundle = $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"};
 
+my $userName = getpwuid($<) or die "cannot figure out user name";
+
 
 sub addRequest {
     my ($storePath, $url, $head) = @_;
@@ -191,11 +193,11 @@ sub getAvailableCaches {
         return map { s/\/+$//; $_ } split(/ /, $s);
     }
 
-    my @urls = strToList ($Nix::Config::config{"binary-caches"} // "");
-    # // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
+    my @urls = strToList($Nix::Config::config{"binary-caches"} //
+        ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
 
     my $urlsFiles = $Nix::Config::config{"binary-cache-files"}
-        // "/nix/var/nix/profiles/per-user/root/channels/binary-caches/*";
+        // "$Nix::Config::stateDir/profiles/per-user/$userName/channels/binary-caches/*";
     foreach my $urlFile (glob $urlsFiles) {
         next unless -f $urlFile;
         open FILE, "<$urlFile" or die "cannot open ‘$urlFile’\n";

src/libexpr/value.hh

@@ -131,7 +131,7 @@ static inline void mkStringNoCopy(Value & v, const char * s)
 
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((string) s).c_str());
+    mkStringNoCopy(v, ((const string &) s).c_str());
 }
 
 

src/libmain/shared.cc

@@ -10,6 +10,7 @@
 #include <cctype>
 #include <exception>
 
+#include <sys/time.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -135,6 +136,11 @@ static void initAndRun(int argc, char * * argv)
        everybody. */
     umask(0022);
 
+    /* Initialise the PRNG. */
+    struct timeval tv;
+    gettimeofday(&tv, 0);
+    srandom(tv.tv_usec);
+
     /* Process the NIX_LOG_TYPE environment variable. */
     string lt = getEnv("NIX_LOG_TYPE");
     if (lt != "") setLogType(lt);

src/libstore/build.cc

@@ -7,7 +7,6 @@
 #include "local-store.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "immutable.hh"
 
 #include <map>
 #include <sstream>
@@ -93,7 +92,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
 class Goal : public boost::enable_shared_from_this<Goal>
 {
 public:
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
+    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
 
 protected:
 
@@ -114,6 +113,10 @@ protected:
        failed because there are no substituters. */
     unsigned int nrNoSubstituters;
 
+    /* Number of substitution goals we are/were waiting for that
+       failed because othey had unsubstitutable references. */
+    unsigned int nrIncompleteClosure;
+
     /* Name of this goal for debugging purposes. */
     string name;
 
@@ -122,7 +125,7 @@ protected:
 
     Goal(Worker & worker) : worker(worker)
     {
-        nrFailed = nrNoSubstituters = 0;
+        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
         exitCode = ecBusy;
     }
 
@@ -307,10 +310,12 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     trace(format("waitee `%1%' done; %2% left") %
         waitee->name % waitees.size());
 
-    if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
+    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
 
     if (result == ecNoSubstituters) ++nrNoSubstituters;
 
+    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
+
     if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
 
         /* If we failed and keepGoing is not set, we remove all
@@ -333,7 +338,7 @@ void Goal::amDone(ExitCode result)
 {
     trace("done");
     assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
+    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
     exitCode = result;
     foreach (WeakGoals::iterator, i, waiters) {
         GoalPtr goal = i->lock();
@@ -710,13 +715,7 @@ HookInstance::HookInstance()
 HookInstance::~HookInstance()
 {
     try {
-        /* Cleanly shut down the hook by closing its stdin if it's not
-           already building.  Otherwise pid's destructor will kill
-           it. */
-        if (pid != -1 && toHook.writeSide != -1) {
-            toHook.writeSide.close();
-            pid.wait(true);
-        }
+        pid.kill();
     } catch (...) {
         ignoreException();
     }
@@ -763,6 +762,10 @@ private:
     /* Whether additional wanted outputs have been added. */
     bool needRestart;
 
+    /* Whether to retry substituting the outputs after building the
+       inputs. */
+    bool retrySubstitution;
+
     /* The derivation stored at drvPath. */
     Derivation drv;
 
@@ -819,7 +822,8 @@ private:
     GoalState state;
 
     /* Stuff we need to pass to initChild(). */
-    PathSet dirsInChroot;
+    typedef map<Path, Path> DirsInChroot; // maps target path to source path
+    DirsInChroot dirsInChroot;
     typedef map<string, string> Environment;
     Environment env;
 
@@ -911,6 +915,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     : Goal(worker)
     , wantedOutputs(wantedOutputs)
     , needRestart(false)
+    , retrySubstitution(false)
     , fLogFile(0)
     , bzLogFile(0)
     , useChroot(false)
@@ -1067,10 +1072,15 @@ void DerivationGoal::outputsSubstituted()
 {
     trace("all outputs substituted (maybe)");
 
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback)
+    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback)
         throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
 
-    nrFailed = nrNoSubstituters = 0;
+    /*  If the substitutes form an incomplete closure, then we should
+        build the dependencies of this derivation, but after that, we
+        can still use the substitutes for this derivation itself. */
+    if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true;
+
+    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
 
     if (needRestart) {
         needRestart = false;
@@ -1176,6 +1186,11 @@ void DerivationGoal::inputsRealised()
         return;
     }
 
+    if (retrySubstitution) {
+        haveDerivation();
+        return;
+    }
+
     /* Gather information necessary for computing the closure and/or
        running the build hook. */
 
@@ -1367,10 +1382,8 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
        way first.  We'd better not be interrupted here, because if
        we're repairing (say) Glibc, we end up with a broken system. */
     Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str();
-    if (pathExists(storePath)) {
-        makeMutable(storePath);
+    if (pathExists(storePath))
         rename(storePath.c_str(), oldPath.c_str());
-    }
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
         throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath);
     if (pathExists(oldPath))
@@ -1869,8 +1882,14 @@ void DerivationGoal::startBuilder()
 
         /* Bind-mount a user-configurable set of directories from the
            host file system. */
-        dirsInChroot = settings.dirsInChroot;
-        dirsInChroot.insert(tmpDir);
+        foreach (StringSet::iterator, i, settings.dirsInChroot) {
+            size_t p = i->find('=');
+            if (p == string::npos)
+                dirsInChroot[*i] = *i;
+            else
+                dirsInChroot[string(*i, 0, p)] = string(*i, p + 1);
+        }
+        dirsInChroot[tmpDir] = tmpDir;
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access
@@ -1887,12 +1906,8 @@ void DerivationGoal::startBuilder()
             if (lstat(i->c_str(), &st))
                 throw SysError(format("getting attributes of path `%1%'") % *i);
             if (S_ISDIR(st.st_mode))
-                dirsInChroot.insert(*i);
+                dirsInChroot[*i] = *i;
             else {
-                /* Creating a hard link to *i is impossible if its
-                   immutable bit is set.  So clear it first. */
-                makeMutable(*i);
-
                 Path p = chrootRootDir + *i;
                 if (link(i->c_str(), p.c_str()) == -1) {
                     /* Hard-linking fails if we exceed the maximum
@@ -2062,9 +2077,9 @@ void DerivationGoal::initChild()
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot
                environment. */
-            foreach (PathSet::iterator, i, dirsInChroot) {
-                Path source = *i;
-                Path target = chrootRootDir + source;
+            foreach (DirsInChroot::iterator, i, dirsInChroot) {
+                Path source = i->second;
+                Path target = chrootRootDir + i->first;
                 if (source == "/proc") continue; // backwards compatibility
                 debug(format("bind mounting `%1%' to `%2%'") % source % target);
                 createDirs(target);
@@ -2638,7 +2653,7 @@ void SubstitutionGoal::referencesValid()
 
     if (nrFailed > 0) {
         debug(format("some references of path `%1%' could not be realised") % storePath);
-        amDone(ecFailed);
+        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
         return;
     }
 

src/libstore/gc.cc

@@ -1,7 +1,6 @@
 #include "globals.hh"
 #include "misc.hh"
 #include "local-store.hh"
-#include "immutable.hh"
 
 #include <boost/shared_ptr.hpp>
 
@@ -396,24 +395,17 @@ struct LocalStore::GCState
     GCResults & results;
     PathSet roots;
     PathSet tempRoots;
-    PathSet deleted;
-    PathSet live;
-    PathSet busy;
-    PathSet invalidated;
+    PathSet dead;
+    PathSet alive;
     bool gcKeepOutputs;
     bool gcKeepDerivations;
     unsigned long long bytesInvalidated;
+    Path trashDir;
+    bool shouldDelete;
     GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
 };
 
 
-static bool shouldDelete(GCOptions::GCAction action)
-{
-    return action == GCOptions::gcDeleteDead
-        || action == GCOptions::gcDeleteSpecific;
-}
-
-
 bool LocalStore::isActiveTempFile(const GCState & state,
     const Path & path, const string & suffix)
 {
@@ -424,152 +416,149 @@ bool LocalStore::isActiveTempFile(const GCState & state,
 
 void LocalStore::deleteGarbage(GCState & state, const Path & path)
 {
-    printMsg(lvlInfo, format("deleting `%1%'") % path);
     unsigned long long bytesFreed;
     deletePathWrapped(path, bytesFreed);
     state.results.bytesFreed += bytesFreed;
 }
 
 
-bool LocalStore::tryToDelete(GCState & state, const Path & path)
+void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 {
     checkInterrupt();
 
-    if (path == linksDir) return true;
+    unsigned long long size = 0;
+
+    if (isValidPath(path)) {
+        PathSet referrers;
+        queryReferrers(path, referrers);
+        foreach (PathSet::iterator, i, referrers)
+            if (*i != path) deletePathRecursive(state, *i);
+        size = queryPathInfo(path).narSize;
+        invalidatePathChecked(path);
+    }
 
     struct stat st;
     if (lstat(path.c_str(), &st)) {
-        if (errno == ENOENT) return true;
+        if (errno == ENOENT) return;
         throw SysError(format("getting status of %1%") % path);
     }
 
-    if (state.deleted.find(path) != state.deleted.end()) return true;
-    if (state.live.find(path) != state.live.end()) return false;
+    printMsg(lvlInfo, format("deleting `%1%'") % path);
+
+    state.results.paths.insert(path);
+
+    /* If the path is not a regular file or symlink, move it to the
+       trash directory.  The move is to ensure that later (when we're
+       not holding the global GC lock) we can delete the path without
+       being afraid that the path has become alive again.  Otherwise
+       delete it right away. */
+    if (S_ISDIR(st.st_mode)) {
+        // Estimate the amount freed using the narSize field.  FIXME:
+        // if the path was not valid, need to determine the actual
+        // size.
+        state.bytesInvalidated += size;
+        // Mac OS X cannot rename directories if they are read-only.
+        if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+            throw SysError(format("making `%1%' writable") % path);
+        Path tmp = state.trashDir + "/" + baseNameOf(path);
+        if (rename(path.c_str(), tmp.c_str()))
+            throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp);
+    } else
+        deleteGarbage(state, path);
+
+    if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
+        printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
+        throw GCLimitReached();
+    }
+}
+
+
+bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
+{
+    if (visited.find(path) != visited.end()) return false;
+
+    if (state.alive.find(path) != state.alive.end()) {
+        return true;
+    }
+
+    if (state.dead.find(path) != state.dead.end()) {
+        return false;
+    }
+
+    if (state.roots.find(path) != state.roots.end()) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+        state.alive.insert(path);
+        return true;
+    }
+
+    visited.insert(path);
+
+    if (!isValidPath(path)) return false;
+
+    PathSet incoming;
+
+    /* Don't delete this path if any of its referrers are alive. */
+    queryReferrers(path, incoming);
+
+    /* If gc-keep-derivations is set and this is a derivation, then
+       don't delete the derivation if any of the outputs are alive. */
+    if (state.gcKeepDerivations && isDerivation(path)) {
+        PathSet outputs = queryDerivationOutputs(path);
+        foreach (PathSet::iterator, i, outputs)
+            if (isValidPath(*i) && queryDeriver(*i) == path)
+                incoming.insert(*i);
+    }
+
+    /* If gc-keep-outputs is set, then don't delete this path if there
+       are derivers of this path that are not garbage. */
+    if (state.gcKeepOutputs) {
+        PathSet derivers = queryValidDerivers(path);
+        foreach (PathSet::iterator, i, derivers)
+            incoming.insert(*i);
+    }
+
+    foreach (PathSet::iterator, i, incoming)
+        if (*i != path)
+            if (canReachRoot(state, visited, *i)) {
+                state.alive.insert(path);
+                return true;
+            }
+
+    return false;
+}
+
+
+void LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+    checkInterrupt();
+
+    if (path == linksDir || path == state.trashDir) return;
 
     startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
 
-    /* If gc-keep-outputs and gc-keep-derivations are both set, we can
-       have cycles in the liveness graph, so we need to treat such
-       strongly connected components as a single unit (‘paths’).  That
-       is, we can delete the elements of ‘paths’ only if all referrers
-       of ‘paths’ are garbage. */
-    PathSet paths, referrers;
-    Paths pathsSorted;
-
-    if (isValidPath(path)) {
-
-        /* Add derivers and outputs of ‘path’ to ‘paths’. */
-        PathSet todo;
-        todo.insert(path);
-        while (!todo.empty()) {
-            Path p = *todo.begin();
-            assertStorePath(p);
-            todo.erase(p);
-            if (paths.find(p) != paths.end()) continue;
-            paths.insert(p);
-            /* If gc-keep-derivations is set and this is a derivation,
-               then don't delete the derivation if any of the outputs
-               are live. */
-            if (state.gcKeepDerivations && isDerivation(p)) {
-                PathSet outputs = queryDerivationOutputs(p);
-                foreach (PathSet::iterator, i, outputs)
-                    if (isValidPath(*i)) todo.insert(*i);
-            }
-            /* If gc-keep-outputs is set, then don't delete this path
-               if there are derivers of this path that are not
-               garbage. */
-            if (state.gcKeepOutputs) {
-                PathSet derivers = queryValidDerivers(p);
-                foreach (PathSet::iterator, i, derivers) todo.insert(*i);
-            }
-        }
-    }
-
-    else {
+    if (!isValidPath(path)) {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
-        if (isActiveTempFile(state, path, ".lock")) return false;
+        if (isActiveTempFile(state, path, ".lock")) return;
 
         /* Don't delete .chroot directories for derivations that are
            currently being built. */
-        if (isActiveTempFile(state, path, ".chroot")) return false;
-
-        paths.insert(path);
+        if (isActiveTempFile(state, path, ".chroot")) return;
     }
 
-    /* Check if any path in ‘paths’ is a root. */
-    foreach (PathSet::iterator, i, paths)
-        if (state.roots.find(*i) != state.roots.end()) {
-            printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % *i);
-            goto isLive;
-        }
+    PathSet visited;
 
-    /* Recursively try to delete the referrers of this strongly
-       connected component.  If any referrer can't be deleted, then
-       these paths can't be deleted either. */
-    foreach (PathSet::iterator, i, paths)
-        if (isValidPath(*i)) queryReferrers(*i, referrers);
-
-    foreach (PathSet::iterator, i, referrers)
-        if (paths.find(*i) == paths.end() && !tryToDelete(state, *i)) {
-            printMsg(lvlDebug, format("cannot delete `%1%' because it has live referrers") % *i);
-            goto isLive;
-        }
-
-    /* The paths are garbage, so delete them. */
-    pathsSorted = topoSortPaths(*this, paths);
-    foreach (Paths::iterator, i, pathsSorted) {
-        if (shouldDelete(state.options.action)) {
-
-            /* If it's a valid path that's not a regular file or
-               symlink, invalidate it, rename it, and schedule it for
-               deletion.  The renaming is to ensure that later (when
-               we're not holding the global GC lock) we can delete the
-               path without being afraid that the path has become
-               alive again.  Otherwise delete it right away. */
-            if (isValidPath(*i)) {
-                if (S_ISDIR(st.st_mode)) {
-                    printMsg(lvlInfo, format("invalidating `%1%'") % *i);
-                    // Estimate the amount freed using the narSize field.
-                    state.bytesInvalidated += queryPathInfo(*i).narSize;
-                    invalidatePathChecked(*i);
-                    makeMutable(i->c_str());
-                    // Mac OS X cannot rename directories if they are read-only.
-                    if (chmod(i->c_str(), st.st_mode | S_IWUSR) == -1)
-                        throw SysError(format("making `%1%' writable") % *i);
-                    Path tmp = (format("%1%-gc-%2%") % *i % getpid()).str();
-                    if (rename(i->c_str(), tmp.c_str()))
-                        throw SysError(format("unable to rename `%1%' to `%2%'") % *i % tmp);
-                    state.invalidated.insert(tmp);
-                } else {
-                    invalidatePathChecked(*i);
-                    deleteGarbage(state, *i);
-                }
-            } else
-                deleteGarbage(state, *i);
-
-            if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
-                printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
-                throw GCLimitReached();
-            }
-
-        } else
-            printMsg(lvlTalkative, format("would delete `%1%'") % *i);
-
-        state.deleted.insert(*i);
-        if (state.options.action != GCOptions::gcReturnLive)
-            state.results.paths.insert(*i);
+    if (canReachRoot(state, visited, path)) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path);
+    } else {
+        /* No path we visited was a root, so everything is garbage.
+           But we only delete ‘path’ and its referrers here so that
+           ‘nix-store --delete’ doesn't have the unexpected effect of
+           recursing into derivations and outputs. */
+        state.dead.insert(visited.begin(), visited.end());
+        if (state.shouldDelete)
+            deletePathRecursive(state, path);
     }
-
-    return true;
-
- isLive:
-    foreach (PathSet::iterator, i, paths) {
-        state.live.insert(*i);
-        if (state.options.action == GCOptions::gcReturnLive)
-            state.results.paths.insert(*i);
-    }
-    return false;
 }
 
 
@@ -625,7 +614,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     GCState state(results);
     state.options = options;
-
+    state.trashDir = settings.nixStore + "/trash";
     state.gcKeepOutputs = settings.gcKeepOutputs;
     state.gcKeepDerivations = settings.gcKeepDerivations;
 
@@ -638,6 +627,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         state.gcKeepDerivations = false;
     }
 
+    state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
@@ -668,6 +659,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        increase, since we hold locks on everything.  So everything
        that is not reachable from `roots'. */
 
+    if (state.shouldDelete) {
+        if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
+        createDirs(state.trashDir);
+    }
+
     /* Now either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
 
@@ -675,13 +671,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         foreach (PathSet::iterator, i, options.pathsToDelete) {
             assertStorePath(*i);
-            if (!tryToDelete(state, *i))
+            tryToDelete(state, *i);
+            if (state.dead.find(*i) == state.dead.end())
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
         }
 
     } else if (options.maxFreed > 0) {
 
-        if (shouldDelete(state.options.action))
+        if (state.shouldDelete)
             printMsg(lvlError, format("deleting garbage..."));
         else
             printMsg(lvlError, format("determining live/dead paths..."));
@@ -727,13 +724,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     }
 
+    if (state.options.action == GCOptions::gcReturnLive) {
+        state.results.paths = state.alive;
+        return;
+    }
+
+    if (state.options.action == GCOptions::gcReturnDead) {
+        state.results.paths = state.dead;
+        return;
+    }
+
     /* Allow other processes to add to the store from here on. */
     fdGCLock.close();
 
-    /* Delete the invalidated paths now that the lock has been
-       released. */
-    foreach (PathSet::iterator, i, state.invalidated)
-        deleteGarbage(state, *i);
+    /* Delete the trash directory. */
+    printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir);
+    deleteGarbage(state, state.trashDir);
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {

src/libstore/globals.cc

@@ -158,7 +158,7 @@ void Settings::get(bool & res, const string & name)
 }
 
 
-void Settings::get(PathSet & res, const string & name)
+void Settings::get(StringSet & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;

src/libstore/globals.hh

@@ -139,7 +139,7 @@ struct Settings {
 
     /* The directories from the host filesystem to be included in the
        chroot. */
-    PathSet dirsInChroot;
+    StringSet dirsInChroot;
 
     /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
     bool impersonateLinux26;
@@ -181,7 +181,7 @@ private:
 
     void get(string & res, const string & name);
     void get(bool & res, const string & name);
-    void get(PathSet & res, const string & name);
+    void get(StringSet & res, const string & name);
     template<class N> void get(N & res, const string & name);
 };
 

src/libstore/local-store.cc

@@ -5,7 +5,6 @@
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
 #include "derivations.hh"
-#include "immutable.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -25,6 +24,12 @@
 #include <sys/mount.h>
 #endif
 
+#if HAVE_LINUX_FS_H
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+#endif
+
 #include <sqlite3.h>
 
 
@@ -48,7 +53,7 @@ static void throwSQLiteError(sqlite3 * db, const format & f)
 #if HAVE_NANOSLEEP
         struct timespec t;
         t.tv_sec = 0;
-        t.tv_nsec = 100 * 1000 * 1000; /* 0.1s */
+        t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */
         nanosleep(&t, 0);
 #else
         sleep(1);
@@ -292,6 +297,7 @@ LocalStore::LocalStore(bool reserveSpace)
         curSchema = getSchema();
 
         if (curSchema < 6) upgradeStore6();
+        else if (curSchema < 7) { upgradeStore7(); openDB(true); }
 
         writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
 
@@ -1787,6 +1793,59 @@ void LocalStore::upgradeStore6()
 }
 
 
+#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
+
+static void makeMutable(const Path & path)
+{
+    checkInterrupt();
+
+    struct stat st = lstat(path);
+
+    if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return;
+
+    if (S_ISDIR(st.st_mode)) {
+        Strings names = readDirectory(path);
+        foreach (Strings::iterator, i, names)
+            makeMutable(path + "/" + *i);
+    }
+
+    /* The O_NOFOLLOW is important to prevent us from changing the
+       mutable bit on the target of a symlink (which would be a
+       security hole). */
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW);
+    if (fd == -1) {
+        if (errno == ELOOP) return; // it's a symlink
+        throw SysError(format("opening file `%1%'") % path);
+    }
+
+    unsigned int flags = 0, old;
+
+    /* Silently ignore errors getting/setting the immutable flag so
+       that we work correctly on filesystems that don't support it. */
+    if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return;
+    old = flags;
+    flags &= ~FS_IMMUTABLE_FL;
+    if (old == flags) return;
+    if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return;
+}
+
+/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */
+void LocalStore::upgradeStore7()
+{
+    if (getuid() != 0) return;
+    printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)...");
+    makeMutable(settings.nixStore);
+}
+
+#else
+
+void LocalStore::upgradeStore7()
+{
+}
+
+#endif
+
+
 void LocalStore::vacuumDB()
 {
     if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)

src/libstore/local-store.hh

@@ -17,8 +17,8 @@ namespace nix {
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
    Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
-   Nix 1.0. */
-const int nixSchemaVersion = 6;
+   Nix 1.0.  Version 7 is Nix 1.3. */
+const int nixSchemaVersion = 7;
 
 
 extern string drvsLogDir;
@@ -111,10 +111,6 @@ public:
 
     Path queryDeriver(const Path & path);
 
-    /* Return all currently valid derivations that have `path' as an
-       output.  (Note that the result of `queryDeriver()' is the
-       derivation that was actually used to produce `path', which may
-       not exist anymore.) */
     PathSet queryValidDerivers(const Path & path);
 
     PathSet queryDerivationOutputs(const Path & path);
@@ -269,6 +265,7 @@ private:
     void updatePathInfo(const ValidPathInfo & info);
 
     void upgradeStore6();
+    void upgradeStore7();
     PathSet queryValidPathsOld();
     ValidPathInfo queryPathInfoOld(const Path & path);
 
@@ -276,7 +273,11 @@ private:
 
     void deleteGarbage(GCState & state, const Path & path);
 
-    bool tryToDelete(GCState & state, const Path & path);
+    void tryToDelete(GCState & state, const Path & path);
+
+    bool canReachRoot(GCState & state, PathSet & visited, const Path & path);
+
+    void deletePathRecursive(GCState & state, const Path & path);
 
     bool isActiveTempFile(const GCState & state,
         const Path & path, const string & suffix);

src/libstore/misc.cc

@@ -15,27 +15,47 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath)
 }
 
 
-void computeFSClosure(StoreAPI & store, const Path & storePath,
-    PathSet & paths, bool flipDirection, bool includeOutputs)
+void computeFSClosure(StoreAPI & store, const Path & path,
+    PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
-    if (paths.find(storePath) != paths.end()) return;
-    paths.insert(storePath);
+    if (paths.find(path) != paths.end()) return;
+    paths.insert(path);
 
-    PathSet references;
-    if (flipDirection)
-        store.queryReferrers(storePath, references);
-    else
-        store.queryReferences(storePath, references);
+    PathSet edges;
 
-    if (includeOutputs && isDerivation(storePath)) {
-        PathSet outputs = store.queryDerivationOutputs(storePath);
-        foreach (PathSet::iterator, i, outputs)
-            if (store.isValidPath(*i))
-                computeFSClosure(store, *i, paths, flipDirection, true);
+    if (flipDirection) {
+        store.queryReferrers(path, edges);
+
+        if (includeOutputs) {
+            PathSet derivers = store.queryValidDerivers(path);
+            foreach (PathSet::iterator, i, derivers)
+                edges.insert(*i);
+        }
+
+        if (includeDerivers && isDerivation(path)) {
+            PathSet outputs = store.queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (store.isValidPath(*i) && store.queryDeriver(*i) == path)
+                    edges.insert(*i);
+        }
+
+    } else {
+        store.queryReferences(path, edges);
+
+        if (includeOutputs && isDerivation(path)) {
+            PathSet outputs = store.queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (store.isValidPath(*i)) edges.insert(*i);
+        }
+
+        if (includeDerivers) {
+            Path deriver = store.queryDeriver(path);
+            if (store.isValidPath(deriver)) edges.insert(deriver);
+        }
     }
 
-    foreach (PathSet::iterator, i, references)
-        computeFSClosure(store, *i, paths, flipDirection, includeOutputs);
+    foreach (PathSet::iterator, i, edges)
+        computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers);
 }
 
 

src/libstore/misc.hh

@@ -17,9 +17,9 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath);
    `storePath' is returned; that is, the closures under the
    `referrers' relation instead of the `references' relation is
    returned. */
-void computeFSClosure(StoreAPI & store, const Path & storePath,
+void computeFSClosure(StoreAPI & store, const Path & path,
     PathSet & paths, bool flipDirection = false,
-    bool includeOutputs = false);
+    bool includeOutputs = false, bool includeDerivers = false);
 
 /* Return the path corresponding to the output identifier `id' in the
    given derivation. */

src/libstore/optimise-store.cc

@@ -2,7 +2,6 @@
 
 #include "util.hh"
 #include "local-store.hh"
-#include "immutable.hh"
 #include "globals.hh"
 
 #include <sys/types.h>
@@ -20,7 +19,6 @@ static void makeWritable(const Path & path)
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path `%1%'") % path);
-    if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
         throw SysError(format("changing writability of `%1%'") % path);
 }
@@ -91,7 +89,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 
     if (!pathExists(linkPath)) {
         /* Nope, create a hard link in the links directory. */
-        makeMutable(path);
         if (link(path.c_str(), linkPath.c_str()) == 0) return;
         if (errno != EEXIST)
             throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
@@ -123,12 +120,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
        its timestamp back to 0. */
     MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
 
-    /* If ‘linkPath’ is immutable, we can't create hard links to it,
-       so make it mutable first.  We also have to make ‘path’ mutable,
-       otherwise rename() will fail to delete it. */
-    makeMutable(path);
-    makeMutable(linkPath);
-
     Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
         % settings.nixStore % getpid() % rand()).str();
 

src/libstore/remote-store.cc

@@ -334,6 +334,16 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
+PathSet RemoteStore::queryValidDerivers(const Path & path)
+{
+    openConnection();
+    writeInt(wopQueryValidDerivers, to);
+    writeString(path, to);
+    processStderr();
+    return readStorePaths<PathSet>(from);
+}
+
+
 PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 {
     openConnection();
@@ -431,7 +441,16 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, bool repair)
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
     openConnection();
     writeInt(wopBuildPaths, to);
-    writeStrings(drvPaths, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 13)
+        writeStrings(drvPaths, to);
+    else {
+        /* For backwards compatibility with old daemons, strip output
+           identifiers. */
+        PathSet drvPaths2;
+        foreach (PathSet::const_iterator, i, drvPaths)
+            drvPaths2.insert(string(*i, 0, i->find('!')));
+        writeStrings(drvPaths2, to);
+    }
     processStderr();
     readInt(from);
 }

src/libstore/remote-store.hh

@@ -40,6 +40,8 @@ public:
 
     Path queryDeriver(const Path & path);
     
+    PathSet queryValidDerivers(const Path & path);
+
     PathSet queryDerivationOutputs(const Path & path);
     
     StringSet queryDerivationOutputNames(const Path & path);

src/libstore/store-api.hh

@@ -133,6 +133,12 @@ public:
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
+    /* Return all currently valid derivations that have `path' as an
+       output.  (Note that the result of `queryDeriver()' is the
+       derivation that was actually used to produce `path', which may
+       not exist anymore.) */
+    virtual PathSet queryValidDerivers(const Path & path) = 0;
+
     /* Query the outputs of the derivation denoted by `path'. */
     virtual PathSet queryDerivationOutputs(const Path & path) = 0;
 

src/libstore/worker-protocol.hh

@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x10c
+#define PROTOCOL_VERSION 0x10d
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -42,6 +42,7 @@ typedef enum {
     wopQuerySubstitutablePathInfos = 30,
     wopQueryValidPaths = 31,
     wopQuerySubstitutablePaths = 32,
+    wopQueryValidDerivers = 33,
 } WorkerOp;
 
 

src/libutil/Makefile.am

@@ -1,12 +1,12 @@
 pkglib_LTLIBRARIES = libutil.la
 
 libutil_la_SOURCES = util.cc hash.cc serialise.cc \
-  archive.cc xml-writer.cc immutable.cc
+  archive.cc xml-writer.cc
 
 libutil_la_LIBADD = ../boost/format/libformat.la
 
 pkginclude_HEADERS = util.hh hash.hh serialise.hh \
-  archive.hh xml-writer.hh types.hh immutable.hh
+  archive.hh xml-writer.hh types.hh
 
 if !HAVE_OPENSSL
 libutil_la_SOURCES += \

src/libutil/immutable.cc

@@ -1,49 +0,0 @@
-#include "config.h"
-
-#include "immutable.hh"
-#include "util.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-#if HAVE_LINUX_FS_H
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <errno.h>
-#endif
-
-namespace nix {
-
-
-void makeMutable(const Path & path)
-{
-#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
-
-    /* Don't even try if we're not root.  One day we should support
-       the CAP_LINUX_IMMUTABLE capability. */
-    if (getuid() != 0) return;
-
-    /* The O_NOFOLLOW is important to prevent us from changing the
-       mutable bit on the target of a symlink (which would be a
-       security hole). */
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW);
-    if (fd == -1) {
-        if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file `%1%'") % path);
-    }
-
-    unsigned int flags = 0, old;
-
-    /* Silently ignore errors getting/setting the immutable flag so
-       that we work correctly on filesystems that don't support it. */
-    if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return;
-    old = flags;
-    flags &= ~FS_IMMUTABLE_FL;
-    if (old == flags) return;
-    if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return;
-#endif
-}
-
-
-}

src/libutil/immutable.hh

@@ -1,10 +0,0 @@
-#pragma once
-
-#include <types.hh>
-
-namespace nix {
-
-/* Make the given path mutable. */
-void makeMutable(const Path & path);
-
-}

src/libutil/util.cc

@@ -13,7 +13,6 @@
 #include <limits.h>
 
 #include "util.hh"
-#include "immutable.hh"
 
 
 extern char * * environ;
@@ -100,7 +99,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
             i++;
 
         /* If `..', delete the last component. */
-        else if (*i == '.' && i + 1 < end && i[1] == '.' && 
+        else if (*i == '.' && i + 1 < end && i[1] == '.' &&
             (i + 2 == end || i[2] == '/'))
         {
             if (!s.empty()) s.erase(s.rfind('/'));
@@ -215,7 +214,7 @@ string readFile(int fd)
     struct stat st;
     if (fstat(fd, &st) == -1)
         throw SysError("statting file");
-    
+
     unsigned char * buf = new unsigned char[st.st_size];
     AutoDeleteArray<unsigned char> d(buf);
     readFull(fd, buf, st.st_size);
@@ -280,9 +279,9 @@ static void _computePathSize(const Path & path,
     blocks += st.st_blocks;
 
     if (S_ISDIR(st.st_mode)) {
-	Strings names = readDirectory(path);
+        Strings names = readDirectory(path);
 
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
             _computePathSize(path + "/" + *i, bytes, blocks);
     }
 }
@@ -305,21 +304,19 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
     struct stat st = lstat(path);
 
-    if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
-
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
         bytesFreed += st.st_blocks * 512;
 
     if (S_ISDIR(st.st_mode)) {
-	Strings names = readDirectory(path);
+        Strings names = readDirectory(path);
 
-	/* Make the directory writable. */
-	if (!(st.st_mode & S_IWUSR)) {
-	    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-		throw SysError(format("making `%1%' writable") % path);
-	}
+        /* Make the directory writable. */
+        if (!(st.st_mode & S_IWUSR)) {
+            if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+                throw SysError(format("making `%1%' writable") % path);
+        }
 
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
             _deletePath(path + "/" + *i, bytesFreed);
     }
 
@@ -351,14 +348,14 @@ void makePathReadOnly(const Path & path)
     struct stat st = lstat(path);
 
     if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) {
-	if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
-	    throw SysError(format("making `%1%' read-only") % path);
+        if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
+            throw SysError(format("making `%1%' read-only") % path);
     }
 
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-	    makePathReadOnly(path + "/" + *i);
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+            makePathReadOnly(path + "/" + *i);
     }
 }
 
@@ -380,25 +377,25 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
     static int globalCounter = 0;
     int localCounter = 0;
     int & counter(useGlobalCounter ? globalCounter : localCounter);
-    
+
     while (1) {
         checkInterrupt();
-	Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-	if (mkdir(tmpDir.c_str(), mode) == 0) {
-	    /* Explicitly set the group of the directory.  This is to
-	       work around around problems caused by BSD's group
-	       ownership semantics (directories inherit the group of
-	       the parent).  For instance, the group of /tmp on
-	       FreeBSD is "wheel", so all directories created in /tmp
-	       will be owned by "wheel"; but if the user is not in
-	       "wheel", then "tar" will fail to unpack archives that
-	       have the setgid bit set on directories. */
-	    if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-		throw SysError(format("setting group of directory `%1%'") % tmpDir);
-	    return tmpDir;
-	}
-	if (errno != EEXIST)
-	    throw SysError(format("creating directory `%1%'") % tmpDir);
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError(format("setting group of directory `%1%'") % tmpDir);
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError(format("creating directory `%1%'") % tmpDir);
     }
 }
 
@@ -418,7 +415,7 @@ Paths createDirs(const Path & path)
     }
 
     if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path);
-    
+
     return created;
 }
 
@@ -727,8 +724,8 @@ AutoCloseDir::operator DIR *()
 void AutoCloseDir::close()
 {
     if (dir) {
-	closedir(dir);
-	dir = 0;
+        closedir(dir);
+        dir = 0;
     }
 }
 
@@ -843,12 +840,12 @@ void killUser(uid_t uid)
             if (setuid(uid) == -1)
                 throw SysError("setting uid");
 
-	    while (true) {
-		if (kill(-1, SIGKILL) == 0) break;
-		if (errno == ESRCH) break; /* no more processes */
-		if (errno != EINTR)
-		    throw SysError(format("cannot kill processes for uid `%1%'") % uid);
-	    }
+            while (true) {
+                if (kill(-1, SIGKILL) == 0) break;
+                if (errno == ESRCH) break; /* no more processes */
+                if (errno != EINTR)
+                    throw SysError(format("cannot kill processes for uid `%1%'") % uid);
+            }
 
         } catch (std::exception & e) {
             writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str());
@@ -856,7 +853,7 @@ void killUser(uid_t uid)
         }
         _exit(0);
     }
-    
+
     /* parent */
     int status = pid.wait(true);
     if (status != 0)
@@ -1043,14 +1040,14 @@ string statusToString(int status)
         if (WIFEXITED(status))
             return (format("failed with exit code %1%") % WEXITSTATUS(status)).str();
         else if (WIFSIGNALED(status)) {
-	    int sig = WTERMSIG(status);
+            int sig = WTERMSIG(status);
 #if HAVE_STRSIGNAL
             const char * description = strsignal(sig);
             return (format("failed due to signal %1% (%2%)") % sig % description).str();
 #else
             return (format("failed due to signal %1%") % sig).str();
 #endif
-	}
+        }
         else
             return "died abnormally";
     } else return "succeeded";

src/libutil/util.hh

@@ -174,7 +174,7 @@ struct AutoDeleteArray
 {
     T * p;
     AutoDeleteArray(T * p) : p(p) { }
-    ~AutoDeleteArray() 
+    ~AutoDeleteArray()
     {
         delete [] p;
     }
@@ -185,7 +185,7 @@ class AutoDelete
 {
     Path path;
     bool del;
-    bool recursive;    
+    bool recursive;
 public:
     AutoDelete(const Path & p, bool recursive = true);
     ~AutoDelete();

src/nix-daemon/nix-daemon.cc

@@ -334,6 +334,7 @@ static void performOp(unsigned int clientVersion,
 
     case wopQueryReferences:
     case wopQueryReferrers:
+    case wopQueryValidDerivers:
     case wopQueryDerivationOutputs: {
         Path path = readStorePath(from);
         startWork();
@@ -342,6 +343,8 @@ static void performOp(unsigned int clientVersion,
             store->queryReferences(path, paths);
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
+        else if (op == wopQueryValidDerivers)
+            paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         stopWork();
         writeStrings(paths, to);

src/nix-env/nix-env.cc

@@ -1350,6 +1350,9 @@ void run(Strings args)
 
     if (!op) throw UsageError("no operation specified");
 
+    if (globals.profile == "")
+        globals.profile = getEnv("NIX_PROFILE", "");
+
     if (globals.profile == "") {
         Path profileLink = getHomeDir() + "/.nix-profile";
         globals.profile = pathExists(profileLink)

src/nix-store/nix-store.cc

@@ -404,7 +404,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
             foreach (Strings::iterator, i, opArgs) {
                 PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
                 foreach (PathSet::iterator, j, paths)
-                    computeFSClosure(*store, *j, referrers, true);
+                    computeFSClosure(*store, *j, referrers, true,
+                        settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
             Roots roots = store->findRoots();
             foreach (Roots::iterator, i, roots)

substitute.mk

@@ -7,6 +7,7 @@
 	 -e "s^@bindir\@^$(bindir)^g" \
 	 -e "s^@datadir\@^$(datadir)^g" \
 	 -e "s^@sysconfdir\@^$(sysconfdir)^g" \
+	 -e "s^@profiledir\@^$(profiledir)^g" \
 	 -e "s^@localstatedir\@^$(localstatedir)^g" \
 	 -e "s^@datadir\@^$(datadir)^g" \
 	 -e "s^@libdir\@^$(libdir)^g" \

tests/Makefile.am

@@ -14,6 +14,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
 
 XFAIL_TESTS =
 
+profiledir = $(sysconfdir)/profile.d
+
 include ../substitute.mk
 
 $(TESTS): common.sh config.nix

tests/binary-cache.sh

@@ -37,3 +37,12 @@ nix-store --option binary-caches "file://$cacheDir" -r $outPath
 nix-store --check-validity $outPath
 nix-store -qR $outPath | grep input-2
 
+
+# Test whether building works if the binary cache contains an
+# incomplete closure.
+clearStore
+
+rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
+
+nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+grep -q "Downloading" $TEST_ROOT/log

tests/common.sh.in

@@ -1,7 +1,7 @@
 set -e
 
 datadir="@datadir@"
-sysconfdir="@sysconfdir@"
+profiledir="@profiledir@"
 
 export TEST_ROOT=$(pwd)/test-tmp
 export NIX_STORE_DIR

tests/nix-profile.sh

@@ -3,8 +3,8 @@ source common.sh
 home=$TEST_ROOT/home
 rm -rf $home
 mkdir -p $home
-HOME=$home $SHELL -e -c ". $sysconfdir/profile.d/nix.sh"
-HOME=$home $SHELL -e -c ". $sysconfdir/profile.d/nix.sh" # test idempotency
+HOME=$home $SHELL -e -c ". $profiledir/nix.sh"
+HOME=$home $SHELL -e -c ". $profiledir/nix.sh" # test idempotency
 
 [ -L $home/.nix-profile ]
 [ -e $home/.nix-channels ]

version

@@ -1 +1 @@
-1.2
+1.3