Bump release date

Otherwise it will set the parent's stdin to non-blocking mode, causing
the subsequent read of the set of inputs/outputs to fail randomly.
That's insane.

configure.ac

@@ -106,6 +106,7 @@ AC_LANG_POP(C++)
 # Check for chroot support (requires chroot() and bind mounts).
 AC_CHECK_FUNCS([chroot])
 AC_CHECK_FUNCS([unshare])
+AC_CHECK_FUNCS([statvfs])
 AC_CHECK_HEADERS([sched.h])
 AC_CHECK_HEADERS([sys/param.h])
 AC_CHECK_HEADERS([sys/mount.h], [], [],

corepkgs/fetchurl.nix

@@ -1,6 +1,6 @@
 with import <nix/config.nix>;
 
-{system ? builtins.currentSystem, url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? ""}:
+{system ? builtins.currentSystem, url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? "", executable ? false}:
 
 assert (outputHash != "" && outputHashAlgo != "")
     || md5 != "" || sha1 != "" || sha256 != "";
@@ -8,10 +8,10 @@ assert (outputHash != "" && outputHashAlgo != "")
 let
 
   builder = builtins.toFile "fetchurl.sh"
-    ''
+    (''
       echo "downloading $url into $out"
       ${curl} --fail --location --max-redirs 20 --insecure "$url" > "$out"
-    '';
+    '' + (if executable then "${coreutils}/chmod +x $out" else ""));
 
 in
     
@@ -25,6 +25,7 @@ derivation {
       if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5";
   outputHash = if outputHash != "" then outputHash else
       if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
+  outputHashMode = if executable then "recursive" else "flat";
   
   inherit system url;
 

doc/dev/release-procedures.txt

@@ -1,33 +0,0 @@
-To produce a `stable' release from the trunk:
-
--1. Update the release notes; make sure that the release date is
-    correct.
-
-0. Make sure that the trunk builds in the release supervisor.
-
-1. Branch the trunk, e.g., `svn cp .../trunk
-   .../branches/0.5-release'.
-
-2. Switch to the branch, e.g., `svn switch .../branches/0.5-release'.
-
-3. In `configure.ac', change `STABLE=0' into `STABLE=1' and commit.
-
-4. In the release supervisor, add a one-time job to build
-   `.../branches/0.5-release'.
-
-5. Make sure that the release succeeds.
-
-6. Move the branch to a tag, e.g., `svn mv .../branches/0.5-release
-   .../tags/0.5'.
-
-   Note that the branch should not be used for maintenance; it should
-   be deleted after the release has been created.  A maintenance
-   branch (e.g., `.../branches/0.5') should be created from the
-   original revision of the trunk (since maintenance releases should
-   also be tested first; hence, we cannot have `STABLE=1').  The same
-   procedure can then be followed to produce maintenance releases;
-   just substitute `.../branches/VERSION' for the trunk.
-
-7. Switch back to the trunk.
-
-8. Bump the version number in `configure.ac' (in AC_INIT).

doc/manual/builtins.xml

@@ -302,6 +302,18 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.hashString</function>
+  <replaceable>type</replaceable> <replaceable>s</replaceable></term>
+
+    <listitem><para>Return a base-16 representation of the
+    cryptographic hash of string <replaceable>s</replaceable>.  The
+    hash algorithm specified by <replaceable>type</replaceable> must
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal> or
+    <literal>"sha256"</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><function>builtins.head</function>
   <replaceable>list</replaceable></term>
 

doc/manual/conf-file.xml

@@ -350,13 +350,25 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     whitespace.  These are not used by default, but can be enabled by
     users of the Nix daemon by specifying <literal>--option
     binary-caches <replaceable>urls</replaceable></literal> on the
-    command line.  Daemon users are only allowed to pass a subset of
-    the URLs listed in <literal>binary-caches</literal> and
+    command line.  Unprivileged users are only allowed to pass a
+    subset of the URLs listed in <literal>binary-caches</literal> and
     <literal>trusted-binary-caches</literal>.</para></listitem>
 
   </varlistentry>
 
 
+  <varlistentry><term><literal>extra-binary-caches</literal></term>
+
+    <listitem><para>Additional binary caches appended to those
+    specified in <option>binary-caches</option> and
+    <option>binary-caches-files</option>.  When used by unprivileged
+    users, untrusted binary caches (i.e. those not listed in
+    <option>trusted-binary-caches</option>) are silently
+    ignored.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
 
     <listitem><para>The maximum number of parallel HTTP connections

doc/manual/installation.xml

@@ -166,11 +166,10 @@ a source distribution.</para>
 
   <listitem><para>Recent versions of Bison and Flex to build the
   parser.  (This is because Nix needs GLR support in Bison and
-  reentrancy support in Flex.)  For Bison, you need version 2.3 or
-  higher (1.875 does <emphasis>not</emphasis> work), which can be
-  obtained from the <link
+  reentrancy support in Flex.)  For Bison, you need version 2.6, which
+  can be obtained from the <link
   xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP
-  server</link>.  For Flex, you need version 2.5.33, which is
+  server</link>.  For Flex, you need version 2.5.35, which is
   available on <link
   xlink:href="http://lex.sourceforge.net/">SourceForge</link>.
   Slightly older versions may also work, but ancient versions like the

doc/manual/opt-common.xml

@@ -343,10 +343,11 @@
 
 <varlistentry><term><option>-I</option> <replaceable>path</replaceable></term>
   
-  <listitem><para>Add a path to the Nix expression search path.  See
-  the <envar>NIX_PATH</envar> environment variable for details.  Paths
-  added through <option>-I</option> take precedence over
-  <envar>NIX_PATH</envar>.</para></listitem>
+  <listitem><para>Add a path to the Nix expression search path.  This
+  option may be given multiple times.  See the <envar>NIX_PATH</envar>
+  environment variable for information on the semantics of the Nix
+  search path.  Paths added through <option>-I</option> take
+  precedence over <envar>NIX_PATH</envar>.</para></listitem>
   
 </varlistentry>
 

doc/manual/quick-start.xml

@@ -14,9 +14,10 @@ to the following chapters.</para>
 <listitem><para>Download a source tarball or RPM or Debian/Ubuntu
 package from <link xlink:href='http://nixos.org/'/>.  Build source
 distributions using the regular sequence:
-        
+
 <screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
+$ cd nix-<replaceable>version</replaceable>
 $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
@@ -69,7 +70,7 @@ libxslt-1.1.0
 </para></listitem>
 
 <listitem><para>Install some packages from the channel:
-        
+
 <screen>
 $ nix-env -i hello firefox <replaceable>...</replaceable> </screen>
 
@@ -87,7 +88,7 @@ $ firefox
 <lineannotation>(read Slashdot or something)</lineannotation></screen>
 
 </para></listitem>
-    
+
 <listitem><para>Uninstall a package:
 
 <screen>

doc/manual/release-notes.xml

@@ -6,6 +6,73 @@
 
 
 
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5.2"><title>Release 1.5.2 (May 13, 2013)</title>
+
+<para>This is primarily a bug fix release.  It has contributions from
+Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5.1"><title>Release 1.5.1 (February 28, 2013)</title>
+
+<para>The bug fix to the bug fix had a bug itself, of course.  But
+this time it will work for sure!</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5"><title>Release 1.5 (February 27, 2013)</title>
+
+<para>This is a brown paper bag release to fix a regression introduced
+by the hard link security fix in 1.4.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.4"><title>Release 1.4 (February 26, 2013)</title>
+
+<para>This release fixes a security bug in multi-user operation.  It
+was possible for derivations to cause the mode of files outside of the
+Nix store to be changed to 444 (read-only but world-readable) by
+creating hard links to those files (<link
+xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a79141ac4">details</link>).</para>
+
+<para>There are also the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem><para>New built-in function:
+  <function>builtins.hashString</function>.</para></listitem>
+
+  <listitem><para>Build logs are now stored in
+  <filename>/nix/var/log/nix/drvs/<replaceable>XX</replaceable>/</filename>,
+  where <replaceable>XX</replaceable> is the first two characters of
+  the derivation.  This is useful on machines that keep a lot of build
+  logs (such as Hydra servers).</para></listitem>
+
+  <listitem><para>The function <function>corepkgs/fetchurl</function>
+  can now make the downloaded file executable.  This will allow
+  getting rid of all bootstrap binaries in the Nixpkgs source
+  tree.</para></listitem>
+
+  <listitem><para>Language change: The expression <literal>"${./path}
+  ..."</literal> now evaluates to a string instead of a
+  path.</para></listitem>
+
+</itemizedlist>
+
+</section>
+
+
 <!--==================================================================-->
 
 <section xml:id="ssec-relnotes-1.3"><title>Release 1.3 (January 4, 2013)</title>

perl/lib/Nix/CopyClosure.pm

@@ -3,16 +3,13 @@ package Nix::CopyClosure;
 use strict;
 use Nix::Config;
 use Nix::Store;
+use List::Util qw(sum);
 
 
 sub copyTo {
     my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
         $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
 
-    $compressor = "$compressor |" if $compressor ne "";
-    $decompressor = "$decompressor |" if $decompressor ne "";
-    $progressViewer = "$progressViewer |" if $progressViewer ne "";
-
     # Get the closure of this path.
     my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
         map { followLinksToStorePath $_ } @{$storePaths})));
@@ -28,21 +25,28 @@ sub copyTo {
     # we'll want to use ‘--from-stdin’, but we can't rely on the
     # target having this option yet.
     my @missing = ();
+    my $missingSize = 0;
     while (scalar(@closure) > 0) {
         my @ps = splice(@closure, 0, 1500);
         open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|");
         while (<READ>) {
             chomp;
             push @missing, $_;
+            my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($_, 1);
+            $missingSize += $narSize;
         }
         close READ or die;
     }
 
+    $compressor = "$compressor |" if $compressor ne "";
+    $decompressor = "$decompressor |" if $decompressor ne "";
+    $progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
+
     # Export the store paths and import them on the remote machine.
     if (scalar @missing > 0) {
         print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
         unless ($dryRun) {
-            open SSH, "| $compressor $progressViewer ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die;
+            open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die;
             exportPaths(fileno(SSH), $sign, @missing);
             close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
         }

perl/lib/Nix/SSH.pm

@@ -44,6 +44,7 @@ sub closeSSHConnection {
     if ($sshStarted) {
         system("ssh $sshHost @sshOpts -O exit 2> /dev/null") == 0
             or warn "unable to stop SSH master: $?";
+        $sshStarted = 0;
     }
 }
 

perl/lib/Nix/Store.pm

@@ -42,50 +42,50 @@ if ($Nix::Config::useBindings) {
     use File::Temp;
     use Fcntl qw/F_SETFD/;
 
-    sub hashFile {
+    *hashFile = sub {
         my ($algo, $base32, $path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
-    
-    sub hashPath {
+    };
+
+    *hashPath = sub {
         my ($algo, $base32, $path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
-    
-    sub hashString {
+    };
+
+    *hashString = sub {
         my ($algo, $base32, $s) = @_;
         my $fh = File::Temp->new();
         print $fh $s;
         my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
-    
-    sub addToStore {
+    };
+
+    *addToStore = sub {
         my ($srcPath, $recursive, $algo) = @_;
         die "not implemented" if $recursive || $algo ne "sha256";
         my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath);
         chomp $res;
         return $res;
-    }
-    
-    sub isValidPath {
+    };
+
+    *isValidPath = sub {
         my ($path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path);
         chomp $res;
         return $res ne $path;
-    }
-    
-    sub queryPathHash {
+    };
+
+    *queryPathHash = sub {
         my ($path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path);
         chomp $res;
         return $res;
-    }
+    };
 }
 
 1;

perl/lib/Nix/Store.xs

@@ -15,7 +15,7 @@
 using namespace nix;
 
 
-void doInit() 
+void doInit()
 {
     if (!store) {
         try {
@@ -237,32 +237,35 @@ SV * derivationFromPath(char * drvPath)
             doInit();
             Derivation drv = derivationFromPath(*store, drvPath);
             hash = newHV();
-            
-            /* TODO: handle drv.outputs */
-            
+
+            HV * outputs = newHV();
+            for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
+                hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0);
+            hv_stores(hash, "outputs", newRV((SV *) outputs));
+
             AV * inputDrvs = newAV();
             for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
                 av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
-            
+
             AV * inputSrcs = newAV();
             for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
                 av_push(inputSrcs, newSVpv(i->c_str(), 0));
             hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs));
-            
+
             hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0));
             hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0));
-            
+
             AV * args = newAV();
             for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i)
                 av_push(args, newSVpv(i->c_str(), 0));
             hv_stores(hash, "args", newRV((SV *) args));
-            
+
             HV * env = newHV();
             for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i)
                 hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0);
             hv_stores(hash, "env", newRV((SV *) env));
-            
+
             RETVAL = newRV_noinc((SV *)hash);
         } catch (Error & e) {
             croak(e.what());

release.nix

@@ -1,15 +1,19 @@
-{ nixpkgs ? <nixpkgs>
-, nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
+{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
 , officialRelease ? false
 }:
 
 let
 
+  pkgs = import <nixpkgs> {};
+
+  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "x86_64-freebsd" "i686-freebsd" "i686-cygwin" "i686-solaris" ];
+
+
   jobs = rec {
 
 
     tarball =
-      with import nixpkgs {};
+      with pkgs;
 
       releaseTools.sourceTarball {
         name = "nix-tarball";
@@ -19,7 +23,7 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison25 flex2535 perl libxml2 libxslt w3m bzip2
+          [ curl bison flex2535 perl libxml2 libxslt w3m bzip2
             tetex dblatex nukeReferences pkgconfig sqlite git
           ];
 
@@ -72,10 +76,9 @@ let
       };
 
 
-    build =
-      { system ? "x86_64-linux" }:
+    build = pkgs.lib.genAttrs systems (system:
 
-      with import nixpkgs { inherit system; };
+      with import <nixpkgs> { inherit system; };
 
       releaseTools.nixBuild {
         name = "nix";
@@ -99,15 +102,15 @@ let
         installFlags = "sysconfdir=$(out)/etc";
 
         doInstallCheck = true;
-      };
+      });
 
-    binaryTarball =
-      { system ? "x86_64-linux" }:
 
-      with import nixpkgs { inherit system; };
+    binaryTarball = pkgs.lib.genAttrs systems (system:
+
+      with import <nixpkgs> { inherit system; };
 
       let
-        toplevel = build { inherit system; };
+        toplevel = builtins.getAttr system jobs.build;
         version = toplevel.src.version;
       in
 
@@ -130,11 +133,11 @@ let
             --transform "s,$TMPDIR/install,/usr/bin/nix-finish-install," \
             --transform "s,$TMPDIR/reginfo,/nix/store/reginfo," \
             $TMPDIR/install $TMPDIR/reginfo $storePaths
-        '';
+        '');
 
 
     coverage =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with import <nixpkgs> { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
@@ -166,14 +169,16 @@ let
       };
 
 
-    rpm_fedora13i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora13i386) 50;
-    rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50;
     rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50;
     rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50;
+    rpm_fedora18i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora18i386) 60;
+    rpm_fedora18x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora18x86_64) 60;
 
 
     deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50;
     deb_debian60x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian60x86_64) 50;
+    deb_debian70i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian70i386) 60;
+    deb_debian70x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian70x86_64) 60;
 
     deb_ubuntu1004i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1004i386) 50;
     deb_ubuntu1004x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1004x86_64) 50;
@@ -185,15 +190,17 @@ let
     deb_ubuntu1204x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1204x86_64) 60;
     deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386) 70;
     deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64) 70;
+    deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386) 80;
+    deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64) 80;
 
 
     # System tests.
     tests.remote_builds = (import ./tests/remote-builds.nix rec {
-      nix = build { inherit system; }; system = "x86_64-linux";
+      nix = build.x86_64-linux; system = "x86_64-linux";
     }).test;
 
     tests.nix_copy_closure = (import ./tests/nix-copy-closure.nix rec {
-      nix = build { inherit system; }; system = "x86_64-linux";
+      nix = build.x86_64-linux; system = "x86_64-linux";
     }).test;
 
   };
@@ -205,7 +212,7 @@ let
   makeRPM =
     system: diskImageFun: prio:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm";
@@ -224,7 +231,7 @@ let
   makeDeb =
     system: diskImageFun: prio:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";

scripts/build-remote.pl.in

@@ -46,7 +46,7 @@ sub all { $_ || return 0 for @_; 1 }
 # Initialisation.
 my $loadIncreased = 0;
 
-my ($localSystem, $maxSilentTime, $printBuildTrace, $buildTimeout) = @ARGV;
+my ($localSystem, $printBuildTrace) = @ARGV;
 
 my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
 my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
@@ -195,7 +195,10 @@ REQ: while (1) {
         # Connect to the selected machine.
         @sshOpts = ("-i", $machine->{sshKeys}, "-x");
         $hostName = $machine->{hostName};
-        last REQ if openSSHConnection $hostName;
+        if (openSSHConnection($hostName)) {
+            last REQ if system("ssh $hostName @sshOpts nix-builds-inhibited < /dev/null > /dev/null 2>&1") != 0;
+            closeSSHConnection;
+        }
 
         warn "unable to open SSH connection to $hostName, trying other available machines...\n";
         $machine->{enabled} = 0;
@@ -258,8 +261,8 @@ close UPLOADLOCK;
 
 # Perform the build.
 my $buildFlags =
-    "--max-silent-time $maxSilentTime --option build-timeout $buildTimeout "
-    . "--fallback --add-root $rootsDir/\$PPID.out --option verbosity 0";
+    " --fallback --add-root $rootsDir/\$PPID.out --quiet"
+    . " --option build-keep-log false";
 
 # We let the remote side kill its process group when the connection is
 # closed unexpectedly.  This is necessary to ensure that no processes
@@ -269,7 +272,7 @@ my $buildFlags =
 # in which case every child receives SIGHUP; however, `-tt' doesn't
 # work on some platforms when connection sharing is used.)
 pipe STDIN, DUMMY; # make sure we have a readable STDIN
-if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
+if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & exec nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
     # Note that if we get exit code 100 from `nix-store -r', it
     # denotes a permanent build failure (as opposed to an SSH problem
     # or a temporary Nix problem).  We propagate this to the caller to

scripts/download-from-binary-cache.pl.in

@@ -27,6 +27,8 @@ my $didExpiration = 0;
 my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
 open(STDERR, ">>/dev/tty") if $debug;
 
+my $cacheFileURLs = ($ENV{"_NIX_CACHE_FILE_URLS"} // "") eq 1; # for testing
+
 my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence, $expireNARExistence);
 
 my $curlm = WWW::Curl::Multi->new;
@@ -206,12 +208,15 @@ sub getAvailableCaches {
         push @urls, strToList($url);
     }
 
+    push @urls, strToList($Nix::Config::config{"extra-binary-caches"} // "");
+
     # Allow Nix daemon users to override the binary caches to a subset
     # of those listed in the config file.  Note that ‘untrusted-*’
     # denotes options passed by the client.
+    my @trustedUrls = uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
+
     if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
         my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
-        my @trustedUrls = uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
         @urls = ();
         foreach my $url (@untrustedUrls) {
             die "binary cache ‘$url’ is not trusted (please add it to ‘trusted-binary-caches’ [@trustedUrls] in $Nix::Config::confDir/nix.conf)\n"
@@ -220,6 +225,12 @@ sub getAvailableCaches {
         }
     }
 
+    my @untrustedUrls = strToList $Nix::Config::config{"untrusted-extra-binary-caches"};
+    foreach my $url (@untrustedUrls) {
+        next unless scalar(grep { $url eq $_ } @trustedUrls) > 0;
+        push @urls, $url;
+    }
+
     foreach my $url (uniq @urls) {
 
         # FIXME: not atomic.
@@ -267,6 +278,12 @@ sub getAvailableCaches {
 }
 
 
+sub shouldCache {
+    my ($url) = @_;
+    return $cacheFileURLs || $url !~ /^file:/;
+}
+
+
 sub processNARInfo {
     my ($storePath, $cache, $request) = @_;
 
@@ -276,7 +293,7 @@ sub processNARInfo {
                 ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
         } else {
             $insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
-                unless $request->{url} =~ /^file:/;
+                if shouldCache $request->{url};
         }
         return undef;
     }
@@ -289,7 +306,7 @@ sub processNARInfo {
         $cache->{id}, basename($storePath), $narInfo->{url}, $narInfo->{compression},
         $narInfo->{fileHash}, $narInfo->{fileSize}, $narInfo->{narHash}, $narInfo->{narSize},
         join(" ", @{$narInfo->{refs}}), $narInfo->{deriver}, $narInfo->{system}, time())
-        unless $request->{url} =~ /^file:/;
+        if shouldCache $request->{url};
 
     return $narInfo;
 }
@@ -455,12 +472,12 @@ sub printSubstitutablePaths {
                         ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
                 } else {
                     $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
-                        unless $request->{url} =~ /^file:/;
+                        if shouldCache $request->{url};
                 }
                 push @left2, $request->{storePath};
             } else {
                 $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
-                    unless $request->{url} =~ /^file:/;
+                    if shouldCache $request->{url};
                 print "$request->{storePath}\n";
             }
         }
@@ -508,6 +525,7 @@ sub downloadBinary {
     }
 
     print STDERR "could not download ‘$storePath’ from any binary cache\n";
+    exit 1;
 }
 
 

scripts/find-runtime-roots.pl.in

@@ -1,22 +1,22 @@
-#! @perl@ -w
+#! @perl@ -w @perlFlags@
 
 use strict;
-
-my $procDir = "/proc";
+use Nix::Utils;
+use Nix::Config;
 
 
 sub readProc {
-    return unless -d $procDir;
+    return unless -d "/proc";
+
+    opendir DIR, "/proc" or return;
 
-    opendir DIR, $procDir or return;
-    
     foreach my $name (readdir DIR) {
         next unless $name =~ /^\d+$/;
 
-        my $process = "$procDir/$name";
+        my $process = "/proc/$name";
 
         #print STDERR "=== $process\n";
-        
+
         my $target;
         print "$target\n" if $target = readlink "$process/exe";
         print "$target\n" if $target = readlink "$process/cwd";
@@ -36,6 +36,13 @@ sub readProc {
             }
             close MAP;
         }
+
+        # Get all store paths that appear in the environment of this process.
+        eval {
+            my $env = Nix::Utils::readFile "$process/environ";
+            my @matches = $env =~ /\Q$Nix::Config::storeDir\E\/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*/g;
+            print "$_\n" foreach @matches;
+        }
     }
 
     closedir DIR;
@@ -58,18 +65,15 @@ readProc;
 lsof;
 
 
-sub readFile {
-    my $path = shift;
-    if (-e $path) {
-        if (open FILE, "$path") {
-            while (<FILE>) {
-                print;
-            }
-            close FILE;
-        }
+sub printFile {
+    my ($fn) = @_;
+    if (-e $fn) {
+        print Nix::Utils::readFile($fn), "\n";
     }
 }
 
+
 # This is rather NixOS-specific, so it probably shouldn't be here.
-readFile "/proc/sys/kernel/modprobe";
-readFile "/proc/sys/kernel/fbsplash";
+printFile "/proc/sys/kernel/modprobe";
+printFile "/proc/sys/kernel/fbsplash";
+printFile "/proc/sys/kernel/poweroff_cmd";

scripts/nix-build.in

@@ -87,7 +87,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") {
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores" or $arg eq "--timeout") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);

scripts/nix-copy-closure.in

@@ -4,6 +4,7 @@ use Nix::SSH;
 use Nix::Config;
 use Nix::Store;
 use Nix::CopyClosure;
+use List::Util qw(sum);
 
 
 if (scalar @ARGV < 1) {
@@ -77,6 +78,8 @@ while (@ARGV) {
     }
 }
 
+die "$0: you did not specify a host name\n" unless defined $sshHost;
+
 
 openSSHConnection $sshHost or die "$0: unable to start SSH\n";
 
@@ -104,6 +107,11 @@ else { # Copy FROM the remote machine.
 
     close READ or die "nix-store on remote machine `$sshHost' failed: $?";
 
+    my $missingSize = 0;
+    if ($progressViewer ne "") {
+        $missingSize = sum (split ' ', `set -f; ssh @sshOpts $sshHost nix-store -q --size @missing`) or die;
+    }
+
     # Export the store paths on the remote machine and import them locally.
     if (scalar @missing > 0) {
         print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
@@ -113,9 +121,9 @@ else { # Copy FROM the remote machine.
             }
             $compressor = "| $compressor" if $compressor ne "";
             $decompressor = "$decompressor |" if $decompressor ne "";
-            $progressViewer = "$progressViewer |" if $progressViewer ne "";
+            $progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
             my $extraOpts = $sign ? "--sign" : "";
-            system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0
+            system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $progressViewer $Nix::Config::binDir/nix-store --import > /dev/null") == 0
                 or die "copying store paths from remote machine `$sshHost' failed: $?";
         }
     }

src/libexpr/eval-inline.hh

@@ -21,13 +21,16 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s2))
 void EvalState::forceValue(Value & v)
 {
     if (v.type == tThunk) {
-        ValueType saved = v.type;
+        Env * env = v.thunk.env;
+        Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
-            v.thunk.expr->eval(*this, *v.thunk.env, v);
+            expr->eval(*this, *env, v);
         } catch (Error & e) {
-            v.type = saved;
+            v.type = tThunk;
+            v.thunk.env = env;
+            v.thunk.expr = expr;
             throw;
         }
     }

src/libexpr/eval.cc

@@ -966,7 +966,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
-            isPath = vStr.type == tPath;
+            isPath = !forceString && vStr.type == tPath;
             first = false;
         }
 

src/libexpr/lexer.l

@@ -9,7 +9,6 @@
 
 %{
 #include "nixexpr.hh"
-#define BISON_HEADER_HACK
 #include "parser-tab.hh"
 
 using namespace nix;

src/libexpr/nixexpr.hh

@@ -277,8 +277,10 @@ MakeBinOp(OpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {
+    bool forceString;
     vector<Expr *> * es;
-    ExprConcatStrings(vector<Expr *> * es) : es(es) { };
+    ExprConcatStrings(bool forceString, vector<Expr *> * es)
+        : forceString(forceString), es(es) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -4,10 +4,10 @@
 %error-verbose
 %defines
 /* %no-lines */
-%parse-param { yyscan_t scanner }
-%parse-param { ParseData * data }
-%lex-param { yyscan_t scanner }
-%lex-param { ParseData * data }
+%parse-param { void * scanner }
+%parse-param { nix::ParseData * data }
+%lex-param { void * scanner }
+%lex-param { nix::ParseData * data }
 %expect 1
 %expect-rr 1
 
@@ -52,7 +52,6 @@ namespace nix {
 
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
-#define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -203,7 +202,7 @@ static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
         es2->push_back(new ExprString(symbols.create(s2)));
     }
 
-    return es2->size() == 1 ? (*es2)[0] : new ExprConcatStrings(es2);
+    return es2->size() == 1 ? (*es2)[0] : new ExprConcatStrings(true, es2);
 }
 
 
@@ -318,7 +317,7 @@ expr_op
     { vector<Expr *> * l = new vector<Expr *>;
       l->push_back($1);
       l->push_back($3);
-      $$ = new ExprConcatStrings(l);
+      $$ = new ExprConcatStrings(false, l);
     }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); }
   | expr_app
@@ -349,7 +348,7 @@ expr_simple
       /* For efficiency, and to simplify parse trees a bit. */
       if ($2->empty()) $$ = new ExprString(data->symbols.create(""));
       else if ($2->size() == 1) $$ = $2->front();
-      else $$ = new ExprConcatStrings($2);
+      else $$ = new ExprConcatStrings(true, $2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(data->symbols, *$2);

src/libexpr/primops.cc

@@ -1107,6 +1107,21 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args
 }
 
 
+/* Return the cryptographic hash of a string in base-16. */
+static void prim_hashString(EvalState & state, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0]);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type `%1%'") % type);
+
+    PathSet context; // discarded
+    string s = state.forceString(*args[1], context);
+
+    mkString(v, printHash(hashString(ht, s)), context);
+};
+
+
 /*************************************************************
  * Versions
  *************************************************************/
@@ -1234,6 +1249,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+    addPrimOp("__hashString", 2, prim_hashString);
 
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);

src/libmain/shared.cc

@@ -168,11 +168,10 @@ static void initAndRun(int argc, char * * argv)
     remaining.clear();
 
     /* Process default options. */
-    int verbosityDelta = lvlInfo;
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
-        if (arg == "--verbose" || arg == "-v") verbosityDelta++;
-        else if (arg == "--quiet") verbosityDelta--;
+        if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1);
+        else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError;
         else if (arg == "--log-type") {
             string s = getArg(arg, i, args.end());
             setLogType(s);
@@ -219,8 +218,6 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
-    verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
-
     settings.update();
 
     run(remaining);

src/libstore/build.cc

@@ -43,6 +43,12 @@
 #include <sched.h>
 #endif
 
+/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but
+   <linux/fs.h> does.  */
+#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H
+#include <linux/fs.h>
+#endif
+
 #define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
 
 #if CHROOT_ENABLED
@@ -166,7 +172,7 @@ public:
     /* Cancel the goal.  It should wake up its waiters, get rid of any
        running child processes that are being monitored by the worker
        (important!), etc. */
-    virtual void cancel() = 0;
+    virtual void cancel(bool timeout) = 0;
 
 protected:
     void amDone(ExitCode result);
@@ -180,9 +186,10 @@ struct Child
 {
     WeakGoalPtr goal;
     set<int> fds;
-    bool monitorForSilence;
+    bool respectTimeouts;
     bool inBuildSlot;
     time_t lastOutput; /* time we last got output on stdout/stderr */
+    time_t timeStarted;
 };
 
 typedef map<pid_t, Child> Children;
@@ -226,9 +233,6 @@ private:
     /* Last time the goals in `waitingForAWhile' where woken up. */
     time_t lastWokenUp;
 
-    /* Last time `waitForInput' was last called.  */
-    time_t lastWait;
-
 public:
 
     /* Set if at least one derivation had a BuildError (i.e. permanent
@@ -260,7 +264,7 @@ public:
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
     void childStarted(GoalPtr goal, pid_t pid,
-        const set<int> & fds, bool inBuildSlot, bool monitorForSilence);
+        const set<int> & fds, bool inBuildSlot, bool respectTimeouts);
 
     /* Unregisters a running child process.  `wakeSleepers' should be
        false if there is no sense in waking up goals that are sleeping
@@ -691,9 +695,7 @@ HookInstance::HookInstance()
                 throw SysError("dupping builder's stdout/stderr");
 
             execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
-                (format("%1%") % settings.maxSilentTime).str().c_str(),
                 (format("%1%") % settings.printBuildTrace).str().c_str(),
-                (format("%1%") % settings.buildTimeout).str().c_str(),
                 NULL);
 
             throw SysError(format("executing `%1%'") % buildHook);
@@ -845,7 +847,7 @@ public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
     ~DerivationGoal();
 
-    void cancel();
+    void cancel(bool timeout);
 
     void work();
 
@@ -971,8 +973,10 @@ void DerivationGoal::killChild()
 }
 
 
-void DerivationGoal::cancel()
+void DerivationGoal::cancel(bool timeout)
 {
+    if (timeout)
+        printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
     killChild();
     amDone(ecFailed);
 }
@@ -1362,8 +1366,8 @@ void DerivationGoal::tryToBuild()
         outputLocks.unlock();
         buildUser.release();
         if (settings.printBuildTrace)
-            printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
-                % drvPath % drv.outputs["out"].path % 0 % e.msg());
+            printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
+                % drvPath % 0 % e.msg());
         worker.permanentFailure = true;
         amDone(ecFailed);
         return;
@@ -1535,11 +1539,11 @@ void DerivationGoal::buildDone()
 
         if (settings.printBuildTrace) {
             if (hook && hookError)
-                printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
-                    % drvPath % drv.outputs["out"].path % status % e.msg());
+                printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
+                    % drvPath % status % e.msg());
             else
-                printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
-                    % drvPath % drv.outputs["out"].path % 1 % e.msg());
+                printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
+                    % drvPath % 1 % e.msg());
         }
 
         /* Register the outputs of this build as "failed" so we won't
@@ -1562,8 +1566,7 @@ void DerivationGoal::buildDone()
     buildUser.release();
 
     if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ build-succeeded %1% %2%")
-            % drvPath % drv.outputs["out"].path);
+        printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath);
     }
 
     amDone(ecSuccess);
@@ -1642,11 +1645,11 @@ HookReply DerivationGoal::tryBuildHook()
     set<int> fds;
     fds.insert(hook->fromHook.readSide);
     fds.insert(hook->builderOut.readSide);
-    worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
+    worker.childStarted(shared_from_this(), hook->pid, fds, false, true);
 
     if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
-            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
+            % drvPath % drv.platform % logFile);
 
     return rpAccept;
 }
@@ -1691,7 +1694,8 @@ void DerivationGoal::startBuilder()
        if HOME is not set, but they will just assume that the settings file
        they are looking for does not exist if HOME is set but points to some
        non-existing path. */
-    env["HOME"] = "/homeless-shelter";
+    Path homeDir = "/homeless-shelter";
+    env["HOME"] = homeDir;
 
     /* Tell the builder where the Nix store is.  Usually they
        shouldn't care, but this is useful for purity checking (e.g.,
@@ -1940,6 +1944,9 @@ void DerivationGoal::startBuilder()
 
     else {
 
+        if (pathExists(homeDir))
+            throw Error(format("directory `%1%' exists; please remove it") % homeDir);
+
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete
            them, we have to do hash rewriting: i.e. in the
@@ -2023,8 +2030,8 @@ void DerivationGoal::startBuilder()
         singleton<set<int> >(builderOut.readSide), true, true);
 
     if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
-            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
+            % drvPath % drv.platform % logFile);
     }
 }
 
@@ -2281,7 +2288,7 @@ void DerivationGoal::computeClosure()
         }
 
         /* Get rid of all weird permissions. */
-        canonicalisePathMetaData(path);
+        canonicalisePathMetaData(path, buildUser.enabled() ? buildUser.getUID() : -1);
 
         /* For this output path, find the references to other paths
            contained in it.  Compute the SHA-256 NAR hash at the same
@@ -2343,13 +2350,15 @@ Path DerivationGoal::openLogFile()
 {
     if (!settings.keepLog) return "";
 
+    string baseName = baseNameOf(drvPath);
+
     /* Create a log file. */
-    Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str();
+    Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
     createDirs(dir);
 
     if (settings.compressLog) {
 
-        Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
+        Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
         AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
         if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
         closeOnExec(fd);
@@ -2364,7 +2373,7 @@ Path DerivationGoal::openLogFile()
         return logFileName;
 
     } else {
-        Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str();
+        Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
         fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
         if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
         closeOnExec(fdLogFile);
@@ -2458,7 +2467,7 @@ bool DerivationGoal::pathFailed(const Path & path)
     printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
 
     if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
+        printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath);
 
     worker.permanentFailure = true;
     amDone(ecFailed);
@@ -2529,7 +2538,7 @@ public:
     SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
     ~SubstitutionGoal();
 
-    void cancel();
+    void cancel(bool timeout);
 
     void work();
 
@@ -2570,8 +2579,10 @@ SubstitutionGoal::~SubstitutionGoal()
 }
 
 
-void SubstitutionGoal::cancel()
+void SubstitutionGoal::cancel(bool timeout)
 {
+    if (timeout)
+        printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath);
     if (pid != -1) {
         pid_t savedPid = pid;
         pid.kill();
@@ -2762,10 +2773,8 @@ void SubstitutionGoal::tryToRun()
 
     state = &SubstitutionGoal::finished;
 
-    if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ substituter-started %1% %2%")
-            % storePath % sub);
-    }
+    if (settings.printBuildTrace)
+        printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub);
 }
 
 
@@ -2831,7 +2840,7 @@ void SubstitutionGoal::finished()
         return;
     }
 
-    canonicalisePathMetaData(destPath);
+    canonicalisePathMetaData(destPath, -1);
 
     worker.store.optimisePath(destPath); // FIXME: combine with hashPath()
 
@@ -2852,9 +2861,8 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path `%1%' succeeded") % storePath);
 
-    if (settings.printBuildTrace) {
+    if (settings.printBuildTrace)
         printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
-    }
 
     amDone(ecSuccess);
 }
@@ -2982,14 +2990,14 @@ unsigned Worker::getNrLocalBuilds()
 
 void Worker::childStarted(GoalPtr goal,
     pid_t pid, const set<int> & fds, bool inBuildSlot,
-    bool monitorForSilence)
+    bool respectTimeouts)
 {
     Child child;
     child.goal = goal;
     child.fds = fds;
-    child.lastOutput = time(0);
+    child.timeStarted = child.lastOutput = time(0);
     child.inBuildSlot = inBuildSlot;
-    child.monitorForSilence = monitorForSilence;
+    child.respectTimeouts = respectTimeouts;
     children[pid] = child;
     if (inBuildSlot) nrLocalBuilds++;
 }
@@ -3105,31 +3113,22 @@ void Worker::waitForInput()
     timeout.tv_usec = 0;
     time_t before = time(0);
 
-    /* If a global timeout has been set, sleep until it's done.  */
-    if (settings.buildTimeout != 0) {
-        useTimeout = true;
-        if (lastWait == 0 || lastWait > before) lastWait = before;
-        timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before);
+    /* If we're monitoring for silence on stdout/stderr, or if there
+       is a build timeout, then wait for input until the first
+       deadline for any child. */
+    assert(sizeof(time_t) >= sizeof(long));
+    time_t nearest = LONG_MAX; // nearest deadline
+    foreach (Children::iterator, i, children) {
+        if (!i->second.respectTimeouts) continue;
+        if (settings.maxSilentTime != 0)
+            nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime);
+        if (settings.buildTimeout != 0)
+            nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout);
     }
-
-    /* If we're monitoring for silence on stdout/stderr, sleep until
-       the first deadline for any child. */
-    if (settings.maxSilentTime != 0) {
-        time_t oldest = 0;
-        foreach (Children::iterator, i, children) {
-            if (i->second.monitorForSilence) {
-                oldest = oldest == 0 || i->second.lastOutput < oldest
-                    ? i->second.lastOutput : oldest;
-            }
-        }
-        if (oldest) {
-            time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before);
-            timeout.tv_sec = useTimeout
-                ? std::min(silenceTimeout, timeout.tv_sec)
-                : silenceTimeout;
-            useTimeout = true;
-            printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
-        }
+    if (nearest != LONG_MAX) {
+        timeout.tv_sec = std::max((time_t) 1, nearest - before);
+        useTimeout = true;
+        printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
     }
 
     /* If we are polling goals that are waiting for a lock, then wake
@@ -3139,7 +3138,7 @@ void Worker::waitForInput()
         if (lastWokenUp == 0)
             printMsg(lvlError, "waiting for locks or build slots...");
         if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max((time_t) 0, (time_t) (lastWokenUp + settings.pollInterval - before));
+        timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before));
     } else lastWokenUp = 0;
 
     using namespace std;
@@ -3163,9 +3162,6 @@ void Worker::waitForInput()
 
     time_t after = time(0);
 
-    /* Keep track of when we were last called.  */
-    lastWait = after;
-
     /* Process all available file descriptors. */
 
     /* Since goals may be canceled from inside the loop below (causing
@@ -3206,22 +3202,23 @@ void Worker::waitForInput()
         }
 
         if (settings.maxSilentTime != 0 &&
-            j->second.monitorForSilence &&
+            j->second.respectTimeouts &&
             after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds of silence")
                 % goal->getName() % settings.maxSilentTime);
-            goal->cancel();
+            goal->cancel(true);
         }
 
         if (settings.buildTimeout != 0 &&
-            after - before >= (time_t) settings.buildTimeout)
+            j->second.respectTimeouts &&
+            after - j->second.timeStarted >= (time_t) settings.buildTimeout)
         {
             printMsg(lvlError,
-                format("%1% timed out after %2% seconds of activity")
+                format("%1% timed out after %2% seconds")
                 % goal->getName() % settings.buildTimeout);
-            goal->cancel();
+            goal->cancel(true);
         }
     }
 

src/libstore/gc.cc

@@ -372,9 +372,9 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
 
     string result = runProgram(rootFinder);
 
-    Strings paths = tokenizeString<Strings>(result, "\n");
+    StringSet paths = tokenizeString<StringSet>(result, "\n");
 
-    foreach (Strings::iterator, i, paths) {
+    foreach (StringSet::iterator, i, paths) {
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store.isValidPath(path)) {

src/libstore/globals.cc

@@ -10,6 +10,14 @@
 namespace nix {
 
 
+/* The default location of the daemon socket, relative to nixStateDir.
+   The socket is in a directory to allow you to control access to the
+   Nix daemon by setting the mode/ownership of the directory
+   appropriately.  (This wouldn't work on the socket itself since it
+   must be deleted and recreated on startup.) */
+#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
+
+
 Settings settings;
 
 
@@ -58,6 +66,7 @@ void Settings::processEnvironment()
     nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+    nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {

src/libstore/globals.hh

@@ -50,6 +50,9 @@ struct Settings {
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
+    /* File name of the socket the daemon listens to.  */
+    Path nixDaemonSocketFile;
+
     /* Whether to keep temporary directories of failed builds. */
     bool keepFailed;
 

src/libstore/local-store.cc

@@ -19,8 +19,9 @@
 #include <stdio.h>
 #include <time.h>
 
-#if HAVE_UNSHARE
+#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
 #include <sched.h>
+#include <sys/statvfs.h>
 #include <sys/mount.h>
 #endif
 
@@ -47,7 +48,11 @@ static void throwSQLiteError(sqlite3 * db, const format & f)
 {
     int err = sqlite3_errcode(db);
     if (err == SQLITE_BUSY) {
-        printMsg(lvlError, "warning: SQLite database is busy");
+        static bool warned = false;
+        if (!warned) {
+            printMsg(lvlError, "warning: SQLite database is busy");
+            warned = true;
+        }
         /* Sleep for a while since retrying the transaction right away
            is likely to fail again. */
 #if HAVE_NANOSLEEP
@@ -430,30 +435,20 @@ void LocalStore::openDB(bool create)
    bind mount.  So make the Nix store writable for this process. */
 void LocalStore::makeStoreWritable()
 {
-#if HAVE_UNSHARE
+#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT)
     if (getuid() != 0) return;
+    /* Check if /nix/store is on a read-only mount. */
+    struct statvfs stat;
+    if (statvfs(settings.nixStore.c_str(), &stat) !=0)
+        throw SysError("Getting info of nix store mountpoint");
 
-    if (!pathExists("/proc/self/mountinfo")) return;
+    if (stat.f_flag & ST_RDONLY) {
+        if (unshare(CLONE_NEWNS) == -1)
+            throw SysError("setting up a private mount namespace");
 
-    /* Check if /nix/store is a read-only bind mount. */
-    bool found = false;
-    Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
-    foreach (Strings::iterator, i, mounts) {
-        vector<string> fields = tokenizeString<vector<string> >(*i, " ");
-        if (fields.at(3) == "/" || fields.at(4) != settings.nixStore) continue;
-        Strings options = tokenizeString<Strings>(fields.at(5), ",");
-        if (std::find(options.begin(), options.end(), "ro") == options.end()) continue;
-        found = true;
-        break;
+        if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1)
+            throw SysError(format("remounting %1% writable") % settings.nixStore);
     }
-
-    if (!found) return;
-
-    if (unshare(CLONE_NEWNS) == -1)
-        throw SysError("setting up a private mount namespace");
-
-    if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1)
-        throw SysError(format("remounting %1% writable") % settings.nixStore);
 #endif
 }
 
@@ -461,36 +456,8 @@ void LocalStore::makeStoreWritable()
 const time_t mtimeStore = 1; /* 1 second into the epoch */
 
 
-void canonicalisePathMetaData(const Path & path, bool recurse)
+static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
 {
-    checkInterrupt();
-
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path `%1%'") % path);
-
-    /* Really make sure that the path is of a supported type.  This
-       has already been checked in dumpPath(). */
-    assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode));
-
-    /* Change ownership to the current uid.  If it's a symlink, use
-       lchown if available, otherwise don't bother.  Wrong ownership
-       of a symlink doesn't matter, since the owning user can't change
-       the symlink and can't delete it because the directory is not
-       writable.  The only exception is top-level paths in the Nix
-       store (since that directory is group-writable for the Nix build
-       users group); we check for this case below. */
-    if (st.st_uid != geteuid()) {
-#if HAVE_LCHOWN
-        if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1)
-#else
-        if (!S_ISLNK(st.st_mode) &&
-            chown(path.c_str(), geteuid(), (gid_t) -1) == -1)
-#endif
-            throw SysError(format("changing owner of `%1%' to %2%")
-                % path % geteuid());
-    }
-
     if (!S_ISLNK(st.st_mode)) {
 
         /* Mask out all type related bits. */
@@ -521,18 +488,84 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
 #endif
             throw SysError(format("changing modification time of `%1%'") % path);
     }
+}
 
-    if (recurse && S_ISDIR(st.st_mode)) {
+
+void canonicaliseTimestampAndPermissions(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError(format("getting attributes of path `%1%'") % path);
+    canonicaliseTimestampAndPermissions(path, st);
+}
+
+
+typedef std::pair<dev_t, ino_t> Inode;
+typedef set<Inode> InodesSeen;
+
+
+static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
+{
+    checkInterrupt();
+
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError(format("getting attributes of path `%1%'") % path);
+
+    /* Really make sure that the path is of a supported type.  This
+       has already been checked in dumpPath(). */
+    assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode));
+
+    /* Fail if the file is not owned by the build user.  This prevents
+       us from messing up the ownership/permissions of files
+       hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
+       However, ignore files that we chown'ed ourselves previously to
+       ensure that we don't fail on hard links within the same build
+       (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
+    if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
+        assert(!S_ISDIR(st.st_mode));
+        if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
+            throw BuildError(format("invalid ownership on file `%1%'") % path);
+        mode_t mode = st.st_mode & ~S_IFMT;
+        assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
+        return;
+    }
+
+    inodesSeen.insert(Inode(st.st_dev, st.st_ino));
+
+    canonicaliseTimestampAndPermissions(path, st);
+
+    /* Change ownership to the current uid.  If it's a symlink, use
+       lchown if available, otherwise don't bother.  Wrong ownership
+       of a symlink doesn't matter, since the owning user can't change
+       the symlink and can't delete it because the directory is not
+       writable.  The only exception is top-level paths in the Nix
+       store (since that directory is group-writable for the Nix build
+       users group); we check for this case below. */
+    if (st.st_uid != geteuid()) {
+#if HAVE_LCHOWN
+        if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1)
+#else
+        if (!S_ISLNK(st.st_mode) &&
+            chown(path.c_str(), geteuid(), (gid_t) -1) == -1)
+#endif
+            throw SysError(format("changing owner of `%1%' to %2%")
+                % path % geteuid());
+    }
+
+    if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
         foreach (Strings::iterator, i, names)
-            canonicalisePathMetaData(path + "/" + *i, true);
+            canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen);
     }
 }
 
 
-void canonicalisePathMetaData(const Path & path)
+void canonicalisePathMetaData(const Path & path, uid_t fromUid)
 {
-    canonicalisePathMetaData(path, true);
+    InodesSeen inodesSeen;
+
+    canonicalisePathMetaData_(path, fromUid, inodesSeen);
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
@@ -974,6 +1007,10 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
     fromPipe.create();
     errorPipe.create();
 
+    /* Hack: prevent substituters that write too much to stderr from
+       deadlocking our read() from stdout. */
+    fcntl(errorPipe.writeSide, F_SETFL, O_NONBLOCK);
+
     setSubstituterEnv();
 
     run.pid = maybeVfork();
@@ -1194,7 +1231,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
             } else
                 writeFile(dstPath, dump);
 
-            canonicalisePathMetaData(dstPath);
+            canonicalisePathMetaData(dstPath, -1);
 
             /* Register the SHA-256 hash of the NAR serialisation of
                the path in the database.  We may just have computed it
@@ -1259,7 +1296,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             writeFile(dstPath, s);
 
-            canonicalisePathMetaData(dstPath);
+            canonicalisePathMetaData(dstPath, -1);
 
             HashResult hash = hashPath(htSHA256, dstPath);
 
@@ -1494,7 +1531,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
                 throw SysError(format("cannot move `%1%' to `%2%'")
                     % unpacked % dstPath);
 
-            canonicalisePathMetaData(dstPath);
+            canonicalisePathMetaData(dstPath, -1);
 
             /* !!! if we were clever, we could prevent the hashPath()
                here. */

src/libstore/local-store.hh

@@ -307,9 +307,9 @@ private:
      without execute permission; setuid bits etc. are cleared)
    - the owner and group are set to the Nix user and group, if we're
      in a setuid Nix installation. */
-void canonicalisePathMetaData(const Path & path);
+void canonicalisePathMetaData(const Path & path, uid_t fromUid);
 
-void canonicalisePathMetaData(const Path & path, bool recurse);
+void canonicaliseTimestampAndPermissions(const Path & path);
 
 MakeError(PathInUse, Error);
 

src/libstore/optimise-store.cc

@@ -32,7 +32,7 @@ struct MakeReadOnly
     {
         try {
             /* This will make the path read-only. */
-            if (path != "") canonicalisePathMetaData(path, false);
+            if (path != "") canonicaliseTimestampAndPermissions(path);
         } catch (...) {
             ignoreException();
         }

src/libstore/remote-store.cc

@@ -91,7 +91,7 @@ void RemoteStore::connectToDaemon()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(fdSocket);
 
-    string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
+    string socketPath = settings.nixDaemonSocketFile;
 
     /* Urgh, sockaddr_un allows path names of only 108 characters.  So
        chdir to the socket directory so that we can pass a relative

src/libstore/worker-protocol.hh

@@ -53,14 +53,6 @@ typedef enum {
 #define STDERR_ERROR 0x63787470
 
 
-/* The default location of the daemon socket, relative to nixStateDir.
-   The socket is in a directory to allow you to control access to the
-   Nix daemon by setting the mode/ownership of the directory
-   appropriately.  (This wouldn't work on the socket itself since it
-   must be deleted and recreated on startup.) */
-#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
-
-
 Path readStorePath(Source & from);
 template<class T> T readStorePaths(Source & from);
 

src/nix-daemon/nix-daemon.cc

@@ -551,7 +551,10 @@ static void performOp(unsigned int clientVersion,
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                settings.set("untrusted-" + name, value);
+                if (name == "build-timeout")
+                    string2Int(value, settings.buildTimeout);
+                else
+                    settings.set("untrusted-" + name, value);
             }
         }
         startWork();
@@ -774,7 +777,7 @@ static void daemonLoop()
         if (fdSocket == -1)
             throw SysError("cannot create Unix domain socket");
 
-        string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
+        string socketPath = settings.nixDaemonSocketFile;
 
         createDirs(dirOf(socketPath));
 
@@ -895,8 +898,6 @@ static void daemonLoop()
 
 void run(Strings args)
 {
-    bool daemon = false;
-
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
         string arg = *i++;
         if (arg == "--daemon") /* ignored for backwards compatibility */;

src/nix-store/nix-store.cc

@@ -238,12 +238,6 @@ static void printTree(const Path & path,
     PathSet references;
     store->queryReferences(path, references);
 
-#if 0
-    for (PathSet::iterator i = drv.inputSrcs.begin();
-         i != drv.inputSrcs.end(); ++i)
-        cout << format("%1%%2%\n") % (tailPad + treeConn) % *i;
-#endif
-
     /* Topologically sort under the relation A < B iff A \in
        closure(B).  That is, if derivation A is an (possibly indirect)
        input of B, then A is printed first.  This has the effect of
@@ -251,7 +245,7 @@ static void printTree(const Path & path,
     Paths sorted = topoSortPaths(*store, references);
     reverse(sorted.begin(), sorted.end());
 
-    for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) {
+    foreach (Paths::iterator, i, sorted) {
         Paths::iterator j = i; ++j;
         printTree(*i, tailPad + treeConn,
             j == sorted.end() ? tailPad + treeNull : tailPad + treeLine,
@@ -293,7 +287,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (*i == "--resolve") query = qResolve;
         else if (*i == "--roots") query = qRoots;
         else if (*i == "--use-output" || *i == "-u") useOutput = true;
-        else if (*i == "--force-realise" || *i == "-f") forceRealise = true;
+        else if (*i == "--force-realise" || *i == "--force-realize" || *i == "-f") forceRealise = true;
         else if (*i == "--include-outputs") includeOutputs = true;
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
@@ -460,35 +454,42 @@ static void opReadLog(Strings opFlags, Strings opArgs)
     foreach (Strings::iterator, i, opArgs) {
         Path path = useDeriver(followLinksToStorePath(*i));
 
-        Path logPath = (format("%1%/%2%/%3%") %
-            settings.nixLogDir % drvsLogDir % baseNameOf(path)).str();
-        Path logBz2Path = logPath + ".bz2";
+        for (int j = 0; j <= 2; j++) {
+            if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path);
 
-        if (pathExists(logPath)) {
-            /* !!! Make this run in O(1) memory. */
-            string log = readFile(logPath);
-            writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size());
+            string baseName = baseNameOf(path);
+            Path logPath =
+                j == 0
+                ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
+                : (format("%1%/%2%/%3%") % settings.nixLogDir % drvsLogDir % baseName).str();
+            Path logBz2Path = logPath + ".bz2";
+
+            if (pathExists(logPath)) {
+                /* !!! Make this run in O(1) memory. */
+                string log = readFile(logPath);
+                writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size());
+                break;
+            }
+
+            else if (pathExists(logBz2Path)) {
+                AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY);
+                FILE * f = 0;
+                if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0)
+                    throw SysError(format("opening file `%1%'") % logBz2Path);
+                int err;
+                BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0);
+                if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path);
+                unsigned char buf[128 * 1024];
+                do {
+                    int n = BZ2_bzRead(&err, bz, buf, sizeof(buf));
+                    if (err != BZ_OK && err != BZ_STREAM_END)
+                        throw Error(format("error reading bzip2 file `%1%'") % logBz2Path);
+                    writeFull(STDOUT_FILENO, buf, n);
+                } while (err != BZ_STREAM_END);
+                BZ2_bzReadClose(&err, bz);
+                break;
+            }
         }
-
-        else if (pathExists(logBz2Path)) {
-            AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY);
-            FILE * f = 0;
-            if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0)
-                throw SysError(format("opening file `%1%'") % logBz2Path);
-            int err;
-            BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0);
-            if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path);
-            unsigned char buf[128 * 1024];
-            do {
-                int n = BZ2_bzRead(&err, bz, buf, sizeof(buf));
-                if (err != BZ_OK && err != BZ_STREAM_END)
-                    throw Error(format("error reading bzip2 file `%1%'") % logBz2Path);
-                writeFull(STDOUT_FILENO, buf, n);
-            } while (err != BZ_STREAM_END);
-            BZ2_bzReadClose(&err, bz);
-        }
-
-        else throw Error(format("build log of derivation `%1%' is not available") % path);
     }
 }
 
@@ -514,7 +515,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
         if (!store->isValidPath(info.path) || reregister) {
             /* !!! races */
             if (canonicalise)
-                canonicalisePathMetaData(info.path);
+                canonicalisePathMetaData(info.path, -1);
             if (!hashGiven) {
                 HashResult hash = hashPath(htSHA256, info.path);
                 info.hash = hash.first;
@@ -842,7 +843,7 @@ void run(Strings args)
 
         Operation oldOp = op;
 
-        if (arg == "--realise" || arg == "-r")
+        if (arg == "--realise" || arg == "--realize" || arg == "-r")
             op = opRealise;
         else if (arg == "--add" || arg == "-A")
             op = opAdd;
@@ -884,7 +885,7 @@ void run(Strings args)
             op = opVerifyPath;
         else if (arg == "--repair-path")
             op = opRepairPath;
-        else if (arg == "--optimise")
+        else if (arg == "--optimise" || arg == "--optimize")
             op = opOptimise;
         else if (arg == "--query-failed-paths")
             op = opQueryFailedPaths;

substitute.mk

@@ -37,3 +37,6 @@
 	 -e "s^@testPath\@^$(coreutils):$$(dirname $$(type -p expr))^g" \
 	 < $< > $@ || rm $@
 	if test -x $<; then chmod +x $@; fi
+
+$(CONFIG_HEADER):
+	true

tests/binary-cache.sh

@@ -5,7 +5,7 @@ clearManifests
 
 # Create the binary cache.
 cacheDir=$TEST_ROOT/binary-cache
-rm -rf $cacheDir
+rm -rf "$cacheDir"
 
 outPath=$(nix-build dependencies.nix --no-out-link)
 
@@ -17,6 +17,8 @@ nix-push --dest $cacheDir $outPath
 clearStore
 rm -f $NIX_STATE_DIR/binary-cache*
 
+export _NIX_CACHE_FILE_URLS=1
+
 nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
 
 nix-store --option binary-caches "file://$cacheDir" -r $outPath
@@ -38,6 +40,20 @@ nix-store --check-validity $outPath
 nix-store -qR $outPath | grep input-2
 
 
+# Test whether fallback works if we have cached info but the
+# corresponding NAR has disappeared.
+clearStore
+
+nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
+
+mkdir $cacheDir/tmp
+mv $cacheDir/*.nar.xz $cacheDir/tmp/
+
+NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
+
+mv $cacheDir/tmp/* $cacheDir/
+
+
 # Test whether building works if the binary cache contains an
 # incomplete closure.
 clearStore

tests/lang/eval-okay-hash.exp

@@ -0,0 +1 @@
+[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" ]

tests/lang/eval-okay-hash.nix

@@ -0,0 +1,7 @@
+let
+  md5 = builtins.hashString "md5";
+  sha1 = builtins.hashString "sha1";
+  sha256 = builtins.hashString "sha256";
+  strings = [ "" "text 1" "text 2" ];
+in
+  (builtins.map md5 strings) ++ (builtins.map sha1 strings) ++ (builtins.map sha256 strings)

tests/nix-copy-closure.nix

@@ -13,14 +13,14 @@ makeTest ({ pkgs, ... }: let pkgA = pkgs.aterm; pkgB = pkgs.wget; in {
           virtualisation.pathsInNixDB = [ pkgA ];
           environment.nix = nix;
         };
-        
+
       server =
         { config, pkgs, ... }:
         { services.openssh.enable = true;
           virtualisation.writableStore = true;
           virtualisation.pathsInNixDB = [ pkgB ];
           environment.nix = nix;
-        };        
+        };
     };
 
   testScript = { nodes }:
@@ -36,8 +36,8 @@ makeTest ({ pkgs, ... }: let pkgA = pkgs.aterm; pkgB = pkgs.wget; in {
       # Install the SSH key on the server.
       $server->succeed("mkdir -m 700 /root/.ssh");
       $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
-      $server->waitForJob("sshd");
-      $client->waitForJob("network-interfaces");
+      $server->waitForUnit("sshd");
+      $client->waitForUnit("network.target");
       $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'");
 
       # Copy the closure of package A from the client to the server.

tests/remote-builds.nix

@@ -72,11 +72,11 @@ in
       $client->succeed("chmod 600 /root/.ssh/id_dsa");
 
       # Install the SSH key on the slaves.
-      $client->waitForJob("network-interfaces");
+      $client->waitForUnit("network.target");
       foreach my $slave ($slave1, $slave2) {
           $slave->succeed("mkdir -m 700 /root/.ssh");
           $slave->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys");
-          $slave->waitForJob("sshd");
+          $slave->waitForUnit("sshd");
           $client->succeed("ssh -o StrictHostKeyChecking=no " . $slave->name() . " 'echo hello world'");
       }
 

tests/timeout.builder.sh

@@ -1,2 +1,2 @@
 echo "\`timeout' builder entering an infinite loop"
-while true ; do : ; done
+while true ; do echo foo; sleep 1; done

tests/timeout.nix

@@ -3,6 +3,4 @@ with import ./config.nix;
 mkDerivation {
   name = "timeout";
   builder = ./timeout.builder.sh;
-  PATH = "";
-  goodPath = path;
 }

version

@@ -1 +1 @@
-1.3
+1.5.2