Fix building without a garbage collector

http://hydra.nixos.org/build/6695350

Makefile.am

@@ -27,11 +27,9 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
-	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
 	-$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
-	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 
 else
 

configure.ac

@@ -68,6 +68,8 @@ if test "$sys_name" = sunos; then
 fi
 
 
+CFLAGS=${CFLAGS:--g -O3 -Wall}
+CXXFLAGS=${CXXFLAGS:--g -O3 -Wall}
 AC_PROG_CC
 AC_PROG_CXX
 
@@ -106,6 +108,7 @@ AC_LANG_POP(C++)
 # Check for chroot support (requires chroot() and bind mounts).
 AC_CHECK_FUNCS([chroot])
 AC_CHECK_FUNCS([unshare])
+AC_CHECK_FUNCS([statvfs])
 AC_CHECK_HEADERS([sched.h])
 AC_CHECK_HEADERS([sys/param.h])
 AC_CHECK_HEADERS([sys/mount.h], [], [],
@@ -124,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
 AC_CHECK_FUNCS([lutimes])
 
 
+# Check for sched_setaffinity.
+AC_CHECK_FUNCS([sched_setaffinity])
+
+
 # Check whether the store optimiser can optimise symlinks.
 AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
 ln -s bla tmp_link
@@ -182,10 +189,10 @@ NEED_PROG(perl, perl)
 NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 NEED_PROG(bzip2, bzip2)
+NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
-AC_PATH_PROG(gzip, gzip)
 AC_PATH_PROG(pv, pv, pv)
 
 
@@ -354,7 +361,6 @@ fi
 AC_SUBST(tarFlags)
 
 
-
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    src/Makefile

corepkgs/buildenv.pl

@@ -36,9 +36,6 @@ sub createLinks {
         if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
             $srcFile =~ /\/perllocal.pod$/ ||
-            $srcFile =~ /\/easy-install.pth$/ ||
-            $srcFile =~ /\/site.py$/ ||
-            $srcFile =~ /\/site.pyc$/ ||
             $srcFile =~ /\/info\/dir$/ ||
             $srcFile =~ /\/log$/)
         {
@@ -56,7 +53,7 @@ sub createLinks {
             elsif (-l _) {
                 my $target = readlink $dstFile or die;
                 if (!-d $target) {
-                    die "collission between directory `$srcFile' and non-directory `$target'";
+                    die "collision between directory `$srcFile' and non-directory `$target'";
                 }
                 unlink $dstFile or die "error unlinking `$dstFile': $!";
                 mkdir $dstFile, 0755 ||
@@ -78,7 +75,7 @@ sub createLinks {
             if (-l $dstFile) {
                 my $target = readlink $dstFile;
                 my $prevPriority = $priorities{$dstFile};
-                die ( "collission between `$srcFile' and `$target'; "
+                die ( "collision between `$srcFile' and `$target'; "
                     . "use `nix-env --set-flag "
                     . "priority NUMBER PKGNAME' to change the priority of "
                     . "one of the conflicting packages\n" )

corepkgs/config.nix.in

@@ -7,6 +7,7 @@ in {
   shell = "@shell@";
   coreutils = "@coreutils@";
   bzip2 = "@bzip2@";
+  gzip = "@gzip@";
   xz = "@xz@";
   tar = "@tar@";
   tarFlags = "@tarFlags@";

corepkgs/fetchurl.nix

@@ -1,6 +1,6 @@
 with import <nix/config.nix>;
 
-{system ? builtins.currentSystem, url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? ""}:
+{system ? builtins.currentSystem, url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? "", executable ? false}:
 
 assert (outputHash != "" && outputHashAlgo != "")
     || md5 != "" || sha1 != "" || sha256 != "";
@@ -8,10 +8,10 @@ assert (outputHash != "" && outputHashAlgo != "")
 let
 
   builder = builtins.toFile "fetchurl.sh"
-    ''
+    (''
       echo "downloading $url into $out"
       ${curl} --fail --location --max-redirs 20 --insecure "$url" > "$out"
-    '';
+    '' + (if executable then "${coreutils}/chmod +x $out" else ""));
 
 in
     
@@ -25,6 +25,7 @@ derivation {
       if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5";
   outputHash = if outputHash != "" then outputHash else
       if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
+  outputHashMode = if executable then "recursive" else "flat";
   
   inherit system url;
 

corepkgs/nar.nix

@@ -6,25 +6,28 @@ let
     ''
       export PATH=${nixBinDir}:${coreutils}
 
-      if [ $compressionType = "xz" ]; then
+      if [ $compressionType = xz ]; then
-        ext=xz
+        ext=.xz
-        compressor="${xz} -9"
+        compressor="| ${xz} -7"
+      elif [ $compressionType = bzip2 ]; then
+        ext=.bz2
+        compressor="| ${bzip2}"
       else
-        ext=bz2
+        ext=
-        compressor="${bzip2}"
+        compressor=
       fi
 
       echo "packing ‘$storePath’..."
       mkdir $out
-      dst=$out/tmp.nar.$ext
+      dst=$out/tmp.nar$ext
 
       set -o pipefail
-      nix-store --dump "$storePath" | $compressor > $dst
+      eval "nix-store --dump \"$storePath\" $compressor > $dst"
 
       hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
       echo -n $hash > $out/nar-compressed-hash
 
-      mv $dst $out/$hash.nar.$ext
+      mv $dst $out/$hash.nar$ext
     '';
 
 in
@@ -40,4 +43,7 @@ derivation {
 
   # Don't build in a chroot because Nix's dependencies may not be there.
   __noChroot = true;
+
+  # Remote machines may not have ${nixBinDir} or ${coreutils} in the same prefixes
+  preferLocalBuild = true;
 }

corepkgs/unpack-channel.nix

@@ -6,9 +6,12 @@ let
     ''
       mkdir $out
       cd $out
-      pat="\.xz\$"
+      xzpat="\.xz\$"
-      if [[ "$src" =~ $pat ]]; then
+      gzpat="\.gz\$"
+      if [[ "$src" =~ $xzpat ]]; then
         ${xz} -d < $src | ${tar} xf - ${tarFlags}
+      elif [[ "$src" =~ $gzpat ]]; then
+        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
       else
         ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
       fi

dev-shell

@@ -0,0 +1,17 @@
+#! /bin/sh
+if [ -e tests/test-tmp ]; then
+    chmod -R u+w tests/test-tmp
+    rm -rf tests/test-tmp
+fi
+
+s=$(type -p nix-shell)
+exec $s release.nix -A tarball --command "
+    export NIX_REMOTE=daemon
+    export NIX_PATH='$NIX_PATH'
+    export NIX_BUILD_SHELL=$(type -p bash)
+    export c=\$configureFlags
+    exec $s release.nix -A build.x86_64-linux --exclude tarball --command '
+        configureFlags+=\" \$c --prefix=$(pwd)/inst --sysconfdir=$(pwd)/inst/etc\"
+        return
+    '" \
+    "$@"

doc/dev/release-procedures.txt

@@ -1,33 +0,0 @@
-To produce a `stable' release from the trunk:
-
--1. Update the release notes; make sure that the release date is
-    correct.
-
-0. Make sure that the trunk builds in the release supervisor.
-
-1. Branch the trunk, e.g., `svn cp .../trunk
-   .../branches/0.5-release'.
-
-2. Switch to the branch, e.g., `svn switch .../branches/0.5-release'.
-
-3. In `configure.ac', change `STABLE=0' into `STABLE=1' and commit.
-
-4. In the release supervisor, add a one-time job to build
-   `.../branches/0.5-release'.
-
-5. Make sure that the release succeeds.
-
-6. Move the branch to a tag, e.g., `svn mv .../branches/0.5-release
-   .../tags/0.5'.
-
-   Note that the branch should not be used for maintenance; it should
-   be deleted after the release has been created.  A maintenance
-   branch (e.g., `.../branches/0.5') should be created from the
-   original revision of the trunk (since maintenance releases should
-   also be tested first; hence, we cannot have `STABLE=1').  The same
-   procedure can then be followed to produce maintenance releases;
-   just substitute `.../branches/VERSION' for the trunk.
-
-7. Switch back to the trunk.
-
-8. Bump the version number in `configure.ac' (in AC_INIT).

doc/manual/Makefile.am

@@ -16,7 +16,7 @@ dblatex_opts = \
 # Note: we use GIF for now, since the PNGs shipped with Docbook aren't
 # transparent.
 
-man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
+man1_MANS = nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
  nix-install-package.1 nix-hash.1 nix-copy-closure.1

doc/manual/builtins.xml

@@ -12,9 +12,9 @@ such as <function>derivation</function>, are always in scope of every
 Nix expression; you can just access them right away.  But to prevent
 polluting the namespace too much, most built-ins are not in scope.
 Instead, you can access them through the <varname>builtins</varname>
-built-in value, which is an attribute set that contains all built-in
+built-in value, which is a set that contains all built-in functions
-functions and values.  For instance, <function>derivation</function>
+and values.  For instance, <function>derivation</function> is also
-is also available as <function>builtins.derivation</function>.</para>
+available as <function>builtins.derivation</function>.</para>
 
 
 <variablelist>
@@ -39,17 +39,17 @@ is also available as <function>builtins.derivation</function>.</para>
 
 
   <varlistentry><term><function>builtins.attrNames</function>
-  <replaceable>attrs</replaceable></term>
+  <replaceable>set</replaceable></term>
 
-    <listitem><para>Return the names of the attributes in the
+    <listitem><para>Return the names of the attributes in the set
-    attribute set <replaceable>attrs</replaceable> in a sorted list.
+    <replaceable>set</replaceable> in a sorted list.  For instance,
-    For instance, <literal>builtins.attrNames { y = 1; x = "foo";
+    <literal>builtins.attrNames { y = 1; x = "foo"; }</literal>
-    }</literal> evaluates to <literal>[ "x" "y" ]</literal>.  There is
+    evaluates to <literal>[ "x" "y" ]</literal>.  There is no built-in
-    no built-in function <function>attrValues</function>, but you can
+    function <function>attrValues</function>, but you can easily
-    easily define it yourself:
+    define it yourself:
 
 <programlisting>
-attrValues = attrs: map (name: builtins.getAttr name attrs) (builtins.attrNames attrs);</programlisting>
+attrValues = set: map (name: builtins.getAttr name set) (builtins.attrNames set);</programlisting>
 
     </para></listitem>
 
@@ -68,8 +68,8 @@ attrValues = attrs: map (name: builtins.getAttr name attrs) (builtins.attrNames
 
   <varlistentry><term><varname>builtins</varname></term>
 
-    <listitem><para>The attribute set <varname>builtins</varname>
+    <listitem><para>The set <varname>builtins</varname> contains all
-    contains all the built-in functions and values.  You can use
+    the built-in functions and values.  You can use
     <varname>builtins</varname> to test for the availability of
     features in the Nix installation, e.g.,
 
@@ -258,11 +258,11 @@ stdenv.mkDerivation {
 
 
   <varlistentry><term><function>builtins.getAttr</function>
-  <replaceable>s</replaceable> <replaceable>attrs</replaceable></term>
+  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>getAttr</function> returns the attribute
-    named <replaceable>s</replaceable> from the attribute set
+    named <replaceable>s</replaceable> from
-    <replaceable>attrs</replaceable>.  Evaluation aborts if the
+    <replaceable>set</replaceable>.  Evaluation aborts if the
     attribute doesn’t exist.  This is a dynamic version of the
     <literal>.</literal> operator, since <replaceable>s</replaceable>
     is an expression rather than an identifier.</para></listitem>
@@ -289,15 +289,27 @@ stdenv.mkDerivation {
 
 
   <varlistentry><term><function>builtins.hasAttr</function>
-  <replaceable>s</replaceable> <replaceable>attrs</replaceable></term>
+  <replaceable>s</replaceable> <replaceable>set</replaceable></term>
 
     <listitem><para><function>hasAttr</function> returns
-    <literal>true</literal> if the attribute set
+    <literal>true</literal> if <replaceable>set</replaceable> has an
-    <replaceable>attrs</replaceable> has an attribute named
+    attribute named <replaceable>s</replaceable>, and
-    <replaceable>s</replaceable>, and <literal>false</literal>
+    <literal>false</literal> otherwise.  This is a dynamic version of
-    otherwise.  This is a dynamic version of the <literal>?</literal>
+    the <literal>?</literal>  operator, since
-    operator, since <replaceable>s</replaceable> is an expression
+    <replaceable>s</replaceable> is an expression rather than an
-    rather than an identifier.</para></listitem>
+    identifier.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><function>builtins.hashString</function>
+  <replaceable>type</replaceable> <replaceable>s</replaceable></term>
+
+    <listitem><para>Return a base-16 representation of the
+    cryptographic hash of string <replaceable>s</replaceable>.  The
+    hash algorithm specified by <replaceable>type</replaceable> must
+    be one of <literal>"md5"</literal>, <literal>"sha1"</literal> or
+    <literal>"sha256"</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -319,12 +331,12 @@ stdenv.mkDerivation {
     <listitem><para>Load, parse and return the Nix expression in the
     file <replaceable>path</replaceable>.  If <replaceable>path
     </replaceable> is a directory, the file <filename>default.nix
-    </filename> in that directory is loaded.  Evaluation aborts if
+    </filename> in that directory is loaded.  Evaluation aborts if the
-    the file doesn’t exist or contains an incorrect Nix
+    file doesn’t exist or contains an incorrect Nix expression.
-    expression.  <function>import</function> implements Nix’s module
+    <function>import</function> implements Nix’s module system: you
-    system: you can put any Nix expression (such as an attribute set
+    can put any Nix expression (such as a set or a function) in a
-    or a function) in a separate file, and use it from Nix expressions
+    separate file, and use it from Nix expressions in other
-    in other files.</para>
+    files.</para>
 
     <para>A Nix expression loaded by <function>import</function> must
     not contain any <emphasis>free variables</emphasis> (identifiers
@@ -371,9 +383,9 @@ x: x + 456</programlisting>
   <varlistentry><term><function>builtins.intersectAttrs</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
-    <listitem><para>Return an attribute set consisting of the
+    <listitem><para>Return a set consisting of the attributes in the
-    attributes in the set <replaceable>e2</replaceable> that also
+    set <replaceable>e2</replaceable> that also exist in the set
-    exist in the set <replaceable>e1</replaceable>.</para></listitem>
+    <replaceable>e1</replaceable>.</para></listitem>
 
   </varlistentry>
 
@@ -382,7 +394,7 @@ x: x + 456</programlisting>
   <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to an attribute set, and
+    <replaceable>e</replaceable> evaluates to a set, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -422,7 +434,7 @@ x: x + 456</programlisting>
   <replaceable>e</replaceable></term>
 
     <listitem><para>Return <literal>true</literal> if
-    <replaceable>e</replaceable> evaluates to a int, and
+    <replaceable>e</replaceable> evaluates to an int, and
     <literal>false</literal> otherwise.</para></listitem>
 
   </varlistentry>
@@ -478,9 +490,9 @@ x: x + 456</programlisting>
   <varlistentry><term><function>builtins.listToAttrs</function>
   <replaceable>e</replaceable></term>
 
-    <listitem><para>Construct an attribute set from a list specifying
+    <listitem><para>Construct a set from a list specifying the names
-    the names and values of each attribute.  Each element of the list
+    and values of each attribute.  Each element of the list should be
-    should be an attribute set consisting of a string-valued attribute
+    a set consisting of a string-valued attribute
     <varname>name</varname> specifying the name of the attribute, and
     an attribute <varname>value</varname> specifying its value.
     Example:
@@ -535,7 +547,7 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting>
     a package name and version.  The package name is everything up to
     but not including the first dash followed by a digit, and the
     version is everything following that dash.  The result is returned
-    in an attribute set <literal>{ name, version }</literal>.  Thus,
+    in a set <literal>{ name, version }</literal>.  Thus,
     <literal>builtins.parseDrvName "nix-0.12pre12876"</literal>
     returns <literal>{ name = "nix"; version = "0.12pre12876";
     }</literal>.</para></listitem>
@@ -586,12 +598,12 @@ in config.someSetting</programlisting>
 
 
   <varlistentry><term><function>removeAttrs</function>
-  <replaceable>attrs</replaceable> <replaceable>list</replaceable></term>
+  <replaceable>set</replaceable> <replaceable>list</replaceable></term>
 
     <listitem><para>Remove the attributes listed in
-    <replaceable>list</replaceable> from the attribute set
+    <replaceable>list</replaceable> from
-    <replaceable>attrs</replaceable>.  The attributes don’t have to
+    <replaceable>set</replaceable>.  The attributes don’t have to
-    exist in <replaceable>attrs</replaceable>. For instance,
+    exist in <replaceable>set</replaceable>. For instance,
 
 <screen>
 removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</screen>
@@ -780,7 +792,7 @@ in foo</programlisting>
     servlet container</link>.  A servlet container contains a number
     of servlets (<filename>*.war</filename> files) each exported under
     a specific URI prefix.  So the servlet configuration is a list of
-    attribute sets containing the <varname>path</varname> and
+    sets containing the <varname>path</varname> and
     <varname>war</varname> of the servlet (<xref
     linkend='ex-toxml-co-servlets' />).  This kind of information is
     difficult to communicate with the normal method of passing
@@ -889,6 +901,19 @@ stdenv.mkDerivation (rec {
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.typeOf</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return a string representing the type of the value
+    <replaceable>e</replaceable>, namely <literal>"int"</literal>,
+    <literal>"bool"</literal>, <literal>"string"</literal>,
+    <literal>"path"</literal>, <literal>"null"</literal>,
+    <literal>"set"</literal>, <literal>"list"</literal> or
+    <literal>"lambda"</literal>.</para></listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 

doc/manual/conf-file.xml

@@ -33,18 +33,18 @@ env-keep-derivations = false
 <para>You can override settings using the <option>--option</option>
 flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
-<para>The following settings are currently available: 
+<para>The following settings are currently available:
 
 <variablelist>
 
-  
+
   <varlistentry xml:id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
 
     <listitem><para>If <literal>true</literal>, the garbage collector
     will keep the outputs of non-garbage derivations.  If
     <literal>false</literal> (default), outputs will be deleted unless
     they are GC roots themselves (or reachable from other roots).</para>
- 
+
     <para>In general, outputs must be registered as roots separately.
     However, even if the output of a derivation is registered as a
     root, the collector will still delete store paths that are used
@@ -53,7 +53,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     this option to <literal>true</literal>.</para></listitem>
 
   </varlistentry>
-  
+
 
   <varlistentry xml:id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
 
@@ -71,7 +71,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
   <varlistentry><term><literal>env-keep-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
@@ -95,14 +95,14 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
   <varlistentry xml:id="conf-build-max-jobs"><term><literal>build-max-jobs</literal></term>
 
     <listitem><para>This option defines the maximum number of jobs
     that Nix will try to build in parallel.  The default is
     <literal>1</literal>.  You should generally set it to the number
-    of CPUs in your system (e.g., <literal>2</literal> on a Athlon 64
+    of CPUs in your system (e.g., <literal>2</literal> on an Athlon 64
-    X2).  It can be overriden using the <option
+    X2).  It can be overridden using the <option
     linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>)
     command line switch.</para></listitem>
 
@@ -119,7 +119,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
     <varname>enableParallelBuilding</varname> is set to
     <literal>true</literal>, the builder passes the
     <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
-    It can be overriden using the <option
+    It can be overridden using the <option
     linkend='opt-cores'>--cores</option> command line switch and
     defaults to <literal>1</literal>.  The value <literal>0</literal>
     means that the builder should use all available CPU cores in the
@@ -134,10 +134,10 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
       <para>This option defines the maximum number of seconds that a
       builder can go without producing any data on standard output or
-      standard error.  This is useful (for instance in a automated
+      standard error.  This is useful (for instance in an automated
       build system) to catch builds that are stuck in an infinite
       loop, or to catch remote builds that are hanging due to network
-      problems.  It can be overriden using the <option
+      problems.  It can be overridden using the <option
       linkend="opt-max-silent-time">--max-silent-time</option> command
       line switch.</para>
 
@@ -148,15 +148,16 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
+
   <varlistentry xml:id="conf-build-timeout"><term><literal>build-timeout</literal></term>
 
     <listitem>
 
       <para>This option defines the maximum number of seconds that a
-      builder can run.  This is useful (for instance in a automated
+      builder can run.  This is useful (for instance in an automated
       build system) to catch builds that are stuck in an infinite loop
       but keep writing to their standard output or standard error.  It
-      can be overriden using the <option
+      can be overridden using the <option
       linkend="opt-timeout">--timeout</option> command line
       switch.</para>
 
@@ -168,6 +169,20 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-build-max-log-size"><term><literal>build-max-log-size</literal></term>
+
+    <listitem>
+
+      <para>This option defines the maximum number of bytes that a
+      builder can write to its stdout/stderr.  If the builder exceeds
+      this limit, it’s killed.  A value of <literal>0</literal> (the
+      default) means that there is no limit.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term>
 
     <listitem><para>This options specifies the Unix group containing
@@ -234,7 +249,27 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-  
+
+  <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
+
+    <listitem><para>When builds are performed in a chroot environment,
+    Nix will mount some directories from the normal file system
+    hierarchy inside the chroot.  These are the Nix store, the
+    temporary build directory (usually
+    <filename>/tmp/nix-build-<replaceable>drvname</replaceable>-<replaceable>number</replaceable></filename>),
+    the <literal>/proc</literal> filesystem, and the directories
+    listed here.  The default is <literal>/dev /dev/pts</literal>,
+    since these contain files needed by many builds (such as
+    <filename>/dev/null</filename>).  You can use the syntax
+    <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal>
+    to mount a path in a different location in the chroot; for
+    instance, <literal>/bin=/nix-bin</literal> will mount the
+    directory <literal>/nix-bin</literal> as <literal>/bin</literal>
+    inside the chroot.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>build-use-substitutes</literal></term>
 
     <listitem><para>If set to <literal>true</literal> (default), Nix
@@ -243,7 +278,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-    
+
   <varlistentry><term><literal>build-fallback</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will fall
@@ -253,34 +288,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
 
   </varlistentry>
 
-    
-  <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term>
 
-    <listitem><para>When builds are performed in a chroot environment,
-    Nix will mount (using <command>mount --bind</command> on Linux)
-    some directories from the normal file system hierarchy inside the
-    chroot.  These are the Nix store, the temporary build directory
-    (usually
-    <filename>/tmp/nix-<replaceable>pid</replaceable>-<replaceable>number</replaceable></filename>)
-    and the directories listed here.  The default is <literal>dev
-    /proc</literal>.  Files in <filename>/dev</filename> (such as
-    <filename>/dev/null</filename>) are needed by many builds, and
-    some files in <filename>/proc</filename> may also be needed
-    occasionally.</para>
-
-    <para>The value used on NixOS is
-    
-<programlisting>
-build-use-chroot = /dev /proc /bin</programlisting>
-
-    to make the <filename>/bin/sh</filename> symlink available (which
-    is still needed by many builders).</para>
-
-    </listitem>
-
-  </varlistentry>
-
-  
   <varlistentry><term><literal>build-cache-failures</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, Nix will
@@ -331,8 +339,8 @@ build-use-chroot = /dev /proc /bin</programlisting>
   <varlistentry><term><literal>binary-caches</literal></term>
 
     <listitem><para>A list of URLs of binary caches, separated by
-    whitespace.  The default is empty.<!-- The default is
+    whitespace.  The default is
-    <literal>http://nixos.org/binary-cache</literal>. --></para></listitem>
+    <literal>http://cache.nixos.org</literal>.</para></listitem>
 
   </varlistentry>
 
@@ -341,10 +349,12 @@ build-use-chroot = /dev /proc /bin</programlisting>
 
     <listitem><para>A list of names of files that will be read to
     obtain additional binary cache URLs.  The default is
-    <literal>/nix/var/nix/profiles/per-user/root/channels/binary-caches/*</literal>,
+    <literal>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels/binary-caches/*</literal>.
-    which ensures that Nix will use the binary caches corresponding to
+    Note that when you’re using the Nix daemon,
-    the channels installed by root.  Do not set this option to read
+    <replaceable>username</replaceable> is always equal to
-    files created by untrusted users!</para></listitem>
+    <literal>root</literal>, so Nix will only use the binary caches
+    provided by the channels installed by root.  Do not set this
+    option to read files created by untrusted users!</para></listitem>
 
   </varlistentry>
 
@@ -355,13 +365,25 @@ build-use-chroot = /dev /proc /bin</programlisting>
     whitespace.  These are not used by default, but can be enabled by
     users of the Nix daemon by specifying <literal>--option
     binary-caches <replaceable>urls</replaceable></literal> on the
-    command line.  Daemon users are only allowed to pass a subset of
+    command line.  Unprivileged users are only allowed to pass a
-    the URLs listed in <literal>binary-caches</literal> and
+    subset of the URLs listed in <literal>binary-caches</literal> and
     <literal>trusted-binary-caches</literal>.</para></listitem>
 
   </varlistentry>
 
 
+  <varlistentry><term><literal>extra-binary-caches</literal></term>
+
+    <listitem><para>Additional binary caches appended to those
+    specified in <option>binary-caches</option> and
+    <option>binary-caches-files</option>.  When used by unprivileged
+    users, untrusted binary caches (i.e. those not listed in
+    <option>trusted-binary-caches</option>) are silently
+    ignored.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
 
     <listitem><para>The maximum number of parallel HTTP connections
@@ -415,15 +437,16 @@ build-use-chroot = /dev /proc /bin</programlisting>
 
   </varlistentry>
 
-  
+
   <varlistentry><term><literal>auto-optimise-store</literal></term>
 
-    <listitem><para>If set to <literal>true</literal> (the default),
+    <listitem><para>If set to <literal>true</literal>, Nix
-    Nix automatically detects files in the store that have identical
+    automatically detects files in the store that have identical
     contents, and replaces them with hard links to a single copy.
-    This saves disk space.  If set to <literal>false</literal>, you
+    This saves disk space.  If set to <literal>false</literal> (the
-    can still run <command>nix-store --optimise</command> to get rid
+    default), you can still run <command>nix-store
-    of duplicate files.</para></listitem>
+    --optimise</command> to get rid of duplicate
+    files.</para></listitem>
 
   </varlistentry>
 

doc/manual/env-common.xml

@@ -283,7 +283,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   <listitem><para>This variable should be set to
   <literal>daemon</literal> if you want to use the Nix daemon to
-  executed Nix operations, which is necessary in <link
+  execute Nix operations. This is necessary in <link
   linkend="ssec-multi-user">multi-user Nix installations</link>.
   Otherwise, it should be left unset.</para></listitem>
 
@@ -293,7 +293,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry xml:id="envar-other-stores"><term><envar>NIX_OTHER_STORES</envar></term>
 
   <listitem><para>This variable contains the paths of remote Nix
-  installations from whichs paths can be copied, separated by colons.
+  installations from which packages can be copied, separated by colons.
   <phrase condition="manual">See <xref linkend="sec-sharing-packages"
   /> for details.</phrase>  Each path should be the
   <filename>/nix</filename> directory of a remote Nix installation
@@ -312,6 +312,24 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
 
+<varlistentry><term><envar>NIX_SHOW_STATS</envar></term>
+
+  <listitem><para>If set to <literal>1</literal>, Nix will print some
+  evaluation statistics, such as the number of values
+  allocated.</para></listitem>
+
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_COUNT_CALLS</envar></term>
+
+  <listitem><para>If set to <literal>1</literal>, Nix will print how
+  often functions were called during Nix expression evaluation.  This
+  is useful for profiling your Nix expressions.</para></listitem>
+
+</varlistentry>
+
+
 <varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term>
 
   <listitem><para>If Nix has been configured to use the Boehm garbage

doc/manual/hacking.xml

@@ -0,0 +1,41 @@
+<appendix xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xml:id="chap-hacking">
+
+<title>Hacking</title>
+
+<para>This section provides some notes on how to hack on Nix.  To get
+the latest version of Nix from GitHub:
+<screen>
+$ git clone git://github.com/NixOS/nix.git
+$ cd hydra
+</screen>
+</para>
+
+<para>To build it and its dependencies:
+<screen>
+$ nix-build release.nix -A build.x86_64-linux
+</screen>
+</para>
+
+<para>To build all dependencies and start a shell in which all
+environment variables are set up so that those dependencies can be
+found:
+<screen>
+$ ./dev-shell
+</screen>
+To build Nix itself in this shell:
+<screen>
+[nix-shell]$ ./bootstrap
+[nix-shell]$ configurePhase
+[nix-shell]$ make
+</screen>
+To test it:
+<screen>
+[nix-shell]$ make install
+[nix-shell]$ make installcheck
+</screen>
+
+</para>
+
+</appendix>

doc/manual/installation.xml

@@ -48,7 +48,7 @@ Debian, Ubuntu, Mac OS X and various other systems from the <link
 xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
 You can also get builds of the latest development release from our
 <link
-xlink:href="http://hydra.nixos.org/view/nix/trunk/latest">continuous
+xlink:href="http://hydra.nixos.org/job/nix/trunk/release/latest-finished#tabs-constituents">continuous
 build system</link>.</para>
 
 <para>For Fedora, RPM packages are available.  These can be installed
@@ -166,11 +166,10 @@ a source distribution.</para>
 
   <listitem><para>Recent versions of Bison and Flex to build the
   parser.  (This is because Nix needs GLR support in Bison and
-  reentrancy support in Flex.)  For Bison, you need version 2.3 or
+  reentrancy support in Flex.)  For Bison, you need version 2.6, which
-  higher (1.875 does <emphasis>not</emphasis> work), which can be
+  can be obtained from the <link
-  obtained from the <link
   xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP
-  server</link>.  For Flex, you need version 2.5.33, which is
+  server</link>.  For Flex, you need version 2.5.35, which is
   available on <link
   xlink:href="http://lex.sourceforge.net/">SourceForge</link>.
   Slightly older versions may also work, but ancient versions like the
@@ -189,7 +188,7 @@ a source distribution.</para>
 downloaded from the <link
 xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>.
 You can also grab the <link
-xlink:href="http://hydra.nixos.org/view/nix/trunk/latest/tarball/download-by-type/file/source-dist">most
+xlink:href="http://hydra.nixos.org/job/nix/trunk/release/latest-finished#tabs-constituents">most
 recent development release</link>.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
@@ -539,7 +538,7 @@ a symbolic link to the current <emphasis>user environment</emphasis>
 installed packages).  The simplest way to set the required environment
 variables is to include the file
 <filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
-in your <filename>~/.bashrc</filename> (or similar), like this:</para>
+in your <filename>~/.profile</filename> (or similar), like this:</para>
 
 <screen>
 source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>

doc/manual/manual.xml

@@ -19,15 +19,15 @@
     </author>
 
     <copyright>
-      <year>2004-2012</year>
+      <year>2004-2013</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>May 2012</date>
+    <date>July 2013</date>
-    
+
   </info>
 
-  
+
   <xi:include href="introduction.xml" />
   <xi:include href="quick-start.xml" />
   <xi:include href="installation.xml" />
@@ -40,17 +40,18 @@
     <title>Command Reference</title>
     <xi:include href="opt-common.xml" />
     <xi:include href="env-common.xml" />
-    
+
     <section>
       <title>Main commands</title>
       <xi:include href="nix-env.xml" />
       <xi:include href="nix-instantiate.xml" />
       <xi:include href="nix-store.xml" />
     </section>
-    
+
     <section>
       <title>Utilities</title>
       <xi:include href="nix-build.xml" />
+      <xi:include href="nix-shell.xml" />
       <xi:include href="nix-channel.xml" />
       <xi:include href="nix-collect-garbage.xml" />
       <xi:include href="nix-copy-closure.xml" />
@@ -66,17 +67,18 @@
       <title>Files</title>
       <xi:include href="conf-file.xml" />
     </section>
-    
+
   </appendix>
 
   <xi:include href="troubleshooting.xml" />
   <!-- <xi:include href="bugs.xml" /> -->
   <xi:include href="glossary.xml" />
+  <xi:include href="hacking.xml" />
 
   <appendix>
     <title>Nix Release Notes</title>
     <xi:include href="release-notes.xml"
                 xpointer="xmlns(x=http://docbook.org/ns/docbook)xpointer(x:article/x:section)" />
   </appendix>
-    
+
 </book>

doc/manual/nix-build.xml

@@ -38,11 +38,6 @@
       </group>
       <replaceable>outlink</replaceable>
     </arg>
-    <arg>
-      <option>--run-env</option>
-      <arg><option>--command</option> <replaceable>cmd</replaceable></arg>
-      <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
-    </arg>
     <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
@@ -75,13 +70,6 @@ a root of the Nix garbage collector.  This root disappears
 automatically when the <filename>result</filename> symlink is deleted
 or renamed.  So don’t rename the symlink.</para></warning>
 
-<para>The subcommand <command>nix-build --run-env</command> will build
-the dependencies of the derivation, but not the derivation itself.  It
-will then start an interactive shell in which all environment
-variables defined by the derivation have been set to their
-corresponding values.  This is useful for reproducing the environment
-of a derivation for development.</para>
-
 </refsection>
 
 
@@ -134,32 +122,12 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
 </variablelist>
 
+<para>The following common options are supported:</para>
+
 <variablelist condition="manpage">
   <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
 </variablelist>
 
-<para>The following options apply to <command>nix-build --run-env</command>.</para>
-
-<variablelist>
-
-  <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term>
-
-    <listitem><para>In the environment of the derivation, executeq the
-    command <replaceable>cmd</replaceable> instead of the default
-    interactive shell.</para></listitem>
-
-  </varlistentry>
-
-  <varlistentry><term><option>--exclude</option> <replaceable>regexp</replaceable></term>
-
-    <listitem><para>Do not build any dependencies whose store path
-    matches the regular expression <replaceable>regexp</replaceable>.
-    This option may be specified multiple times.</para></listitem>
-
-  </varlistentry>
-
-</variablelist>
-
 </refsection>
 
 
@@ -176,20 +144,6 @@ lrwxrwxrwx  <replaceable>...</replaceable>  result -> /nix/store/d18hyl92g30l...
 $ ls ./result/bin/
 firefox  firefox-config</screen>
 
-<para>To build the dependencies of the package Pan, and start an
-interactive shell in which to build it:
-
-<screen>
-$ nix-build '&lt;nixpkgs>' --run-env -A pan
-$ tar xf $src
-$ cd pan-*
-$ ./configure
-$ make
-$ ./pan/gui/pan
-</screen>
-
-</para>
-
 <para>If a derivation has multiple outputs,
 <command>nix-build</command> will build the default (first) output.
 You can also build all outputs:

doc/manual/nix-env.xml

@@ -93,11 +93,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
     <listitem><para>Specifies the profile to be used by those
     operations that operate on a profile (designated below as the
-    <emphasis>active profile</emphasis>).  A profile is sequence of
+    <emphasis>active profile</emphasis>).  A profile is a sequence of
     user environments called <emphasis>generations</emphasis>, one of
-    which is the <emphasis>current generation</emphasis>.  The default
+    which is the <emphasis>current
-    profile is the target of the symbolic link
+    generation</emphasis>.</para></listitem>
-    <filename>~/.nix-profile</filename> (see below).</para></listitem>
 
   </varlistentry>
 
@@ -154,11 +153,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     default.</para>
 
     <para>The Nix expressions in this directory are combined into a
-    single attribute set, with each file as an attribute that has the
+    single set, with each file as an attribute that has the name of
-    name of the file.  Thus, if <filename>~/.nix-defexpr</filename>
+    the file.  Thus, if <filename>~/.nix-defexpr</filename> contains
-    contains two files, <filename>foo</filename> and
+    two files, <filename>foo</filename> and <filename>bar</filename>,
-    <filename>bar</filename>, then the default Nix expression will
+    then the default Nix expression will essentially be
-    essentially be
 
 <programlisting>
 {
@@ -210,6 +208,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     <arg choice='plain'><option>--preserve-installed</option></arg>
     <arg choice='plain'><option>-P</option></arg>
   </group>
+  <group choice='opt'>
+    <arg choice='plain'><option>--remove-all</option></arg>
+    <arg choice='plain'><option>-r</option></arg>
+  </group>
   <arg choice='plain' rep='repeat'><replaceable>args</replaceable></arg>
 </cmdsynopsis>
 
@@ -273,7 +275,7 @@ number of possible ways:
   linkend="ss-functions">functions</link> that are called with the
   active Nix expression as their single argument.  The derivations
   returned by those function calls are installed.  This allows
-  derivations to be specified in a unambiguous way, which is necessary
+  derivations to be specified in an unambiguous way, which is necessary
   if there are multiple derivations with the same
   name.</para></listitem>
 
@@ -319,6 +321,16 @@ number of possible ways:
 
   </varlistentry>
 
+  <varlistentry><term><option>--remove-all</option></term>
+    <term><option>-r</option></term>
+
+    <listitem><para>Remove all previously installed packages first.
+    This is equivalent to running <literal>nix-env -e '*'</literal>
+    first, except that everything happens in a single
+    transaction.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </refsection>
@@ -392,7 +404,7 @@ $ nix-env -f ./foo.nix -i -E \
 I.e., this evaluates to <literal>(f: (f {system =
 "i686-linux";}).subversionWithJava) (import ./foo.nix)</literal>, thus
 selecting the <literal>subversionWithJava</literal> attribute from the
-attribute set returned by calling the function defined in
+set returned by calling the function defined in
 <filename>./foo.nix</filename>.</para>
 
 <para>A dry-run tells you which paths will be downloaded or built from
@@ -693,7 +705,7 @@ firefox-2.0.0.9 <lineannotation>(the current one)</lineannotation>
 $ nix-env --preserve-installed -i firefox-2.0.0.11
 installing `firefox-2.0.0.11'
 building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment'
-Collission between `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.11/bin/firefox'
+collision between `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.11/bin/firefox'
   and `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.9/bin/firefox'.
 <lineannotation>(i.e., can’t have two active at the same time)</lineannotation>
 
@@ -1245,7 +1257,7 @@ wrapper around <option>--list-generations</option> and
 $ nix-env --rollback
 switching from generation 92 to 91
 
-$ nix-env --rolback
+$ nix-env --rollback
 error: no generation older than the current (91) exists</screen>
 
 </refsection>
@@ -1256,6 +1268,16 @@ error: no generation older than the current (91) exists</screen>
 <refsection condition="manpage"><title>Environment variables</title>
 
 <variablelist>
+  
+  <varlistentry><term><envar>NIX_PROFILE</envar></term>
+
+    <listitem><para>Location of the Nix profile.  Defaults to the
+    target of the symlink <filename>~/.nix-profile</filename>, if it
+    exists, or <filename>/nix/var/nix/profiles/default</filename>
+    otherwise.</para></listitem>
+
+  </varlistentry>
+
   <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
 </variablelist>
 

doc/manual/nix-push.xml

@@ -20,6 +20,7 @@
     <command>nix-push</command>
     <arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg>
     <arg><option>--bzip2</option></arg>
+    <arg><option>--none</option></arg>
     <arg><option>--force</option></arg>
     <arg><option>--link</option></arg>
     <arg><option>--manifest</option></arg>
@@ -106,6 +107,12 @@ automatically.</para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--none</option></term>
+
+    <listitem><para>Do not compress NARs.</para></listitem>
+
+  </varlistentry>
+
   <varlistentry><term><option>--force</option></term>
 
     <listitem><para>Overwrite <filename>.narinfo</filename> files if
@@ -252,7 +259,7 @@ The properties that are currently supported are:
     <listitem><para>Each binary cache has a priority (defaulting to
     50).  Binary caches are checked for binaries in order of ascending
     priority; thus a higher number denotes a lower priority.  The
-    binary cache <uri>http://nixos.org/binary-cache</uri> has priority
+    binary cache <uri>http://cache.nixos.org</uri> has priority
     40.</para></listitem>
 
   </varlistentry>
@@ -270,14 +277,14 @@ URL <replaceable>url</replaceable> has a binary for
 <replaceable>p</replaceable>, Nix fetches
 <replaceable>url/h</replaceable>, where <replaceable>h</replaceable>
 is the hash part of <replaceable>p</replaceable>.  Thus, if we have a
-cache <uri>http://nixos.org/binary-cache</uri> and we want to obtain
+cache <uri>http://cache.nixos.org</uri> and we want to obtain
 the store path
 <screen>
 /nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7
 </screen>
 then Nix will attempt to fetch
 <screen>
-http://nixos.org/binary-cache/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo
+http://cache.nixos.org/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo
 </screen>
 (Commands such as <command>nix-env -qas</command> will issue an HTTP
 HEAD request, since it only needs to know if the
@@ -381,7 +388,7 @@ The fields are as follows:
 references exist (e.g.,
 <filename>/nix/store/2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16</filename>),
 Nix will fetch <screen>
-http://nixos.org/binary-cache/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2
+http://cache.nixos.org/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2
 </screen> and decompress and unpack it to
 <filename>/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7</filename>.</para>
 

doc/manual/nix-shell.xml

@@ -0,0 +1,142 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-shell">
+
+<refmeta>
+  <refentrytitle>nix-shell</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-shell</refname>
+  <refpurpose>start an interactive shell based on a Nix expression</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-shell</command>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
+    <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
+    <arg>
+      <group choice='req'>
+        <arg choice='plain'><option>--attr</option></arg>
+        <arg choice='plain'><option>-A</option></arg>
+      </group>
+      <replaceable>attrPath</replaceable>
+    </arg>
+    <arg><option>--command</option> <replaceable>cmd</replaceable></arg>
+    <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
+    <arg><option>--pure</option></arg>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>The command <command>nix-shell</command> will build
+the dependencies of the specified derivation, but not the derivation
+itself.  It will then start an interactive shell in which all
+environment variables defined by the derivation have been set to their
+corresponding values, and the script <literal>$stdenv/setup</literal>
+has been sourced.  This is useful for reproducing the environment of a
+derivation for development.</para>
+
+</refsection>
+
+
+<refsection><title>Options</title>
+
+<para>All options not listed here are passed to <command>nix-store
+--realise</command>, except for <option>--arg</option> and
+<option>--attr</option> / <option>-A</option> which are passed to
+<command>nix-instantiate</command>.  <phrase condition="manual">See
+also <xref linkend="sec-common-options" />.</phrase></para>
+
+<variablelist>
+
+  <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term>
+
+    <listitem><para>In the environment of the derivation, run the
+    shell command <replaceable>cmd</replaceable> instead of starting
+    an interactive shell.  However, if you end the shell command with
+    <literal>return</literal>, you still get an interactive shell.
+    This can be useful for doing any additional
+    initialisation.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--exclude</option> <replaceable>regexp</replaceable></term>
+
+    <listitem><para>Do not build any dependencies whose store path
+    matches the regular expression <replaceable>regexp</replaceable>.
+    This option may be specified multiple times.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--pure</option></term>
+
+    <listitem><para>If this flag is specified, the environment is
+    almost entirely cleared before the interactive shell is started,
+    so you get an environment that more closely corresponds to the
+    “real” Nix build.  A few variables, in particular
+    <envar>HOME</envar>, <envar>USER</envar> and
+    <envar>DISPLAY</envar>, are retained.  Note that
+    <filename>~/.bashrc</filename> and (depending on your Bash
+    installation) <filename>/etc/bashrc</filename> are still sourced,
+    so any variables set there will affect the interactive
+    shell.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+<para>The following common options are supported:</para>
+
+<variablelist condition="manpage">
+  <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" />
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<para>To build the dependencies of the package Pan, and start an
+interactive shell in which to build it:
+
+<screen>
+$ nix-shell '&lt;nixpkgs>' -A pan
+$ unpackPhase
+$ cd pan-*
+$ configurePhase
+$ buildPhase
+$ ./pan/gui/pan
+</screen>
+
+To clear the environment first, and do some additional automatic
+initialisation of the interactive shell:
+
+<screen>
+$ nix-shell '&lt;nixpkgs>' -A pan --pure \
+    --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
+</screen>
+
+</para>
+
+</refsection>
+
+
+<refsection condition="manpage"><title>Environment variables</title>
+
+<variablelist>
+  <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" />
+</variablelist>
+
+</refsection>
+
+
+</refentry>

doc/manual/nix-store.xml

@@ -164,7 +164,7 @@ the specified store paths.  Realisation is a somewhat overloaded term:
   exist in the file system).  If the path is already valid, we are
   done immediately.  Otherwise, the path and any missing paths in its
   closure may be produced through substitutes.  If there are no
-  (succesful) subsitutes, realisation fails.</para></listitem>
+  (successful) subsitutes, realisation fails.</para></listitem>
 
 </itemizedlist>
 
@@ -852,7 +852,7 @@ in Nix itself.</para>
 
 <refsection><title>Description</title>
             
-<para>The operation <option>--verify-paths</option> compares the
+<para>The operation <option>--verify-path</option> compares the
 contents of the given store paths to their cryptographic hashes stored
 in Nix’s database.  For every changed path, it prints a warning
 message.  The exit status is 0 if no path has changed, and 1
@@ -1242,9 +1242,9 @@ variable <envar>_args</envar>.</para>
 $ nix-store --print-env $(nix-instantiate '&lt;nixpkgs>' -A firefox)
 <replaceable>…</replaceable>
 export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2'
-export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnm-stdenv'
+export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv'
 export system; system='x86_64-linux'
-export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh'
+export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh'
 </screen>
 
 </refsection>

doc/manual/opt-common.xml

@@ -181,7 +181,7 @@
   from, say, a network repository.  If the repository is down, the
   realisation of the derivation will fail.  When this option is
   specified, Nix will build the derivation instead.  Thus,
-  installation from binaries falls back on nstallation from source.
+  installation from binaries falls back on installation from source.
   This option is not the default since it is generally not desirable
   for a transient failure in obtaining the substitutes to lead to a
   full build from source (with the related consumption of
@@ -236,7 +236,7 @@
       interpreted by the <command>nix-log2xml</command> tool in the
       Nix source distribution.  The resulting XML file can be fed into
       the <command>log2html.xsl</command> stylesheet to create an HTML
-      file that can be browsed interactively, using Javascript to
+      file that can be browsed interactively, using JavaScript to
       expand and collapse parts of the output.</para></listitem>
 
     </varlistentry>
@@ -343,10 +343,11 @@
 
 <varlistentry><term><option>-I</option> <replaceable>path</replaceable></term>
   
-  <listitem><para>Add a path to the Nix expression search path.  See
+  <listitem><para>Add a path to the Nix expression search path.  This
-  the <envar>NIX_PATH</envar> environment variable for details.  Paths
+  option may be given multiple times.  See the <envar>NIX_PATH</envar>
-  added through <option>-I</option> take precedence over
+  environment variable for information on the semantics of the Nix
-  <envar>NIX_PATH</envar>.</para></listitem>
+  search path.  Paths added through <option>-I</option> take
+  precedence over <envar>NIX_PATH</envar>.</para></listitem>
   
 </varlistentry>
 

doc/manual/package-management.xml

@@ -129,7 +129,7 @@ downloading binaries from <systemitem
 class='fqdomainname'>nixos.org</systemitem>, instead of building
 them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
-such as an DSL line it’s on the order of a few minutes.</para>
+such as a DSL line it’s on the order of a few minutes.</para>
 
 <para>Naturally, packages can also be uninstalled:
 
@@ -225,7 +225,7 @@ uses is to create directory trees of symlinks to
 themselves (though automatically generated by
 <command>nix-env</command>), so they too reside in the Nix store.  For
 instance, in <xref linkend='fig-user-environments' /> the user
-environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
+environment <filename>/nix/store/0c1p5z4kda11...-user-env</filename>
 contains a symlink to just Subversion 1.1.2 (arrows in the figure
 indicate symlinks).  This would be what we would obtain if we had done
 
@@ -538,7 +538,7 @@ dependencies) to a file, and then unpack that file into another Nix
 store.  For example,
 
 <screen>
-$ nix-store --export $(type -p firefox) > firefox.closure</screen>
+$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen>
 
 writes the closure of Firefox to a file.  You can then copy this file
 to another machine and install the closure:
@@ -552,7 +552,7 @@ another command, e.g. to copy and install a closure directly to/on
 another machine:
 
 <screen>
-$ nix-store --export $(type -p firefox) | bzip2 | \
+$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \
     ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
 
 But note that <command>nix-copy-closure</command> is generally more
@@ -583,7 +583,7 @@ $ nix-env -i firefox</screen>
 and Nix has to build a path that it sees is already present in
 <filename>/mnt/nix</filename>, then it will just copy from there
 instead of building it from source.</para>
- 
+
 
 </section>
 

doc/manual/quick-start.xml

@@ -14,9 +14,10 @@ to the following chapters.</para>
 <listitem><para>Download a source tarball or RPM or Debian/Ubuntu
 package from <link xlink:href='http://nixos.org/'/>.  Build source
 distributions using the regular sequence:
-        
+
 <screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
+$ cd nix-<replaceable>version</replaceable>
 $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
@@ -35,7 +36,7 @@ changed using
 
 <listitem><para>You should add
 <filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
-to your <filename>~/.bashrc</filename> (or some other login
+to your <filename>~/.profile</filename> (or some other login
 file).</para></listitem>
 
 <listitem><para>Subscribe to the Nix Packages channel.
@@ -69,7 +70,7 @@ libxslt-1.1.0
 </para></listitem>
 
 <listitem><para>Install some packages from the channel:
-        
+
 <screen>
 $ nix-env -i hello firefox <replaceable>...</replaceable> </screen>
 
@@ -87,7 +88,7 @@ $ firefox
 <lineannotation>(read Slashdot or something)</lineannotation></screen>
 
 </para></listitem>
-    
+
 <listitem><para>Uninstall a package:
 
 <screen>

doc/manual/release-notes.xml

@@ -5,6 +5,329 @@
 <title>Nix Release Notes</title>
 
 
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.6.1"><title>Release 1.6.1 (October 28, 2013)</title>
+
+<para>This is primarily a bug fix release.  Changes of interest
+are:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix 1.6 accidentally changed the semantics of antiquoted
+    paths in strings, such as <literal>"${/foo}/bar"</literal>.  This
+    release reverts to the Nix 1.5.3 behaviour.</para>
+  </listitem>
+
+  <listitem>
+    <para>Previously, Nix optimised expressions such as
+    <literal>"${<replaceable>expr</replaceable>}"</literal> to
+    <replaceable>expr</replaceable>.  Thus it neither checked whether
+    <replaceable>expr</replaceable> could be coerced to a string, nor
+    applied such coercions.  This meant that
+    <literal>"${123}"</literal> evaluatued to <literal>123</literal>,
+    and <literal>"${./foo}"</literal> evaluated to
+    <literal>./foo</literal> (even though
+    <literal>"${./foo} "</literal> evaluates to
+    <literal>"/nix/store/<replaceable>hash</replaceable>-foo "</literal>).
+    Nix now checks the type of antiquoted expressions and
+    applies coercions.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now shows the exact position of undefined variables.  In
+    particular, undefined variable errors in a <literal>with</literal>
+    previously didn't show <emphasis>any</emphasis> position
+    information, so this makes it a lot easier to fix such
+    errors.</para>
+  </listitem>
+
+  <listitem>
+    <para>Undefined variables are now treated consistently.
+    Previously, the <function>tryEval</function> function would catch
+    undefined variables inside a <literal>with</literal> but not
+    outside.  Now <function>tryEval</function> never catches undefined
+    variables.</para>
+  </listitem>
+
+  <listitem>
+    <para>Bash completion in <command>nix-shell</command> now works
+    correctly.</para>
+  </listitem>
+
+  <listitem>
+    <para>Stack traces are less verbose: they no longer show calls to
+    builtin functions and only show a single line for each derivation
+    on the call stack.</para>
+  </listitem>
+
+  <listitem>
+    <para>New built-in function: <function>builtins.typeOf</function>,
+    which returns the type of its argument as a string.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.6.0"><title>Release 1.6 (September 10, 2013)</title>
+
+<para>In addition to the usual bug fixes, this release has several new
+features:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The command <command>nix-build --run-env</command> has been
+    renamed to <command>nix-shell</command>.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-shell</command> now sources
+    <filename>$stdenv/setup</filename> <emphasis>inside</emphasis> the
+    interactive shell, rather than in a parent shell.  This ensures
+    that shell functions defined by <literal>stdenv</literal> can be
+    used in the interactive shell.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-shell</command> has a new flag
+    <option>--pure</option> to clear the environment, so you get an
+    environment that more closely corresponds to the “real” Nix build.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-shell</command> now sets the shell prompt
+    (<envar>PS1</envar>) to ensure that Nix shells are distinguishable
+    from your regular shells.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-env</command> no longer requires a
+    <literal>*</literal> argument to match all packages, so
+    <literal>nix-env -qa</literal> is equivalent to <literal>nix-env
+    -qa '*'</literal>.</para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-env -i</command> has a new flag
+    <option>--remove-all</option> (<option>-r</option>) to remove all
+    previous packages from the profile.  This makes it easier to do
+    declarative package management similar to NixOS’s
+    <option>environment.systemPackages</option>.  For instance, if you
+    have a specification <filename>my-packages.nix</filename> like this:
+
+<programlisting>
+with import &lt;nixpkgs> {};
+[ thunderbird
+  geeqie
+  ...
+]
+</programlisting>
+
+    then after any change to this file, you can run:
+
+<screen>
+$ nix-env -f my-packages.nix -ir
+</screen>
+
+    to update your profile to match the specification.</para>
+  </listitem>
+
+  <listitem>
+    <para>The ‘<literal>with</literal>’ language construct is now more
+    lazy.  It only evaluates its argument if a variable might actually
+    refer to an attribute in the argument.  For instance, this now
+    works:
+
+<programlisting>
+let
+  pkgs = with pkgs; { foo = "old"; bar = foo; } // overrides;
+  overrides = { foo = "new"; };
+in pkgs.bar
+</programlisting>
+
+    This evaluates to <literal>"new"</literal>, while previously it
+    gave an “infinite recursion” error.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix now has proper integer arithmetic operators. For
+    instance, you can write <literal>x + y</literal> instead of
+    <literal>builtins.add x y</literal>, or <literal>x &lt;
+    y</literal> instead of <literal>builtins.lessThan x y</literal>.
+    The comparison operators also work on strings.</para>
+  </listitem>
+
+  <listitem>
+    <para>On 64-bit systems, Nix integers are now 64 bits rather than
+    32 bits.</para>
+  </listitem>
+
+  <listitem>
+    <para>When using the Nix daemon, the <command>nix-daemon</command>
+    worker process now runs on the same CPU as the client, on systems
+    that support setting CPU affinity.  This gives a significant speedup
+    on some systems.</para>
+  </listitem>
+
+  <listitem>
+    <para>If a stack overflow occurs in the Nix evaluator, you now get
+    a proper error message (rather than “Segmentation fault”) on some
+    systems.</para>
+  </listitem>
+
+  <listitem>
+    <para>In addition to directories, you can now bind-mount regular
+    files in chroots through the (now misnamed) option
+    <option>build-chroot-dirs</option>.</para>
+  </listitem>
+
+</itemizedlist>
+
+<para>This release has contributions from Domen Kožar, Eelco Dolstra,
+Florian Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea
+Levy.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5.3"><title>Release 1.5.3 (June 17, 2013)</title>
+
+<para>This is primarily a bug fix release.  The following changes are
+noteworthy:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Yet another security bug involving hard links to files
+    outside the store was fixed.  This bug only affected multi-user
+    installations that do not have hard link restrictions
+    enabled.  (NixOS is thus not vulnerable.)</para>
+  </listitem>
+
+  <listitem>
+    <para>The default binary cache URL has changed from
+    <uri>http://nixos.org/binary-cache</uri> to
+    <uri>http://cache.nixos.org</uri>.  The latter is hosted on Amazon
+    CloudFront (courtesy of <link
+    xlink:href="http://www.logicblox.com/">LogicBlox</link>) and
+    should provide better performance for users in both Europe and
+    North America.</para>
+  </listitem>
+
+  <listitem>
+    <para>The binary cache substituter now prints a warning message if
+    fetching information from the cache takes more than five seconds.
+    Thus network or server problems no longer make Nix appear to just
+    hang.</para>
+  </listitem>
+
+  <listitem>
+    <para>Stack traces now show function names, e.g.
+<screen>
+while evaluating `concatMapStrings' at `<replaceable>...</replaceable>/nixpkgs/pkgs/lib/strings.nix:18:25':
+</screen>
+      Also, if a function is called with an unexpected argument, Nix
+      now shows the name of the argument.
+    </para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5.2"><title>Release 1.5.2 (May 13, 2013)</title>
+
+<para>This is primarily a bug fix release.  It has contributions from
+Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5.1"><title>Release 1.5.1 (February 28, 2013)</title>
+
+<para>The bug fix to the bug fix had a bug itself, of course.  But
+this time it will work for sure!</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.5"><title>Release 1.5 (February 27, 2013)</title>
+
+<para>This is a brown paper bag release to fix a regression introduced
+by the hard link security fix in 1.4.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.4"><title>Release 1.4 (February 26, 2013)</title>
+
+<para>This release fixes a security bug in multi-user operation.  It
+was possible for derivations to cause the mode of files outside of the
+Nix store to be changed to 444 (read-only but world-readable) by
+creating hard links to those files (<link
+xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a79141ac4">details</link>).</para>
+
+<para>There are also the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem><para>New built-in function:
+  <function>builtins.hashString</function>.</para></listitem>
+
+  <listitem><para>Build logs are now stored in
+  <filename>/nix/var/log/nix/drvs/<replaceable>XX</replaceable>/</filename>,
+  where <replaceable>XX</replaceable> is the first two characters of
+  the derivation.  This is useful on machines that keep a lot of build
+  logs (such as Hydra servers).</para></listitem>
+
+  <listitem><para>The function <function>corepkgs/fetchurl</function>
+  can now make the downloaded file executable.  This will allow
+  getting rid of all bootstrap binaries in the Nixpkgs source
+  tree.</para></listitem>
+
+  <listitem><para>Language change: The expression <literal>"${./path}
+  ..."</literal> now evaluates to a string instead of a
+  path.</para></listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.3"><title>Release 1.3 (January 4, 2013)</title>
+
+<para>This is primarily a bug fix release.  When this version is first
+run on Linux, it removes any immutable bits from the Nix store and
+increases the schema version of the Nix store.  (The previous release
+removed support for setting the immutable bit; this release clears any
+remaining immutable bits to make certain operations more
+efficient.)</para>
+
+<para>This release has contributions from Eelco Dolstra and Stuart
+Pernsteiner.</para>
+
+</section>
+
 
 <!--==================================================================-->
 
@@ -23,10 +346,10 @@
     configuration setting <option>binary-caches</option> contains a
     list of URLs of binary caches.  For instance, doing
 <screen>
-$ nix-env -i thunderbird --option binary-caches http://nixos.org/binary-cache
+$ nix-env -i thunderbird --option binary-caches http://cache.nixos.org
 </screen>
     will install Thunderbird and its dependencies, using the available
-    pre-built binaries in <uri>http://nixos.org/binary-cache</uri>.
+    pre-built binaries in <uri>http://cache.nixos.org</uri>.
     The main advantage over the old “manifest”-based method of getting
     pre-built binaries is that you don’t have to worry about your
     manifest being in sync with the Nix expressions you’re installing
@@ -39,7 +362,7 @@ $ nix-env -i thunderbird --option binary-caches http://nixos.org/binary-cache
     used automatically if you subscribe to that channel.  If you use
     the Nixpkgs or NixOS channels
     (<uri>http://nixos.org/channels</uri>) you automatically get the
-    cache <uri>http://nixos.org/binary-cache</uri>.</para>
+    cache <uri>http://cache.nixos.org</uri>.</para>
 
     <para>Binary caches are created using <command>nix-push</command>.
     For details on the operation and format of binary caches, see the
@@ -1189,7 +1512,7 @@ irreversible.</para></warning>
   </para></listitem>
   
 
-  <listitem><para>You can now unambigously specify which derivation to
+  <listitem><para>You can now unambiguously specify which derivation to
   build or install in <command>nix-env</command>,
   <command>nix-instantiate</command> and <command>nix-build</command>
   using the <option>--attr</option> / <option>-A</option> flags, which
@@ -1329,7 +1652,7 @@ irreversible.</para></warning>
     <itemizedlist>
 
       <listitem><para>Make sure that the garbage collector can run
-      succesfully when the disk is full
+      successfully when the disk is full
       (<literal>NIX-18</literal>).</para></listitem>
 
       <listitem><para><command>nix-env</command> now locks the profile
@@ -1684,7 +2007,7 @@ $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'</screen>
   </itemizedlist></para></listitem>
 
   <listitem><para>A hash of the contents of a store path is now stored
-  in the database after a succesful build.  This allows you to check
+  in the database after a successful build.  This allows you to check
   whether store paths have been tampered with: <command>nix-store
   --verify --check-contents</command>.</para></listitem>
 
@@ -1848,7 +2171,7 @@ $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'</screen>
     <itemizedlist>
 
       <listitem><para>Derivations for other platforms are filtered out
-      (which can be overriden using
+      (which can be overridden using
       <option>--system-filter</option>).</para></listitem>
 
       <listitem><para><option>--install</option> by default now

doc/manual/troubleshooting.xml

@@ -17,7 +17,7 @@ bug tracker</link> for a list of currently known issues.</para>
 $ nix-env -i docbook-xml
 ...
 adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
-collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
+collision between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
   and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
   at /nix/store/...-builder.pl line 62.</screen>
 
@@ -68,7 +68,7 @@ in <filename>/nix/store</filename>, as can be seen using <command>ls
 $ ls -l /nix/store
 drwxrwxrwt 32000 nix nix 4620288 Sep 8 15:08 store</screen>
 
-The <literal>ext2</literal> file system is limited to a inode link
+The <literal>ext2</literal> file system is limited to an inode link
 count of 32,000 (each subdirectory increasing the count by one).
 Furthermore, the <literal>st_nlink</literal> field of the
 <function>stat</function> system call is a 16-bit value.</para>

doc/manual/writing-nix-expressions.xml

@@ -15,7 +15,7 @@ on to a more in-depth look at the Nix expression language.</para>
 For more extensive information on adding packages to the Nix Packages
 collection (such as functions in the standard environment and coding
 conventions), please consult <link
-xlink:href="http://hydra.nixos.org/job/nixpkgs/trunk/tarball/latest/download-by-type/doc/manual">its
+xlink:href="http://nixos.org/nixpkgs/manual/">its
 manual</link>.</para></note>
 
 
@@ -118,10 +118,10 @@ the single Nix expression in that directory
     <varname>stdenv.mkDerivation</varname>.
     <varname>mkDerivation</varname> is a function provided by
     <varname>stdenv</varname> that builds a package from a set of
-    <emphasis>attributes</emphasis>.  An attribute set is just a list
+    <emphasis>attributes</emphasis>.  A set is just a list of
-    of key/value pairs where each value is an arbitrary Nix
+    key/value pairs where each key is a string and each value is an
-    expression.  They take the general form
+    arbitrary Nix expression.  They take the general form <literal>{
-    <literal>{ <replaceable>name1</replaceable> =
+    <replaceable>name1</replaceable> =
     <replaceable>expr1</replaceable>; <replaceable>...</replaceable>
     <replaceable>nameN</replaceable> =
     <replaceable>exprN</replaceable>; }</literal>.</para>
@@ -384,9 +384,9 @@ some fragments of
 
     <para>This is where the actual composition takes place.  Here we
     <emphasis>call</emphasis> the function imported from
-    <filename>../applications/misc/hello/ex-1</filename> with an
+    <filename>../applications/misc/hello/ex-1</filename> with a set
-    attribute set containing the things that the function expects,
+    containing the things that the function expects, namely
-    namely <varname>fetchurl</varname>, <varname>stdenv</varname>, and
+    <varname>fetchurl</varname>, <varname>stdenv</varname>, and
     <varname>perl</varname>.  We use inherit again to use the
     attributes defined in the surrounding scope (we could also have
     written <literal>fetchurl = fetchurl;</literal>, etc.).</para>
@@ -617,7 +617,7 @@ Laziness means that arguments to functions are evaluated only when
 they are needed.  Functional means that functions are
 <quote>normal</quote> values that can be passed around and manipulated
 in interesting ways.  The language is not a full-featured, general
-purpose language.  It's main job is to describe packages,
+purpose language.  Its main job is to describe packages,
 compositions of packages, and the variability within
 packages.</para>
 
@@ -701,7 +701,7 @@ configureFlags = "
     number of spaces equal to the minimal indentation of the string as
     a whole (disregarding the indentation of empty lines).  For
     instance, the first and second line are indented two space, while
-    the third line is indented three spaces.  Thus, two spaces are
+    the third line is indented four spaces.  Thus, two spaces are
     stripped from each line, so the resulting string is
 
 <programlisting>
@@ -714,18 +714,18 @@ configureFlags = "
     text on the initial line.</para>
 
     <para>Antiquotation
-    (<literal>${<replaceable>expr</replaceable>}}</literal>) is
+    (<literal>${<replaceable>expr</replaceable>}</literal>) is
     supported in indented strings.</para>
 
     <para>Since <literal>${</literal> and <literal>''</literal> have
     special meaning in indented strings, you need a way to quote them.
     <literal>${</literal> can be escaped by prefixing it with
-    <literal>''</literal>, i.e., <literal>''${</literal>.
+    <literal>''</literal> (that is, two single quotes), i.e.,
-    <literal>''</literal> can be escaped by prefixing it with
+    <literal>''${</literal>.  <literal>''</literal> can be escaped by
-    <literal>'</literal>, i.e., <literal>'''</literal>.  Finally,
+    prefixing it with <literal>'</literal>, i.e.,
-    linefeed, carriage-return and tab characters can be writted as
+    <literal>'''</literal>.  Finally, linefeed, carriage-return and
-    <literal>''\n</literal>, <literal>''\r</literal>,
+    tab characters can be written as <literal>''\n</literal>,
-    <literal>''\t</literal>.</para>
+    <literal>''\r</literal>, <literal>''\t</literal>.</para>
 
     <para>Indented strings are primarily useful in that they allow
     multi-line string literals to follow the indentation of the
@@ -775,7 +775,7 @@ stdenv.mkDerivation {
   absolute at parse time relative to the directory of the Nix
   expression that contained it.  For instance, if a Nix expression in
   <filename>/foo/bar/bla.nix</filename> refers to
-  <filename>../xyzzy/fnord.nix</filename>, the absolutised path is
+  <filename>../xyzzy/fnord.nix</filename>, the absolute path is
   <filename>/foo/xyzzy/fnord.nix</filename>.</para></listitem>
 
   <listitem><para><emphasis>Booleans</emphasis> with values
@@ -805,20 +805,21 @@ to be enclosed in parentheses.  If they had been omitted, e.g.,
 [ 123 ./foo.nix "abc" f { x = y; } ]</programlisting>
 
 the result would be a list of five elements, the fourth one being a
-function and the fifth being an attribute set.</para>
+function and the fifth being a set.</para>
 
 </simplesect>
 
 
-<simplesect><title>Attribute sets</title>
+<simplesect><title>Sets</title>
 
-<para>Attribute sets are really the core of the language, since
+<para>Sets are really the core of the language, since ultimately the
-ultimately it's all about creating derivations, which are really just
+Nix language is all about creating derivations, which are really just
 sets of attributes to be passed to build scripts.</para>
 
-<para>Attribute sets are just a list of name/value pairs enclosed in
+<para>Sets are just a list of name/value pairs (called
-curly brackets, where each value is an arbitrary expression terminated
+<emphasis>attributes</emphasis>) enclosed in curly brackets, where
-by a semicolon.  For example:
+each value is an arbitrary expression terminated by a semicolon.  For
+example:
 
 <programlisting>
 { x = 123;
@@ -826,12 +827,12 @@ by a semicolon.  For example:
   y = f { bla = 456; };
 }</programlisting>
 
-This defines an attribute set with attributes named
+This defines a set with attributes named <varname>x</varname>,
-<varname>x</varname>, <varname>test</varname>, <varname>y</varname>.
+<varname>text</varname>, <varname>y</varname>.  The order of the
-The order of the attributes is irrelevant.  An attribute name may only
+attributes is irrelevant.  An attribute name may only occur
-occur once.</para>
+once.</para>
 
-<para>Attributes can be selected from an attribute set using the
+<para>Attributes can be selected from a set using the
 <literal>.</literal> operator.  For instance,
 
 <programlisting>
@@ -864,10 +865,10 @@ This will evaluate to <literal>123</literal>.</para>
 <section><title>Language constructs</title>
 
 
-<simplesect><title>Recursive attribute sets</title>
+<simplesect><title>Recursive sets</title>
 
-<para>Recursive attribute sets are just normal attribute sets, but the
+<para>Recursive sets are just normal sets, but the attributes can
-attributes can refer to each other.  For example,
+refer to each other.  For example,
 
 <programlisting>
 rec {
@@ -880,11 +881,11 @@ evaluates to <literal>123</literal>.  Note that without
 <literal>rec</literal> the binding <literal>x = y;</literal> would
 refer to the variable <varname>y</varname> in the surrounding scope,
 if one exists, and would be invalid if no such variable exists.  That
-is, in a normal (non-recursive) attribute set, attributes are not
+is, in a normal (non-recursive) set, attributes are not added to the
-added to the lexical scope; in a recursive set, they are.</para>
+lexical scope; in a recursive set, they are.</para>
 
-<para>Recursive attribute sets of course introduce the danger of
+<para>Recursive sets of course introduce the danger of infinite
-infinite recursion.  For example,
+recursion.  For example,
 
 <programlisting>
 rec {
@@ -901,8 +902,8 @@ encountered</quote>).</para></footnote>.</para>
 
 <simplesect><title>Let-expressions</title>
 
-<para>A let-expression allows you define local
+<para>A let-expression allows you define local variables for an
-variables for an expression.  For instance,
+expression.  For instance,
 
 <programlisting>
 let
@@ -914,20 +915,14 @@ evaluates to <literal>"foobar"</literal>.
 
 </para>
 
-<note><para>There is also an obsolete form of let-expression,
-<literal>let { <replaceable>attrs</replaceable> }</literal>, which is
-translated to <literal>rec { <replaceable>attrs</replaceable>
-}.body</literal>.  That is, the body of the let-expression is the
-<literal>body</literal> attribute of the attribute set.</para></note>
-
 </simplesect>
 
 
 <simplesect><title>Inheriting attributes</title>
 
-<para>When defining an attribute set it is often convenient to copy
+<para>When defining a set it is often convenient to copy variables
-variables from the surrounding lexical scope (e.g., when you want to
+from the surrounding lexical scope (e.g., when you want to propagate
-propagate attributes).  This can be shortened using the
+attributes).  This can be shortened using the
 <literal>inherit</literal> keyword.  For instance,
 
 <programlisting>
@@ -936,10 +931,10 @@ let x = 123; in
   y = 456;
 }</programlisting>
 
-evaluates to <literal>{ x = 123; y = 456; }</literal>.  (Note that this
+evaluates to <literal>{ x = 123; y = 456; }</literal>.  (Note that
-works because <varname>x</varname> is added to the lexical scope by
+this works because <varname>x</varname> is added to the lexical scope
-the <literal>let</literal> construct.)  It is also possible to inherit
+by the <literal>let</literal> construct.)  It is also possible to
-attributes from another attribute set.  For instance, in this fragment
+inherit attributes from another set.  For instance, in this fragment
 from <filename>all-packages.nix</filename>,
 
 <programlisting>
@@ -958,13 +953,12 @@ from <filename>all-packages.nix</filename>,
   libjpg = ...;
   ...</programlisting>
 
-the attribute set used in the function call to the function defined in
+the set used in the function call to the function defined in
 <filename>../tools/graphics/graphviz</filename> inherits a number of
 variables from the surrounding scope (<varname>fetchurl</varname>
 ... <varname>yacc</varname>), but also inherits
 <varname>libXaw</varname> (the X Athena Widgets) from the
-<varname>xlibs</varname> (X11 client-side libraries) attribute
+<varname>xlibs</varname> (X11 client-side libraries) set.</para>
-set.</para>
 
 </simplesect>
 
@@ -1003,11 +997,11 @@ map (concat "foo") [ "bar" "bla" "abc" ]</programlisting>
   "fooabc" ]</literal>.</para></listitem>
 
 
-  <listitem><para>An <emphasis>attribute set pattern</emphasis> of the
+  <listitem><para>A <emphasis>set pattern</emphasis> of the form
-  form <literal>{ name1, name2, …, nameN }</literal>
+  <literal>{ name1, name2, …, nameN }</literal> matches a set
-  matches an attribute set containing the listed attributes, and binds
+  containing the listed attributes, and binds the values of those
-  the values of those attributes to variables in the function body.
+  attributes to variables in the function body.  For example, the
-  For example, the function
+  function
 
 <programlisting>
 { x, y, z }: z + y + x</programlisting>
@@ -1174,9 +1168,8 @@ used in the Nix expression for Subversion.</para>
 <programlisting>
 with <replaceable>e1</replaceable>; <replaceable>e2</replaceable></programlisting>
 
-introduces the attribute set <replaceable>e1</replaceable> into the
+introduces the set <replaceable>e1</replaceable> into the lexical
-lexical scope of the expression <replaceable>e2</replaceable>.  For
+scope of the expression <replaceable>e2</replaceable>.  For instance,
-instance,
 
 <programlisting>
 let as = { x = "foo"; y = "bar"; };
@@ -1235,7 +1228,7 @@ weakest binding).</para>
         </entry>
         <entry>none</entry>
         <entry>Select attribute denoted by the attribute path
-        <replaceable>attrpath</replaceable> from attribute set
+        <replaceable>attrpath</replaceable> from set
         <replaceable>e</replaceable>.  (An attribute path is a
         dot-separated list of attribute names.)  If the attribute
         doesn’t exist, return <replaceable>def</replaceable> if
@@ -1251,8 +1244,8 @@ weakest binding).</para>
         <entry><replaceable>e</replaceable> <literal>?</literal>
         <replaceable>attrpath</replaceable></entry>
         <entry>none</entry>
-        <entry>Test whether attribute set <replaceable>e</replaceable>
+        <entry>Test whether set <replaceable>e</replaceable> contains
-        contains the attribute denoted by <replaceable>attrpath</replaceable>;
+        the attribute denoted by <replaceable>attrpath</replaceable>;
         return <literal>true</literal> or
         <literal>false</literal>.</entry>
       </row>
@@ -1275,10 +1268,11 @@ weakest binding).</para>
         <entry><replaceable>e1</replaceable> <literal>//</literal>
         <replaceable>e2</replaceable></entry>
         <entry>right</entry>
-        <entry>Return an attribute set consisting of the attributes in
+        <entry>Return a set consisting of the attributes in
         <replaceable>e1</replaceable> and
         <replaceable>e2</replaceable> (with the latter taking
-        precedence over the former in case of equally named attributes).</entry>
+        precedence over the former in case of equally named
+        attributes).</entry>
       </row>
       <row>
         <entry><replaceable>e1</replaceable> <literal>==</literal>
@@ -1322,9 +1316,9 @@ weakest binding).</para>
 <section xml:id="ssec-derivation"><title>Derivations</title>
 
 <para>The most important built-in function is
-<function>derivation</function>, which is used to describe a
+<function>derivation</function>, which is used to describe a single
-single derivation (a build action).  It takes as input an attribute
+derivation (a build action).  It takes as input a set, the attributes
-set, the attributes of which specify the inputs of the build.</para>
+of which specify the inputs of the build.</para>
 
 <itemizedlist>
 

nix.spec.in

@@ -22,6 +22,7 @@ Requires: /usr/bin/perl
 Requires: curl
 Requires: perl-DBD-SQLite
 Requires: bzip2
+Requires: gzip
 Requires: xz
 BuildRequires: bzip2-devel
 BuildRequires: sqlite-devel

perl/lib/Nix/CopyClosure.pm

@@ -3,16 +3,13 @@ package Nix::CopyClosure;
 use strict;
 use Nix::Config;
 use Nix::Store;
+use List::Util qw(sum);
 
 
 sub copyTo {
     my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
         $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_;
 
-    $compressor = "$compressor |" if $compressor ne "";
-    $decompressor = "$decompressor |" if $decompressor ne "";
-    $progressViewer = "$progressViewer |" if $progressViewer ne "";
-
     # Get the closure of this path.
     my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
         map { followLinksToStorePath $_ } @{$storePaths})));
@@ -28,21 +25,28 @@ sub copyTo {
     # we'll want to use ‘--from-stdin’, but we can't rely on the
     # target having this option yet.
     my @missing = ();
+    my $missingSize = 0;
     while (scalar(@closure) > 0) {
         my @ps = splice(@closure, 0, 1500);
         open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|");
         while (<READ>) {
             chomp;
             push @missing, $_;
+            my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($_, 1);
+            $missingSize += $narSize;
         }
         close READ or die;
     }
 
+    $compressor = "$compressor |" if $compressor ne "";
+    $decompressor = "$decompressor |" if $decompressor ne "";
+    $progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
+
     # Export the store paths and import them on the remote machine.
     if (scalar @missing > 0) {
         print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n";
         unless ($dryRun) {
-            open SSH, "| $compressor $progressViewer ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die;
+            open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die;
             exportPaths(fileno(SSH), $sign, @missing);
             close SSH or die "copying store paths to remote machine `$sshHost' failed: $?";
         }

perl/lib/Nix/Manifest.pm

@@ -2,6 +2,7 @@ package Nix::Manifest;
 
 use strict;
 use DBI;
+use DBD::SQLite;
 use Cwd;
 use File::stat;
 use File::Path;
@@ -227,6 +228,9 @@ sub writeManifest {
 sub updateManifestDB {
     my $manifestDir = $Nix::Config::manifestDir;
 
+    my @manifests = glob "$manifestDir/*.nixmanifest";
+    return undef if scalar @manifests == 0;
+
     mkpath($manifestDir);
 
     unlink "$manifestDir/cache.sqlite"; # remove obsolete cache
@@ -311,7 +315,7 @@ EOF
     # unless we've already done so on a previous run.
     my %seen;
 
-    for my $manifestLink (glob "$manifestDir/*.nixmanifest") {
+    for my $manifestLink (@manifests) {
         my $manifest = Cwd::abs_path($manifestLink);
         next unless -f $manifest;
         my $timestamp = lstat($manifest)->mtime;

perl/lib/Nix/SSH.pm

@@ -44,6 +44,7 @@ sub closeSSHConnection {
     if ($sshStarted) {
         system("ssh $sshHost @sshOpts -O exit 2> /dev/null") == 0
             or warn "unable to stop SSH master: $?";
+        $sshStarted = 0;
     }
 }
 

perl/lib/Nix/Store.pm

@@ -42,50 +42,50 @@ if ($Nix::Config::useBindings) {
     use File::Temp;
     use Fcntl qw/F_SETFD/;
 
-    sub hashFile {
+    *hashFile = sub {
         my ($algo, $base32, $path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
+    };
-    
+
-    sub hashPath {
+    *hashPath = sub {
         my ($algo, $base32, $path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
+    };
-    
+
-    sub hashString {
+    *hashString = sub {
         my ($algo, $base32, $s) = @_;
         my $fh = File::Temp->new();
         print $fh $s;
         my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ());
         chomp $res;
         return $res;
-    }
+    };
-    
+
-    sub addToStore {
+    *addToStore = sub {
         my ($srcPath, $recursive, $algo) = @_;
         die "not implemented" if $recursive || $algo ne "sha256";
         my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath);
         chomp $res;
         return $res;
-    }
+    };
-    
+
-    sub isValidPath {
+    *isValidPath = sub {
         my ($path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path);
         chomp $res;
         return $res ne $path;
-    }
+    };
-    
+
-    sub queryPathHash {
+    *queryPathHash = sub {
         my ($path) = @_;
         my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path);
         chomp $res;
         return $res;
-    }
+    };
 }
 
 1;

perl/lib/Nix/Store.xs

@@ -15,11 +15,12 @@
 using namespace nix;
 
 
-void doInit() 
+void doInit()
 {
     if (!store) {
         try {
             settings.processEnvironment();
+            settings.lockCPU = false;
             store = openStore();
         } catch (Error & e) {
             croak(e.what());
@@ -237,32 +238,35 @@ SV * derivationFromPath(char * drvPath)
             doInit();
             Derivation drv = derivationFromPath(*store, drvPath);
             hash = newHV();
-            
+
-            /* TODO: handle drv.outputs */
+            HV * outputs = newHV();
-            
+            for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
+                hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0);
+            hv_stores(hash, "outputs", newRV((SV *) outputs));
+
             AV * inputDrvs = newAV();
             for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
                 av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
-            
+
             AV * inputSrcs = newAV();
             for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
                 av_push(inputSrcs, newSVpv(i->c_str(), 0));
             hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs));
-            
+
             hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0));
             hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0));
-            
+
             AV * args = newAV();
             for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i)
                 av_push(args, newSVpv(i->c_str(), 0));
             hv_stores(hash, "args", newRV((SV *) args));
-            
+
             HV * env = newHV();
             for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i)
                 hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0);
             hv_stores(hash, "env", newRV((SV *) env));
-            
+
             RETVAL = newRV_noinc((SV *)hash);
         } catch (Error & e) {
             croak(e.what());

perl/lib/Nix/Utils.pm

@@ -23,7 +23,7 @@ sub uniq {
 
 sub writeFile {
     my ($fn, $s) = @_;
-    open TMP, ">$fn" or die;
+    open TMP, ">$fn" or die "cannot create file `$fn': $!";
     print TMP "$s" or die;
     close TMP or die;
 }
@@ -31,7 +31,7 @@ sub writeFile {
 sub readFile {
     local $/ = undef;
     my ($fn) = @_;
-    open TMP, "<$fn" or die;
+    open TMP, "<$fn" or die "cannot open file `$fn': $!";
     my $s = <TMP>;
     close TMP or die;
     return $s;

release.nix

@@ -1,15 +1,19 @@
-{ nixpkgs ? <nixpkgs>
+{ nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
-, nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; }
 , officialRelease ? false
 }:
 
 let
 
+  pkgs = import <nixpkgs> {};
+
+  systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "x86_64-freebsd" "i686-freebsd" ];
+
+
   jobs = rec {
 
 
     tarball =
-      with import nixpkgs {};
+      with pkgs;
 
       releaseTools.sourceTarball {
         name = "nix-tarball";
@@ -19,7 +23,7 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison25 flex2535 perl libxml2 libxslt w3m bzip2
+          [ curl bison flex2535 perl libxml2 libxslt w3m bzip2
             tetex dblatex nukeReferences pkgconfig sqlite git
           ];
 
@@ -72,10 +76,9 @@ let
       };
 
 
-    build =
+    build = pkgs.lib.genAttrs systems (system:
-      { system ? "x86_64-linux" }:
 
-      with import nixpkgs { inherit system; };
+      with import <nixpkgs> { inherit system; };
 
       releaseTools.nixBuild {
         name = "nix";
@@ -89,20 +92,25 @@ let
           --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}
           --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix}
           --enable-gc
+          --sysconfdir=/etc
         '';
 
         enableParallelBuilding = true;
 
+        makeFlags = "profiledir=$(out)/etc/profile.d";
+
+        installFlags = "sysconfdir=$(out)/etc";
+
         doInstallCheck = true;
-      };
+      });
 
-    binaryTarball =
-      { system ? "x86_64-linux" }:
 
-      with import nixpkgs { inherit system; };
+    binaryTarball = pkgs.lib.genAttrs systems (system:
+
+      with import <nixpkgs> { inherit system; };
 
       let
-        toplevel = build { inherit system; };
+        toplevel = builtins.getAttr system jobs.build;
         version = toplevel.src.version;
       in
 
@@ -125,11 +133,11 @@ let
             --transform "s,$TMPDIR/install,/usr/bin/nix-finish-install," \
             --transform "s,$TMPDIR/reginfo,/nix/store/reginfo," \
             $TMPDIR/install $TMPDIR/reginfo $storePaths
-        '';
+        '');
 
 
     coverage =
-      with import nixpkgs { system = "x86_64-linux"; };
+      with import <nixpkgs> { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
@@ -161,17 +169,19 @@ let
       };
 
 
-    rpm_fedora13i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora13i386) 50;
-    rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50;
     rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50;
     rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50;
+    rpm_fedora18i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora18i386) 60;
+    rpm_fedora18x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora18x86_64) 60;
+    rpm_fedora19i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora19i386) 70;
+    rpm_fedora19x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora19x86_64) 70;
 
 
     deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50;
     deb_debian60x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian60x86_64) 50;
+    deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386) 60;
+    deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64) 60;
 
-    deb_ubuntu1004i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1004i386) 50;
-    deb_ubuntu1004x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1004x86_64) 50;
     deb_ubuntu1010i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1010i386) 50;
     deb_ubuntu1010x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1010x86_64) 50;
     deb_ubuntu1110i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1110i386) 60;
@@ -180,17 +190,51 @@ let
     deb_ubuntu1204x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1204x86_64) 60;
     deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386) 70;
     deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64) 70;
+    deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386) 80;
+    deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64) 80;
+    deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) 90;
+    deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) 90;
 
 
     # System tests.
     tests.remote_builds = (import ./tests/remote-builds.nix rec {
-      nix = build { inherit system; }; system = "x86_64-linux";
+      nix = build.x86_64-linux; system = "x86_64-linux";
     }).test;
 
     tests.nix_copy_closure = (import ./tests/nix-copy-closure.nix rec {
-      nix = build { inherit system; }; system = "x86_64-linux";
+      nix = build.x86_64-linux; system = "x86_64-linux";
     }).test;
 
+
+    # Aggregate job containing the release-critical jobs.
+    release = pkgs.releaseTools.aggregate {
+      name = "nix-${tarball.version}";
+      meta.description = "Release-critical builds";
+      constituents =
+        [ tarball
+          build.i686-freebsd
+          build.i686-linux
+          build.x86_64-darwin
+          build.x86_64-freebsd
+          build.x86_64-linux
+          binaryTarball.i686-freebsd
+          binaryTarball.i686-linux
+          binaryTarball.x86_64-darwin
+          binaryTarball.x86_64-freebsd
+          binaryTarball.x86_64-linux
+          deb_debian7i386
+          deb_debian7x86_64
+          deb_ubuntu1304i386
+          deb_ubuntu1304x86_64
+          deb_ubuntu1310i386
+          deb_ubuntu1310x86_64
+          rpm_fedora19i386
+          rpm_fedora19x86_64
+          tests.remote_builds
+          tests.nix_copy_closure
+        ];
+    };
+
   };
 
 
@@ -200,7 +244,7 @@ let
   makeRPM =
     system: diskImageFun: prio:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm";
@@ -219,7 +263,7 @@ let
   makeDeb =
     system: diskImageFun: prio:
 
-    with import nixpkgs { inherit system; };
+    with import <nixpkgs> { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";
@@ -229,7 +273,7 @@ let
       memSize = 1024;
       meta.schedulingPriority = prio;
       configureFlags = "--sysconfdir=/etc";
-      debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" ];
+      debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" ];
       doInstallCheck = true;
     };
 

scripts/Makefile.am

@@ -7,15 +7,18 @@ noinst_SCRIPTS = nix-profile.sh \
   find-runtime-roots.pl build-remote.pl nix-reduce-build \
   copy-from-other-stores.pl nix-http-export.cgi
 
+profiledir = $(sysconfdir)/profile.d
+
 install-exec-local: download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl find-runtime-roots.pl
-	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
+	$(INSTALL) -d $(DESTDIR)$(profiledir)
-	$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
+	$(INSTALL_DATA) nix-profile.sh $(DESTDIR)$(profiledir)/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL_PROGRAM) download-using-manifests.pl copy-from-other-stores.pl download-from-binary-cache.pl $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
+	ln -sf nix-build $(DESTDIR)$(bindir)/nix-shell
 
 include ../substitute.mk
 

scripts/build-remote.pl.in

@@ -195,9 +195,13 @@ REQ: while (1) {
         # Connect to the selected machine.
         @sshOpts = ("-i", $machine->{sshKeys}, "-x");
         $hostName = $machine->{hostName};
-        last REQ if openSSHConnection $hostName;
+        if (openSSHConnection($hostName)) {
-
+            last REQ if system("ssh $hostName @sshOpts nix-builds-inhibited < /dev/null > /dev/null 2>&1") != 0;
-        warn "unable to open SSH connection to $hostName, trying other available machines...\n";
+            warn "machine `$hostName' is refusing builds, trying other available machines...\n";
+            closeSSHConnection;
+        } else {
+            warn "unable to open SSH connection to `$hostName', trying other available machines...\n";
+        }
         $machine->{enabled} = 0;
     }
 }
@@ -209,7 +213,6 @@ my @inputs = split /\s/, readline(STDIN);
 my @outputs = split /\s/, readline(STDIN);
 
 
-print STDERR "building `$drvPath' on `$hostName'\n";
 print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace;
 
 
@@ -258,8 +261,9 @@ close UPLOADLOCK;
 
 # Perform the build.
 my $buildFlags =
-    "--max-silent-time $maxSilentTime --option build-timeout $buildTimeout "
+    "--max-silent-time $maxSilentTime --option build-timeout $buildTimeout"
-    . "--fallback --add-root $rootsDir/\$PPID.out --option verbosity 0";
+    . " --fallback --add-root $rootsDir/\$PPID.out --quiet"
+    . " --option build-keep-log false --option build-use-substitutes false";
 
 # We let the remote side kill its process group when the connection is
 # closed unexpectedly.  This is necessary to ensure that no processes
@@ -268,8 +272,9 @@ my $buildFlags =
 # is interrupted unless the `-tt' flag is used to force a pseudo-tty,
 # in which case every child receives SIGHUP; however, `-tt' doesn't
 # work on some platforms when connection sharing is used.)
+print STDERR "building `$drvPath' on `$hostName'\n";
 pipe STDIN, DUMMY; # make sure we have a readable STDIN
-if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
+if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & exec nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
     # Note that if we get exit code 100 from `nix-store -r', it
     # denotes a permanent build failure (as opposed to an SSH problem
     # or a temporary Nix problem).  We propagate this to the caller to
@@ -285,13 +290,11 @@ if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store
 
 
 # Copy the output from the build machine.
-foreach my $output (@outputs) {
+my @outputs2 = grep { !isValidPath($_) } @outputs;
-    my $maybeSignRemote = "";
+if (scalar @outputs2 > 0) {
-    $maybeSignRemote = "--sign" if $UID != 0;
+    system("exec ssh $hostName @sshOpts 'nix-store --export @outputs2'" .
-    next if isValidPath($output);
+           "| NIX_HELD_LOCKS='@outputs2' @bindir@/nix-store --import > /dev/null") == 0
-    system("exec ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
+        or die("cannot copy paths " . join(", ", @outputs) . " from `$hostName': $?");
-           "| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
-        or die "cannot copy $output from $hostName: $?";
 }
 
 

scripts/copy-from-other-stores.pl.in

@@ -16,6 +16,9 @@ foreach my $dir (@remoteStoresAll) {
     push @remoteStores, glob($dir);
 }
 
+exit if scalar @remoteStores == 0;
+print "\n";
+
 
 $ENV{"NIX_REMOTE"} = "";
 
@@ -27,7 +30,7 @@ sub findStorePath {
         next unless -e $sourcePath || -l $sourcePath;
         $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
         return ($store, $sourcePath) if
-            system("@bindir@/nix-store --check-validity $storePath") == 0;
+            system("$binDir/nix-store --check-validity $storePath") == 0;
     }
     return undef;
 }
@@ -53,16 +56,16 @@ if ($ARGV[0] eq "--query") {
 
                 $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
 
-                my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
+                my $deriver = `$binDir/nix-store --query --deriver $storePath`;
                 die "cannot query deriver of `$storePath'" if $? != 0;
                 chomp $deriver;
                 $deriver = "" if $deriver eq "unknown-deriver";
 
                 my @references = split "\n",
-                    `@bindir@/nix-store --query --references $storePath`;
+                    `$binDir/nix-store --query --references $storePath`;
                 die "cannot query references of `$storePath'" if $? != 0;
 
-                my $narSize = `@bindir@/nix-store --query --size $storePath`;
+                my $narSize = `$binDir/nix-store --query --size $storePath`;
                 die "cannot query size of `$storePath'" if $? != 0;
                 chomp $narSize;
 
@@ -70,7 +73,7 @@ if ($ARGV[0] eq "--query") {
                 print "$deriver\n";
                 print scalar @references, "\n";
                 print "$_\n" foreach @references;
-                print "$narSize\n";
+                print "0\n";
                 print "$narSize\n";
             }
 

scripts/download-from-binary-cache.pl.in

@@ -1,6 +1,7 @@
 #! @perl@ -w @perlFlags@
 
 use DBI;
+use DBD::SQLite;
 use File::Basename;
 use IO::Select;
 use Nix::Config;
@@ -24,8 +25,11 @@ my $ttlNegative = 24 * 3600; # when to purge negative lookups from the database
 my $ttlNegativeUse = 3600; # how long negative lookups are valid for non-"have" lookups
 my $didExpiration = 0;
 
-my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
+my $showAfter = 5; # show that we're waiting for a request after this many seconds
-open(STDERR, ">>/dev/tty") if $debug;
+
+my $debug = ($Nix::Config::config{"debug-subst"} // "") eq 1 || ($Nix::Config::config{"untrusted-debug-subst"} // "") eq 1;
+
+my $cacheFileURLs = ($ENV{"_NIX_CACHE_FILE_URLS"} // "") eq 1; # for testing
 
 my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence, $expireNARExistence);
 
@@ -36,13 +40,16 @@ my %requests;
 my %scheduled;
 my $caBundle = $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"};
 
+my $userName = getpwuid($<) or die "cannot figure out user name";
+
 
 sub addRequest {
     my ($storePath, $url, $head) = @_;
 
     my $curl = WWW::Curl::Easy->new;
     my $curlId = $curlIdCount++;
-    $requests{$curlId} = { storePath => $storePath, url => $url, handle => $curl, content => "", type => $head ? "HEAD" : "GET" };
+    $requests{$curlId} = { storePath => $storePath, url => $url, handle => $curl, content => "", type => $head ? "HEAD" : "GET"
+                         , shown => 0, started => time() };
 
     $curl->setopt(CURLOPT_PRIVATE, $curlId);
     $curl->setopt(CURLOPT_URL, $url);
@@ -53,6 +60,7 @@ sub addRequest {
     $curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version");
     $curl->setopt(CURLOPT_NOBODY, 1) if $head;
     $curl->setopt(CURLOPT_FAILONERROR, 1);
+    $curl->setopt(CURLOPT_TIMEOUT, int($ENV{"NIX_CONNECT_TIMEOUT"} // 0));
 
     if ($activeRequests >= $maxParallelRequests) {
         $scheduled{$curlId} = 1;
@@ -72,7 +80,7 @@ sub processRequests {
 
         # Sleep until we can read or write some data.
         if (scalar @{$rfds} + scalar @{$wfds} + scalar @{$efds} > 0) {
-            IO::Select->select(IO::Select->new(@{$rfds}), IO::Select->new(@{$wfds}), IO::Select->new(@{$efds}), 0.1);
+            IO::Select->select(IO::Select->new(@{$rfds}), IO::Select->new(@{$wfds}), IO::Select->new(@{$efds}), 1.0);
         }
 
         if ($curlm->perform() != $activeRequests) {
@@ -97,6 +105,16 @@ sub processRequests {
                 }
             }
         }
+
+        my $time = time();
+        while (my ($key, $request) = each %requests) {
+            next unless defined $request->{handle};
+            next if $request->{shown};
+            if ($time > $request->{started} + $showAfter) {
+                print STDERR "still waiting for ‘$request->{url}’ after $showAfter seconds...\n";
+                $request->{shown} = 1;
+            }
+        }
     }
 }
 
@@ -182,20 +200,16 @@ sub getAvailableCaches {
     return if $gotCaches;
     $gotCaches = 1;
 
-    return if
-        ($Nix::Config::config{"use-binary-caches"} // "true") eq "false" ||
-        ($Nix::Config::config{"untrusted-use-binary-caches"} // "true") eq "false";
-
     sub strToList {
         my ($s) = @_;
         return map { s/\/+$//; $_ } split(/ /, $s);
     }
 
-    my @urls = strToList ($Nix::Config::config{"binary-caches"} // "");
+    my @urls = strToList($Nix::Config::config{"binary-caches"} //
-    # // ($Nix::Config::storeDir eq "/nix/store" ? "http://nixos.org/binary-cache" : ""));
+        ($Nix::Config::storeDir eq "/nix/store" ? "http://cache.nixos.org" : ""));
 
     my $urlsFiles = $Nix::Config::config{"binary-cache-files"}
-        // "/nix/var/nix/profiles/per-user/root/channels/binary-caches/*";
+        // "$Nix::Config::stateDir/profiles/per-user/$userName/channels/binary-caches/*";
     foreach my $urlFile (glob $urlsFiles) {
         next unless -f $urlFile;
         open FILE, "<$urlFile" or die "cannot open ‘$urlFile’\n";
@@ -204,12 +218,15 @@ sub getAvailableCaches {
         push @urls, strToList($url);
     }
 
+    push @urls, strToList($Nix::Config::config{"extra-binary-caches"} // "");
+
     # Allow Nix daemon users to override the binary caches to a subset
     # of those listed in the config file.  Note that ‘untrusted-*’
     # denotes options passed by the client.
+    my @trustedUrls = uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
+
     if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
         my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
-        my @trustedUrls = uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
         @urls = ();
         foreach my $url (@untrustedUrls) {
             die "binary cache ‘$url’ is not trusted (please add it to ‘trusted-binary-caches’ [@trustedUrls] in $Nix::Config::confDir/nix.conf)\n"
@@ -218,6 +235,12 @@ sub getAvailableCaches {
         }
     }
 
+    my @untrustedUrls = strToList $Nix::Config::config{"untrusted-extra-binary-caches"} // "";
+    foreach my $url (@untrustedUrls) {
+        next unless scalar(grep { $url eq $_ } @trustedUrls) > 0;
+        push @urls, $url;
+    }
+
     foreach my $url (uniq @urls) {
 
         # FIXME: not atomic.
@@ -252,11 +275,12 @@ sub getAvailableCaches {
             elsif ($1 eq "Priority") { $priority = int($2); }
         }
 
-        $dbh->do("insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)",
+        $dbh->do("insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)",
                  {}, $url, time(), $storeDir, $wantMassQuery, $priority);
-        my $id = $dbh->last_insert_id("", "", "", "");
+        $queryCache->execute($url);
+        $res = $queryCache->fetchrow_hashref() or die;
         next if $storeDir ne $Nix::Config::storeDir;
-        push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery, priority => $priority };
+        push @caches, { id => $res->{id}, url => $url, wantMassQuery => $wantMassQuery, priority => $priority };
     }
 
     @caches = sort { $a->{priority} <=> $b->{priority} } @caches;
@@ -265,16 +289,22 @@ sub getAvailableCaches {
 }
 
 
+sub shouldCache {
+    my ($url) = @_;
+    return $cacheFileURLs || $url !~ /^file:/;
+}
+
+
 sub processNARInfo {
     my ($storePath, $cache, $request) = @_;
 
     if ($request->{result} != 0) {
-        if ($request->{result} != 37 && $request->{httpStatus} != 404) {
+        if ($request->{result} != 37 && $request->{httpStatus} != 404 && $request->{httpStatus} != 403) {
             print STDERR "could not download ‘$request->{url}’ (" .
                 ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
         } else {
             $insertNARExistence->execute($cache->{id}, basename($storePath), 0, time())
-                unless $request->{url} =~ /^file:/;
+                if shouldCache $request->{url};
         }
         return undef;
     }
@@ -287,7 +317,7 @@ sub processNARInfo {
         $cache->{id}, basename($storePath), $narInfo->{url}, $narInfo->{compression},
         $narInfo->{fileHash}, $narInfo->{fileSize}, $narInfo->{narHash}, $narInfo->{narSize},
         join(" ", @{$narInfo->{refs}}), $narInfo->{deriver}, $narInfo->{system}, time())
-        unless $request->{url} =~ /^file:/;
+        if shouldCache $request->{url};
 
     return $narInfo;
 }
@@ -448,17 +478,17 @@ sub printSubstitutablePaths {
 
         foreach my $request (values %requests) {
             if ($request->{result} != 0) {
-                if ($request->{result} != 37 && $request->{httpStatus} != 404) {
+                if ($request->{result} != 37 && $request->{httpStatus} != 404 && $request->{httpStatus} != 403) {
                     print STDERR "could not check ‘$request->{url}’ (" .
                         ($request->{result} != 0 ? "Curl error $request->{result}" : "HTTP status $request->{httpStatus}") . ")\n";
                 } else {
                     $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 0, time())
-                        unless $request->{url} =~ /^file:/;
+                        if shouldCache $request->{url};
                 }
                 push @left2, $request->{storePath};
             } else {
                 $insertNARExistence->execute($cache->{id}, basename($request->{storePath}), 1, time())
-                    unless $request->{url} =~ /^file:/;
+                    if shouldCache $request->{url};
                 print "$request->{storePath}\n";
             }
         }
@@ -484,8 +514,9 @@ sub downloadBinary {
         next unless defined $info;
 
         my $decompressor;
-        if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
+        if ($info->{compression} eq "bzip2") { $decompressor = "| $Nix::Config::bzip2 -d"; }
-        elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
+        elsif ($info->{compression} eq "xz") { $decompressor = "| $Nix::Config::xz -d"; }
+        elsif ($info->{compression} eq "none") { $decompressor = ""; }
         else {
             print STDERR "unknown compression method ‘$info->{compression}’\n";
             next;
@@ -493,7 +524,7 @@ sub downloadBinary {
         my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
         print STDERR "\n*** Downloading ‘$url’ to ‘$storePath’...\n";
         checkURL $url;
-        if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
+        if (system("$Nix::Config::curl --fail --location --insecure '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
             warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n";
             next;
         }
@@ -506,9 +537,17 @@ sub downloadBinary {
     }
 
     print STDERR "could not download ‘$storePath’ from any binary cache\n";
+    exit 1;
 }
 
 
+# Bail out right away if binary caches are disabled.
+exit 0 if
+    ($Nix::Config::config{"use-binary-caches"} // "true") eq "false" ||
+    ($Nix::Config::config{"untrusted-use-binary-caches"} // "true") eq "false";
+print "\n";
+flush STDOUT;
+
 initCache();
 
 

scripts/download-using-manifests.pl.in

@@ -22,6 +22,8 @@ my $curl = "$Nix::Config::curl --fail --location --insecure";
 
 # Open the manifest cache and update it if necessary.
 my $dbh = updateManifestDB();
+exit 0 unless defined $dbh; # exit if there are no manifests
+print "\n";
 
 
 # $hashCache->{$algo}->{$path} yields the $algo-hash of $path.
@@ -342,17 +344,18 @@ while (scalar @path > 0) {
         checkURL $narFile->{url};
 
         my $decompressor =
-            $narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" :
+            $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" :
-            $narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" :
+            $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" :
+            $narFile->{compressionType} eq "none" ? "" :
             die "unknown compression type `$narFile->{compressionType}'";
 
         if ($curStep < $maxStep) {
             # The archive will be used a base to a patch.
-            system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0
+            system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0
                 or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
         } else {
             # Unpack the archive to the target path.
-            system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
+            system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
                 or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
         }
 

scripts/find-runtime-roots.pl.in

@@ -1,22 +1,22 @@
-#! @perl@ -w
+#! @perl@ -w @perlFlags@
 
 use strict;
-
+use Nix::Utils;
-my $procDir = "/proc";
+use Nix::Config;
 
 
 sub readProc {
-    return unless -d $procDir;
+    return unless -d "/proc";
+
+    opendir DIR, "/proc" or return;
 
-    opendir DIR, $procDir or return;
-    
     foreach my $name (readdir DIR) {
         next unless $name =~ /^\d+$/;
 
-        my $process = "$procDir/$name";
+        my $process = "/proc/$name";
 
         #print STDERR "=== $process\n";
-        
+
         my $target;
         print "$target\n" if $target = readlink "$process/exe";
         print "$target\n" if $target = readlink "$process/cwd";
@@ -36,6 +36,13 @@ sub readProc {
             }
             close MAP;
         }
+
+        # Get all store paths that appear in the environment of this process.
+        eval {
+            my $env = Nix::Utils::readFile "$process/environ";
+            my @matches = $env =~ /\Q$Nix::Config::storeDir\E\/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*/g;
+            print "$_\n" foreach @matches;
+        }
     }
 
     closedir DIR;
@@ -58,18 +65,15 @@ readProc;
 lsof;
 
 
-sub readFile {
+sub printFile {
-    my $path = shift;
+    my ($fn) = @_;
-    if (-e $path) {
+    if (-e $fn) {
-        if (open FILE, "$path") {
+        print Nix::Utils::readFile($fn), "\n";
-            while (<FILE>) {
-                print;
-            }
-            close FILE;
-        }
     }
 }
 
+
 # This is rather NixOS-specific, so it probably shouldn't be here.
-readFile "/proc/sys/kernel/modprobe";
+printFile "/proc/sys/kernel/modprobe";
-readFile "/proc/sys/kernel/fbsplash";
+printFile "/proc/sys/kernel/fbsplash";
+printFile "/proc/sys/kernel/poweroff_cmd";

scripts/nix-build.in

@@ -3,23 +3,27 @@
 use strict;
 use Nix::Config;
 use Nix::Store;
+use Nix::Utils;
 use File::Temp qw(tempdir);
 
 
 my $dryRun = 0;
 my $verbose = 0;
-my $runEnv = 0;
+my $runEnv = $0 =~ /nix-shell$/;
+my $pure = 0;
 
 my @instArgs = ();
 my @buildArgs = ();
 my @exprs = ();
 
 my $shell = $ENV{SHELL} || "/bin/sh";
-my $envCommand = "p=\$PATH; source \$stdenv/setup; PATH=\$PATH:\$p; exec $shell";
+my $envCommand = ""; # interactive shell
 my @envExclude = ();
 
+my $myName = $runEnv ? "nix-shell" : "nix-build";
 
-my $tmpDir = tempdir("nix-build.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+
+my $tmpDir = tempdir("$myName.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
 my $outLink = "./result";
@@ -33,11 +37,11 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
     my $arg = $ARGV[$n];
 
     if ($arg eq "--help") {
-        exec "man nix-build" or die;
+        exec "man $myName" or die;
     }
 
     elsif ($arg eq "--version") {
-        print "nix-build (Nix) $Nix::Config::version\n";
+        print "$myName (Nix) $Nix::Config::version\n";
         exit 0;
     }
 
@@ -87,7 +91,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") {
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores" or $arg eq "--timeout") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
@@ -117,14 +121,14 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         push @instArgs, $arg;
     }
 
-    elsif ($arg eq "--run-env") {
+    elsif ($arg eq "--run-env") { # obsolete
         $runEnv = 1;
     }
 
     elsif ($arg eq "--command") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
-        $envCommand = $ARGV[$n];
+        $envCommand = "$ARGV[$n]\nexit $!";
     }
 
     elsif ($arg eq "--exclude") {
@@ -133,6 +137,10 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         push @envExclude, $ARGV[$n];
     }
 
+    elsif ($arg eq "--pure") {
+        $pure = 1;
+    }
+
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -149,17 +157,22 @@ foreach my $expr (@exprs) {
 
     # Instantiate.
     my @drvPaths;
-    # !!! would prefer the perl 5.8.0 pipe open feature here.
+    if ($expr !~ /^\/.*\.drv$/) {
-    my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
+        # !!! would prefer the perl 5.8.0 pipe open feature here.
-    while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
+        my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
-    if (!close DRVPATHS) {
+        while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
-        die "nix-instantiate killed by signal " . ($? & 127) . "\n" if ($? & 127);
+        if (!close DRVPATHS) {
-        exit 1;
+            die "nix-instantiate killed by signal " . ($? & 127) . "\n" if ($? & 127);
+            exit 1;
+        }
+    } else {
+        push @drvPaths, $expr;
     }
 
     if ($runEnv) {
-        die "$0: ‘--run-env’ requires a single derivation\n" if scalar @drvPaths != 1;
+        die "$0: a single derivation is required\n" if scalar @drvPaths != 1;
-        my $drvPath = readlink $drvPaths[0] or die "cannot read symlink `$drvPaths[0]'";
+        my $drvPath = $drvPaths[0];
+        $drvPath = readlink $drvPath or die "cannot read symlink `$drvPath'" if -l $drvPath;
         my $drv = derivationFromPath($drvPath);
 
         # Build or fetch all dependencies of the derivation.
@@ -168,16 +181,40 @@ foreach my $expr (@exprs) {
             or die "$0: failed to build all dependencies\n";
 
         # Set the environment.
-        $ENV{'NIX_BUILD_TOP'} = $ENV{'TMPDIR'} || "/tmp";
+        if ($pure) {
-        foreach (keys %{$drv->{env}}) {
+            foreach my $name (keys %ENV) {
-            $ENV{$_} = $drv->{env}->{$_};
+                next if $name eq "HOME" || $name eq "USER" || $name eq "LOGNAME" || $name eq "DISPLAY" || $name eq "PATH";
+                delete $ENV{$name};
+            }
+            # NixOS hack: prevent /etc/bashrc from sourcing /etc/profile.
+            $ENV{'__ETC_PROFILE_SOURCED'} = 1;
         }
+        $ENV{'NIX_BUILD_TOP'} = $ENV{'TMPDIR'} = $ENV{'TEMPDIR'} = $ENV{'TMP'} = $ENV{'TEMP'} = $ENV{'TMPDIR'} // "/tmp";
+        $ENV{'NIX_STORE'} = $Nix::Config::storeDir;
+        $ENV{$_} = $drv->{env}->{$_} foreach keys %{$drv->{env}};
+
+        $ENV{'IN_NIX_SHELL'} = 1;
 
         # Run a shell using the derivation's environment.  For
         # convenience, source $stdenv/setup to setup additional
-        # environment variables.  Also don't lose the current $PATH
+        # environment variables and shell functions.  Also don't lose
-        # directories.
+        # the current $PATH directories.
-        exec($ENV{SHELL}, "-c", $envCommand);
+        my $rcfile = "$tmpDir/rc";
+        writeFile(
+            $rcfile,
+            'unset BASH_ENV; ' .
+            '[ -n "$PS1" ] && [ -e ~/.bashrc ] && source ~/.bashrc; ' .
+            ($pure ? '' : 'p=$PATH; ' ) .
+            'dontAddDisableDepTrack=1; ' .
+            '[ -e $stdenv/setup ] && source $stdenv/setup; ' .
+            ($pure ? '' : 'PATH=$PATH:$p; ') .
+            'set +e; ' .
+            '[ -n "$PS1" ] && PS1="\n\[\033[1;32m\][nix-shell:\w]$\[\033[0m\] "; ' .
+            'unset NIX_ENFORCE_PURITY; ' .
+            'shopt -u nullglob; ' .
+            $envCommand);
+        $ENV{BASH_ENV} = $rcfile;
+        exec($ENV{NIX_BUILD_SHELL} // "bash", "--rcfile", $rcfile);
         die;
     }
 
@@ -206,7 +243,7 @@ foreach my $expr (@exprs) {
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "$Nix::Config::binDir/nix-store", "--add-root", $outLink, "--indirect", "-r",
+    my $pid = open(OUTPATHS, "-|") || exec "$Nix::Config::binDir/nix-store", "--add-root", $outLink, "--indirect", "-r",
         @buildArgs, @drvPaths2;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
     if (!close OUTPATHS) {

scripts/nix-copy-closure.in

@@ -4,6 +4,7 @@ use Nix::SSH;
 use Nix::Config;
 use Nix::Store;
 use Nix::CopyClosure;
+use List::Util qw(sum);
 
 
 if (scalar @ARGV < 1) {
@@ -77,6 +78,8 @@ while (@ARGV) {
     }
 }
 
+die "$0: you did not specify a host name\n" unless defined $sshHost;
+
 
 openSSHConnection $sshHost or die "$0: unable to start SSH\n";
 
@@ -104,6 +107,11 @@ else { # Copy FROM the remote machine.
 
     close READ or die "nix-store on remote machine `$sshHost' failed: $?";
 
+    my $missingSize = 0;
+    if ($progressViewer ne "") {
+        $missingSize = sum (split ' ', `set -f; ssh @sshOpts $sshHost nix-store -q --size @missing`) or die;
+    }
+
     # Export the store paths on the remote machine and import them locally.
     if (scalar @missing > 0) {
         print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
@@ -113,9 +121,9 @@ else { # Copy FROM the remote machine.
             }
             $compressor = "| $compressor" if $compressor ne "";
             $decompressor = "$decompressor |" if $decompressor ne "";
-            $progressViewer = "$progressViewer |" if $progressViewer ne "";
+            $progressViewer = "$progressViewer -s $missingSize |" if $progressViewer ne "";
             my $extraOpts = $sign ? "--sign" : "";
-            system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0
+            system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $progressViewer $Nix::Config::binDir/nix-store --import > /dev/null") == 0
                 or die "copying store paths from remote machine `$sshHost' failed: $?";
         }
     }

scripts/nix-push.in

@@ -34,6 +34,8 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         exec "man nix-push" or die;
     } elsif ($arg eq "--bzip2") {
         $compressionType = "bzip2";
+    } elsif ($arg eq "--none") {
+        $compressionType = "none";
     } elsif ($arg eq "--force") {
         $force = 1;
     } elsif ($arg eq "--dest") {
@@ -202,7 +204,7 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) {
     $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
     close HASH;
 
-    my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
+    my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
 
     my $narFile = "$narDir/$narName";
     (-f $narFile) or die "NAR file for $storePath not found";

src/libexpr/attr-path.cc

@@ -6,9 +6,8 @@
 namespace nix {
 
 
-// !!! Shouldn't we return a pointer to a Value?
+Value * findAlongAttrPath(EvalState & state, const string & attrPath,
-void findAlongAttrPath(EvalState & state, const string & attrPath,
+    Bindings & autoArgs, Value & vIn)
-    Bindings & autoArgs, Expr * e, Value & v)
 {
     Strings tokens = tokenizeString<Strings>(attrPath, ".");
 
@@ -17,8 +16,8 @@ void findAlongAttrPath(EvalState & state, const string & attrPath,
 
     string curPath;
 
-    state.mkThunk_(v, e);
+    Value * v = &vIn;
-    
+
     foreach (Strings::iterator, i, tokens) {
 
         if (!curPath.empty()) curPath += ".";
@@ -27,47 +26,48 @@ void findAlongAttrPath(EvalState & state, const string & attrPath,
         /* Is *i an index (integer) or a normal attribute name? */
         enum { apAttr, apIndex } apType = apAttr;
         string attr = *i;
-        int attrIndex = -1;
+        unsigned int attrIndex;
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        Value vTmp;
+        Value * vNew = state.allocValue();
-        state.autoCallFunction(autoArgs, v, vTmp);
+        state.autoCallFunction(autoArgs, *v, *vNew);
-        v = vTmp;
+        v = vNew;
-        state.forceValue(v);
+        state.forceValue(*v);
 
-        /* It should evaluate to either an attribute set or an
+        /* It should evaluate to either a set or an expression,
-           expression, according to what is specified in the
+           according to what is specified in the attrPath. */
-           attrPath. */
 
         if (apType == apAttr) {
 
-            if (v.type != tAttrs)
+            if (v->type != tAttrs)
                 throw TypeError(
-                    format("the expression selected by the selection path `%1%' should be an attribute set but is %2%")
+                    format("the expression selected by the selection path `%1%' should be a set but is %2%")
-                    % curPath % showType(v));
+                    % curPath % showType(*v));
 
-            Bindings::iterator a = v.attrs->find(state.symbols.create(attr));
+            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v.attrs->end())
+            if (a == v->attrs->end())
                 throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
-            v = *a->value;
+            v = &*a->value;
         }
 
         else if (apType == apIndex) {
 
-            if (v.type != tList)
+            if (v->type != tList)
                 throw TypeError(
                     format("the expression selected by the selection path `%1%' should be a list but is %2%")
-                    % curPath % showType(v));
+                    % curPath % showType(*v));
 
-            if (attrIndex >= v.list.length)
+            if (attrIndex >= v->list.length)
                 throw Error(format("list index %1% in selection path `%2%' is out of range") % attrIndex % curPath);
 
-            v = *v.list.elems[attrIndex];
+            v = v->list.elems[attrIndex];
         }
-        
+
     }
+
+    return v;
 }
 
- 
+
 }

src/libexpr/attr-path.hh

@@ -7,7 +7,7 @@
 
 namespace nix {
 
-void findAlongAttrPath(EvalState & state, const string & attrPath,
+Value * findAlongAttrPath(EvalState & state, const string & attrPath,
-    Bindings & autoArgs, Expr * e, Value & v);
+    Bindings & autoArgs, Value & vIn);
 
 }

src/libexpr/eval-inline.hh

@@ -21,13 +21,16 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s2))
 void EvalState::forceValue(Value & v)
 {
     if (v.type == tThunk) {
-        ValueType saved = v.type;
+        Env * env = v.thunk.env;
+        Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
-            v.thunk.expr->eval(*this, *v.thunk.env, v);
+            expr->eval(*this, *env, v);
         } catch (Error & e) {
-            v.type = saved;
+            v.type = tThunk;
+            v.thunk.env = env;
+            v.thunk.expr = expr;
             throw;
         }
     }
@@ -42,7 +45,7 @@ inline void EvalState::forceAttrs(Value & v)
 {
     forceValue(v);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while an attribute set was expected", showType(v));
+        throwTypeError("value is %1% while a set was expected", showType(v));
 }
 
 

src/libexpr/eval.cc

@@ -44,7 +44,7 @@ void Bindings::sort()
 {
     std::sort(begin(), end());
 }
-    
+
 
 std::ostream & operator << (std::ostream & str, const Value & v)
 {
@@ -116,7 +116,7 @@ string showType(const Value & v)
         case tString: return "a string";
         case tPath: return "a path";
         case tNull: return "null";
-        case tAttrs: return "an attribute set";
+        case tAttrs: return "a set";
         case tList: return "a list";
         case tThunk: return "a thunk";
         case tApp: return "a function application";
@@ -136,15 +136,16 @@ EvalState::EvalState()
     , sType(symbols.create("type"))
     , sMeta(symbols.create("meta"))
     , sName(symbols.create("name"))
+    , sValue(symbols.create("value"))
     , sSystem(symbols.create("system"))
     , sOverrides(symbols.create("__overrides"))
     , sOutputs(symbols.create("outputs"))
     , sOutputName(symbols.create("outputName"))
     , sIgnoreNulls(symbols.create("__ignoreNulls"))
-    , baseEnv(allocEnv(128))
-    , baseEnvDispl(0)
-    , staticBaseEnv(false, 0)
     , repair(false)
+    , baseEnv(allocEnv(128))
+    , staticBaseEnv(false, 0)
+    , baseEnvDispl(0)
 {
     nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
     nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
@@ -158,7 +159,7 @@ EvalState::EvalState()
            physical RAM, up to a maximum of 384 MiB) so that in most
            cases we don't need to garbage collect at all.  (Collection
            has a fairly significant overhead.)  The heap size can be
-           overriden through libgc's GC_INITIAL_HEAP_SIZE environment
+           overridden through libgc's GC_INITIAL_HEAP_SIZE environment
            variable.  We should probably also provide a nix.conf
            setting for this.  Note that GC_expand_hp() causes a lot of
            virtual, but not physical (resident) memory to be
@@ -222,6 +223,12 @@ void EvalState::addPrimOp(const string & name,
 }
 
 
+void EvalState::getBuiltin(const string & name, Value & v)
+{
+    v = *baseEnv.values[0]->attrs->find(symbols.create(name))->value;
+}
+
+
 /* Every "format" object (even temporary) takes up a few hundred bytes
    of stack space, which is a real killer in the recursive
    evaluator.  So here are some helper functions for throwing
@@ -242,14 +249,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s))
     throw TypeError(s);
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos, const string & s2))
+LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1, const string & s2))
 {
-    throw TypeError(format(s) % pos % s2);
+    throw TypeError(format(s) % s1 % s2);
-}
-
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos))
-{
-    throw TypeError(format(s) % pos);
 }
 
 LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos))
@@ -257,16 +259,16 @@ LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos))
     throw AssertionError(format(s) % pos);
 }
 
+LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
+{
+    throw UndefinedVarError(format(s) % s1 % pos);
+}
+
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
 {
     e.addPrefix(format(s) % s2);
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const Pos & pos))
-{
-    e.addPrefix(format(s) % pos);
-}
-
 LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
 {
     e.addPrefix(format(s) % s2 % pos);
@@ -286,7 +288,7 @@ void mkString(Value & v, const string & s, const PathSet & context)
         unsigned int n = 0;
         v.string.context = (const char * *)
             GC_MALLOC((context.size() + 1) * sizeof(char *));
-        foreach (PathSet::const_iterator, i, context)  
+        foreach (PathSet::const_iterator, i, context)
             v.string.context[n++] = GC_STRDUP(i->c_str());
         v.string.context[n] = 0;
     }
@@ -299,23 +301,29 @@ void mkPath(Value & v, const char * s)
 }
 
 
-inline Value * EvalState::lookupVar(Env * env, const VarRef & var)
+inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
     for (unsigned int l = var.level; l; --l, env = env->up) ;
-    
+
-    if (var.fromWith) {
+    if (!var.fromWith) return env->values[var.displ];
-        while (1) {
+
-            Bindings::iterator j = env->values[0]->attrs->find(var.name);
+    while (1) {
-            if (j != env->values[0]->attrs->end()) {
+        if (!env->haveWithAttrs) {
-                if (countCalls && j->pos) attrSelects[*j->pos]++;
+            if (noEval) return 0;
-                return j->value;
+            Value * v = allocValue();
-            }
+            evalAttrs(*env->up, (Expr *) env->values[0], *v);
-            if (env->prevWith == 0)
+            env->values[0] = v;
-                throwEvalError("undefined variable `%1%'", var.name);
+            env->haveWithAttrs = true;
-            for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
         }
-    } else
+        Bindings::iterator j = env->values[0]->attrs->find(var.name);
-        return env->values[var.displ];
+        if (j != env->values[0]->attrs->end()) {
+            if (countCalls && j->pos) attrSelects[*j->pos]++;
+            return j->value;
+        }
+        if (!env->prevWith)
+            throwUndefinedVarError("undefined variable `%1%' at %2%", var.name, var.pos);
+        for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
+    }
 }
 
 
@@ -332,10 +340,10 @@ Env & EvalState::allocEnv(unsigned int size)
     nrValuesInEnvs += size;
     Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *));
 
-    /* Clear the values because maybeThunk() expects this. */
+    /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */
     for (unsigned i = 0; i < size; ++i)
         env->values[i] = 0;
-    
+
     return *env;
 }
 
@@ -347,7 +355,7 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
     return v;
 }
 
-    
+
 void EvalState::mkList(Value & v, unsigned int length)
 {
     v.type = tList;
@@ -400,7 +408,7 @@ unsigned long nrAvoided = 0;
 
 Value * ExprVar::maybeThunk(EvalState & state, Env & env)
 {
-    Value * v = state.lookupVar(&env, info);
+    Value * v = state.lookupVar(&env, *this, true);
     /* The value might not be initialised in the environment yet.
        In that case, ignore it. */
     if (v) { nrAvoided++; return v; }
@@ -429,19 +437,35 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 
 void EvalState::evalFile(const Path & path, Value & v)
 {
-    FileEvalCache::iterator i = fileEvalCache.find(path);
+    FileEvalCache::iterator i;
-    if (i == fileEvalCache.end()) {
+    if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
-        startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path);
-        Expr * e = parseExprFromFile(path);
-        try {
-            eval(e, v);
-        } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating the file `%1%':\n", path);
-            throw;
-        }
-        fileEvalCache[path] = v;
-    } else
         v = i->second;
+        return;
+    }
+
+    Path path2 = resolveExprPath(path);
+    if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) {
+        v = i->second;
+        return;
+    }
+
+    startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path2);
+    Expr * e = parseExprFromFile(path2);
+    try {
+        eval(e, v);
+    } catch (Error & e) {
+        addErrorPrefix(e, "while evaluating the file `%1%':\n", path2);
+        throw;
+    }
+
+    fileEvalCache[path2] = v;
+    if (path != path2) fileEvalCache[path] = v;
+}
+
+
+void EvalState::resetFileCache()
+{
+    fileEvalCache.clear();
 }
 
 
@@ -465,7 +489,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
 {
     e->eval(*this, env, v);
     if (v.type != tAttrs)
-        throwTypeError("value is %1% while an attribute set was expected", showType(v));
+        throwTypeError("value is %1% while a set was expected", showType(v));
 }
 
 
@@ -510,23 +534,17 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
            environment, while the inherited attributes are evaluated
            in the original environment. */
         unsigned int displ = 0;
-        foreach (AttrDefs::iterator, i, attrs)
+        foreach (AttrDefs::iterator, i, attrs) {
-            if (i->second.inherited) {
+            Value * vAttr;
-                /* !!! handle overrides? */
+            if (hasOverrides && !i->second.inherited) {
-                Value * vAttr = state.lookupVar(&env, i->second.var);
+                vAttr = state.allocValue();
-                env2.values[displ++] = vAttr;
+                mkThunk(*vAttr, env2, i->second.e);
-                v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos));
+            } else
-            } else {
+                vAttr = i->second.e->maybeThunk(state, i->second.inherited ? env : env2);
-                Value * vAttr;
+            env2.values[displ++] = vAttr;
-                if (hasOverrides) {
+            v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos));
-                    vAttr = state.allocValue();
+        }
-                    mkThunk(*vAttr, env2, i->second.e);
+
-                } else
-                    vAttr = i->second.e->maybeThunk(state, env2);
-                env2.values[displ++] = vAttr;
-                v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos));
-            }
-        
         /* If the rec contains an attribute called `__overrides', then
            evaluate it, and add the attributes in that set to the rec.
            This allows overriding of recursive attributes, which is
@@ -552,10 +570,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
     else {
         foreach (AttrDefs::iterator, i, attrs)
-            if (i->second.inherited)
+            v.attrs->push_back(Attr(i->first, i->second.e->maybeThunk(state, env), &i->second.pos));
-                v.attrs->push_back(Attr(i->first, state.lookupVar(&env, i->second.var), &i->second.pos));
-            else
-                v.attrs->push_back(Attr(i->first, i->second.e->maybeThunk(state, env), &i->second.pos));
     }
 }
 
@@ -572,10 +587,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
        environment. */
     unsigned int displ = 0;
     foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
-        if (i->second.inherited)
+        env2.values[displ++] = i->second.e->maybeThunk(state, i->second.inherited ? env : env2);
-            env2.values[displ++] = state.lookupVar(&env, i->second.var);
-        else
-            env2.values[displ++] = i->second.e->maybeThunk(state, env2);
 
     body->eval(state, env2, v);
 }
@@ -591,7 +603,7 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
 
 void ExprVar::eval(EvalState & state, Env & env, Value & v)
 {
-    Value * v2 = state.lookupVar(&env, info);
+    Value * v2 = state.lookupVar(&env, *this, false);
     state.forceValue(*v2);
     v = *v2;
 }
@@ -608,7 +620,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
     e->eval(state, env, vTmp);
 
     try {
-        
+
         foreach (AttrPath::const_iterator, i, attrPath) {
             nrLookups++;
             Bindings::iterator j;
@@ -629,17 +641,16 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             pos = j->pos;
             if (state.countCalls && pos) state.attrSelects[*pos]++;
         }
-    
 
         state.forceValue(*vAttrs);
-        
+
     } catch (Error & e) {
-        if (pos)
+        if (pos && pos->file != state.sDerivationNix)
             addErrorPrefix(e, "while evaluating the attribute `%1%' at %2%:\n",
                 showAttrPath(attrPath), *pos);
         throw;
     }
-    
+
     v = *vAttrs;
 }
 
@@ -663,7 +674,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
             vAttrs = j->value;
         }
     }
-    
+
     mkBool(v, true);
 }
 
@@ -698,10 +709,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
         assert(primOp->type == tPrimOp);
         unsigned int arity = primOp->primOp->arity;
         unsigned int argsLeft = arity - argsDone;
-        
+
         if (argsLeft == 1) {
             /* We have all the arguments, so call the primop. */
-                
+
             /* Put all the arguments in an array. */
             Value * vArgs[arity];
             unsigned int n = arity - 1;
@@ -712,12 +723,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
             /* And call the primop. */
             nrPrimOpCalls++;
             if (countCalls) primOpCalls[primOp->primOp->name]++;
-            try {
+            primOp->primOp->fun(*this, vArgs, v);
-                primOp->primOp->fun(*this, vArgs, v);
-            } catch (Error & e) {
-                addErrorPrefix(e, "while evaluating the builtin function `%1%':\n", primOp->primOp->name);
-                throw;
-            }
         } else {
             v.type = tPrimOpApp;
             v.primOpApp.left = allocValue();
@@ -726,7 +732,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
         }
         return;
     }
-    
+
     if (fun.type != tLambda)
         throwTypeError("attempt to call something which is not a function but %1%",
             showType(fun));
@@ -744,7 +750,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
 
     else {
         forceAttrs(arg);
-        
+
         if (!fun.lambda.fun->arg.empty())
             env2.values[displ++] = &arg;
 
@@ -755,8 +761,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
         foreach (Formals::Formals_::iterator, i, fun.lambda.fun->formals->formals) {
             Bindings::iterator j = arg.attrs->find(i->name);
             if (j == arg.attrs->end()) {
-                if (!i->def) throwTypeError("function at %1% called without required argument `%2%'",
+                if (!i->def) throwTypeError("%1% called without required argument `%2%'",
-                    fun.lambda.fun->pos, i->name);
+                    fun.lambda.fun->showNamePos(), i->name);
                 env2.values[displ++] = i->def->maybeThunk(*this, env2);
             } else {
                 attrsUsed++;
@@ -765,20 +771,24 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
         }
 
         /* Check that each actual argument is listed as a formal
-           argument (unless the attribute match specifies a `...').
+           argument (unless the attribute match specifies a `...'). */
-           TODO: show the names of the expected/unexpected
+        if (!fun.lambda.fun->formals->ellipsis && attrsUsed != arg.attrs->size()) {
-           arguments. */
+            /* Nope, so show the first unexpected argument to the
-        if (!fun.lambda.fun->formals->ellipsis && attrsUsed != arg.attrs->size()) 
+               user. */
-            throwTypeError("function at %1% called with unexpected argument", fun.lambda.fun->pos);
+            foreach (Bindings::iterator, i, *arg.attrs)
+                if (fun.lambda.fun->formals->argNames.find(i->name) == fun.lambda.fun->formals->argNames.end())
+                    throwTypeError("%1% called with unexpected argument `%2%'", fun.lambda.fun->showNamePos(), i->name);
+            abort(); // can't happen
+        }
     }
 
     nrFunctionCalls++;
-    if (countCalls) functionCalls[fun.lambda.fun->pos]++;
+    if (countCalls) functionCalls[fun.lambda.fun]++;
 
     try {
         fun.lambda.fun->body->eval(*this, env2, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the function at %1%:\n", fun.lambda.fun->pos);
+        addErrorPrefix(e, "while evaluating %1%:\n", fun.lambda.fun->showNamePos());
         throw;
     }
 }
@@ -815,9 +825,8 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(1));
     env2.up = &env;
     env2.prevWith = prevWith;
-
+    env2.haveWithAttrs = false;
-    env2.values[0] = state.allocValue();
+    env2.values[0] = (Value *) attrs;
-    state.evalAttrs(env, attrs, *env2.values[0]);
 
     body->eval(state, env2, v);
 }
@@ -828,7 +837,7 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
     (state.evalBool(env, cond) ? then : else_)->eval(state, env, v);
 }
 
-    
+
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
     if (!state.evalBool(env, cond))
@@ -836,7 +845,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     body->eval(state, env, v);
 }
 
-    
+
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
     mkBool(v, !state.evalBool(env, e));
@@ -890,9 +899,8 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 
     state.mkAttrs(v, v1.attrs->size() + v2.attrs->size());
 
-    /* Merge the attribute sets, preferring values from the second
+    /* Merge the sets, preferring values from the second set.  Make
-       set.  Make sure to keep the resulting vector in sorted
+       sure to keep the resulting vector in sorted order. */
-       order. */
     Bindings::iterator i = v1.attrs->begin();
     Bindings::iterator j = v2.attrs->begin();
 
@@ -909,7 +917,7 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 
     while (i != v1.attrs->end()) v.attrs->push_back(*i++);
     while (j != v2.attrs->end()) v.attrs->push_back(*j++);
-    
+
     state.nrOpUpdateValuesCopied += v.attrs->size();
 }
 
@@ -954,31 +962,39 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
     PathSet context;
     std::ostringstream s;
-        
+    NixInt n = 0;
-    bool first = true, isPath = false;
+
-    Value vStr;
+    bool first = !forceString;
+    ValueType firstType = tString;
 
     foreach (vector<Expr *>::iterator, i, *es) {
-        (*i)->eval(state, env, vStr);
+        Value vTmp;
+        (*i)->eval(state, env, vTmp);
 
         /* If the first element is a path, then the result will also
            be a path, we don't copy anything (yet - that's done later,
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
-            isPath = vStr.type == tPath;
+            firstType = vTmp.type;
             first = false;
         }
 
-        s << state.coerceToString(vStr, context, false, !isPath);
+        if (firstType == tInt) {
+            if (vTmp.type != tInt)
+                throwEvalError("cannot add %1% to an integer", showType(vTmp));
+            n += vTmp.integer;
+        } else
+            s << state.coerceToString(vTmp, context, false, firstType == tString);
     }
-        
-    if (isPath && !context.empty())
-        throwEvalError("a string that refers to a store path cannot be appended to a path, in `%1%'", s.str());
 
-    if (isPath)
+    if (firstType == tInt)
+        mkInt(v, n);
+    else if (firstType == tPath) {
+        if (!context.empty())
+            throwEvalError("a string that refers to a store path cannot be appended to a path, in `%1%'", s.str());
         mkPath(v, s.str().c_str());
-    else
+    } else
         mkString(v, s.str(), context);
 }
 
@@ -986,12 +1002,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 void EvalState::strictForceValue(Value & v)
 {
     forceValue(v);
-    
+
     if (v.type == tAttrs) {
         foreach (Bindings::iterator, i, *v.attrs)
             strictForceValue(*i->value);
     }
-    
+
     else if (v.type == tList) {
         for (unsigned int n = 0; n < v.list.length; ++n)
             strictForceValue(*v.list.elems[n]);
@@ -999,7 +1015,7 @@ void EvalState::strictForceValue(Value & v)
 }
 
 
-int EvalState::forceInt(Value & v)
+NixInt EvalState::forceInt(Value & v)
 {
     forceValue(v);
     if (v.type != tInt)
@@ -1037,7 +1053,7 @@ string EvalState::forceString(Value & v)
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
-        for (const char * * p = v.string.context; *p; ++p) 
+        for (const char * * p = v.string.context; *p; ++p)
             context.insert(*p);
 }
 
@@ -1067,7 +1083,7 @@ bool EvalState::isDerivation(Value & v)
     if (i == v.attrs->end()) return false;
     forceValue(*i->value);
     if (i->value->type != tString) return false;
-    return forceStringNoCtx(*i->value) == "derivation";
+    return strcmp(i->value->string.s, "derivation") == 0;
 }
 
 
@@ -1087,7 +1103,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
         Path path(canonPath(v.path));
 
         if (!copyToStore) return path;
-        
+
         if (nix::isDerivation(path))
             throwEvalError("file names are not allowed to end in `%1%'", drvExtension);
 
@@ -1109,8 +1125,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
 
     if (v.type == tAttrs) {
         Bindings::iterator i = v.attrs->find(sOutPath);
-        if (i == v.attrs->end())
+        if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string");
-            throwTypeError("cannot coerce an attribute set (except a derivation) to a string");
         return coerceToString(*i->value, context, coerceMore, copyToStore);
     }
 
@@ -1136,7 +1151,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
             return result;
         }
     }
-    
+
     throwTypeError("cannot coerce %1% to a string", showType(v));
 }
 
@@ -1156,9 +1171,8 @@ bool EvalState::eqValues(Value & v1, Value & v2)
     forceValue(v2);
 
     /* !!! Hack to support some old broken code that relies on pointer
-       equality tests between attribute sets.  (Specifically,
+       equality tests between sets.  (Specifically, builderDefs calls
-       builderDefs calls uniqList on a list of attribute sets.)  Will
+       uniqList on a list of sets.)  Will remove this eventually. */
-       remove this eventually. */
     if (&v1 == &v2) return true;
 
     if (v1.type != v2.type) return false;
@@ -1196,8 +1210,8 @@ bool EvalState::eqValues(Value & v1, Value & v2)
             return true;
 
         case tAttrs: {
-            /* If both attribute sets denote a derivation (type =
+            /* If both sets denote a derivation (type = "derivation"),
-               "derivation"), then compare their outPaths. */
+               then compare their outPaths. */
             if (isDerivation(v1) && isDerivation(v2)) {
                 Bindings::iterator i = v1.attrs->find(sOutPath);
                 Bindings::iterator j = v2.attrs->find(sOutPath);
@@ -1212,7 +1226,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
             for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j)
                 if (i->name != j->name || !eqValues(*i->value, *j->value))
                     return false;
-            
+
             return true;
         }
 
@@ -1247,10 +1261,11 @@ void EvalState::printStats()
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
     printMsg(v, format("  values allocated: %1% (%2% bytes)")
         % nrValues % (nrValues * sizeof(Value)));
-    printMsg(v, format("  attribute sets allocated: %1%") % nrAttrsets);
+    printMsg(v, format("  sets allocated: %1%") % nrAttrsets);
     printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
     printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
     printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
+    printMsg(v, format("  size of symbol table: %1%") % symbols.totalSize());
     printMsg(v, format("  number of thunks: %1%") % nrThunks);
     printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
     printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
@@ -1258,26 +1273,27 @@ void EvalState::printStats()
     printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
 
     if (countCalls) {
+        v = lvlInfo;
 
         printMsg(v, format("calls to %1% primops:") % primOpCalls.size());
         typedef std::multimap<unsigned int, Symbol> PrimOpCalls_;
-        std::multimap<unsigned int, Symbol> primOpCalls_;
+        PrimOpCalls_ primOpCalls_;
         foreach (PrimOpCalls::iterator, i, primOpCalls)
             primOpCalls_.insert(std::pair<unsigned int, Symbol>(i->second, i->first));
         foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_)
             printMsg(v, format("%1$10d %2%") % i->first % i->second);
 
         printMsg(v, format("calls to %1% functions:") % functionCalls.size());
-        typedef std::multimap<unsigned int, Pos> FunctionCalls_;
+        typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_;
-        std::multimap<unsigned int, Pos> functionCalls_;
+        FunctionCalls_ functionCalls_;
         foreach (FunctionCalls::iterator, i, functionCalls)
-            functionCalls_.insert(std::pair<unsigned int, Pos>(i->second, i->first));
+            functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i->second, i->first));
         foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_)
-            printMsg(v, format("%1$10d %2%") % i->first % i->second);
+            printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos());
 
         printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size());
         typedef std::multimap<unsigned int, Pos> AttrSelects_;
-        std::multimap<unsigned int, Pos> attrSelects_;
+        AttrSelects_ attrSelects_;
         foreach (AttrSelects::iterator, i, attrSelects)
             attrSelects_.insert(std::pair<unsigned int, Pos>(i->second, i->first));
         foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_)

src/libexpr/eval.hh

@@ -19,8 +19,8 @@ class EvalState;
 struct Attr;
 
 
-/* Attribute sets are represented as a vector of attributes, sorted by
+/* Sets are represented as a vector of attributes, sorted by symbol
-   symbol (i.e. pointer to the attribute name in the symbol table). */
+   (i.e. pointer to the attribute name in the symbol table). */
 #if HAVE_BOEHMGC
 typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase;
 #else
@@ -52,7 +52,8 @@ struct PrimOp
 struct Env
 {
     Env * up;
-    unsigned int prevWith; // nr of levels up to next `with' environment
+    unsigned short prevWith; // nr of levels up to next `with' environment
+    bool haveWithAttrs;
     Value * values[0];
 };
 
@@ -87,23 +88,21 @@ struct EvalState;
 std::ostream & operator << (std::ostream & str, const Value & v);
 
 
-class EvalState 
+class EvalState
 {
 public:
     SymbolTable symbols;
 
-    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName,
+    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls;
+    Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
        already exist there. */
     bool repair;
 
 private:
-    SrcToStore srcToStore; 
+    SrcToStore srcToStore;
-
-    /* A cache from path names to parse trees. */
-    std::map<Path, Expr *> parseTrees;
 
     /* A cache from path names to values. */
 #if HAVE_BOEHMGC
@@ -118,23 +117,25 @@ private:
     SearchPath::iterator searchPathInsertionPoint;
 
 public:
-    
+
     EvalState();
     ~EvalState();
 
     void addToSearchPath(const string & s);
 
-    /* Parse a Nix expression from the specified file.  If `path'
+    /* Parse a Nix expression from the specified file. */
-       refers to a directory, then "/default.nix" is appended. */
+    Expr * parseExprFromFile(const Path & path);
-    Expr * parseExprFromFile(Path path);
 
     /* Parse a Nix expression from the specified string. */
+    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
     Expr * parseExprFromString(const string & s, const Path & basePath);
-    
+
     /* Evaluate an expression read from the given file to normal
        form. */
     void evalFile(const Path & path, Value & v);
 
+    void resetFileCache();
+
     /* Look up a file in the search path. */
     Path findFile(const string & path);
 
@@ -158,7 +159,7 @@ public:
     void strictForceValue(Value & v);
 
     /* Force `v', and then verify that it has the expected type. */
-    int forceInt(Value & v);
+    NixInt forceInt(Value & v);
     bool forceBool(Value & v);
     inline void forceAttrs(Value & v);
     inline void forceList(Value & v);
@@ -174,7 +175,7 @@ public:
     /* String coercion.  Converts strings, paths and derivations to a
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
-       referenced paths are copied to the Nix store as a side effect.q */
+       referenced paths are copied to the Nix store as a side effect. */
     string coerceToString(Value & v, PathSet & context,
         bool coerceMore = false, bool copyToStore = true);
 
@@ -183,39 +184,43 @@ public:
        path.  Nothing is copied to the store. */
     Path coerceToPath(Value & v, PathSet & context);
 
-private:
+public:
 
     /* The base environment, containing the builtin functions and
        values. */
     Env & baseEnv;
 
-    unsigned int baseEnvDispl;
-
-public:
-    
     /* The same, but used during parsing to resolve variables. */
     StaticEnv staticBaseEnv; // !!! should be private
 
 private:
-    
+
+    unsigned int baseEnvDispl;
+
     void createBaseEnv();
-    
+
     void addConstant(const string & name, Value & v);
 
     void addPrimOp(const string & name,
         unsigned int arity, PrimOpFun primOp);
 
-    inline Value * lookupVar(Env * env, const VarRef & var);
+public:
-    
+
+    void getBuiltin(const string & name, Value & v);
+
+private:
+
+    inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
+
     friend class ExprVar;
     friend class ExprAttrs;
     friend class ExprLet;
 
-    Expr * parse(const char * text,
+    Expr * parse(const char * text, const Path & path,
-        const Path & path, const Path & basePath);
+        const Path & basePath, StaticEnv & staticEnv);
 
 public:
-    
+
     /* Do a deep equality test between two values.  That is, list
        elements and attributes are compared recursively. */
     bool eqValues(Value & v1, Value & v2);
@@ -225,7 +230,7 @@ public:
     /* Automatically call a function for which each argument has a
        default value or has a binding in the `args' map. */
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
-    
+
     /* Allocation primitives. */
     Value * allocValue();
     Env & allocEnv(unsigned int size);
@@ -259,7 +264,7 @@ private:
     typedef std::map<Symbol, unsigned int> PrimOpCalls;
     PrimOpCalls primOpCalls;
 
-    typedef std::map<Pos, unsigned int> FunctionCalls;
+    typedef std::map<ExprLambda *, unsigned int> FunctionCalls;
     FunctionCalls functionCalls;
 
     typedef std::map<Pos, unsigned int> AttrSelects;
@@ -276,4 +281,8 @@ private:
 string showType(const Value & v);
 
 
+/* If `path' refers to a directory, then append "/default.nix". */
+Path resolveExprPath(Path path);
+
+
 }

src/libexpr/get-drvs.cc

@@ -41,7 +41,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(EvalState & state)
 
             /* For each output... */
             for (unsigned int j = 0; j < i->value->list.length; ++j) {
-                /* Evaluate the corresponding attribute set. */
+                /* Evaluate the corresponding set. */
                 string name = state.forceStringNoCtx(*i->value->list.elems[j]);
                 Bindings::iterator out = attrs->find(state.symbols.create(name));
                 if (out == attrs->end()) continue; // FIXME: throw error?
@@ -119,11 +119,10 @@ void DrvInfo::setMetaInfo(const MetaInfo & meta)
 typedef set<Bindings *> Done;
 
 
-/* Evaluate value `v'.  If it evaluates to an attribute set of type
+/* Evaluate value `v'.  If it evaluates to a set of type `derivation',
-   `derivation', then put information about it in `drvs' (unless it's
+   then put information about it in `drvs' (unless it's already in
-   already in `doneExprs').  The result boolean indicates whether it
+   `doneExprs').  The result boolean indicates whether it makes sense
-   makes sense for the caller to recursively search for derivations in
+   for the caller to recursively search for derivations in `v'. */
-   `v'. */
 static bool getDerivation(EvalState & state, Value & v,
     const string & attrPath, DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
@@ -132,8 +131,8 @@ static bool getDerivation(EvalState & state, Value & v,
         state.forceValue(v);
         if (!state.isDerivation(v)) return true;
 
-        /* Remove spurious duplicates (e.g., an attribute set like
+        /* Remove spurious duplicates (e.g., a set like `rec { x =
-           `rec { x = derivation {...}; y = x;}'. */
+           derivation {...}; y = x;}'. */
         if (done.find(v.attrs) != done.end()) return false;
         done.insert(v.attrs);
 
@@ -218,10 +217,9 @@ static void getDerivations(EvalState & state, Value & vIn,
             if (combineChannels)
                 getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
             else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
-                /* If the value of this attribute is itself an
+                /* If the value of this attribute is itself a set,
-                   attribute set, should we recurse into it?  => Only
+                   should we recurse into it?  => Only if it has a
-                   if it has a `recurseForDerivations = true'
+                   `recurseForDerivations = true' attribute. */
-                   attribute. */
                 if (v2.type == tAttrs) {
                     Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations"));
                     if (j != v2.attrs->end() && state.forceBool(*j->value))

src/libexpr/lexer.l

@@ -9,21 +9,20 @@
 
 %{
 #include "nixexpr.hh"
-#define BISON_HEADER_HACK
 #include "parser-tab.hh"
 
 using namespace nix;
 
 namespace nix {
 
-    
+
 static void initLoc(YYLTYPE * loc)
 {
     loc->first_line = loc->last_line = 1;
     loc->first_column = loc->last_column = 1;
 }
 
-    
+
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
     loc->first_line = loc->last_line;
@@ -35,7 +34,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
            if (*s == '\n') /* cr/lf */
                s++;
            /* fall through */
-       case '\n': 
+       case '\n':
            ++loc->last_line;
            loc->last_column = 1;
            break;
@@ -69,7 +68,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s)
     return new ExprString(symbols.create(t));
 }
 
- 
+
 }
 
 #define YY_USER_INIT initLoc(yylloc)
@@ -102,6 +101,8 @@ or          { return OR_KW; }
 
 \=\=        { return EQ; }
 \!\=        { return NEQ; }
+\<\=        { return LEQ; }
+\>\=        { return GEQ; }
 \&\&        { return AND; }
 \|\|        { return OR; }
 \-\>        { return IMPL; }
@@ -109,8 +110,10 @@ or          { return OR_KW; }
 \+\+        { return CONCAT; }
 
 {ID}        { yylval->id = strdup(yytext); return ID; }
-{INT}       { int n = atoi(yytext); /* !!! overflow */
+{INT}       { errno = 0;
-              yylval->n = n;
+              yylval->n = strtol(yytext, 0, 10);
+              if (errno != 0)
+                  throw ParseError(format("invalid integer `%1%'") % yytext);
               return INT;
             }
 
@@ -168,7 +171,7 @@ or          { return OR_KW; }
 
 
 namespace nix {
-    
+
 /* Horrible, disgusting hack: allow the parser to set the scanner
    start condition back to STRING.  Necessary in interpolations like
    "foo${expr}bar"; after the close brace we have to go back to the

src/libexpr/names.cc

@@ -55,7 +55,7 @@ static string nextComponent(string::const_iterator & p,
     else
         while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
             s += *p++;
-    
+
     return s;
 }
 
@@ -80,7 +80,7 @@ int compareVersions(const string & v1, const string & v2)
 {
     string::const_iterator p1 = v1.begin();
     string::const_iterator p2 = v2.begin();
-    
+
     while (p1 != v1.end() || p2 != v2.end()) {
         string c1 = nextComponent(p1, v1.end());
         string c2 = nextComponent(p2, v2.end());
@@ -95,11 +95,10 @@ int compareVersions(const string & v1, const string & v2)
 DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;
-    for (Strings::const_iterator i = opArgs.begin();
+    foreach (Strings::const_iterator, i, opArgs)
-         i != opArgs.end(); ++i)
         result.push_back(DrvName(*i));
     return result;
 }
 
- 
+
 }

src/libexpr/nixexpr.cc

@@ -38,7 +38,7 @@ void ExprPath::show(std::ostream & str)
 
 void ExprVar::show(std::ostream & str)
 {
-    str << info.name;
+    str << name;
 }
 
 void ExprSelect::show(std::ostream & str)
@@ -174,7 +174,7 @@ void ExprPath::bindVars(const StaticEnv & env)
 {
 }
 
-void VarRef::bind(const StaticEnv & env)
+void ExprVar::bindVars(const StaticEnv & env)
 {
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
@@ -198,17 +198,12 @@ void VarRef::bind(const StaticEnv & env)
     /* Otherwise, the variable must be obtained from the nearest
        enclosing `with'.  If there is no `with', then we can issue an
        "undefined variable" error now. */
-    if (withLevel == -1) throw EvalError(format("undefined variable `%1%'") % name);
+    if (withLevel == -1) throw UndefinedVarError(format("undefined variable `%1%' at %2%") % name % pos);
 
     fromWith = true;
     this->level = withLevel;
 }
 
-void ExprVar::bindVars(const StaticEnv & env)
-{
-    info.bind(env);
-}
-
 void ExprSelect::bindVars(const StaticEnv & env)
 {
     e->bindVars(env);
@@ -224,20 +219,18 @@ void ExprAttrs::bindVars(const StaticEnv & env)
 {
     if (recursive) {
         StaticEnv newEnv(false, &env);
-    
+
         unsigned int displ = 0;
         foreach (AttrDefs::iterator, i, attrs)
             newEnv.vars[i->first] = i->second.displ = displ++;
-        
+
         foreach (AttrDefs::iterator, i, attrs)
-            if (i->second.inherited) i->second.var.bind(env);
+            i->second.e->bindVars(i->second.inherited ? env : newEnv);
-            else i->second.e->bindVars(newEnv);
     }
 
     else
         foreach (AttrDefs::iterator, i, attrs)
-            if (i->second.inherited) i->second.var.bind(env);
+            i->second.e->bindVars(env);
-            else i->second.e->bindVars(env);
 }
 
 void ExprList::bindVars(const StaticEnv & env)
@@ -249,9 +242,9 @@ void ExprList::bindVars(const StaticEnv & env)
 void ExprLambda::bindVars(const StaticEnv & env)
 {
     StaticEnv newEnv(false, &env);
-    
+
     unsigned int displ = 0;
-    
+
     if (!arg.empty()) newEnv.vars[arg] = displ++;
 
     if (matchAttrs) {
@@ -268,15 +261,14 @@ void ExprLambda::bindVars(const StaticEnv & env)
 void ExprLet::bindVars(const StaticEnv & env)
 {
     StaticEnv newEnv(false, &env);
-    
+
     unsigned int displ = 0;
     foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
         newEnv.vars[i->first] = i->second.displ = displ++;
-    
+
     foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
-        if (i->second.inherited) i->second.var.bind(env);
+        i->second.e->bindVars(i->second.inherited ? env : newEnv);
-        else i->second.e->bindVars(newEnv);
+
-    
     body->bindVars(newEnv);
 }
 
@@ -293,8 +285,8 @@ void ExprWith::bindVars(const StaticEnv & env)
             prevWith = level;
             break;
         }
-    
+
-    attrs->bindVars(env);    
+    attrs->bindVars(env);
     StaticEnv newEnv(true, &env);
     body->bindVars(newEnv);
 }
@@ -324,4 +316,36 @@ void ExprConcatStrings::bindVars(const StaticEnv & env)
 }
 
 
+/* Storing function names. */
+
+void Expr::setName(Symbol & name)
+{
+}
+
+
+void ExprLambda::setName(Symbol & name)
+{
+    this->name = name;
+    body->setName(name);
+}
+
+
+string ExprLambda::showNamePos()
+{
+    return (format("%1% at %2%") % (name.set() ? "`" + (string) name + "'" : "an anonymous function") % pos).str();
+}
+
+
+
+/* Symbol table. */
+
+size_t SymbolTable::totalSize() const
+{
+    size_t n = 0;
+    foreach (Symbols::const_iterator, i, symbols)
+        n += i->size();
+    return n;
+}
+
+
 }

src/libexpr/nixexpr.hh

@@ -16,20 +16,23 @@ MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
 MakeError(ImportError, EvalError) // error building an imported derivation
+MakeError(UndefinedVarError, Error)
 
 
 /* Position objects. */
 
 struct Pos
 {
-    string file;
+    Symbol file;
     unsigned int line, column;
     Pos() : line(0), column(0) { };
-    Pos(const string & file, unsigned int line, unsigned int column)
+    Pos(const Symbol & file, unsigned int line, unsigned int column)
         : file(file), line(line), column(column) { };
     bool operator < (const Pos & p2) const
     {
-        int d = file.compare(p2.file);
+        if (!line) return p2.line;
+        if (!p2.line) return false;
+        int d = ((string) file).compare((string) p2.file);
         if (d < 0) return true;
         if (d > 0) return false;
         if (line < p2.line) return true;
@@ -63,6 +66,7 @@ struct Expr
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
     virtual Value * maybeThunk(EvalState & state, Env & env);
+    virtual void setName(Symbol & name);
 };
 
 std::ostream & operator << (std::ostream & str, Expr & e);
@@ -74,9 +78,9 @@ std::ostream & operator << (std::ostream & str, Expr & e);
 
 struct ExprInt : Expr
 {
-    int n;
+    NixInt n;
     Value v;
-    ExprInt(int n) : n(n) { mkInt(v, n); };
+    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -106,32 +110,25 @@ struct ExprPath : Expr
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
-struct VarRef
+struct ExprVar : Expr
 {
+    Pos pos;
     Symbol name;
 
     /* Whether the variable comes from an environment (e.g. a rec, let
        or function argument) or from a "with". */
     bool fromWith;
-    
+
     /* In the former case, the value is obtained by going `level'
        levels up from the current environment and getting the
        `displ'th value in that environment.  In the latter case, the
        value is obtained by getting the attribute named `name' from
-       the attribute set stored in the environment that is `level'
+       the set stored in the environment that is `level' levels up
-       levels up from the current one.*/
+       from the current one.*/
     unsigned int level;
     unsigned int displ;
 
-    VarRef() { };
+    ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
-    VarRef(const Symbol & name) : name(name) { };
-    void bind(const StaticEnv & env);
-};
-
-struct ExprVar : Expr
-{
-    VarRef info;
-    ExprVar(const Symbol & name) : info(name) { };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -158,12 +155,10 @@ struct ExprAttrs : Expr
     bool recursive;
     struct AttrDef {
         bool inherited;
-        Expr * e; // if not inherited
+        Expr * e;
-        VarRef var; // if inherited
         Pos pos;
         unsigned int displ; // displacement
-        AttrDef(Expr * e, const Pos & pos) : inherited(false), e(e), pos(pos) { };
+        AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { };
-        AttrDef(const Symbol & name, const Pos & pos) : inherited(true), var(name), pos(pos) { };
         AttrDef() { };
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
@@ -197,6 +192,7 @@ struct Formals
 struct ExprLambda : Expr
 {
     Pos pos;
+    Symbol name;
     Symbol arg;
     bool matchAttrs;
     Formals * formals;
@@ -208,6 +204,8 @@ struct ExprLambda : Expr
             throw ParseError(format("duplicate formal function argument `%1%' at %2%")
                 % arg % pos);
     };
+    void setName(Symbol & name);
+    string showNamePos();
     COMMON_METHODS
 };
 
@@ -277,8 +275,10 @@ MakeBinOp(OpConcatLists, "++")
 
 struct ExprConcatStrings : Expr
 {
+    bool forceString;
     vector<Expr *> * es;
-    ExprConcatStrings(vector<Expr *> * es) : es(es) { };
+    ExprConcatStrings(bool forceString, vector<Expr *> * es)
+        : forceString(forceString), es(es) { };
     COMMON_METHODS
 };
 

src/libexpr/parser.y

@@ -4,32 +4,32 @@
 %error-verbose
 %defines
 /* %no-lines */
-%parse-param { yyscan_t scanner }
+%parse-param { void * scanner }
-%parse-param { ParseData * data }
+%parse-param { nix::ParseData * data }
-%lex-param { yyscan_t scanner }
+%lex-param { void * scanner }
-%lex-param { ParseData * data }
+%lex-param { nix::ParseData * data }
 %expect 1
 %expect-rr 1
 
 %code requires {
-    
+
 #ifndef BISON_HEADER
 #define BISON_HEADER
-    
+
 #include "util.hh"
-    
+
 #include "nixexpr.hh"
 #include "eval.hh"
 
 namespace nix {
 
-    struct ParseData 
+    struct ParseData
     {
         EvalState & state;
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Path path;
+        Symbol path;
         string error;
         Symbol sLetBody;
         ParseData(EvalState & state)
@@ -38,7 +38,7 @@ namespace nix {
             , sLetBody(symbols.create("<let-body>"))
             { };
     };
-    
+
 }
 
 #define YY_DECL int yylex \
@@ -52,7 +52,6 @@ namespace nix {
 
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
-#define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -64,14 +63,14 @@ using namespace nix;
 
 
 namespace nix {
-    
+
 
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
         % showAttrPath(attrPath) % pos % prevPos);
 }
- 
+
 
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
@@ -79,7 +78,7 @@ static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
     throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
         % showAttrPath(attrPath) % pos % prevPos);
 }
- 
+
 
 static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
     Expr * e, const Pos & pos)
@@ -105,6 +104,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
             }
         }
     }
+    e->setName(attrPath.back());
 }
 
 
@@ -121,7 +121,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
-    
+
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
        the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */
@@ -170,7 +170,7 @@ static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
             es2->push_back(*i);
             continue;
         }
-        
+
         string s2;
         for (unsigned int j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
@@ -203,7 +203,8 @@ static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
         es2->push_back(new ExprString(symbols.create(s2)));
     }
 
-    return es2->size() == 1 ? (*es2)[0] : new ExprConcatStrings(es2);
+    /* If this is a single string, then don't do a concatenation. */
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(true, es2);
 }
 
 
@@ -232,13 +233,13 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %}
 
 %union {
-  // !!! We're probably leaking stuff here.  
+  // !!! We're probably leaking stuff here.
   nix::Expr * e;
   nix::ExprList * list;
   nix::ExprAttrs * attrs;
   nix::Formals * formals;
   nix::Formal * formal;
-  int n;
+  nix::NixInt n;
   char * id; // !!! -> Symbol
   char * path;
   char * uri;
@@ -269,12 +270,15 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %left OR
 %left AND
 %nonassoc EQ NEQ
+%left '<' '>' LEQ GEQ
 %right UPDATE
-%left NEG
+%left NOT
-%left '+'
+%left '+' '-'
+%left '*' '/'
 %right CONCAT
 %nonassoc '?'
 %nonassoc '~'
+%nonassoc NEGATE
 
 %%
 
@@ -306,9 +310,14 @@ expr_if
   ;
 
 expr_op
-  : '!' expr_op %prec NEG { $$ = new ExprOpNot($2); }
+  : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
+| '-' expr_op %prec NEGATE { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__sub")), new ExprInt(0)), $2); }
   | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
   | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
+  | expr_op '<' expr_op { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__lessThan")), $1), $3); }
+  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__lessThan")), $3), $1)); }
+  | expr_op '>' expr_op { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__lessThan")), $3), $1); }
+  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__lessThan")), $1), $3)); }
   | expr_op AND expr_op { $$ = new ExprOpAnd($1, $3); }
   | expr_op OR expr_op { $$ = new ExprOpOr($1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl($1, $3); }
@@ -318,8 +327,11 @@ expr_op
     { vector<Expr *> * l = new vector<Expr *>;
       l->push_back($1);
       l->push_back($3);
-      $$ = new ExprConcatStrings(l);
+      $$ = new ExprConcatStrings(false, l);
     }
+  | expr_op '-' expr_op { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__sub")), $1), $3); }
+  | expr_op '*' expr_op { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__mul")), $1), $3); }
+  | expr_op '/' expr_op { $$ = new ExprApp(new ExprApp(new ExprVar(noPos, data->symbols.create("__div")), $1), $3); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); }
   | expr_app
   ;
@@ -338,18 +350,18 @@ expr_select
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
-    { $$ = new ExprApp($1, new ExprVar(data->symbols.create("or"))); }
+    { $$ = new ExprApp($1, new ExprVar(CUR_POS, data->symbols.create("or"))); }
   | expr_simple { $$ = $1; }
   ;
 
 expr_simple
-  : ID { $$ = new ExprVar(data->symbols.create($1)); }
+  : ID { $$ = new ExprVar(CUR_POS, data->symbols.create($1)); }
   | INT { $$ = new ExprInt($1); }
   | '"' string_parts '"' {
       /* For efficiency, and to simplify parse trees a bit. */
       if ($2->empty()) $$ = new ExprString(data->symbols.create(""));
-      else if ($2->size() == 1) $$ = $2->front();
+      else if ($2->size() == 1 && dynamic_cast<ExprString *>($2->front())) $$ = $2->front();
-      else $$ = new ExprConcatStrings($2);
+      else $$ = new ExprConcatStrings(true, $2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(data->symbols, *$2);
@@ -361,10 +373,10 @@ expr_simple
       /* The file wasn't found in the search path.  However, we can't
          throw an error here, because the expression might never be
          evaluated.  So return an expression that lazily calls
-         ‘abort’. */
+         ‘throw’. */
       $$ = path2 == ""
           ? (Expr * ) new ExprApp(
-              new ExprVar(data->symbols.create("throw")),
+              new ExprVar(noPos, data->symbols.create("throw")),
               new ExprString(data->symbols.create(
                       (format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % path).str())))
           : (Expr * ) new ExprPath(path2);
@@ -402,7 +414,7 @@ binds
           if ($$->attrs.find(*i) != $$->attrs.end())
               dupAttr(*i, makeCurPos(@3, data), $$->attrs[*i].pos);
           Pos pos = makeCurPos(@3, data);
-          $$->attrs[*i] = ExprAttrs::AttrDef(*i, pos);
+          $$->attrs[*i] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, *i), pos, true);
       }
     }
   | binds INHERIT '(' expr ')' attrs ';'
@@ -454,7 +466,7 @@ formal
   : ID { $$ = new Formal(data->symbols.create($1), 0); }
   | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
   ;
-  
+
 %%
 
 
@@ -467,34 +479,30 @@ formal
 
 
 namespace nix {
-      
+
 
 Expr * EvalState::parse(const char * text,
-    const Path & path, const Path & basePath)
+    const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
     data.basePath = basePath;
-    data.path = path;
+    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
-    
+
     if (res) throw ParseError(data.error);
 
-    try {
+    data.result->bindVars(staticEnv);
-        data.result->bindVars(staticBaseEnv);
+
-    } catch (Error & e) {
-        throw ParseError(format("%1%, in `%2%'") % e.msg() % path);
-    }
-    
     return data.result;
 }
 
 
-Expr * EvalState::parseExprFromFile(Path path)
+Path resolveExprPath(Path path)
 {
     assert(path[0] == '/');
 
@@ -512,21 +520,25 @@ Expr * EvalState::parseExprFromFile(Path path)
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 
-    /* Read and parse the input file, unless it's already in the parse
+    return path;
-       tree cache. */
+}
-    Expr * e = parseTrees[path];
-    if (!e) {
-        e = parse(readFile(path).c_str(), path, dirOf(path));
-        parseTrees[path] = e;
-    }
 
-    return e;
+
+Expr * EvalState::parseExprFromFile(const Path & path)
+{
+    return parse(readFile(path).c_str(), path, dirOf(path), staticBaseEnv);
+}
+
+
+Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+{
+    return parse(s.c_str(), "(string)", basePath, staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 {
-    return parse(s.c_str(), "(string)", basePath);
+    return parseExprFromString(s, basePath, staticBaseEnv);
 }
 
 
@@ -541,7 +553,7 @@ void EvalState::addToSearchPath(const string & s)
         prefix = string(s, 0, pos);
         path = string(s, pos + 1);
     }
-    
+
     path = absPath(path);
     if (pathExists(path)) {
         debug(format("adding path `%1%' to the search path") % path);
@@ -554,7 +566,7 @@ Path EvalState::findFile(const string & path)
 {
     foreach (SearchPath::iterator, i, searchPath) {
         Path res;
-        if (i->first.empty()) 
+        if (i->first.empty())
             res = i->second + "/" + path;
         else {
             if (path.compare(0, i->first.size(), i->first) != 0 ||

src/libexpr/primops.cc

@@ -37,7 +37,7 @@ std::pair<string, string> decodeContext(const string & s)
 
 
 /* Load and evaluate an expression from path specified by the
-   argument. */ 
+   argument. */
 static void prim_import(EvalState & state, Value * * args, Value & v)
 {
     PathSet context;
@@ -56,7 +56,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
                 unsigned long long downloadSize, narSize;
                 queryMissing(*store, singleton<PathSet>(ctx),
                     willBuild, willSubstitute, unknown, downloadSize, narSize);
-                  
+
                 /* !!! If using a substitute, we only need to fetch
                    the selected output of this derivation. */
                 store->buildPaths(singleton<PathSet>(ctx));
@@ -82,8 +82,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
         }
         w.attrs->sort();
         Value fun;
-        state.mkThunk_(fun,
+        state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun);
-            state.parseExprFromFile(state.findFile("nix/imported-drv-to-derivation.nix")));
         state.forceFunction(fun);
         mkApp(v, fun, w);
         state.forceAttrs(v);
@@ -93,6 +92,30 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
 }
 
 
+/* Return a string representing the type of the expression. */
+static void prim_typeOf(EvalState & state, Value * * args, Value & v)
+{
+    state.forceValue(*args[0]);
+    string t;
+    switch (args[0]->type) {
+        case tInt: t = "int"; break;
+        case tBool: t = "bool"; break;
+        case tString: t = "string"; break;
+        case tPath: t = "path"; break;
+        case tNull: t = "null"; break;
+        case tAttrs: t = "set"; break;
+        case tList: t = "list"; break;
+        case tLambda:
+        case tPrimOp:
+        case tPrimOpApp:
+            t = "lambda";
+            break;
+        default: abort();
+    }
+    mkString(v, state.symbols.create(t));
+}
+
+
 /* Determine whether the argument is the null value. */
 static void prim_isNull(EvalState & state, Value * * args, Value & v)
 {
@@ -109,7 +132,7 @@ static void prim_isFunction(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Determine whether the argument is an Int. */
+/* Determine whether the argument is an integer. */
 static void prim_isInt(EvalState & state, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
@@ -117,7 +140,7 @@ static void prim_isInt(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Determine whether the argument is an String. */
+/* Determine whether the argument is a string. */
 static void prim_isString(EvalState & state, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
@@ -125,7 +148,7 @@ static void prim_isString(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Determine whether the argument is an Bool. */
+/* Determine whether the argument is a Boolean. */
 static void prim_isBool(EvalState & state, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
@@ -135,24 +158,31 @@ static void prim_isBool(EvalState & state, Value * * args, Value & v)
 
 struct CompareValues
 {
-    bool operator () (const Value & v1, const Value & v2) const
+    bool operator () (const Value * v1, const Value * v2) const
     {
-        if (v1.type != v2.type)
+        if (v1->type != v2->type)
             throw EvalError("cannot compare values of different types");
-        switch (v1.type) {
+        switch (v1->type) {
             case tInt:
-                return v1.integer < v2.integer;
+                return v1->integer < v2->integer;
             case tString:
-                return strcmp(v1.string.s, v2.string.s) < 0;
+                return strcmp(v1->string.s, v2->string.s) < 0;
             case tPath:
-                return strcmp(v1.path, v2.path) < 0;
+                return strcmp(v1->path, v2->path) < 0;
             default:
-                throw EvalError(format("cannot compare %1% with %2%") % showType(v1) % showType(v2));
+                throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         }
     }
 };
 
 
+#if HAVE_BOEHMGC
+typedef list<Value *, gc_allocator<Value *> > ValueList;
+#else
+typedef list<Value *> ValueList;
+#endif
+
+
 static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
 {
     startNest(nest, lvlDebug, "finding dependencies");
@@ -166,7 +196,7 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
         throw EvalError("attribute `startSet' required");
     state.forceList(*startSet->value);
 
-    list<Value *> workSet;
+    ValueList workSet;
     for (unsigned int n = 0; n < startSet->value->list.length; ++n)
         workSet.push_back(startSet->value->list.elems[n]);
 
@@ -180,11 +210,13 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
     /* Construct the closure by applying the operator to element of
        `workSet', adding the result to `workSet', continuing until
        no new elements are found. */
-    list<Value> res;
+    ValueList res;
-    set<Value, CompareValues> doneKeys; // !!! use Value *?
+    // `doneKeys' doesn't need to be a GC root, because its values are
+    // reachable from res.
+    set<Value *, CompareValues> doneKeys;
     while (!workSet.empty()) {
-	Value * e = *(workSet.begin());
+        Value * e = *(workSet.begin());
-	workSet.pop_front();
+        workSet.pop_front();
 
         state.forceAttrs(*e);
 
@@ -194,10 +226,10 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
             throw EvalError("attribute `key' required");
         state.forceValue(*key->value);
 
-        if (doneKeys.find(*key->value) != doneKeys.end()) continue;
+        if (doneKeys.find(key->value) != doneKeys.end()) continue;
-        doneKeys.insert(*key->value);
+        doneKeys.insert(key->value);
-        res.push_back(*e);
+        res.push_back(e);
-        
+
         /* Call the `operator' function with `e' as argument. */
         Value call;
         mkApp(call, *op->value, *e);
@@ -213,8 +245,8 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
     /* Create the result list. */
     state.mkList(v, res.size());
     unsigned int n = 0;
-    foreach (list<Value>::iterator, i, res)
+    foreach (ValueList::iterator, i, res)
-        *(v.list.elems[n++] = state.allocValue()) = *i;
+        v.list.elems[n++] = *i;
 }
 
 
@@ -247,17 +279,17 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Try evaluating the argument. Success => {success=true; value=something;}, 
+/* Try evaluating the argument. Success => {success=true; value=something;},
  * else => {success=false; value=false;} */
 static void prim_tryEval(EvalState & state, Value * * args, Value & v)
 {
     state.mkAttrs(v, 2);
     try {
         state.forceValue(*args[0]);
-        v.attrs->push_back(Attr(state.symbols.create("value"), args[0]));
+        v.attrs->push_back(Attr(state.sValue, args[0]));
         mkBool(*state.allocAttr(v, state.symbols.create("success")), true);
     } catch (AssertionError & e) {
-        mkBool(*state.allocAttr(v, state.symbols.create("value")), false);
+        mkBool(*state.allocAttr(v, state.sValue), false);
         mkBool(*state.allocAttr(v, state.symbols.create("success")), false);
     }
     v.attrs->sort();
@@ -395,10 +427,8 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the derivation attribute `%1%' at %2%:\n")
+            e.addPrefix(format("while evaluating the attribute `%1%' of the derivation `%2%' at %3%:\n")
-                % key % *i->pos);
+                % key % drvName % posDrvName);
-            e.addPrefix(format("while instantiating the derivation named `%1%' at %2%:\n")
-                % drvName % posDrvName);
             throw;
         }
     }
@@ -634,7 +664,7 @@ static void prim_toFile(EvalState & state, Value * * args, Value & v)
             throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name);
         refs.insert(path);
     }
-    
+
     Path storePath = settings.readOnlyMode
         ? computeStorePathForText(name, contents, refs)
         : store->addTextToStore(name, contents, refs, state.repair);
@@ -651,7 +681,7 @@ struct FilterFromExpr : PathFilter
 {
     EvalState & state;
     Value & filter;
-    
+
     FilterFromExpr(EvalState & state, Value & filter)
         : state(state), filter(filter)
     {
@@ -672,12 +702,12 @@ struct FilterFromExpr : PathFilter
         state.callFunction(filter, arg1, fun2);
 
         Value arg2;
-        mkString(arg2, 
+        mkString(arg2,
             S_ISREG(st.st_mode) ? "regular" :
             S_ISDIR(st.st_mode) ? "directory" :
             S_ISLNK(st.st_mode) ? "symlink" :
             "unknown" /* not supported, will fail! */);
-        
+
         Value res;
         state.callFunction(fun2, arg2, res);
 
@@ -708,12 +738,12 @@ static void prim_filterSource(EvalState & state, Value * * args, Value & v)
 
 
 /*************************************************************
- * Attribute sets
+ * Sets
  *************************************************************/
 
 
-/* Return the names of the attributes in an attribute set as a sorted
+/* Return the names of the attributes in a set as a sorted list of
-   list of strings. */
+   strings. */
 static void prim_attrNames(EvalState & state, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0]);
@@ -755,7 +785,7 @@ static void prim_hasAttr(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Determine whether the argument is an attribute set. */
+/* Determine whether the argument is a set. */
 static void prim_isAttrs(EvalState & state, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
@@ -786,10 +816,11 @@ static void prim_removeAttrs(EvalState & state, Value * * args, Value & v)
 }
 
 
-/* Builds an attribute set from a list specifying (name, value)
+/* Builds a set from a list specifying (name, value) pairs.  To be
-   pairs.  To be precise, a list [{name = "name1"; value = value1;}
+   precise, a list [{name = "name1"; value = value1;} ... {name =
-   ... {name = "nameN"; value = valueN;}] is transformed to {name1 =
+   "nameN"; value = valueN;}] is transformed to {name1 = value1;
-   value1; ... nameN = valueN;}. */
+   ... nameN = valueN;}.  In case of duplicate occurences of the same
+   name, the first takes precedence. */
 static void prim_listToAttrs(EvalState & state, Value * * args, Value & v)
 {
     state.forceList(*args[0]);
@@ -801,36 +832,35 @@ static void prim_listToAttrs(EvalState & state, Value * * args, Value & v)
     for (unsigned int i = 0; i < args[0]->list.length; ++i) {
         Value & v2(*args[0]->list.elems[i]);
         state.forceAttrs(v2);
-        
+
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
             throw TypeError("`name' attribute missing in a call to `listToAttrs'");
         string name = state.forceStringNoCtx(*j->value);
-        
-        Bindings::iterator j2 = v2.attrs->find(state.symbols.create("value"));
-        if (j2 == v2.attrs->end())
-            throw TypeError("`value' attribute missing in a call to `listToAttrs'");
 
         Symbol sym = state.symbols.create(name);
         if (seen.find(sym) == seen.end()) {
+            Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
+            if (j2 == v2.attrs->end())
+                throw TypeError("`value' attribute missing in a call to `listToAttrs'");
+
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
             seen.insert(sym);
         }
-        /* !!! Throw an error if `name' already exists? */
     }
 
     v.attrs->sort();
 }
 
 
-/* Return the right-biased intersection of two attribute sets as1 and
+/* Return the right-biased intersection of two sets as1 and as2,
-   as2, i.e. a set that contains every attribute from as2 that is also
+   i.e. a set that contains every attribute from as2 that is also a
-   a member of as1. */
+   member of as1. */
 static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v)
 {
     state.forceAttrs(*args[0]);
     state.forceAttrs(*args[1]);
-        
+
     state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size()));
 
     foreach (Bindings::iterator, i, *args[0]->attrs) {
@@ -933,7 +963,7 @@ static void prim_map(EvalState & state, Value * * args, Value & v)
     state.mkList(v, args[1]->list.length);
 
     for (unsigned int n = 0; n < v.list.length; ++n)
-        mkApp(*(v.list.elems[n] = state.allocValue()), 
+        mkApp(*(v.list.elems[n] = state.allocValue()),
             *args[0], *args[1]->list.elems[n]);
 }
 
@@ -1024,7 +1054,7 @@ static void prim_mul(EvalState & state, Value * * args, Value & v)
 
 static void prim_div(EvalState & state, Value * * args, Value & v)
 {
-    int i2 = state.forceInt(*args[1]);
+    NixInt i2 = state.forceInt(*args[1]);
     if (i2 == 0) throw EvalError("division by zero");
     mkInt(v, state.forceInt(*args[0]) / i2);
 }
@@ -1032,7 +1062,10 @@ static void prim_div(EvalState & state, Value * * args, Value & v)
 
 static void prim_lessThan(EvalState & state, Value * * args, Value & v)
 {
-    mkBool(v, state.forceInt(*args[0]) < state.forceInt(*args[1]));
+    state.forceValue(*args[0]);
+    state.forceValue(*args[1]);
+    CompareValues comp;
+    mkBool(v, comp(args[0], args[1]));
 }
 
 
@@ -1102,11 +1135,26 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, Value * * args
         if (p.at(0) == '=') p = "~" + string(p, 1);
         context2.insert(p);
     }
-    
+
     mkString(v, s, context2);
 }
 
 
+/* Return the cryptographic hash of a string in base-16. */
+static void prim_hashString(EvalState & state, Value * * args, Value & v)
+{
+    string type = state.forceStringNoCtx(*args[0]);
+    HashType ht = parseHashType(type);
+    if (ht == htUnknown)
+      throw Error(format("unknown hash type `%1%'") % type);
+
+    PathSet context; // discarded
+    string s = state.forceString(*args[1], context);
+
+    mkString(v, printHash(hashString(ht, s)), context);
+};
+
+
 /*************************************************************
  * Versions
  *************************************************************/
@@ -1149,10 +1197,10 @@ void EvalState::createBaseEnv()
 
     mkBool(v, true);
     addConstant("true", v);
-    
+
     mkBool(v, false);
     addConstant("false", v);
-    
+
     v.type = tNull;
     addConstant("null", v);
 
@@ -1174,6 +1222,7 @@ void EvalState::createBaseEnv()
 
     // Miscellaneous
     addPrimOp("import", 1, prim_import);
+    addPrimOp("__typeOf", 1, prim_typeOf);
     addPrimOp("isNull", 1, prim_isNull);
     addPrimOp("__isFunction", 1, prim_isFunction);
     addPrimOp("__isString", 1, prim_isString);
@@ -1200,7 +1249,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__toFile", 2, prim_toFile);
     addPrimOp("__filterSource", 2, prim_filterSource);
 
-    // Attribute sets
+    // Sets
     addPrimOp("__attrNames", 1, prim_attrNames);
     addPrimOp("__getAttr", 2, prim_getAttr);
     addPrimOp("__hasAttr", 2, prim_hasAttr);
@@ -1234,6 +1283,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
     addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
+    addPrimOp("__hashString", 2, prim_hashString);
 
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);
@@ -1244,11 +1294,13 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    mkThunk_(v, parseExprFromFile(findFile("nix/derivation.nix")));
+    string path = findFile("nix/derivation.nix");
+    sDerivationNix = symbols.create(path);
+    evalFile(path, v);
     addConstant("derivation", v);
 
-    /* Now that we've added all primops, sort the `builtins' attribute
+    /* Now that we've added all primops, sort the `builtins' set,
-       set, because attribute lookups expect it to be sorted. */
+       because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
 }
 

src/libexpr/symbol-table.hh

@@ -13,11 +13,10 @@
 namespace nix {
 
 /* Symbol table used by the parser and evaluator to represent and look
-   up identifiers and attribute sets efficiently.
+   up identifiers and attributes efficiently.  SymbolTable::create()
-   SymbolTable::create() converts a string into a symbol.  Symbols
+   converts a string into a symbol.  Symbols have the property that
-   have the property that they can be compared efficiently (using a
+   they can be compared efficiently (using a pointer equality test),
-   pointer equality test), because the symbol table stores only one
+   because the symbol table stores only one copy of each string. */
-   copy of each string. */
 
 class Symbol
 {
@@ -28,17 +27,17 @@ private:
 
 public:
     Symbol() : s(0) { };
-    
+
     bool operator == (const Symbol & s2) const
     {
         return s == s2.s;
     }
-    
+
     bool operator != (const Symbol & s2) const
     {
         return s != s2.s;
     }
-    
+
     bool operator < (const Symbol & s2) const
     {
         return s < s2.s;
@@ -49,6 +48,11 @@ public:
         return *s;
     }
 
+    bool set() const
+    {
+        return s;
+    }
+
     bool empty() const
     {
         return s->empty();
@@ -66,7 +70,7 @@ inline std::ostream & operator << (std::ostream & str, const Symbol & sym)
 class SymbolTable
 {
 private:
-#if HAVE_TR1_UNORDERED_SET 
+#if HAVE_TR1_UNORDERED_SET
     typedef std::tr1::unordered_set<string> Symbols;
 #else
     typedef std::set<string> Symbols;
@@ -84,6 +88,8 @@ public:
     {
         return symbols.size();
     }
+
+    size_t totalSize() const;
 };
 
 }

src/libexpr/value.hh

@@ -31,17 +31,20 @@ struct PrimOp;
 struct Symbol;
 
 
+typedef long NixInt;
+
+
 struct Value
 {
     ValueType type;
-    union 
+    union
     {
-        int integer;
+        NixInt integer;
         bool boolean;
-        
+
-        /* Strings in the evaluator carry a so-called `context' (the
+        /* Strings in the evaluator carry a so-called `context' which
-           ATermList) which is a list of strings representing store
+           is a list of strings representing store paths.  This is to
-           paths.  This is to allow users to write things like
+           allow users to write things like
 
              "--with-freetype2-library=" + freetype + "/lib"
 
@@ -63,7 +66,7 @@ struct Value
             const char * s;
             const char * * context; // must be in sorted order
         } string;
-        
+
         const char * path;
         Bindings * attrs;
         struct {
@@ -97,7 +100,7 @@ static inline void clearValue(Value & v)
 }
 
 
-static inline void mkInt(Value & v, int n)
+static inline void mkInt(Value & v, NixInt n)
 {
     clearValue(v);
     v.type = tInt;
@@ -131,7 +134,7 @@ static inline void mkStringNoCopy(Value & v, const char * s)
 
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((string) s).c_str());
+    mkStringNoCopy(v, ((const string &) s).c_str());
 }
 
 

src/libmain/Makefile.am

@@ -1,6 +1,6 @@
 pkglib_LTLIBRARIES = libmain.la
 
-libmain_la_SOURCES = shared.cc
+libmain_la_SOURCES = shared.cc stack.cc
 
 libmain_la_LIBADD = ../libstore/libstore.la @BDW_GC_LIBS@
 

src/libmain/shared.cc

@@ -10,8 +10,10 @@
 #include <cctype>
 #include <exception>
 
+#include <sys/time.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <signal.h>
 
 #if HAVE_BOEHMGC
 #include <gc/gc.h>
@@ -99,6 +101,9 @@ string getArg(const string & opt,
 }
 
 
+void detectStackOverflow();
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
@@ -130,11 +135,19 @@ static void initAndRun(int argc, char * * argv)
     if (sigaction(SIGCHLD, &act, 0))
         throw SysError("resetting SIGCHLD");
 
+    /* Register a SIGSEGV handler to detect stack overflows. */
+    detectStackOverflow();
+
     /* There is no privacy in the Nix system ;-)  At least not for
        now.  In particular, store objects should be readable by
        everybody. */
     umask(0022);
 
+    /* Initialise the PRNG. */
+    struct timeval tv;
+    gettimeofday(&tv, 0);
+    srandom(tv.tv_usec);
+
     /* Process the NIX_LOG_TYPE environment variable. */
     string lt = getEnv("NIX_LOG_TYPE");
     if (lt != "") setLogType(lt);
@@ -162,11 +175,10 @@ static void initAndRun(int argc, char * * argv)
     remaining.clear();
 
     /* Process default options. */
-    int verbosityDelta = lvlInfo;
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
-        if (arg == "--verbose" || arg == "-v") verbosityDelta++;
+        if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1);
-        else if (arg == "--quiet") verbosityDelta--;
+        else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError;
         else if (arg == "--log-type") {
             string s = getArg(arg, i, args.end());
             setLogType(s);
@@ -213,8 +225,6 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
-    verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
-
     settings.update();
 
     run(remaining);
@@ -293,6 +303,9 @@ int main(int argc, char * * argv)
         if (e.prefix() != "" && !showTrace)
             printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
         return e.status;
+    } catch (std::bad_alloc & e) {
+        printMsg(lvlError, "error: out of memory");
+        return 1;
     } catch (std::exception & e) {
         printMsg(lvlError, format("error: %1%") % e.what());
         return 1;

src/libmain/stack.cc

@@ -0,0 +1,72 @@
+#include "config.h"
+
+#include "types.hh"
+
+#include <cstring>
+#include <cstddef>
+#include <cstdlib>
+
+#include <unistd.h>
+#include <signal.h>
+
+namespace nix {
+
+
+static void sigsegvHandler(int signo, siginfo_t * info, void * ctx)
+{
+    /* Detect stack overflows by comparing the faulting address with
+       the stack pointer.  Unfortunately, getting the stack pointer is
+       not portable. */
+    bool haveSP = true;
+    char * sp;
+#if defined(__x86_64__) && defined(REG_RSP)
+    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP];
+#elif defined(REG_ESP)
+    sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_ESP];
+#else
+    haveSP = false;
+#endif
+
+    if (haveSP) {
+        ptrdiff_t diff = (char *) info->si_addr - sp;
+        if (diff < 0) diff = -diff;
+        if (diff < 4096) {
+            char msg[] = "error: stack overflow (possible infinite recursion)\n";
+            write(2, msg, strlen(msg));
+            _exit(1); // maybe abort instead?
+        }
+    }
+
+    /* Restore default behaviour (i.e. segfault and dump core). */
+    struct sigaction act;
+    sigfillset(&act.sa_mask);
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGSEGV, &act, 0)) abort();
+}
+
+
+void detectStackOverflow()
+{
+#if defined(SA_SIGINFO) && defined (SA_ONSTACK)
+    /* Install a SIGSEGV handler to detect stack overflows.  This
+       requires an alternative stack, otherwise the signal cannot be
+       delivered when we're out of stack space. */
+    stack_t stack;
+    stack.ss_size = 4096 * 4 + MINSIGSTKSZ;
+    stack.ss_sp = new char[stack.ss_size];
+    if (!stack.ss_sp) throw Error("cannot allocate alternative stack");
+    stack.ss_flags = 0;
+    if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack");
+
+    struct sigaction act;
+    sigfillset(&act.sa_mask);
+    act.sa_sigaction = sigsegvHandler;
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK;
+    if (sigaction(SIGSEGV, &act, 0))
+        throw SysError("resetting SIGCHLD");
+#endif
+}
+
+
+}

src/libstore/build.cc

@@ -7,7 +7,7 @@
 #include "local-store.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "immutable.hh"
+#include "affinity.hh"
 
 #include <map>
 #include <sstream>
@@ -21,6 +21,7 @@
 #include <sys/wait.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/utsname.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
@@ -44,6 +45,12 @@
 #include <sched.h>
 #endif
 
+/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but
+   <linux/fs.h> does.  */
+#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H
+#include <linux/fs.h>
+#endif
+
 #define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
 
 #if CHROOT_ENABLED
@@ -93,7 +100,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
 class Goal : public boost::enable_shared_from_this<Goal>
 {
 public:
-    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
+    typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
 
 protected:
 
@@ -114,6 +121,10 @@ protected:
        failed because there are no substituters. */
     unsigned int nrNoSubstituters;
 
+    /* Number of substitution goals we are/were waiting for that
+       failed because othey had unsubstitutable references. */
+    unsigned int nrIncompleteClosure;
+
     /* Name of this goal for debugging purposes. */
     string name;
 
@@ -122,7 +133,7 @@ protected:
 
     Goal(Worker & worker) : worker(worker)
     {
-        nrFailed = nrNoSubstituters = 0;
+        nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
         exitCode = ecBusy;
     }
 
@@ -163,7 +174,7 @@ public:
     /* Cancel the goal.  It should wake up its waiters, get rid of any
        running child processes that are being monitored by the worker
        (important!), etc. */
-    virtual void cancel() = 0;
+    virtual void cancel(bool timeout) = 0;
 
 protected:
     void amDone(ExitCode result);
@@ -177,9 +188,10 @@ struct Child
 {
     WeakGoalPtr goal;
     set<int> fds;
-    bool monitorForSilence;
+    bool respectTimeouts;
     bool inBuildSlot;
     time_t lastOutput; /* time we last got output on stdout/stderr */
+    time_t timeStarted;
 };
 
 typedef map<pid_t, Child> Children;
@@ -223,9 +235,6 @@ private:
     /* Last time the goals in `waitingForAWhile' where woken up. */
     time_t lastWokenUp;
 
-    /* Last time `waitForInput' was last called.  */
-    time_t lastWait;
-
 public:
 
     /* Set if at least one derivation had a BuildError (i.e. permanent
@@ -257,7 +266,7 @@ public:
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
     void childStarted(GoalPtr goal, pid_t pid,
-        const set<int> & fds, bool inBuildSlot, bool monitorForSilence);
+        const set<int> & fds, bool inBuildSlot, bool respectTimeouts);
 
     /* Unregisters a running child process.  `wakeSleepers' should be
        false if there is no sense in waking up goals that are sleeping
@@ -307,10 +316,12 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
     trace(format("waitee `%1%' done; %2% left") %
         waitee->name % waitees.size());
 
-    if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
+    if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed;
 
     if (result == ecNoSubstituters) ++nrNoSubstituters;
 
+    if (result == ecIncompleteClosure) ++nrIncompleteClosure;
+
     if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) {
 
         /* If we failed and keepGoing is not set, we remove all
@@ -333,7 +344,7 @@ void Goal::amDone(ExitCode result)
 {
     trace("done");
     assert(exitCode == ecBusy);
-    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
+    assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure);
     exitCode = result;
     foreach (WeakGoals::iterator, i, waiters) {
         GoalPtr goal = i->lock();
@@ -357,6 +368,8 @@ void Goal::trace(const format & f)
 /* Common initialisation performed in child processes. */
 static void commonChildInit(Pipe & logPipe)
 {
+    restoreAffinity();
+
     /* Put the child in a separate session (and thus a separate
        process group) so that it has no controlling terminal (meaning
        that e.g. ssh cannot open /dev/tty) and it doesn't receive
@@ -559,6 +572,7 @@ static void runSetuidHelper(const string & command,
             args.push_back(0);
 
             restoreSIGPIPE();
+            restoreAffinity();
 
             execve(program.c_str(), (char * *) &args[0], 0);
             throw SysError(format("executing `%1%'") % program);
@@ -710,13 +724,7 @@ HookInstance::HookInstance()
 HookInstance::~HookInstance()
 {
     try {
-        /* Cleanly shut down the hook by closing its stdin if it's not
+        pid.kill();
-           already building.  Otherwise pid's destructor will kill
-           it. */
-        if (pid != -1 && toHook.writeSide != -1) {
-            toHook.writeSide.close();
-            pid.wait(true);
-        }
     } catch (...) {
         ignoreException();
     }
@@ -763,6 +771,10 @@ private:
     /* Whether additional wanted outputs have been added. */
     bool needRestart;
 
+    /* Whether to retry substituting the outputs after building the
+       inputs. */
+    bool retrySubstitution;
+
     /* The derivation stored at drvPath. */
     Derivation drv;
 
@@ -781,6 +793,12 @@ private:
     /* Outputs that are already valid. */
     PathSet validPaths;
 
+    /* Outputs that are corrupt or not valid. */
+    PathSet missingPaths;
+
+    /* Paths that have been subject to hash rewriting. */
+    PathSet rewrittenPaths;
+
     /* User selected for running the builder. */
     UserLock buildUser;
 
@@ -795,6 +813,9 @@ private:
     BZFILE * bzLogFile;
     AutoCloseFD fdLogFile;
 
+    /* Number of bytes received from the builder's stdout/stderr. */
+    unsigned long logSize;
+
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
 
@@ -819,7 +840,8 @@ private:
     GoalState state;
 
     /* Stuff we need to pass to initChild(). */
-    PathSet dirsInChroot;
+    typedef map<Path, Path> DirsInChroot; // maps target path to source path
+    DirsInChroot dirsInChroot;
     typedef map<string, string> Environment;
     Environment env;
 
@@ -832,6 +854,11 @@ private:
     bool repair;
     map<Path, Path> redirectedBadOutputs;
 
+    /* Set of inodes seen during calls to canonicalisePathMetaData()
+       for this build's outputs.  This needs to be shared between
+       outputs to allow hard links between outputs. */
+    InodesSeen inodesSeen;
+
     /* Magic exit code denoting that setting up the child environment
        failed.  (It's possible that the child actually returns the
        exit code, but ah well.) */
@@ -841,7 +868,7 @@ public:
     DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
     ~DerivationGoal();
 
-    void cancel();
+    void cancel(bool timeout);
 
     void work();
 
@@ -911,6 +938,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
     : Goal(worker)
     , wantedOutputs(wantedOutputs)
     , needRestart(false)
+    , retrySubstitution(false)
     , fLogFile(0)
     , bzLogFile(0)
     , useChroot(false)
@@ -966,8 +994,10 @@ void DerivationGoal::killChild()
 }
 
 
-void DerivationGoal::cancel()
+void DerivationGoal::cancel(bool timeout)
 {
+    if (settings.printBuildTrace && timeout)
+        printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
     killChild();
     amDone(ecFailed);
 }
@@ -1052,7 +1082,7 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes)
+    if (settings.useSubstitutes && !willBuildLocally(drv))
         foreach (PathSet::iterator, i, invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(*i, repair));
 
@@ -1067,10 +1097,15 @@ void DerivationGoal::outputsSubstituted()
 {
     trace("all outputs substituted (maybe)");
 
-    if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback)
+    if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback)
         throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
 
-    nrFailed = nrNoSubstituters = 0;
+    /*  If the substitutes form an incomplete closure, then we should
+        build the dependencies of this derivation, but after that, we
+        can still use the substitutes for this derivation itself. */
+    if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true;
+
+    nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
 
     if (needRestart) {
         needRestart = false;
@@ -1176,6 +1211,11 @@ void DerivationGoal::inputsRealised()
         return;
     }
 
+    if (retrySubstitution) {
+        haveDerivation();
+        return;
+    }
+
     /* Gather information necessary for computing the closure and/or
        running the build hook. */
 
@@ -1233,6 +1273,13 @@ PathSet outputPaths(const DerivationOutputs & outputs)
 }
 
 
+static string get(const StringPairs & map, const string & key)
+{
+    StringPairs::const_iterator i = map.find(key);
+    return i == map.end() ? (string) "" : i->second;
+}
+
+
 static bool canBuildLocally(const string & platform)
 {
     return platform == settings.thisSystem
@@ -1243,6 +1290,12 @@ static bool canBuildLocally(const string & platform)
 }
 
 
+bool willBuildLocally(const Derivation & drv)
+{
+    return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
+}
+
+
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
@@ -1286,6 +1339,9 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
+    missingPaths = outputPaths(drv.outputs);
+    foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i);
+
     /* If any of the outputs already exist but are not valid, delete
        them. */
     foreach (DerivationOutputs::iterator, i, drv.outputs) {
@@ -1304,11 +1360,10 @@ void DerivationGoal::tryToBuild()
 
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set. */
-    bool preferLocalBuild =
+    bool buildLocally = willBuildLocally(drv);
-        drv.env["preferLocalBuild"] == "1" && canBuildLocally(drv.platform);
 
     /* Is the build hook willing to accept this job? */
-    if (!preferLocalBuild) {
+    if (!buildLocally) {
         switch (tryBuildHook()) {
             case rpAccept:
                 /* Yes, it has started doing so.  Wait until we get
@@ -1331,7 +1386,7 @@ void DerivationGoal::tryToBuild()
        derivation prefers to be done locally, do it even if
        maxBuildJobs is 0. */
     unsigned int curBuilds = worker.getNrLocalBuilds();
-    if (curBuilds >= settings.maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
+    if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
         worker.waitForBuildSlot(shared_from_this());
         outputLocks.unlock();
         return;
@@ -1347,8 +1402,8 @@ void DerivationGoal::tryToBuild()
         outputLocks.unlock();
         buildUser.release();
         if (settings.printBuildTrace)
-            printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
+            printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
-                % drvPath % drv.outputs["out"].path % 0 % e.msg());
+                % drvPath % 0 % e.msg());
         worker.permanentFailure = true;
         amDone(ecFailed);
         return;
@@ -1367,10 +1422,8 @@ void replaceValidPath(const Path & storePath, const Path tmpPath)
        way first.  We'd better not be interrupted here, because if
        we're repairing (say) Glibc, we end up with a broken system. */
     Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str();
-    if (pathExists(storePath)) {
+    if (pathExists(storePath))
-        makeMutable(storePath);
         rename(storePath.c_str(), oldPath.c_str());
-    }
     if (rename(tmpPath.c_str(), storePath.c_str()) == -1)
         throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath);
     if (pathExists(oldPath))
@@ -1425,8 +1478,8 @@ void DerivationGoal::buildDone()
         /* Some cleanup per path.  We do this here and not in
            computeClosure() for convenience when the build has
            failed. */
-        foreach (DerivationOutputs::iterator, i, drv.outputs) {
+        foreach (PathSet::iterator, i, missingPaths) {
-            Path path = i->second.path;
+            Path path = *i;
 
             /* If the output was already valid, just skip (discard) it. */
             if (validPaths.find(path) != validPaths.end()) continue;
@@ -1463,6 +1516,12 @@ void DerivationGoal::buildDone()
             /* Apply hash rewriting if necessary. */
             if (!rewritesFromTmp.empty()) {
                 printMsg(lvlError, format("warning: rewriting hashes in `%1%'; cross fingers") % path);
+
+                /* Canonicalise first.  This ensures that the path
+                   we're rewriting doesn't contain a hard link to
+                   /etc/shadow or something like that. */
+                canonicalisePathMetaData(path, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen);
+
                 /* FIXME: this is in-memory. */
                 StringSink sink;
                 dumpPath(path, sink);
@@ -1470,6 +1529,8 @@ void DerivationGoal::buildDone()
                 sink.s = rewriteHashes(sink.s, rewritesFromTmp);
                 StringSource source(sink.s);
                 restorePath(path, source);
+
+                rewrittenPaths.insert(path);
             }
 
             /* Gain ownership of the build result using the setuid
@@ -1522,11 +1583,11 @@ void DerivationGoal::buildDone()
 
         if (settings.printBuildTrace) {
             if (hook && hookError)
-                printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
+                printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
-                    % drvPath % drv.outputs["out"].path % status % e.msg());
+                    % drvPath % status % e.msg());
             else
-                printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
+                printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
-                    % drvPath % drv.outputs["out"].path % 1 % e.msg());
+                    % drvPath % 1 % e.msg());
         }
 
         /* Register the outputs of this build as "failed" so we won't
@@ -1548,10 +1609,8 @@ void DerivationGoal::buildDone()
     /* Release the build user, if applicable. */
     buildUser.release();
 
-    if (settings.printBuildTrace) {
+    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-succeeded %1% %2%")
+        printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath);
-            % drvPath % drv.outputs["out"].path);
-    }
 
     amDone(ecSuccess);
 }
@@ -1567,7 +1626,7 @@ HookReply DerivationGoal::tryBuildHook()
     /* Tell the hook about system features (beyond the system type)
        required from the build machine.  (The hook could parse the
        drv file itself, but this is easier.) */
-    Strings features = tokenizeString<Strings>(drv.env["requiredSystemFeatures"]);
+    Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures"));
     foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
 
     /* Send the request to the hook. */
@@ -1632,8 +1691,8 @@ HookReply DerivationGoal::tryBuildHook()
     worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
 
     if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
+        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+            % drvPath % drv.platform % logFile);
 
     return rpAccept;
 }
@@ -1655,9 +1714,7 @@ int childEntry(void * arg)
 
 void DerivationGoal::startBuilder()
 {
-    PathSet missing = outputPaths(drv.outputs);
+    startNest(nest, lvlInfo, format(repair ? "repairing path(s) %1%" : "building path(s) %1%") % showPaths(missingPaths));
-    foreach (PathSet::iterator, i, validPaths) missing.erase(*i);
-    startNest(nest, lvlInfo, format("building path(s) %1%") % showPaths(missing));
 
     /* Right platform? */
     if (!canBuildLocally(drv.platform))
@@ -1678,7 +1735,8 @@ void DerivationGoal::startBuilder()
        if HOME is not set, but they will just assume that the settings file
        they are looking for does not exist if HOME is set but points to some
        non-existing path. */
-    env["HOME"] = "/homeless-shelter";
+    Path homeDir = "/homeless-shelter";
+    env["HOME"] = homeDir;
 
     /* Tell the builder where the Nix store is.  Usually they
        shouldn't care, but this is useful for purity checking (e.g.,
@@ -1726,7 +1784,7 @@ void DerivationGoal::startBuilder()
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
     if (fixedOutput) {
-        Strings varNames = tokenizeString<Strings>(drv.env["impureEnvVars"]);
+        Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars"));
         foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i);
     }
 
@@ -1737,7 +1795,7 @@ void DerivationGoal::startBuilder()
        temporary build directory.  The text files have the format used
        by `nix-store --register-validity'.  However, the deriver
        fields are left empty. */
-    string s = drv.env["exportReferencesGraph"];
+    string s = get(drv.env, "exportReferencesGraph");
     Strings ss = tokenizeString<Strings>(s);
     if (ss.size() % 2 != 0)
         throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s);
@@ -1823,7 +1881,7 @@ void DerivationGoal::startBuilder()
     if (fixedOutput) useChroot = false;
 
     /* Hack to allow derivations to disable chroot builds. */
-    if (drv.env["__noChroot"] == "1") useChroot = false;
+    if (get(drv.env, "__noChroot") == "1") useChroot = false;
 
     if (useChroot) {
 #if CHROOT_ENABLED
@@ -1869,8 +1927,14 @@ void DerivationGoal::startBuilder()
 
         /* Bind-mount a user-configurable set of directories from the
            host file system. */
-        dirsInChroot = settings.dirsInChroot;
+        foreach (StringSet::iterator, i, settings.dirsInChroot) {
-        dirsInChroot.insert(tmpDir);
+            size_t p = i->find('=');
+            if (p == string::npos)
+                dirsInChroot[*i] = *i;
+            else
+                dirsInChroot[string(*i, 0, p)] = string(*i, p + 1);
+        }
+        dirsInChroot[tmpDir] = tmpDir;
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access
@@ -1887,12 +1951,8 @@ void DerivationGoal::startBuilder()
             if (lstat(i->c_str(), &st))
                 throw SysError(format("getting attributes of path `%1%'") % *i);
             if (S_ISDIR(st.st_mode))
-                dirsInChroot.insert(*i);
+                dirsInChroot[*i] = *i;
             else {
-                /* Creating a hard link to *i is impossible if its
-                   immutable bit is set.  So clear it first. */
-                makeMutable(*i);
-
                 Path p = chrootRootDir + *i;
                 if (link(i->c_str(), p.c_str()) == -1) {
                     /* Hard-linking fails if we exceed the maximum
@@ -1925,6 +1985,9 @@ void DerivationGoal::startBuilder()
 
     else {
 
+        if (pathExists(homeDir))
+            throw Error(format("directory `%1%' exists; please remove it") % homeDir);
+
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete
            them, we have to do hash rewriting: i.e. in the
@@ -1942,7 +2005,7 @@ void DerivationGoal::startBuilder()
         /* If we're repairing, then we don't want to delete the
            corrupt outputs in advance.  So rewrite them as well. */
         if (repair)
-            foreach (PathSet::iterator, i, missing)
+            foreach (PathSet::iterator, i, missingPaths)
                 if (worker.store.isValidPath(*i) && pathExists(*i))
                     redirectedBadOutputs[*i] = addHashRewrite(*i);
     }
@@ -2008,8 +2071,8 @@ void DerivationGoal::startBuilder()
         singleton<set<int> >(builderOut.readSide), true, true);
 
     if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
+        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+            % drvPath % drv.platform % logFile);
     }
 }
 
@@ -2062,12 +2125,20 @@ void DerivationGoal::initChild()
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot
                environment. */
-            foreach (PathSet::iterator, i, dirsInChroot) {
+            foreach (DirsInChroot::iterator, i, dirsInChroot) {
-                Path source = *i;
+                struct stat st;
-                Path target = chrootRootDir + source;
+                Path source = i->second;
+                Path target = chrootRootDir + i->first;
                 if (source == "/proc") continue; // backwards compatibility
                 debug(format("bind mounting `%1%' to `%2%'") % source % target);
-                createDirs(target);
+                if (stat(source.c_str(), &st) == -1)
+                    throw SysError(format("getting attributes of path `%1%'") % source);
+                if (S_ISDIR(st.st_mode))
+                    createDirs(target);
+                else {
+                    createDirs(dirOf(target));
+                    writeFile(target, "");
+                }
                 if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
                     throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
             }
@@ -2105,7 +2176,11 @@ void DerivationGoal::initChild()
 #ifdef CAN_DO_LINUX32_BUILDS
         /* Change the personality to 32-bit if we're doing an
            i686-linux build on an x86_64-linux machine. */
-        if (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") {
+        struct utsname utsbuf;
+        uname(&utsbuf);
+        if (drv.platform == "i686-linux" &&
+            (settings.thisSystem == "x86_64-linux" ||
+             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
             if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1)
                 throw SysError("cannot set i686-linux personality");
         }
@@ -2227,6 +2302,8 @@ void DerivationGoal::computeClosure()
        output paths read-only. */
     foreach (DerivationOutputs::iterator, i, drv.outputs) {
         Path path = i->second.path;
+        if (missingPaths.find(path) == missingPaths.end()) continue;
+
         if (!pathExists(path)) {
             throw BuildError(
                 format("builder for `%1%' failed to produce output path `%2%'")
@@ -2265,8 +2342,10 @@ void DerivationGoal::computeClosure()
                     % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2));
         }
 
-        /* Get rid of all weird permissions. */
+        /* Get rid of all weird permissions.  This also checks that
-        canonicalisePathMetaData(path);
+           all files are owned by the build user, if applicable. */
+        canonicalisePathMetaData(path,
+            buildUser.enabled() && rewrittenPaths.find(path) == rewrittenPaths.end() ? buildUser.getUID() : -1, inodesSeen);
 
         /* For this output path, find the references to other paths
            contained in it.  Compute the SHA-256 NAR hash at the same
@@ -2293,7 +2372,7 @@ void DerivationGoal::computeClosure()
            refer to), check that all references are in that list.  !!!
            allowedReferences should really be per-output. */
         if (drv.env.find("allowedReferences") != drv.env.end()) {
-            PathSet allowed = parseReferenceSpecifiers(drv, drv.env["allowedReferences"]);
+            PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences"));
             foreach (PathSet::iterator, i, references)
                 if (allowed.find(*i) == allowed.end())
                     throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
@@ -2308,12 +2387,12 @@ void DerivationGoal::computeClosure()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     ValidPathInfos infos;
-    foreach (DerivationOutputs::iterator, i, drv.outputs) {
+    foreach (PathSet::iterator, i, missingPaths) {
         ValidPathInfo info;
-        info.path = i->second.path;
+        info.path = *i;
-        info.hash = contentHashes[i->second.path].first;
+        info.hash = contentHashes[*i].first;
-        info.narSize = contentHashes[i->second.path].second;
+        info.narSize = contentHashes[*i].second;
-        info.references = allReferences[i->second.path];
+        info.references = allReferences[*i];
         info.deriver = drvPath;
         infos.push_back(info);
     }
@@ -2326,15 +2405,19 @@ string drvsLogDir = "drvs";
 
 Path DerivationGoal::openLogFile()
 {
+    logSize = 0;
+
     if (!settings.keepLog) return "";
 
+    string baseName = baseNameOf(drvPath);
+
     /* Create a log file. */
-    Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str();
+    Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
     createDirs(dir);
 
     if (settings.compressLog) {
 
-        Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str();
+        Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
         AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
         if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
         closeOnExec(fd);
@@ -2349,7 +2432,7 @@ Path DerivationGoal::openLogFile()
         return logFileName;
 
     } else {
-        Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str();
+        Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
         fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
         if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
         closeOnExec(fdLogFile);
@@ -2399,6 +2482,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     if ((hook && fd == hook->builderOut.readSide) ||
         (!hook && fd == builderOut.readSide))
     {
+        logSize += data.size();
+        if (settings.maxLogSize && logSize > settings.maxLogSize) {
+            printMsg(lvlError,
+                format("%1% killed after writing more than %2% bytes of log output")
+                % getName() % settings.maxLogSize);
+            cancel(true); // not really a timeout, but close enough
+            return;
+        }
         if (verbosity >= settings.buildVerbosity)
             writeToStderr(data);
         if (bzLogFile) {
@@ -2443,7 +2534,7 @@ bool DerivationGoal::pathFailed(const Path & path)
     printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path);
 
     if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path);
+        printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath);
 
     worker.permanentFailure = true;
     amDone(ecFailed);
@@ -2514,7 +2605,7 @@ public:
     SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
     ~SubstitutionGoal();
 
-    void cancel();
+    void cancel(bool timeout);
 
     void work();
 
@@ -2555,8 +2646,10 @@ SubstitutionGoal::~SubstitutionGoal()
 }
 
 
-void SubstitutionGoal::cancel()
+void SubstitutionGoal::cancel(bool timeout)
 {
+    if (settings.printBuildTrace && timeout)
+        printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath);
     if (pid != -1) {
         pid_t savedPid = pid;
         pid.kill();
@@ -2638,7 +2731,7 @@ void SubstitutionGoal::referencesValid()
 
     if (nrFailed > 0) {
         debug(format("some references of path `%1%' could not be realised") % storePath);
-        amDone(ecFailed);
+        amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
         return;
     }
 
@@ -2747,10 +2840,8 @@ void SubstitutionGoal::tryToRun()
 
     state = &SubstitutionGoal::finished;
 
-    if (settings.printBuildTrace) {
+    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ substituter-started %1% %2%")
+        printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub);
-            % storePath % sub);
-    }
 }
 
 
@@ -2816,7 +2907,7 @@ void SubstitutionGoal::finished()
         return;
     }
 
-    canonicalisePathMetaData(destPath);
+    canonicalisePathMetaData(destPath, -1);
 
     worker.store.optimisePath(destPath); // FIXME: combine with hashPath()
 
@@ -2837,9 +2928,8 @@ void SubstitutionGoal::finished()
     printMsg(lvlChatty,
         format("substitution of path `%1%' succeeded") % storePath);
 
-    if (settings.printBuildTrace) {
+    if (settings.printBuildTrace)
         printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
-    }
 
     amDone(ecSuccess);
 }
@@ -2967,14 +3057,14 @@ unsigned Worker::getNrLocalBuilds()
 
 void Worker::childStarted(GoalPtr goal,
     pid_t pid, const set<int> & fds, bool inBuildSlot,
-    bool monitorForSilence)
+    bool respectTimeouts)
 {
     Child child;
     child.goal = goal;
     child.fds = fds;
-    child.lastOutput = time(0);
+    child.timeStarted = child.lastOutput = time(0);
     child.inBuildSlot = inBuildSlot;
-    child.monitorForSilence = monitorForSilence;
+    child.respectTimeouts = respectTimeouts;
     children[pid] = child;
     if (inBuildSlot) nrLocalBuilds++;
 }
@@ -3090,31 +3180,22 @@ void Worker::waitForInput()
     timeout.tv_usec = 0;
     time_t before = time(0);
 
-    /* If a global timeout has been set, sleep until it's done.  */
+    /* If we're monitoring for silence on stdout/stderr, or if there
-    if (settings.buildTimeout != 0) {
+       is a build timeout, then wait for input until the first
-        useTimeout = true;
+       deadline for any child. */
-        if (lastWait == 0 || lastWait > before) lastWait = before;
+    assert(sizeof(time_t) >= sizeof(long));
-        timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before);
+    time_t nearest = LONG_MAX; // nearest deadline
+    foreach (Children::iterator, i, children) {
+        if (!i->second.respectTimeouts) continue;
+        if (settings.maxSilentTime != 0)
+            nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime);
+        if (settings.buildTimeout != 0)
+            nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout);
     }
-
+    if (nearest != LONG_MAX) {
-    /* If we're monitoring for silence on stdout/stderr, sleep until
+        timeout.tv_sec = std::max((time_t) 1, nearest - before);
-       the first deadline for any child. */
+        useTimeout = true;
-    if (settings.maxSilentTime != 0) {
+        printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
-        time_t oldest = 0;
-        foreach (Children::iterator, i, children) {
-            if (i->second.monitorForSilence) {
-                oldest = oldest == 0 || i->second.lastOutput < oldest
-                    ? i->second.lastOutput : oldest;
-            }
-        }
-        if (oldest) {
-            time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before);
-            timeout.tv_sec = useTimeout
-                ? std::min(silenceTimeout, timeout.tv_sec)
-                : silenceTimeout;
-            useTimeout = true;
-            printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
-        }
     }
 
     /* If we are polling goals that are waiting for a lock, then wake
@@ -3124,7 +3205,7 @@ void Worker::waitForInput()
         if (lastWokenUp == 0)
             printMsg(lvlError, "waiting for locks or build slots...");
         if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max((time_t) 0, (time_t) (lastWokenUp + settings.pollInterval - before));
+        timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before));
     } else lastWokenUp = 0;
 
     using namespace std;
@@ -3148,9 +3229,6 @@ void Worker::waitForInput()
 
     time_t after = time(0);
 
-    /* Keep track of when we were last called.  */
-    lastWait = after;
-
     /* Process all available file descriptors. */
 
     /* Since goals may be canceled from inside the loop below (causing
@@ -3191,22 +3269,23 @@ void Worker::waitForInput()
         }
 
         if (settings.maxSilentTime != 0 &&
-            j->second.monitorForSilence &&
+            j->second.respectTimeouts &&
             after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds of silence")
                 % goal->getName() % settings.maxSilentTime);
-            goal->cancel();
+            goal->cancel(true);
         }
 
         if (settings.buildTimeout != 0 &&
-            after - before >= (time_t) settings.buildTimeout)
+            j->second.respectTimeouts &&
+            after - j->second.timeStarted >= (time_t) settings.buildTimeout)
         {
             printMsg(lvlError,
-                format("%1% timed out after %2% seconds of activity")
+                format("%1% timed out after %2% seconds")
                 % goal->getName() % settings.buildTimeout);
-            goal->cancel();
+            goal->cancel(true);
         }
     }
 

src/libstore/gc.cc

@@ -1,7 +1,6 @@
 #include "globals.hh"
 #include "misc.hh"
 #include "local-store.hh"
-#include "immutable.hh"
 
 #include <boost/shared_ptr.hpp>
 
@@ -259,8 +258,7 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
         if (lockFile(*fd, ltWrite, false)) {
-            printMsg(lvlError, format("removing stale temporary roots file `%1%'")
+            printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path);
-                % path);
             unlink(path.c_str());
             writeFull(*fd, (const unsigned char *) "d", 1);
             continue;
@@ -291,46 +289,47 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
 }
 
 
-static void findRoots(StoreAPI & store, const Path & path,
+static void foundRoot(StoreAPI & store,
-    bool recurseSymlinks, bool deleteStale, Roots & roots)
+    const Path & path, const Path & target, Roots & roots)
+{
+    Path storePath = toStorePath(target);
+    if (store.isValidPath(storePath))
+        roots[path] = storePath;
+    else
+        printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") % path % storePath);
+}
+
+
+static void findRoots(StoreAPI & store, const Path & path, Roots & roots)
 {
     try {
 
-        struct stat st;
+        struct stat st = lstat(path);
-        if (lstat(path.c_str(), &st) == -1)
-            throw SysError(format("statting `%1%'") % path);
-
-        printMsg(lvlVomit, format("looking at `%1%'") % path);
 
         if (S_ISDIR(st.st_mode)) {
             Strings names = readDirectory(path);
             foreach (Strings::iterator, i, names)
-                findRoots(store, path + "/" + *i, recurseSymlinks, deleteStale, roots);
+                findRoots(store, path + "/" + *i, roots);
         }
 
         else if (S_ISLNK(st.st_mode)) {
-            Path target = absPath(readLink(path), dirOf(path));
+            Path target = readLink(path);
+            if (isInStore(target))
+                foundRoot(store, path, target, roots);
 
-            if (isInStore(target)) {
+            /* Handle indirect roots. */
-                debug(format("found root `%1%' in `%2%'")
+            else {
-                    % target % path);
+                target = absPath(target, dirOf(path));
-                Path storePath = toStorePath(target);
+                if (!pathExists(target)) {
-                if (store.isValidPath(storePath))
+                    if (isInDir(path, settings.nixStateDir + "/" + gcRootsDir + "/auto")) {
-                    roots[path] = storePath;
+                        printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target);
-                else
+                        unlink(path.c_str());
-                    printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
+                    }
-                        % path % storePath);
+                } else {
-            }
+                    struct stat st2 = lstat(target);
-
+                    if (!S_ISLNK(st2.st_mode)) return;
-            else if (recurseSymlinks) {
+                    Path target2 = readLink(target);
-                if (pathExists(target))
+                    if (isInStore(target2)) foundRoot(store, target, target2, roots);
-                    findRoots(store, target, false, deleteStale, roots);
-                else if (deleteStale) {
-                    printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target);
-                    /* Note that we only delete when recursing, i.e.,
-                       when we are still in the `gcroots' tree.  We
-                       never delete stuff outside that tree. */
-                    unlink(path.c_str());
                 }
             }
         }
@@ -347,18 +346,16 @@ static void findRoots(StoreAPI & store, const Path & path,
 }
 
 
-static Roots findRoots(StoreAPI & store, bool deleteStale)
-{
-    Roots roots;
-    Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str());
-    findRoots(store, rootsDir, true, deleteStale, roots);
-    return roots;
-}
-
-
 Roots LocalStore::findRoots()
 {
-    return nix::findRoots(*this, false);
+    Roots roots;
+
+    /* Process direct roots in {gcroots,manifests,profiles}. */
+    nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots);
+    nix::findRoots(*this, settings.nixStateDir + "/manifests", roots);
+    nix::findRoots(*this, settings.nixStateDir + "/profiles", roots);
+
+    return roots;
 }
 
 
@@ -373,9 +370,9 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
 
     string result = runProgram(rootFinder);
 
-    Strings paths = tokenizeString<Strings>(result, "\n");
+    StringSet paths = tokenizeString<StringSet>(result, "\n");
 
-    foreach (Strings::iterator, i, paths) {
+    foreach (StringSet::iterator, i, paths) {
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store.isValidPath(path)) {
@@ -396,24 +393,17 @@ struct LocalStore::GCState
     GCResults & results;
     PathSet roots;
     PathSet tempRoots;
-    PathSet deleted;
+    PathSet dead;
-    PathSet live;
+    PathSet alive;
-    PathSet busy;
-    PathSet invalidated;
     bool gcKeepOutputs;
     bool gcKeepDerivations;
     unsigned long long bytesInvalidated;
+    Path trashDir;
+    bool shouldDelete;
     GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
 };
 
 
-static bool shouldDelete(GCOptions::GCAction action)
-{
-    return action == GCOptions::gcDeleteDead
-        || action == GCOptions::gcDeleteSpecific;
-}
-
-
 bool LocalStore::isActiveTempFile(const GCState & state,
     const Path & path, const string & suffix)
 {
@@ -424,152 +414,149 @@ bool LocalStore::isActiveTempFile(const GCState & state,
 
 void LocalStore::deleteGarbage(GCState & state, const Path & path)
 {
-    printMsg(lvlInfo, format("deleting `%1%'") % path);
     unsigned long long bytesFreed;
     deletePathWrapped(path, bytesFreed);
     state.results.bytesFreed += bytesFreed;
 }
 
 
-bool LocalStore::tryToDelete(GCState & state, const Path & path)
+void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 {
     checkInterrupt();
 
-    if (path == linksDir) return true;
+    unsigned long long size = 0;
+
+    if (isValidPath(path)) {
+        PathSet referrers;
+        queryReferrers(path, referrers);
+        foreach (PathSet::iterator, i, referrers)
+            if (*i != path) deletePathRecursive(state, *i);
+        size = queryPathInfo(path).narSize;
+        invalidatePathChecked(path);
+    }
 
     struct stat st;
     if (lstat(path.c_str(), &st)) {
-        if (errno == ENOENT) return true;
+        if (errno == ENOENT) return;
         throw SysError(format("getting status of %1%") % path);
     }
 
-    if (state.deleted.find(path) != state.deleted.end()) return true;
+    printMsg(lvlInfo, format("deleting `%1%'") % path);
-    if (state.live.find(path) != state.live.end()) return false;
+
+    state.results.paths.insert(path);
+
+    /* If the path is not a regular file or symlink, move it to the
+       trash directory.  The move is to ensure that later (when we're
+       not holding the global GC lock) we can delete the path without
+       being afraid that the path has become alive again.  Otherwise
+       delete it right away. */
+    if (S_ISDIR(st.st_mode)) {
+        // Estimate the amount freed using the narSize field.  FIXME:
+        // if the path was not valid, need to determine the actual
+        // size.
+        state.bytesInvalidated += size;
+        // Mac OS X cannot rename directories if they are read-only.
+        if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+            throw SysError(format("making `%1%' writable") % path);
+        Path tmp = state.trashDir + "/" + baseNameOf(path);
+        if (rename(path.c_str(), tmp.c_str()))
+            throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp);
+    } else
+        deleteGarbage(state, path);
+
+    if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
+        printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
+        throw GCLimitReached();
+    }
+}
+
+
+bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
+{
+    if (visited.find(path) != visited.end()) return false;
+
+    if (state.alive.find(path) != state.alive.end()) {
+        return true;
+    }
+
+    if (state.dead.find(path) != state.dead.end()) {
+        return false;
+    }
+
+    if (state.roots.find(path) != state.roots.end()) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+        state.alive.insert(path);
+        return true;
+    }
+
+    visited.insert(path);
+
+    if (!isValidPath(path)) return false;
+
+    PathSet incoming;
+
+    /* Don't delete this path if any of its referrers are alive. */
+    queryReferrers(path, incoming);
+
+    /* If gc-keep-derivations is set and this is a derivation, then
+       don't delete the derivation if any of the outputs are alive. */
+    if (state.gcKeepDerivations && isDerivation(path)) {
+        PathSet outputs = queryDerivationOutputs(path);
+        foreach (PathSet::iterator, i, outputs)
+            if (isValidPath(*i) && queryDeriver(*i) == path)
+                incoming.insert(*i);
+    }
+
+    /* If gc-keep-outputs is set, then don't delete this path if there
+       are derivers of this path that are not garbage. */
+    if (state.gcKeepOutputs) {
+        PathSet derivers = queryValidDerivers(path);
+        foreach (PathSet::iterator, i, derivers)
+            incoming.insert(*i);
+    }
+
+    foreach (PathSet::iterator, i, incoming)
+        if (*i != path)
+            if (canReachRoot(state, visited, *i)) {
+                state.alive.insert(path);
+                return true;
+            }
+
+    return false;
+}
+
+
+void LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+    checkInterrupt();
+
+    if (path == linksDir || path == state.trashDir) return;
 
     startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
 
-    /* If gc-keep-outputs and gc-keep-derivations are both set, we can
+    if (!isValidPath(path)) {
-       have cycles in the liveness graph, so we need to treat such
-       strongly connected components as a single unit (‘paths’).  That
-       is, we can delete the elements of ‘paths’ only if all referrers
-       of ‘paths’ are garbage. */
-    PathSet paths, referrers;
-    Paths pathsSorted;
-
-    if (isValidPath(path)) {
-
-        /* Add derivers and outputs of ‘path’ to ‘paths’. */
-        PathSet todo;
-        todo.insert(path);
-        while (!todo.empty()) {
-            Path p = *todo.begin();
-            assertStorePath(p);
-            todo.erase(p);
-            if (paths.find(p) != paths.end()) continue;
-            paths.insert(p);
-            /* If gc-keep-derivations is set and this is a derivation,
-               then don't delete the derivation if any of the outputs
-               are live. */
-            if (state.gcKeepDerivations && isDerivation(p)) {
-                PathSet outputs = queryDerivationOutputs(p);
-                foreach (PathSet::iterator, i, outputs)
-                    if (isValidPath(*i)) todo.insert(*i);
-            }
-            /* If gc-keep-outputs is set, then don't delete this path
-               if there are derivers of this path that are not
-               garbage. */
-            if (state.gcKeepOutputs) {
-                PathSet derivers = queryValidDerivers(p);
-                foreach (PathSet::iterator, i, derivers) todo.insert(*i);
-            }
-        }
-    }
-
-    else {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
-        if (isActiveTempFile(state, path, ".lock")) return false;
+        if (isActiveTempFile(state, path, ".lock")) return;
 
         /* Don't delete .chroot directories for derivations that are
            currently being built. */
-        if (isActiveTempFile(state, path, ".chroot")) return false;
+        if (isActiveTempFile(state, path, ".chroot")) return;
-
-        paths.insert(path);
     }
 
-    /* Check if any path in ‘paths’ is a root. */
+    PathSet visited;
-    foreach (PathSet::iterator, i, paths)
-        if (state.roots.find(*i) != state.roots.end()) {
-            printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % *i);
-            goto isLive;
-        }
 
-    /* Recursively try to delete the referrers of this strongly
+    if (canReachRoot(state, visited, path)) {
-       connected component.  If any referrer can't be deleted, then
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path);
-       these paths can't be deleted either. */
+    } else {
-    foreach (PathSet::iterator, i, paths)
+        /* No path we visited was a root, so everything is garbage.
-        if (isValidPath(*i)) queryReferrers(*i, referrers);
+           But we only delete ‘path’ and its referrers here so that
-
+           ‘nix-store --delete’ doesn't have the unexpected effect of
-    foreach (PathSet::iterator, i, referrers)
+           recursing into derivations and outputs. */
-        if (paths.find(*i) == paths.end() && !tryToDelete(state, *i)) {
+        state.dead.insert(visited.begin(), visited.end());
-            printMsg(lvlDebug, format("cannot delete `%1%' because it has live referrers") % *i);
+        if (state.shouldDelete)
-            goto isLive;
+            deletePathRecursive(state, path);
-        }
-
-    /* The paths are garbage, so delete them. */
-    pathsSorted = topoSortPaths(*this, paths);
-    foreach (Paths::iterator, i, pathsSorted) {
-        if (shouldDelete(state.options.action)) {
-
-            /* If it's a valid path that's not a regular file or
-               symlink, invalidate it, rename it, and schedule it for
-               deletion.  The renaming is to ensure that later (when
-               we're not holding the global GC lock) we can delete the
-               path without being afraid that the path has become
-               alive again.  Otherwise delete it right away. */
-            if (isValidPath(*i)) {
-                if (S_ISDIR(st.st_mode)) {
-                    printMsg(lvlInfo, format("invalidating `%1%'") % *i);
-                    // Estimate the amount freed using the narSize field.
-                    state.bytesInvalidated += queryPathInfo(*i).narSize;
-                    invalidatePathChecked(*i);
-                    makeMutable(i->c_str());
-                    // Mac OS X cannot rename directories if they are read-only.
-                    if (chmod(i->c_str(), st.st_mode | S_IWUSR) == -1)
-                        throw SysError(format("making `%1%' writable") % *i);
-                    Path tmp = (format("%1%-gc-%2%") % *i % getpid()).str();
-                    if (rename(i->c_str(), tmp.c_str()))
-                        throw SysError(format("unable to rename `%1%' to `%2%'") % *i % tmp);
-                    state.invalidated.insert(tmp);
-                } else {
-                    invalidatePathChecked(*i);
-                    deleteGarbage(state, *i);
-                }
-            } else
-                deleteGarbage(state, *i);
-
-            if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
-                printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
-                throw GCLimitReached();
-            }
-
-        } else
-            printMsg(lvlTalkative, format("would delete `%1%'") % *i);
-
-        state.deleted.insert(*i);
-        if (state.options.action != GCOptions::gcReturnLive)
-            state.results.paths.insert(*i);
     }
-
-    return true;
-
- isLive:
-    foreach (PathSet::iterator, i, paths) {
-        state.live.insert(*i);
-        if (state.options.action == GCOptions::gcReturnLive)
-            state.results.paths.insert(*i);
-    }
-    return false;
 }
 
 
@@ -625,7 +612,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     GCState state(results);
     state.options = options;
-
+    state.trashDir = settings.nixStore + "/trash";
     state.gcKeepOutputs = settings.gcKeepOutputs;
     state.gcKeepDerivations = settings.gcKeepDerivations;
 
@@ -638,6 +625,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         state.gcKeepDerivations = false;
     }
 
+    state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
@@ -646,7 +635,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
     printMsg(lvlError, format("finding garbage collector roots..."));
-    Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(*this, true);
+    Roots rootMap = options.ignoreLiveness ? Roots() : findRoots();
 
     foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second);
 
@@ -668,6 +657,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
        increase, since we hold locks on everything.  So everything
        that is not reachable from `roots'. */
 
+    if (state.shouldDelete) {
+        if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
+        createDirs(state.trashDir);
+    }
+
     /* Now either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
 
@@ -675,13 +669,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         foreach (PathSet::iterator, i, options.pathsToDelete) {
             assertStorePath(*i);
-            if (!tryToDelete(state, *i))
+            tryToDelete(state, *i);
+            if (state.dead.find(*i) == state.dead.end())
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
         }
 
     } else if (options.maxFreed > 0) {
 
-        if (shouldDelete(state.options.action))
+        if (state.shouldDelete)
             printMsg(lvlError, format("deleting garbage..."));
         else
             printMsg(lvlError, format("determining live/dead paths..."));
@@ -727,13 +722,22 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     }
 
+    if (state.options.action == GCOptions::gcReturnLive) {
+        state.results.paths = state.alive;
+        return;
+    }
+
+    if (state.options.action == GCOptions::gcReturnDead) {
+        state.results.paths = state.dead;
+        return;
+    }
+
     /* Allow other processes to add to the store from here on. */
     fdGCLock.close();
 
-    /* Delete the invalidated paths now that the lock has been
+    /* Delete the trash directory. */
-       released. */
+    printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir);
-    foreach (PathSet::iterator, i, state.invalidated)
+    deleteGarbage(state, state.trashDir);
-        deleteGarbage(state, *i);
 
     /* Clean up the links directory. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {

src/libstore/globals.cc

@@ -10,6 +10,14 @@
 namespace nix {
 
 
+/* The default location of the daemon socket, relative to nixStateDir.
+   The socket is in a directory to allow you to control access to the
+   Nix daemon by setting the mode/ownership of the directory
+   appropriately.  (This wouldn't work on the socket itself since it
+   must be deleted and recreated on startup.) */
+#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
+
+
 Settings settings;
 
 
@@ -38,6 +46,7 @@ Settings::Settings()
     impersonateLinux26 = false;
     keepLog = true;
     compressLog = true;
+    maxLogSize = 0;
     cacheFailure = false;
     pollInterval = 5;
     checkRootReachability = false;
@@ -45,6 +54,7 @@ Settings::Settings()
     gcKeepDerivations = true;
     autoOptimiseStore = false;
     envKeepDerivations = false;
+    lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
 }
 
 
@@ -58,11 +68,14 @@ void Settings::processEnvironment()
     nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+    nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH);
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {
+#if 0
         if (getEnv("NIX_OTHER_STORES") != "")
             substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+#endif
         substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
         substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
     } else
@@ -129,6 +142,7 @@ void Settings::update()
     get(impersonateLinux26, "build-impersonate-linux-26");
     get(keepLog, "build-keep-log");
     get(compressLog, "build-compress-log");
+    get(maxLogSize, "build-max-log-size");
     get(cacheFailure, "build-cache-failure");
     get(pollInterval, "build-poll-interval");
     get(checkRootReachability, "gc-check-reachability");
@@ -158,7 +172,7 @@ void Settings::get(bool & res, const string & name)
 }
 
 
-void Settings::get(PathSet & res, const string & name)
+void Settings::get(StringSet & res, const string & name)
 {
     SettingsMap::iterator i = settings.find(name);
     if (i == settings.end()) return;

src/libstore/globals.hh

@@ -50,6 +50,9 @@ struct Settings {
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
+    /* File name of the socket the daemon listens to.  */
+    Path nixDaemonSocketFile;
+
     /* Whether to keep temporary directories of failed builds. */
     bool keepFailed;
 
@@ -139,7 +142,7 @@ struct Settings {
 
     /* The directories from the host filesystem to be included in the
        chroot. */
-    PathSet dirsInChroot;
+    StringSet dirsInChroot;
 
     /* Whether to impersonate a Linux 2.6 machine on newer kernels. */
     bool impersonateLinux26;
@@ -150,6 +153,10 @@ struct Settings {
     /* Whether to compress logs. */
     bool compressLog;
 
+    /* Maximum number of bytes a builder can write to stdout/stderr
+       before being killed (0 means no limit). */
+    unsigned long maxLogSize;
+
     /* Whether to cache build failures. */
     bool cacheFailure;
 
@@ -176,12 +183,15 @@ struct Settings {
        (to prevent them from being GCed). */
     bool envKeepDerivations;
 
+    /* Whether to lock the Nix client and worker to the same CPU. */
+    bool lockCPU;
+
 private:
     SettingsMap settings, overrides;
 
     void get(string & res, const string & name);
     void get(bool & res, const string & name);
-    void get(PathSet & res, const string & name);
+    void get(StringSet & res, const string & name);
     template<class N> void get(N & res, const string & name);
 };
 

src/libstore/local-store.hh

@@ -17,8 +17,8 @@ namespace nix {
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
    Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
-   Nix 1.0. */
+   Nix 1.0.  Version 7 is Nix 1.3. */
-const int nixSchemaVersion = 6;
+const int nixSchemaVersion = 7;
 
 
 extern string drvsLogDir;
@@ -44,8 +44,12 @@ struct OptimiseStats
 
 struct RunningSubstituter
 {
+    Path program;
     Pid pid;
     AutoCloseFD to, from, error;
+    FdSource fromBuf;
+    bool disabled;
+    RunningSubstituter() : disabled(false) { };
 };
 
 
@@ -111,10 +115,6 @@ public:
 
     Path queryDeriver(const Path & path);
 
-    /* Return all currently valid derivations that have `path' as an
-       output.  (Note that the result of `queryDeriver()' is the
-       derivation that was actually used to produce `path', which may
-       not exist anymore.) */
     PathSet queryValidDerivers(const Path & path);
 
     PathSet queryDerivationOutputs(const Path & path);
@@ -177,7 +177,7 @@ public:
 
     /* Register the validity of a path, i.e., that `path' exists, that
        the paths referenced by it exists, and in the case of an output
-       path of a derivation, that it has been produced by a succesful
+       path of a derivation, that it has been produced by a successful
        execution of the derivation (or something equivalent).  Also
        register the hash of the file system contents of the path.  The
        hash must be a SHA-256 hash. */
@@ -269,6 +269,7 @@ private:
     void updatePathInfo(const ValidPathInfo & info);
 
     void upgradeStore6();
+    void upgradeStore7();
     PathSet queryValidPathsOld();
     ValidPathInfo queryPathInfoOld(const Path & path);
 
@@ -276,7 +277,11 @@ private:
 
     void deleteGarbage(GCState & state, const Path & path);
 
-    bool tryToDelete(GCState & state, const Path & path);
+    void tryToDelete(GCState & state, const Path & path);
+
+    bool canReachRoot(GCState & state, PathSet & visited, const Path & path);
+
+    void deletePathRecursive(GCState & state, const Path & path);
 
     bool isActiveTempFile(const GCState & state,
         const Path & path, const string & suffix);
@@ -288,6 +293,10 @@ private:
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
 
+    string getLineFromSubstituter(RunningSubstituter & run);
+
+    template<class T> T getIntLineFromSubstituter(RunningSubstituter & run);
+
     Path createTempDirInStore();
 
     Path importPath(bool requireSignature, Source & source);
@@ -295,9 +304,17 @@ private:
     void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
 
     void optimisePath_(OptimiseStats & stats, const Path & path);
+
+    // Internal versions that are not wrapped in retry_sqlite.
+    bool isValidPath_(const Path & path);
+    void queryReferrers_(const Path & path, PathSet & referrers);
 };
 
 
+typedef std::pair<dev_t, ino_t> Inode;
+typedef set<Inode> InodesSeen;
+
+
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 1 (i.e.,
@@ -306,9 +323,10 @@ private:
      without execute permission; setuid bits etc. are cleared)
    - the owner and group are set to the Nix user and group, if we're
      in a setuid Nix installation. */
-void canonicalisePathMetaData(const Path & path);
+void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen);
+void canonicalisePathMetaData(const Path & path, uid_t fromUid);
 
-void canonicalisePathMetaData(const Path & path, bool recurse);
+void canonicaliseTimestampAndPermissions(const Path & path);
 
 MakeError(PathInUse, Error);
 

src/libstore/misc.cc

@@ -15,27 +15,47 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath)
 }
 
 
-void computeFSClosure(StoreAPI & store, const Path & storePath,
+void computeFSClosure(StoreAPI & store, const Path & path,
-    PathSet & paths, bool flipDirection, bool includeOutputs)
+    PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
-    if (paths.find(storePath) != paths.end()) return;
+    if (paths.find(path) != paths.end()) return;
-    paths.insert(storePath);
+    paths.insert(path);
 
-    PathSet references;
+    PathSet edges;
-    if (flipDirection)
-        store.queryReferrers(storePath, references);
-    else
-        store.queryReferences(storePath, references);
 
-    if (includeOutputs && isDerivation(storePath)) {
+    if (flipDirection) {
-        PathSet outputs = store.queryDerivationOutputs(storePath);
+        store.queryReferrers(path, edges);
-        foreach (PathSet::iterator, i, outputs)
+
-            if (store.isValidPath(*i))
+        if (includeOutputs) {
-                computeFSClosure(store, *i, paths, flipDirection, true);
+            PathSet derivers = store.queryValidDerivers(path);
+            foreach (PathSet::iterator, i, derivers)
+                edges.insert(*i);
+        }
+
+        if (includeDerivers && isDerivation(path)) {
+            PathSet outputs = store.queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (store.isValidPath(*i) && store.queryDeriver(*i) == path)
+                    edges.insert(*i);
+        }
+
+    } else {
+        store.queryReferences(path, edges);
+
+        if (includeOutputs && isDerivation(path)) {
+            PathSet outputs = store.queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (store.isValidPath(*i)) edges.insert(*i);
+        }
+
+        if (includeDerivers) {
+            Path deriver = store.queryDeriver(path);
+            if (store.isValidPath(deriver)) edges.insert(deriver);
+        }
     }
 
-    foreach (PathSet::iterator, i, references)
+    foreach (PathSet::iterator, i, edges)
-        computeFSClosure(store, *i, paths, flipDirection, includeOutputs);
+        computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers);
 }
 
 
@@ -100,7 +120,8 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
                 if (invalid.empty()) continue;
 
                 todoDrv.insert(*i);
-                if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
+                if (settings.useSubstitutes && !willBuildLocally(drv))
+                    query.insert(invalid.begin(), invalid.end());
             }
 
             else {
@@ -123,7 +144,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
             PathSet outputs;
             bool mustBuild = false;
-            if (settings.useSubstitutes) {
+            if (settings.useSubstitutes && !willBuildLocally(drv)) {
                 foreach (DerivationOutputs::iterator, j, drv.outputs) {
                     if (!wantOutput(j->first, i2.second)) continue;
                     if (!store.isValidPath(j->second.path)) {

src/libstore/misc.hh

@@ -17,9 +17,9 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath);
    `storePath' is returned; that is, the closures under the
    `referrers' relation instead of the `references' relation is
    returned. */
-void computeFSClosure(StoreAPI & store, const Path & storePath,
+void computeFSClosure(StoreAPI & store, const Path & path,
     PathSet & paths, bool flipDirection = false,
-    bool includeOutputs = false);
+    bool includeOutputs = false, bool includeDerivers = false);
 
 /* Return the path corresponding to the output identifier `id' in the
    given derivation. */
@@ -32,5 +32,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
     PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
     unsigned long long & downloadSize, unsigned long long & narSize);
 
+bool willBuildLocally(const Derivation & drv);
+
 
 }

src/libstore/optimise-store.cc

@@ -2,7 +2,6 @@
 
 #include "util.hh"
 #include "local-store.hh"
-#include "immutable.hh"
 #include "globals.hh"
 
 #include <sys/types.h>
@@ -20,7 +19,6 @@ static void makeWritable(const Path & path)
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path `%1%'") % path);
-    if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
         throw SysError(format("changing writability of `%1%'") % path);
 }
@@ -34,7 +32,7 @@ struct MakeReadOnly
     {
         try {
             /* This will make the path read-only. */
-            if (path != "") canonicalisePathMetaData(path, false);
+            if (path != "") canonicaliseTimestampAndPermissions(path);
         } catch (...) {
             ignoreException();
         }
@@ -91,7 +89,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
 
     if (!pathExists(linkPath)) {
         /* Nope, create a hard link in the links directory. */
-        makeMutable(path);
         if (link(path.c_str(), linkPath.c_str()) == 0) return;
         if (errno != EEXIST)
             throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
@@ -123,12 +120,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
        its timestamp back to 0. */
     MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
 
-    /* If ‘linkPath’ is immutable, we can't create hard links to it,
-       so make it mutable first.  We also have to make ‘path’ mutable,
-       otherwise rename() will fail to delete it. */
-    makeMutable(path);
-    makeMutable(linkPath);
-
     Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
         % settings.nixStore % getpid() % rand()).str();
 

src/libstore/remote-store.cc

@@ -3,6 +3,7 @@
 #include "remote-store.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
+#include "affinity.hh"
 #include "globals.hh"
 
 #include <sys/types.h>
@@ -15,7 +16,6 @@
 #include <unistd.h>
 #include <cstring>
 
-
 namespace nix {
 
 
@@ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace)
         if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
             throw Error("Nix daemon protocol version not supported");
         writeInt(PROTOCOL_VERSION, to);
+
+        if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
+            int cpu = settings.lockCPU ? lockToCurrentCPU() : -1;
+            if (cpu != -1) {
+                writeInt(1, to);
+                writeInt(cpu, to);
+            } else
+                writeInt(0, to);
+        }
+
         if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
             writeInt(reserveSpace, to);
+
         processStderr();
     }
     catch (Error & e) {
@@ -91,7 +102,7 @@ void RemoteStore::connectToDaemon()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(fdSocket);
 
-    string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
+    string socketPath = settings.nixDaemonSocketFile;
 
     /* Urgh, sockaddr_un allows path names of only 108 characters.  So
        chdir to the socket directory so that we can pass a relative
@@ -334,6 +345,16 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
+PathSet RemoteStore::queryValidDerivers(const Path & path)
+{
+    openConnection();
+    writeInt(wopQueryValidDerivers, to);
+    writeString(path, to);
+    processStderr();
+    return readStorePaths<PathSet>(from);
+}
+
+
 PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 {
     openConnection();
@@ -431,7 +452,16 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, bool repair)
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
     openConnection();
     writeInt(wopBuildPaths, to);
-    writeStrings(drvPaths, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 13)
+        writeStrings(drvPaths, to);
+    else {
+        /* For backwards compatibility with old daemons, strip output
+           identifiers. */
+        PathSet drvPaths2;
+        foreach (PathSet::const_iterator, i, drvPaths)
+            drvPaths2.insert(string(*i, 0, i->find('!')));
+        writeStrings(drvPaths2, to);
+    }
     processStderr();
     readInt(from);
 }

src/libstore/remote-store.hh

@@ -40,6 +40,8 @@ public:
 
     Path queryDeriver(const Path & path);
     
+    PathSet queryValidDerivers(const Path & path);
+
     PathSet queryDerivationOutputs(const Path & path);
     
     StringSet queryDerivationOutputNames(const Path & path);

src/libstore/store-api.cc

@@ -18,10 +18,7 @@ GCOptions::GCOptions()
 
 bool isInStore(const Path & path)
 {
-    return path[0] == '/'
+    return isInDir(path, settings.nixStore);
-        && string(path, 0, settings.nixStore.size()) == settings.nixStore
-        && path.size() >= settings.nixStore.size() + 2
-        && path[settings.nixStore.size()] == '/';
 }
 
 

src/libstore/store-api.hh

@@ -133,6 +133,12 @@ public:
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
+    /* Return all currently valid derivations that have `path' as an
+       output.  (Note that the result of `queryDeriver()' is the
+       derivation that was actually used to produce `path', which may
+       not exist anymore.) */
+    virtual PathSet queryValidDerivers(const Path & path) = 0;
+
     /* Query the outputs of the derivation denoted by `path'. */
     virtual PathSet queryDerivationOutputs(const Path & path) = 0;
 
@@ -322,7 +328,7 @@ Path addPermRoot(StoreAPI & store, const Path & storePath,
 
 
 /* Sort a set of paths topologically under the references relation.
-   If p refers to q, then p follows q in this list. */
+   If p refers to q, then p preceeds q in this list. */
 Paths topoSortPaths(StoreAPI & store, const PathSet & paths);
 
 

src/libstore/worker-protocol.hh

@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x10c
+#define PROTOCOL_VERSION 0x10e
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -42,6 +42,7 @@ typedef enum {
     wopQuerySubstitutablePathInfos = 30,
     wopQueryValidPaths = 31,
     wopQuerySubstitutablePaths = 32,
+    wopQueryValidDerivers = 33,
 } WorkerOp;
 
 
@@ -52,14 +53,6 @@ typedef enum {
 #define STDERR_ERROR 0x63787470
 
 
-/* The default location of the daemon socket, relative to nixStateDir.
-   The socket is in a directory to allow you to control access to the
-   Nix daemon by setting the mode/ownership of the directory
-   appropriately.  (This wouldn't work on the socket itself since it
-   must be deleted and recreated on startup.) */
-#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
-
-
 Path readStorePath(Source & from);
 template<class T> T readStorePaths(Source & from);
 

src/libutil/Makefile.am

@@ -1,12 +1,12 @@
 pkglib_LTLIBRARIES = libutil.la
 
 libutil_la_SOURCES = util.cc hash.cc serialise.cc \
-  archive.cc xml-writer.cc immutable.cc
+  archive.cc xml-writer.cc affinity.cc
 
 libutil_la_LIBADD = ../boost/format/libformat.la
 
 pkginclude_HEADERS = util.hh hash.hh serialise.hh \
-  archive.hh xml-writer.hh types.hh immutable.hh
+  archive.hh xml-writer.hh types.hh affinity.hh
 
 if !HAVE_OPENSSL
 libutil_la_SOURCES += \

src/libutil/affinity.cc

@@ -0,0 +1,55 @@
+#include "types.hh"
+#include "util.hh"
+#include "affinity.hh"
+
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+namespace nix {
+
+
+#if HAVE_SCHED_SETAFFINITY
+static bool didSaveAffinity = false;
+static cpu_set_t savedAffinity;
+#endif
+
+
+void setAffinityTo(int cpu)
+{
+#if HAVE_SCHED_SETAFFINITY
+    if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
+    didSaveAffinity = true;
+    printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu);
+    cpu_set_t newAffinity;
+    CPU_ZERO(&newAffinity);
+    CPU_SET(cpu, &newAffinity);
+    if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
+        printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu);
+#endif
+}
+
+
+int lockToCurrentCPU()
+{
+#if HAVE_SCHED_SETAFFINITY
+    int cpu = sched_getcpu();
+    if (cpu != -1) setAffinityTo(cpu);
+    return cpu;
+#else
+    return -1;
+#endif
+}
+
+
+void restoreAffinity()
+{
+#if HAVE_SCHED_SETAFFINITY
+    if (!didSaveAffinity) return;
+    if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
+        printMsg(lvlError, "failed to restore affinity %1%");
+#endif
+}
+
+
+}

src/libutil/affinity.hh

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix {
+
+void setAffinityTo(int cpu);
+int lockToCurrentCPU();
+void restoreAffinity();
+
+}

src/libutil/archive.cc

@@ -282,6 +282,7 @@ struct RestoreSink : ParseSink
     void createRegularFile(const Path & path)
     {
         Path p = dstPath + path;
+        fd.close();
         fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666);
         if (fd == -1) throw SysError(format("creating file `%1%'") % p);
     }

src/libutil/immutable.cc

@@ -1,49 +0,0 @@
-#include "config.h"
-
-#include "immutable.hh"
-#include "util.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-#if HAVE_LINUX_FS_H
-#include <linux/fs.h>
-#include <sys/ioctl.h>
-#include <errno.h>
-#endif
-
-namespace nix {
-
-
-void makeMutable(const Path & path)
-{
-#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
-
-    /* Don't even try if we're not root.  One day we should support
-       the CAP_LINUX_IMMUTABLE capability. */
-    if (getuid() != 0) return;
-
-    /* The O_NOFOLLOW is important to prevent us from changing the
-       mutable bit on the target of a symlink (which would be a
-       security hole). */
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW);
-    if (fd == -1) {
-        if (errno == ELOOP) return; // it's a symlink
-        throw SysError(format("opening file `%1%'") % path);
-    }
-
-    unsigned int flags = 0, old;
-
-    /* Silently ignore errors getting/setting the immutable flag so
-       that we work correctly on filesystems that don't support it. */
-    if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return;
-    old = flags;
-    flags &= ~FS_IMMUTABLE_FL;
-    if (old == flags) return;
-    if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return;
-#endif
-}
-
-
-}

src/libutil/immutable.hh

@@ -1,10 +0,0 @@
-#pragma once
-
-#include <types.hh>
-
-namespace nix {
-
-/* Make the given path mutable. */
-void makeMutable(const Path & path);
-
-}

src/libutil/serialise.cc

@@ -90,6 +90,12 @@ size_t BufferedSource::read(unsigned char * data, size_t len)
 }
 
 
+bool BufferedSource::hasData()
+{
+    return bufPosOut < bufPosIn;
+}
+
+
 size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
 {
     ssize_t n;

src/libutil/serialise.hh

@@ -61,8 +61,10 @@ struct BufferedSource : Source
     
     size_t read(unsigned char * data, size_t len);
     
-    /* Underlying read call, to be overriden. */
+    /* Underlying read call, to be overridden. */
     virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
+
+    bool hasData();
 };
 
 

src/libutil/util.cc

@@ -12,8 +12,11 @@
 #include <fcntl.h>
 #include <limits.h>
 
+#ifdef __APPLE__
+#include <sys/syscall.h>
+#endif
+
 #include "util.hh"
-#include "immutable.hh"
 
 
 extern char * * environ;
@@ -100,7 +103,7 @@ Path canonPath(const Path & path, bool resolveSymlinks)
             i++;
 
         /* If `..', delete the last component. */
-        else if (*i == '.' && i + 1 < end && i[1] == '.' && 
+        else if (*i == '.' && i + 1 < end && i[1] == '.' &&
             (i + 2 == end || i[2] == '/'))
         {
             if (!s.empty()) s.erase(s.rfind('/'));
@@ -149,6 +152,15 @@ string baseNameOf(const Path & path)
 }
 
 
+bool isInDir(const Path & path, const Path & dir)
+{
+    return path[0] == '/'
+        && string(path, 0, dir.size()) == dir
+        && path.size() >= dir.size() + 2
+        && path[dir.size()] == '/';
+}
+
+
 struct stat lstat(const Path & path)
 {
     struct stat st;
@@ -215,7 +227,7 @@ string readFile(int fd)
     struct stat st;
     if (fstat(fd, &st) == -1)
         throw SysError("statting file");
-    
+
     unsigned char * buf = new unsigned char[st.st_size];
     AutoDeleteArray<unsigned char> d(buf);
     readFull(fd, buf, st.st_size);
@@ -280,9 +292,9 @@ static void _computePathSize(const Path & path,
     blocks += st.st_blocks;
 
     if (S_ISDIR(st.st_mode)) {
-	Strings names = readDirectory(path);
+        Strings names = readDirectory(path);
 
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
             _computePathSize(path + "/" + *i, bytes, blocks);
     }
 }
@@ -305,21 +317,19 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
     struct stat st = lstat(path);
 
-    if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path);
-
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
         bytesFreed += st.st_blocks * 512;
 
     if (S_ISDIR(st.st_mode)) {
-	Strings names = readDirectory(path);
+        Strings names = readDirectory(path);
 
-	/* Make the directory writable. */
+        /* Make the directory writable. */
-	if (!(st.st_mode & S_IWUSR)) {
+        if (!(st.st_mode & S_IWUSR)) {
-	    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+            if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-		throw SysError(format("making `%1%' writable") % path);
+                throw SysError(format("making `%1%' writable") % path);
-	}
+        }
 
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
             _deletePath(path + "/" + *i, bytesFreed);
     }
 
@@ -351,14 +361,14 @@ void makePathReadOnly(const Path & path)
     struct stat st = lstat(path);
 
     if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) {
-	if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
+        if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
-	    throw SysError(format("making `%1%' read-only") % path);
+            throw SysError(format("making `%1%' read-only") % path);
     }
 
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+        for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-	    makePathReadOnly(path + "/" + *i);
+            makePathReadOnly(path + "/" + *i);
     }
 }
 
@@ -380,25 +390,25 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
     static int globalCounter = 0;
     int localCounter = 0;
     int & counter(useGlobalCounter ? globalCounter : localCounter);
-    
+
     while (1) {
         checkInterrupt();
-	Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-	if (mkdir(tmpDir.c_str(), mode) == 0) {
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
-	    /* Explicitly set the group of the directory.  This is to
+            /* Explicitly set the group of the directory.  This is to
-	       work around around problems caused by BSD's group
+               work around around problems caused by BSD's group
-	       ownership semantics (directories inherit the group of
+               ownership semantics (directories inherit the group of
-	       the parent).  For instance, the group of /tmp on
+               the parent).  For instance, the group of /tmp on
-	       FreeBSD is "wheel", so all directories created in /tmp
+               FreeBSD is "wheel", so all directories created in /tmp
-	       will be owned by "wheel"; but if the user is not in
+               will be owned by "wheel"; but if the user is not in
-	       "wheel", then "tar" will fail to unpack archives that
+               "wheel", then "tar" will fail to unpack archives that
-	       have the setgid bit set on directories. */
+               have the setgid bit set on directories. */
-	    if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-		throw SysError(format("setting group of directory `%1%'") % tmpDir);
+                throw SysError(format("setting group of directory `%1%'") % tmpDir);
-	    return tmpDir;
+            return tmpDir;
-	}
+        }
-	if (errno != EEXIST)
+        if (errno != EEXIST)
-	    throw SysError(format("creating directory `%1%'") % tmpDir);
+            throw SysError(format("creating directory `%1%'") % tmpDir);
     }
 }
 
@@ -418,7 +428,7 @@ Paths createDirs(const Path & path)
     }
 
     if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path);
-    
+
     return created;
 }
 
@@ -619,8 +629,8 @@ AutoCloseFD::AutoCloseFD(int fd)
 
 AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd)
 {
-    /* Copying a AutoCloseFD isn't allowed (who should get to close
+    /* Copying an AutoCloseFD isn't allowed (who should get to close
-       it?).  But as a edge case, allow copying of closed
+       it?).  But as an edge case, allow copying of closed
        AutoCloseFDs.  This is necessary due to tiresome reasons
        involving copy constructor use on default object values in STL
        containers (like when you do `map[value]' where value isn't in
@@ -727,8 +737,8 @@ AutoCloseDir::operator DIR *()
 void AutoCloseDir::close()
 {
     if (dir) {
-	closedir(dir);
+        closedir(dir);
-	dir = 0;
+        dir = 0;
     }
 }
 
@@ -793,6 +803,7 @@ void Pid::kill()
 
 int Pid::wait(bool block)
 {
+    assert(pid != -1);
     while (1) {
         int status;
         int res = waitpid(pid, &status, block ? 0 : WNOHANG);
@@ -843,12 +854,21 @@ void killUser(uid_t uid)
             if (setuid(uid) == -1)
                 throw SysError("setting uid");
 
-	    while (true) {
+            while (true) {
-		if (kill(-1, SIGKILL) == 0) break;
+#ifdef __APPLE__
-		if (errno == ESRCH) break; /* no more processes */
+                /* OSX's kill syscall takes a third parameter that, among other
-		if (errno != EINTR)
+                   things, determines if kill(-1, signo) affects the calling
-		    throw SysError(format("cannot kill processes for uid `%1%'") % uid);
+                   process. In the OSX libc, it's set to true, which means
-	    }
+                   "follow POSIX", which we don't want here
+                 */
+                if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
+#else
+                if (kill(-1, SIGKILL) == 0) break;
+#endif
+                if (errno == ESRCH) break; /* no more processes */
+                if (errno != EINTR)
+                    throw SysError(format("cannot kill processes for uid `%1%'") % uid);
+            }
 
         } catch (std::exception & e) {
             writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str());
@@ -856,7 +876,7 @@ void killUser(uid_t uid)
         }
         _exit(0);
     }
-    
+
     /* parent */
     int status = pid.wait(true);
     if (status != 0)
@@ -1043,14 +1063,14 @@ string statusToString(int status)
         if (WIFEXITED(status))
             return (format("failed with exit code %1%") % WEXITSTATUS(status)).str();
         else if (WIFSIGNALED(status)) {
-	    int sig = WTERMSIG(status);
+            int sig = WTERMSIG(status);
 #if HAVE_STRSIGNAL
             const char * description = strsignal(sig);
             return (format("failed due to signal %1% (%2%)") % sig % description).str();
 #else
             return (format("failed due to signal %1%") % sig).str();
 #endif
-	}
+        }
         else
             return "died abnormally";
     } else return "succeeded";
@@ -1063,14 +1083,6 @@ bool statusOk(int status)
 }
 
 
-string int2String(int n)
-{
-    std::ostringstream str;
-    str << n;
-    return str.str();
-}
-
-
 bool hasSuffix(const string & s, const string & suffix)
 {
     return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix;

src/libutil/util.hh

@@ -45,6 +45,10 @@ Path dirOf(const Path & path);
    following the final `/'. */
 string baseNameOf(const Path & path);
 
+/* Check whether a given path is a descendant of the given
+   directory. */
+bool isInDir(const Path & path, const Path & dir);
+
 /* Get status of `path'. */
 struct stat lstat(const Path & path);
 
@@ -174,7 +178,7 @@ struct AutoDeleteArray
 {
     T * p;
     AutoDeleteArray(T * p) : p(p) { }
-    ~AutoDeleteArray() 
+    ~AutoDeleteArray()
     {
         delete [] p;
     }
@@ -185,7 +189,7 @@ class AutoDelete
 {
     Path path;
     bool del;
-    bool recursive;    
+    bool recursive;
 public:
     AutoDelete(const Path & p, bool recursive = true);
     ~AutoDelete();
@@ -315,7 +319,12 @@ template<class N> bool string2Int(const string & s, N & n)
     return str && str.get() == EOF;
 }
 
-string int2String(int n);
+template<class N> string int2String(N n)
+{
+    std::ostringstream str;
+    str << n;
+    return str.str();
+}
 
 
 /* Return true iff `s' ends in `suffix'. */

src/nix-daemon/nix-daemon.cc

@@ -4,6 +4,7 @@
 #include "serialise.hh"
 #include "worker-protocol.hh"
 #include "archive.hh"
+#include "affinity.hh"
 #include "globals.hh"
 
 #include <cstring>
@@ -25,8 +26,13 @@ using namespace nix;
    that lack it, we only notice the disconnection the next time we try
    to write to the client.  So if you have a builder that never
    generates output on stdout/stderr, the daemon will never notice
-   that the client has disconnected until the builder terminates. */
+   that the client has disconnected until the builder terminates.
-#ifdef O_ASYNC
+
+   GNU/Hurd does have O_ASYNC, but its Unix-domain socket translator
+   (pflocal) does not implement F_SETOWN.  See
+   <http://lists.gnu.org/archive/html/bug-guix/2013-07/msg00021.html> for
+   details.*/
+#if defined O_ASYNC && !defined __GNU__
 #define HAVE_HUP_NOTIFICATION
 #ifndef SIGPOLL
 #define SIGPOLL SIGIO
@@ -273,7 +279,7 @@ struct SavingSourceAdapter : Source
 };
 
 
-static void performOp(unsigned int clientVersion,
+static void performOp(bool trusted, unsigned int clientVersion,
     Source & from, Sink & to, unsigned int op)
 {
     switch (op) {
@@ -334,6 +340,7 @@ static void performOp(unsigned int clientVersion,
 
     case wopQueryReferences:
     case wopQueryReferrers:
+    case wopQueryValidDerivers:
     case wopQueryDerivationOutputs: {
         Path path = readStorePath(from);
         startWork();
@@ -342,6 +349,8 @@ static void performOp(unsigned int clientVersion,
             store->queryReferences(path, paths);
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
+        else if (op == wopQueryValidDerivers)
+            paths = store->queryValidDerivers(path);
         else paths = store->queryDerivationOutputs(path);
         stopWork();
         writeStrings(paths, to);
@@ -548,7 +557,10 @@ static void performOp(unsigned int clientVersion,
             for (unsigned int i = 0; i < n; i++) {
                 string name = readString(from);
                 string value = readString(from);
-                settings.set("untrusted-" + name, value);
+                if (name == "build-timeout")
+                    string2Int(value, settings.buildTimeout);
+                else
+                    settings.set(trusted ? name : "untrusted-" + name, value);
             }
         }
         startWork();
@@ -637,7 +649,7 @@ static void performOp(unsigned int clientVersion,
 }
 
 
-static void processConnection()
+static void processConnection(bool trusted)
 {
     canSendStderr = false;
     myPid = getpid();
@@ -660,6 +672,9 @@ static void processConnection()
     to.flush();
     unsigned int clientVersion = readInt(from);
 
+    if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
+        setAffinityTo(readInt(from));
+
     bool reserveSpace = true;
     if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
         reserveSpace = readInt(from) != 0;
@@ -705,9 +720,9 @@ static void processConnection()
         opCount++;
 
         try {
-            performOp(clientVersion, from, to, op);
+            performOp(trusted, clientVersion, from, to, op);
         } catch (Error & e) {
-            /* If we're not in a state were we can send replies, then
+            /* If we're not in a state where we can send replies, then
                something went wrong processing the input of the
                client.  This can happen especially if I/O errors occur
                during addTextToStore() / importPath().  If that
@@ -716,6 +731,10 @@ static void processConnection()
             if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg());
             stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0);
             if (!errorAllowed) break;
+        } catch (std::bad_alloc & e) {
+            if (canSendStderr)
+                stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
+            throw;
         }
 
         to.flush();
@@ -771,7 +790,7 @@ static void daemonLoop()
         if (fdSocket == -1)
             throw SysError("cannot create Unix domain socket");
 
-        string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH;
+        string socketPath = settings.nixDaemonSocketFile;
 
         createDirs(dirOf(socketPath));
 
@@ -833,6 +852,7 @@ static void daemonLoop()
             /* Get the identity of the caller, if possible. */
             uid_t clientUid = -1;
             pid_t clientPid = -1;
+            bool trusted = false;
 
 #if defined(SO_PEERCRED)
             ucred cred;
@@ -840,6 +860,7 @@ static void daemonLoop()
             if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) != -1) {
                 clientPid = cred.pid;
                 clientUid = cred.uid;
+                if (clientUid == 0) trusted = true;
             }
 #endif
 
@@ -873,10 +894,10 @@ static void daemonLoop()
                     /* Handle the connection. */
                     from.fd = remote;
                     to.fd = remote;
-                    processConnection();
+                    processConnection(trusted);
 
                 } catch (std::exception & e) {
-                    writeToStderr("child error: " + string(e.what()) + "\n");
+                    writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n");
                 }
                 exit(0);
             }
@@ -892,8 +913,6 @@ static void daemonLoop()
 
 void run(Strings args)
 {
-    bool daemon = false;
-
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
         string arg = *i++;
         if (arg == "--daemon") /* ignored for backwards compatibility */;

src/nix-env/nix-env.cc

@@ -54,6 +54,7 @@ struct Globals
     EvalState state;
     bool dryRun;
     bool preserveInstalled;
+    bool removeAll;
     string forceName;
     bool prebuiltOnly;
 };
@@ -94,18 +95,14 @@ static bool parseInstallSourceOptions(Globals & globals,
 }
 
 
-static bool isNixExpr(const Path & path)
+static bool isNixExpr(const Path & path, struct stat & st)
 {
-    struct stat st;
+    return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix"));
-    if (stat(path.c_str(), &st) == -1)
-        throw SysError(format("getting information about `%1%'") % path);
-
-    return !S_ISDIR(st.st_mode) || pathExists(path + "/default.nix");
 }
 
 
 static void getAllExprs(EvalState & state,
-    const Path & path, ExprAttrs & attrs)
+    const Path & path, StringSet & attrs, Value & v)
 {
     Strings names = readDirectory(path);
     StringSet namesSorted(names.begin(), names.end());
@@ -122,7 +119,7 @@ static void getAllExprs(EvalState & state,
         if (stat(path2.c_str(), &st) == -1)
             continue; // ignore dangling symlinks in ~/.nix-defexpr
 
-        if (isNixExpr(path2)) {
+        if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) {
             /* Strip off the `.nix' filename suffix (if applicable),
                otherwise the attribute cannot be selected with the
                `-A' option.  Useful if you want to stick a Nix
@@ -130,32 +127,50 @@ static void getAllExprs(EvalState & state,
             string attrName = *i;
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
-            attrs.attrs[state.symbols.create(attrName)] =
+            if (attrs.find(attrName) != attrs.end()) {
-                ExprAttrs::AttrDef(state.parseExprFromFile(absPath(path2)), noPos);
+                printMsg(lvlError, format("warning: name collision in input Nix expressions, skipping `%1%'") % path2);
+                continue;
+            }
+            attrs.insert(attrName);
+            /* Load the expression on demand. */
+            Value & vFun(*state.allocValue());
+            Value & vArg(*state.allocValue());
+            state.getBuiltin("import", vFun);
+            mkString(vArg, path2);
+            mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg);
         }
-        else
+        else if (S_ISDIR(st.st_mode))
             /* `path2' is a directory (with no default.nix in it);
                recurse into it. */
-            getAllExprs(state, path2, attrs);
+            getAllExprs(state, path2, attrs, v);
     }
 }
 
 
-static Expr * loadSourceExpr(EvalState & state, const Path & path)
+static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
 {
-    if (isNixExpr(path)) return state.parseExprFromFile(absPath(path));
+    struct stat st;
+    if (stat(path.c_str(), &st) == -1)
+        throw SysError(format("getting information about `%1%'") % path);
+
+    if (isNixExpr(path, st)) {
+        state.evalFile(path, v);
+        return;
+    }
 
     /* The path is a directory.  Put the Nix expressions in the
-       directory in an attribute set, with the file name of each
+       directory in a set, with the file name of each expression as
-       expression as the attribute name.  Recurse into subdirectories
+       the attribute name.  Recurse into subdirectories (but keep the
-       (but keep the attribute set flat, not nested, to make it easier
+       set flat, not nested, to make it easier for a user to have a
-       for a user to have a ~/.nix-defexpr directory that includes
+       ~/.nix-defexpr directory that includes some system-wide
-       some system-wide directory). */
+       directory). */
-    ExprAttrs * attrs = new ExprAttrs;
+    if (S_ISDIR(st.st_mode)) {
-    attrs->attrs[state.symbols.create("_combineChannels")] =
+        state.mkAttrs(v, 16);
-        ExprAttrs::AttrDef(new ExprList(), noPos);
+        state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0);
-    getAllExprs(state, path, *attrs);
+        StringSet attrs;
-    return attrs;
+        getAllExprs(state, path, attrs, v);
+        v.attrs->sort();
+    }
 }
 
 
@@ -163,8 +178,10 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
     string systemFilter, Bindings & autoArgs,
     const string & pathPrefix, DrvInfos & elems)
 {
-    Value v;
+    Value vRoot;
-    findAlongAttrPath(state, pathPrefix, autoArgs, loadSourceExpr(state, nixExprPath), v);
+    loadSourceExpr(state, nixExprPath, vRoot);
+
+    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot));
 
     getDerivations(state, v, pathPrefix, autoArgs, elems, true);
 
@@ -227,13 +244,13 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
     const Strings & args, bool newestOnly)
 {
     DrvNames selectors = drvNamesFromArgs(args);
+    if (selectors.empty())
+        selectors.push_back(DrvName("*"));
 
     DrvInfos elems;
     set<unsigned int> done;
 
-    for (DrvNames::iterator i = selectors.begin();
+    foreach (DrvNames::iterator, i, selectors) {
-         i != selectors.end(); ++i)
-    {
         typedef list<std::pair<DrvInfo, unsigned int> > Matches;
         Matches matches;
         unsigned int n = 0;
@@ -305,8 +322,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
     }
 
     /* Check that all selectors have been used. */
-    for (DrvNames::iterator i = selectors.begin();
+    foreach (DrvNames::iterator, i, selectors)
-         i != selectors.end(); ++i)
         if (i->hits == 0 && i->fullName != "*")
             throw Error(format("selector `%1%' matches no derivations")
                 % i->fullName);
@@ -356,13 +372,15 @@ static void queryInstSources(EvalState & state,
            (import ./foo.nix)' = `(import ./foo.nix).bar'. */
         case srcNixExprs: {
 
-            Expr * e1 = loadSourceExpr(state, instSource.nixExprPath);
+            Value vArg;
+            loadSourceExpr(state, instSource.nixExprPath, vArg);
 
             foreach (Strings::const_iterator, i, args) {
-                Expr * e2 = state.parseExprFromString(*i, absPath("."));
+                Expr * eFun = state.parseExprFromString(*i, absPath("."));
-                Expr * call = new ExprApp(e2, e1);
+                Value vFun, vTmp;
-                Value v; state.eval(call, v);
+                state.eval(eFun, vFun);
-                getDerivations(state, v, "", instSource.autoArgs, elems, true);
+                mkApp(vTmp, vFun, vArg);
+                getDerivations(state, vTmp, "", instSource.autoArgs, elems, true);
             }
 
             break;
@@ -413,10 +431,10 @@ static void queryInstSources(EvalState & state,
         }
 
         case srcAttrPath: {
+            Value vRoot;
+            loadSourceExpr(state, instSource.nixExprPath, vRoot);
             foreach (Strings::const_iterator, i, args) {
-                Value v;
+                Value & v(*findAlongAttrPath(state, *i, instSource.autoArgs, vRoot));
-                findAlongAttrPath(state, *i, instSource.autoArgs,
-                    loadSourceExpr(state, instSource.nixExprPath), v);
                 getDerivations(state, v, "", instSource.autoArgs, elems, true);
             }
             break;
@@ -472,29 +490,31 @@ static void installDerivations(Globals & globals,
         newNames.insert(DrvName(i->name).name);
     }
 
-    /* Add in the already installed derivations, unless they have the
-       same name as a to-be-installed element. */
 
     while (true) {
         string lockToken = optimisticLockProfile(profile);
 
-        DrvInfos installedElems = queryInstalled(globals.state, profile);
-
         DrvInfos allElems(newElems);
-        foreach (DrvInfos::iterator, i, installedElems) {
-            DrvName drvName(i->name);
-            MetaInfo meta = i->queryMetaInfo(globals.state);
-            if (!globals.preserveInstalled &&
-                newNames.find(drvName.name) != newNames.end() &&
-                !keep(meta))
-                printMsg(lvlInfo,
-                    format("replacing old `%1%'") % i->name);
-            else
-                allElems.push_back(*i);
-        }
 
-        foreach (DrvInfos::iterator, i, newElems)
+        /* Add in the already installed derivations, unless they have
-            printMsg(lvlInfo, format("installing `%1%'") % i->name);
+           the same name as a to-be-installed element. */
+        if (!globals.removeAll) {
+            DrvInfos installedElems = queryInstalled(globals.state, profile);
+
+            foreach (DrvInfos::iterator, i, installedElems) {
+                DrvName drvName(i->name);
+                MetaInfo meta = i->queryMetaInfo(globals.state);
+                if (!globals.preserveInstalled &&
+                    newNames.find(drvName.name) != newNames.end() &&
+                    !keep(meta))
+                    printMsg(lvlInfo, format("replacing old `%1%'") % i->name);
+                else
+                    allElems.push_back(*i);
+            }
+
+            foreach (DrvInfos::iterator, i, newElems)
+                printMsg(lvlInfo, format("installing `%1%'") % i->name);
+        }
 
         printMissing(globals.state, newElems);
 
@@ -514,6 +534,8 @@ static void opInstall(Globals & globals,
         if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
         else if (arg == "--preserve-installed" || arg == "-P")
             globals.preserveInstalled = true;
+        else if (arg == "--remove-all" || arg == "-r")
+            globals.removeAll = true;
         else throw UsageError(format("unknown flag `%1%'") % arg);
     }
 
@@ -804,7 +826,7 @@ void printTable(Table & table)
 }
 
 
-/* This function compares the version of a element against the
+/* This function compares the version of an element against the
    versions in the given set of elements.  `cvLess' means that only
    lower versions are in the set, `cvEqual' means that at most an
    equal version is in the set, and `cvGreater' means that there is at
@@ -895,9 +917,6 @@ static void opQuery(Globals & globals,
         else remaining.push_back(arg);
     }
 
-    if (remaining.size() == 0)
-        printMsg(lvlInfo, "warning: you probably meant to specify the argument '*' to show all packages");
-
 
     /* Obtain derivation information from the specified source. */
     DrvInfos availElems, installedElems;
@@ -1284,6 +1303,7 @@ void run(Strings args)
 
     globals.dryRun = false;
     globals.preserveInstalled = false;
+    globals.removeAll = false;
     globals.prebuiltOnly = false;
 
     for (Strings::iterator i = args.begin(); i != args.end(); ) {
@@ -1350,6 +1370,9 @@ void run(Strings args)
 
     if (!op) throw UsageError("no operation specified");
 
+    if (globals.profile == "")
+        globals.profile = getEnv("NIX_PROFILE", "");
+
     if (globals.profile == "") {
         Path profileLink = getHomeDir() + "/.nix-profile";
         globals.profile = pathExists(profileLink)

src/nix-instantiate/nix-instantiate.cc

@@ -39,44 +39,48 @@ void processExpr(EvalState & state, const Strings & attrPaths,
     bool parseOnly, bool strict, Bindings & autoArgs,
     bool evalOnly, bool xmlOutput, bool location, Expr * e)
 {
-    if (parseOnly)
+    if (parseOnly) {
         std::cout << format("%1%\n") % *e;
-    else
+        return;
-        foreach (Strings::const_iterator, i, attrPaths) {
+    }
-            Value v;
-            findAlongAttrPath(state, *i, autoArgs, e, v);
-            state.forceValue(v);
 
-            PathSet context;
+    Value vRoot;
-            if (evalOnly)
+    state.eval(e, vRoot);
-                if (xmlOutput)
+
-                    printValueAsXML(state, strict, location, v, std::cout, context);
+    foreach (Strings::const_iterator, i, attrPaths) {
-                else {
+        Value & v(*findAlongAttrPath(state, *i, autoArgs, vRoot));
-                    if (strict) state.strictForceValue(v);
+        state.forceValue(v);
-                    std::cout << v << std::endl;
+
-                }
+        PathSet context;
+        if (evalOnly)
+            if (xmlOutput)
+                printValueAsXML(state, strict, location, v, std::cout, context);
             else {
-                DrvInfos drvs;
+                if (strict) state.strictForceValue(v);
-                getDerivations(state, v, "", autoArgs, drvs, false);
+                std::cout << v << std::endl;
-                foreach (DrvInfos::iterator, i, drvs) {
+            }
-                    Path drvPath = i->queryDrvPath(state);
+        else {
+            DrvInfos drvs;
+            getDerivations(state, v, "", autoArgs, drvs, false);
+            foreach (DrvInfos::iterator, i, drvs) {
+                Path drvPath = i->queryDrvPath(state);
 
-                    /* What output do we want? */
+                /* What output do we want? */
-                    string outputName = i->queryOutputName(state);
+                string outputName = i->queryOutputName(state);
-                    if (outputName == "")
+                if (outputName == "")
-                        throw Error(format("derivation `%1%' lacks an `outputName' attribute ") % drvPath);
+                    throw Error(format("derivation `%1%' lacks an `outputName' attribute ") % drvPath);
 
-                    if (gcRoot == "")
+                if (gcRoot == "")
-                        printGCWarning();
+                    printGCWarning();
-                    else {
+                else {
-                        Path rootName = gcRoot;
+                    Path rootName = gcRoot;
-                        if (++rootNr > 1) rootName += "-" + int2String(rootNr);
+                    if (++rootNr > 1) rootName += "-" + int2String(rootNr);
-                        drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot);
+                    drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot);
-                    }
-                    std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : "");
                 }
+                std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : "");
             }
         }
+    }
 }
 
 
@@ -160,7 +164,7 @@ void run(Strings args)
         files.push_back("./default.nix");
 
     foreach (Strings::iterator, i, files) {
-        Expr * e = state.parseExprFromFile(lookupFileArg(state, *i));
+        Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i)));
         processExpr(state, attrPaths, parseOnly, strict, autoArgs,
             evalOnly, xmlOutput, xmlOutputSourceLocation, e);
     }

src/nix-store/nix-store.cc

@@ -238,12 +238,6 @@ static void printTree(const Path & path,
     PathSet references;
     store->queryReferences(path, references);
 
-#if 0
-    for (PathSet::iterator i = drv.inputSrcs.begin();
-         i != drv.inputSrcs.end(); ++i)
-        cout << format("%1%%2%\n") % (tailPad + treeConn) % *i;
-#endif
-
     /* Topologically sort under the relation A < B iff A \in
        closure(B).  That is, if derivation A is an (possibly indirect)
        input of B, then A is printed first.  This has the effect of
@@ -251,7 +245,7 @@ static void printTree(const Path & path,
     Paths sorted = topoSortPaths(*store, references);
     reverse(sorted.begin(), sorted.end());
 
-    for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) {
+    foreach (Paths::iterator, i, sorted) {
         Paths::iterator j = i; ++j;
         printTree(*i, tailPad + treeConn,
             j == sorted.end() ? tailPad + treeNull : tailPad + treeLine,
@@ -293,7 +287,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (*i == "--resolve") query = qResolve;
         else if (*i == "--roots") query = qRoots;
         else if (*i == "--use-output" || *i == "-u") useOutput = true;
-        else if (*i == "--force-realise" || *i == "-f") forceRealise = true;
+        else if (*i == "--force-realise" || *i == "--force-realize" || *i == "-f") forceRealise = true;
         else if (*i == "--include-outputs") includeOutputs = true;
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
@@ -404,7 +398,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
             foreach (Strings::iterator, i, opArgs) {
                 PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
                 foreach (PathSet::iterator, j, paths)
-                    computeFSClosure(*store, *j, referrers, true);
+                    computeFSClosure(*store, *j, referrers, true,
+                        settings.gcKeepOutputs, settings.gcKeepDerivations);
             }
             Roots roots = store->findRoots();
             foreach (Roots::iterator, i, roots)
@@ -459,35 +454,42 @@ static void opReadLog(Strings opFlags, Strings opArgs)
     foreach (Strings::iterator, i, opArgs) {
         Path path = useDeriver(followLinksToStorePath(*i));
 
-        Path logPath = (format("%1%/%2%/%3%") %
+        for (int j = 0; j <= 2; j++) {
-            settings.nixLogDir % drvsLogDir % baseNameOf(path)).str();
+            if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path);
-        Path logBz2Path = logPath + ".bz2";
 
-        if (pathExists(logPath)) {
+            string baseName = baseNameOf(path);
-            /* !!! Make this run in O(1) memory. */
+            Path logPath =
-            string log = readFile(logPath);
+                j == 0
-            writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size());
+                ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
+                : (format("%1%/%2%/%3%") % settings.nixLogDir % drvsLogDir % baseName).str();
+            Path logBz2Path = logPath + ".bz2";
+
+            if (pathExists(logPath)) {
+                /* !!! Make this run in O(1) memory. */
+                string log = readFile(logPath);
+                writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size());
+                break;
+            }
+
+            else if (pathExists(logBz2Path)) {
+                AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY);
+                FILE * f = 0;
+                if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0)
+                    throw SysError(format("opening file `%1%'") % logBz2Path);
+                int err;
+                BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0);
+                if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path);
+                unsigned char buf[128 * 1024];
+                do {
+                    int n = BZ2_bzRead(&err, bz, buf, sizeof(buf));
+                    if (err != BZ_OK && err != BZ_STREAM_END)
+                        throw Error(format("error reading bzip2 file `%1%'") % logBz2Path);
+                    writeFull(STDOUT_FILENO, buf, n);
+                } while (err != BZ_STREAM_END);
+                BZ2_bzReadClose(&err, bz);
+                break;
+            }
         }
-
-        else if (pathExists(logBz2Path)) {
-            AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY);
-            FILE * f = 0;
-            if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0)
-                throw SysError(format("opening file `%1%'") % logBz2Path);
-            int err;
-            BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0);
-            if (!bz) throw Error(format("cannot open bzip2 file `%1%'") % logBz2Path);
-            unsigned char buf[128 * 1024];
-            do {
-                int n = BZ2_bzRead(&err, bz, buf, sizeof(buf));
-                if (err != BZ_OK && err != BZ_STREAM_END)
-                    throw Error(format("error reading bzip2 file `%1%'") % logBz2Path);
-                writeFull(STDOUT_FILENO, buf, n);
-            } while (err != BZ_STREAM_END);
-            BZ2_bzReadClose(&err, bz);
-        }
-
-        else throw Error(format("build log of derivation `%1%' is not available") % path);
     }
 }
 
@@ -513,7 +515,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
         if (!store->isValidPath(info.path) || reregister) {
             /* !!! races */
             if (canonicalise)
-                canonicalisePathMetaData(info.path);
+                canonicalisePathMetaData(info.path, -1);
             if (!hashGiven) {
                 HashResult hash = hashPath(htSHA256, info.path);
                 info.hash = hash.first;
@@ -692,7 +694,9 @@ static void opExport(Strings opFlags, Strings opArgs)
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
     FdSink sink(STDOUT_FILENO);
-    exportPaths(*store, opArgs, sign, sink);
+    Paths sorted = topoSortPaths(*store, PathSet(opArgs.begin(), opArgs.end()));
+    reverse(sorted.begin(), sorted.end());
+    exportPaths(*store, sorted, sign, sink);
 }
 
 
@@ -841,7 +845,7 @@ void run(Strings args)
 
         Operation oldOp = op;
 
-        if (arg == "--realise" || arg == "-r")
+        if (arg == "--realise" || arg == "--realize" || arg == "-r")
             op = opRealise;
         else if (arg == "--add" || arg == "-A")
             op = opAdd;
@@ -883,7 +887,7 @@ void run(Strings args)
             op = opVerifyPath;
         else if (arg == "--repair-path")
             op = opRepairPath;
-        else if (arg == "--optimise")
+        else if (arg == "--optimise" || arg == "--optimize")
             op = opOptimise;
         else if (arg == "--query-failed-paths")
             op = opQueryFailedPaths;

substitute.mk

@@ -7,6 +7,7 @@
 	 -e "s^@bindir\@^$(bindir)^g" \
 	 -e "s^@datadir\@^$(datadir)^g" \
 	 -e "s^@sysconfdir\@^$(sysconfdir)^g" \
+	 -e "s^@profiledir\@^$(profiledir)^g" \
 	 -e "s^@localstatedir\@^$(localstatedir)^g" \
 	 -e "s^@datadir\@^$(datadir)^g" \
 	 -e "s^@libdir\@^$(libdir)^g" \
@@ -36,3 +37,6 @@
 	 -e "s^@testPath\@^$(coreutils):$$(dirname $$(type -p expr))^g" \
 	 < $< > $@ || rm $@
 	if test -x $<; then chmod +x $@; fi
+
+$(CONFIG_HEADER):
+	true

tests/Makefile.am

@@ -14,6 +14,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
 
 XFAIL_TESTS =
 
+profiledir = $(sysconfdir)/profile.d
+
 include ../substitute.mk
 
 $(TESTS): common.sh config.nix

tests/binary-cache.sh

@@ -5,7 +5,7 @@ clearManifests
 
 # Create the binary cache.
 cacheDir=$TEST_ROOT/binary-cache
-rm -rf $cacheDir
+rm -rf "$cacheDir"
 
 outPath=$(nix-build dependencies.nix --no-out-link)
 
@@ -17,6 +17,8 @@ nix-push --dest $cacheDir $outPath
 clearStore
 rm -f $NIX_STATE_DIR/binary-cache*
 
+export _NIX_CACHE_FILE_URLS=1
+
 nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---"
 
 nix-store --option binary-caches "file://$cacheDir" -r $outPath
@@ -37,3 +39,26 @@ nix-store --option binary-caches "file://$cacheDir" -r $outPath
 nix-store --check-validity $outPath
 nix-store -qR $outPath | grep input-2
 
+
+# Test whether fallback works if we have cached info but the
+# corresponding NAR has disappeared.
+clearStore
+
+nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
+
+mkdir $cacheDir/tmp
+mv $cacheDir/*.nar* $cacheDir/tmp/
+
+NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback
+
+mv $cacheDir/tmp/* $cacheDir/
+
+
+# Test whether building works if the binary cache contains an
+# incomplete closure.
+clearStore
+
+rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
+
+nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+grep -q "Downloading" $TEST_ROOT/log

tests/build-hook.nix

@@ -17,5 +17,6 @@ in
   mkDerivation {
     name = "build-hook";
     builder = ./dependencies.builder0.sh;
-    inherit input1 input2;
+    input1 = " " + input1 + "/.";
+    input2 = " ${input2}/.";
   }

tests/common.sh.in

@@ -1,7 +1,7 @@
 set -e
 
 datadir="@datadir@"
-sysconfdir="@sysconfdir@"
+profiledir="@profiledir@"
 
 export TEST_ROOT=$(pwd)/test-tmp
 export NIX_STORE_DIR

tests/dependencies.builder0.sh

@@ -1,3 +1,6 @@
+[ "${input1: -2}" = /. ]
+[ "${input2: -2}" = /. ]
+
 mkdir $out
 echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
 

tests/dependencies.nix

@@ -9,14 +9,14 @@ let {
 
   input2 = mkDerivation {
     name = "dependencies-input-2";
-    builder = ./dependencies.builder2.sh;
+    builder = "${./dependencies.builder2.sh}";
   };
 
   body = mkDerivation {
     name = "dependencies";
-    builder = ./dependencies.builder0.sh  + "/FOOBAR/../.";
+    builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
     input1 = input1 + "/.";
-    inherit input2;
+    input2 = "${input2}/.";
     meta.description = "Random test package";
   };
 

tests/lang.sh

@@ -40,7 +40,7 @@ for i in lang/eval-okay-*.nix; do
         if test -e lang/$i.flags; then
             flags=$(cat lang/$i.flags)
         fi
-        if ! NIX_PATH=lang/dir3:lang/dir4:$NIX_PATH nix-instantiate $flags --eval-only --strict lang/$i.nix > lang/$i.out; then
+        if ! NIX_PATH=lang/dir3:lang/dir4_PATH nix-instantiate $flags --eval-only --strict lang/$i.nix > lang/$i.out; then
             echo "FAIL: $i should evaluate"
             fail=1
         elif ! diff lang/$i.out lang/$i.exp; then

tests/lang/eval-fail-antiquoted-path.nix

@@ -0,0 +1,4 @@
+# This must fail to evaluate, since ./fnord doesn't exist.  If it did
+# exist, it would produce "/nix/store/<hash>-fnord/xyzzy" (with an
+# appropriate context).
+"${./fnord}/xyzzy"

tests/lang/eval-fail-bad-antiquote-1.nix

@@ -0,0 +1 @@
+"${x: x}"