Typos

This is useful for the new hydra-queue-runner.

.gitignore

@@ -112,6 +112,8 @@ Makefile.config
 *.a
 *.o
 *.so
+*.dll
+*.exe
 *.dep
 *~
 *.pc

Makefile

@@ -10,6 +10,7 @@ makefiles = \
   src/nix-instantiate/local.mk \
   src/nix-env/local.mk \
   src/nix-daemon/local.mk \
+  src/nix-collect-garbage/local.mk \
   src/download-via-ssh/local.mk \
   src/nix-log2xml/local.mk \
   src/bsdiff-4.3/local.mk \
@@ -25,7 +26,7 @@ makefiles = \
 
 GLOBAL_CXXFLAGS += -std=c++0x -g -Wall
 
-include Makefile.config
+-include Makefile.config
 
 OPTIMIZE = 1
 

Makefile.config.in

@@ -4,9 +4,11 @@ CFLAGS = @CFLAGS@
 CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 HAVE_OPENSSL = @HAVE_OPENSSL@
+HAVE_SODIUM = @HAVE_SODIUM@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
+SODIUM_LIBS = @SODIUM_LIBS@
 bash = @bash@
 bindir = @bindir@
 bsddiff_compat_include = @bsddiff_compat_include@

configure.ac

@@ -87,6 +87,7 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
 # include <sys/param.h>
 # endif
 ])
+AC_CHECK_HEADERS([sys/syscall.h])
 
 
 # Check for lutimes, optionally used for changing the mtime of
@@ -205,6 +206,18 @@ AC_CHECK_HEADERS([bzlib.h], [true],
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
 
+# Look for libcurl, a required dependency.
+PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
+
+
+# Look for libsodium, an optional dependency.
+PKG_CHECK_MODULES([SODIUM], [libsodium],
+  [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.])
+   CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"
+   have_sodium=1], [have_sodium=])
+AC_SUBST(HAVE_SODIUM, [$have_sodium])
+
+
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
   [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]),

corepkgs/buildenv.nix

@@ -23,6 +23,5 @@ derivation {
   # network traffic, so don't do that.
   preferLocalBuild = true;
 
-  # Don't build in a chroot because Nix's dependencies may not be there.
-  __noChroot = true;
+  inherit chrootDeps;
 }

corepkgs/config.nix.in

@@ -2,7 +2,7 @@ let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
     if val != "" then val else def;
-in {
+in rec {
   perl = "@perl@";
   shell = "@bash@";
   coreutils = "@coreutils@";
@@ -14,4 +14,14 @@ in {
   tr = "@tr@";
   curl = "@curl@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
+  nixPrefix = "@prefix@";
+
+  # If Nix is installed in the Nix store, then automatically add it as
+  # a dependency to the core packages. This ensures that they work
+  # properly in a chroot.
+  chrootDeps =
+    if dirOf nixPrefix == builtins.storeDir then
+      [ (builtins.storePath nixPrefix) ]
+    else
+      [ ];
 }

corepkgs/fetchurl.nix

@@ -14,7 +14,7 @@ let
     '' + (if executable then "${coreutils}/chmod +x $out" else ""));
 
 in
-    
+
 derivation {
   name = baseNameOf (toString url);
   builder = shell;
@@ -26,15 +26,12 @@ derivation {
   outputHash = if outputHash != "" then outputHash else
       if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5;
   outputHashMode = if executable then "recursive" else "flat";
-  
+
   inherit system url;
 
   # No need to double the amount of network traffic
   preferLocalBuild = true;
 
-  # Don't build in a chroot because Nix's dependencies may not be there.
-  __noChroot = true;
-
   impureEnvVars = [
     # We borrow these environment variables from the caller to allow
     # easy proxy configuration.  This is impure, but a fixed-output
@@ -42,4 +39,6 @@ derivation {
     # by definition pure.
     "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
   ];
+
+  inherit chrootDeps;
 }

corepkgs/nar.nix

@@ -41,9 +41,8 @@ derivation {
   args = [ "-e" builder ];
   inherit storePath hashAlgo compressionType;
 
-  # Don't build in a chroot because Nix's dependencies may not be there.
-  __noChroot = true;
-
   # Remote machines may not have ${nixBinDir} or ${coreutils} in the same prefixes
   preferLocalBuild = true;
+
+  inherit chrootDeps;
 }

corepkgs/unpack-channel.nix

@@ -37,6 +37,5 @@ derivation {
   # No point in doing this remotely.
   preferLocalBuild = true;
 
-  # Don't build in a chroot because Nix's dependencies may not be there.
-  __noChroot = true;
+  inherit chrootDeps;
 }

doc/manual/command-ref/conf-file.xml

@@ -227,23 +227,32 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   <varlistentry><term><literal>build-use-chroot</literal></term>
 
     <listitem><para>If set to <literal>true</literal>, builds will be
-    performed in a <emphasis>chroot environment</emphasis>, i.e., the
-    build will be isolated from the normal file system hierarchy and
-    will only see its dependencies in the Nix store, the temporary
-    build directory, private versions of <filename>/proc</filename>,
+    performed in a <emphasis>chroot environment</emphasis>, i.e.,
+    they’re isolated from the normal file system hierarchy and will
+    only see their dependencies in the Nix store, the temporary build
+    directory, private versions of <filename>/proc</filename>,
     <filename>/dev</filename>, <filename>/dev/shm</filename> and
     <filename>/dev/pts</filename>, and the paths configured with the
     <link linkend='conf-build-chroot-dirs'><literal>build-chroot-dirs</literal>
     option</link>. This is useful to prevent undeclared dependencies
-    on files in directories such as
-    <filename>/usr/bin</filename>.</para>
+    on files in directories such as <filename>/usr/bin</filename>. In
+    addition, on Linux, builds run in private PID, mount, network, IPC
+    and UTS namespaces to isolate them from other processes in the
+    system (except that fixed-output derivations do not run in private
+    network namespace to ensure they can access the network).</para>
 
-    <para>The use of a chroot requires that Nix is run as root (so you
-    should use the <link linkend='conf-build-users-group'>“build
-    users” feature</link> to perform the actual builds under different
-    users than root).  Currently, chroot builds only work on Linux
-    because Nix uses “bind mounts” to make the Nix store and other
-    directories available inside the chroot.</para>
+    <para>Currently, chroots only work on Linux and Mac OS X. The use
+    of a chroot requires that Nix is run as root (so you should use
+    the <link linkend='conf-build-users-group'>“build users”
+    feature</link> to perform the actual builds under different users
+    than root).</para>
+
+    <para>If this option is set to <literal>relaxed</literal>, then
+    fixed-output derivations and derivations that have the
+    <varname>__noChroot</varname> attribute set to
+    <literal>true</literal> do not run in chroots.</para>
+
+    <para>The default is <literal>false</literal>.</para>
 
     </listitem>
 
@@ -392,6 +401,26 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry><term><literal>signed-binary-caches</literal></term>
+
+    <listitem><para>If set to <literal>*</literal>, Nix will only
+    download binaries if they are signed using one of the keys listed
+    in <option>binary-cache-public-keys</option>.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><literal>binary-cache-public-keys</literal></term>
+
+    <listitem><para>A whitespace-separated list of public keys
+    corresponding to the secret keys trusted to sign binary
+    caches. For example:
+    <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+    hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><literal>binary-caches-parallel-connections</literal></term>
 
     <listitem><para>The maximum number of parallel HTTP connections
@@ -538,6 +567,55 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term>
+
+    <listitem>
+
+      <para>If set to <literal>true</literal>, the Nix evaluator will
+      not allow access to any files outside of the Nix search path (as
+      set via the <envar>NIX_PATH</envar> environment variable or the
+      <option>-I</option> option). The default is
+      <literal>false</literal>.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
+  <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term>
+
+    <listitem>
+
+
+      <para>If set, the path to a program that can set extra
+      derivation-specific settings for this system. This is used for settings
+      that can't be captured by the derivation model itself and are too variable
+      between different versions of the same system to be hard-coded into nix.
+      </para>
+
+      <para>The hook is passed the derivation path and, if chroots are enabled,
+      the chroot directory. It can then modify the chroot and send a series of
+      commands to modify various settings to stdout. The currently recognized
+      commands are:</para>
+
+      <variablelist>
+        <varlistentry xml:id="extra-chroot-dirs"><term><literal>extra-chroot-dirs</literal></term>
+
+          <listitem>
+
+            <para>Pass a list of files and directories to be included in the
+            chroot for this build. One entry per line, terminated by an empty
+            line. Entries have the same format as build-chroot-dirs.</para>
+
+          </listitem>
+
+        </varlistentry>
+      </variablelist>
+    </listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>

doc/manual/command-ref/env-common.xml

@@ -36,8 +36,20 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
     <literal>&lt;nixpkgs/<replaceable>path</replaceable>></literal> in
     <filename>/home/eelco/Dev/nixpkgs-branch/<replaceable>path</replaceable></filename>
     and
-    <filename>/etc/nixos/nixpkgs/<replaceable>path</replaceable></filename>.
-    </para>
+    <filename>/etc/nixos/nixpkgs/<replaceable>path</replaceable></filename>.</para>
+
+    <para>If a path in the Nix search path starts with
+    <literal>http://</literal> or <literal>https://</literal>, it is
+    interpreted as the URL of a tarball that will be downloaded and
+    unpacked to a temporary location. The tarball must consist of a
+    single top-level directory. For example, setting
+    <envar>NIX_PATH</envar> to
+
+    <screen>
+nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz</screen>
+
+    tells Nix to download the latest revision in the Nixpkgs/NixOS
+    14.12 channel.</para>
 
     <para>The search path can be extended using the <option
     linkend="opt-I">-I</option> option, which takes precedence over

doc/manual/command-ref/nix-build.xml

@@ -58,6 +58,13 @@ and so on).</para>
 <command>nix-build</command> will use <filename>default.nix</filename>
 in the current directory, if it exists.</para>
 
+<para>If an element of <replaceable>paths</replaceable> starts with
+<literal>http://</literal> or <literal>https://</literal>, it is
+interpreted as the URL of a tarball that will be downloaded and
+unpacked to a temporary location. The tarball must include a single
+top-level directory containing at least a file named
+<filename>default.nix</filename>.</para>
+
 <para><command>nix-build</command> is essentially a wrapper around
 <link
 linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
@@ -175,6 +182,15 @@ bar
 
 </para>
 
+<para>Build the GNU Hello package from the latest revision of the
+master branch of Nixpkgs:
+
+<screen>
+$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+</screen>
+
+</para>
+
 </refsection>
 
 

doc/manual/command-ref/nix-channel.xml

@@ -21,7 +21,7 @@
     <command>nix-channel</command>
     <group choice='req'>
       <arg choice='plain'><option>--add</option> <replaceable>url</replaceable> <arg choice='opt'><replaceable>name</replaceable></arg></arg>
-      <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--remove</option> <replaceable>name</replaceable></arg>
       <arg choice='plain'><option>--list</option></arg>
       <arg choice='plain'><option>--update</option> <arg rep='repeat'><replaceable>names</replaceable></arg></arg>
       <arg choice='plain'><option>--rollback</option> <arg choice='opt'><replaceable>generation</replaceable></arg></arg>
@@ -129,7 +129,7 @@ $ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
 
 <variablelist>
 
-  <varlistentry><term><filename>/nix/var/nix/profiles/<replaceable>username</replaceable>/channels</filename></term>
+  <varlistentry><term><filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename></term>
 
     <listitem><para><command>nix-channel</command> uses a
     <command>nix-env</command> profile to keep track of previous
@@ -144,7 +144,7 @@ $ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
   <varlistentry><term><filename>~/.nix-defexpr/channels</filename></term>
 
     <listitem><para>This is a symlink to
-    <filename>/nix/var/nix/profiles/<replaceable>username</replaceable>/channels</filename>. It
+    <filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename>. It
     ensures that <command>nix-env</command> can find your channels. In
     a multi-user installation, you may also have
     <filename>~/.nix-defexpr/channels_root</filename>, which links to

doc/manual/command-ref/nix-env.xml

@@ -154,7 +154,15 @@ also <xref linkend="sec-common-options" />.</phrase></para>
     <option>--install</option>, <option>--upgrade</option>, and
     <option>--query --available</option> operations to obtain
     derivations.  The default is
-    <filename>~/.nix-defexpr</filename>.</para></listitem>
+    <filename>~/.nix-defexpr</filename>.</para>
+
+    <para>If the argument starts with <literal>http://</literal> or
+    <literal>https://</literal>, it is interpreted as the URL of a
+    tarball that will be downloaded and unpacked to a temporary
+    location. The tarball must include a single top-level directory
+    containing at least a file named <filename>default.nix</filename>.</para>
+
+    </listitem>
 
   </varlistentry>
 
@@ -416,7 +424,7 @@ $ nix-env --install gcc-3.3.2
 installing `gcc-3.3.2'
 uninstalling `gcc-3.1'</screen>
 
-Note the the previously installed version is removed, since
+Note the previously installed version is removed, since
 <option>--preserve-installed</option> was not specified.</para>
 
 <para>To install an arbitrary version:
@@ -496,6 +504,18 @@ the following paths will be substituted:
 
 </para>
 
+<para>To install Firefox from the latest revision in the Nixpkgs/NixOS
+14.12 channel:
+
+<screen>
+$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+</screen>
+
+(The GitHub repository <literal>nixpkgs-channels</literal> is updated
+automatically from the main <literal>nixpkgs</literal> repository
+after certain tests have succeeded and binaries have been built and
+uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
+
 </refsection>
 
 </refsection>
@@ -701,6 +721,44 @@ $ nix-env -e '.*' <lineannotation>(remove everything)</lineannotation></screen>
 
 
 
+<!--######################################################################-->
+
+<refsection xml:id="rsec-nix-env-set"><title>Operation <option>--set</option></title>
+
+<refsection><title>Synopsis</title>
+
+<cmdsynopsis>
+  <command>nix-env</command>
+  <arg choice='plain'><option>--set</option></arg>
+  <arg choice='plain'><replaceable>drvname</replaceable></arg>
+</cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+
+<para>The <option>--set</option> operation modifies the current generation of a
+profile so that it contains exactly the specified derivation, and nothing else.
+</para>
+
+</refsection>
+
+<refsection><title>Examples</title>
+
+<para>
+The following updates a profile such that its current generation will contain
+just Firefox:
+
+<screen>
+$ nix-env -p /nix/var/nix/profiles/browser --set firefox</screen>
+
+</para>
+
+</refsection>
+
+</refsection>
+
+
+
 <!--######################################################################-->
 
 <refsection xml:id="rsec-nix-env-set-flag"><title>Operation <option>--set-flag</option></title>
@@ -1091,54 +1149,103 @@ user environment elements, etc. -->
 
 <refsection><title>Examples</title>
 
+<para>To show installed packages:
+
 <screen>
-$ nix-env -q <lineannotation>(show installed derivations)</lineannotation>
+$ nix-env -q
 bison-1.875c
 docbook-xml-4.2
 firefox-1.0.4
 MPlayer-1.0pre7
 ORBit2-2.8.3
-...
+<replaceable>…</replaceable>
+</screen>
 
-$ nix-env -qa <lineannotation>(show available derivations)</lineannotation>
+</para>
+
+<para>To show available packages:
+
+<screen>
+$ nix-env -qa
 firefox-1.0.7
 GConf-2.4.0.1
 MPlayer-1.0pre7
 ORBit2-2.8.3
-...
+<replaceable>…</replaceable>
+</screen>
 
-$ nix-env -qas      <lineannotation>(show status of available derivations)</lineannotation>
+</para>
+
+<para>To show the status of available packages:
+
+<screen>
+$ nix-env -qas
 -P- firefox-1.0.7   <lineannotation>(not installed but present)</lineannotation>
 --S GConf-2.4.0.1   <lineannotation>(not present, but there is a substitute for fast installation)</lineannotation>
 --S MPlayer-1.0pre3 <lineannotation>(i.e., this is not the installed MPlayer, even though the version is the same!)</lineannotation>
 IP- ORBit2-2.8.3    <lineannotation>(installed and by definition present)</lineannotation>
-...
+<replaceable>…</replaceable>
+</screen>
 
-<lineannotation>(show available derivations in the Nix expression <!-- !!! <filename>-->foo.nix<!-- </filename> -->)</lineannotation>
+</para>
+
+<para>To show available packages in the Nix expression <filename>foo.nix</filename>:
+
+<screen>
 $ nix-env -f ./foo.nix -qa
 foo-1.2.3
+</screen>
 
-$ nix-env -qc <lineannotation>(compare installed versions to what’s available)</lineannotation>
+</para>
+
+<para>To compare installed versions to what’s available:
+
+<screen>
+$ nix-env -qc
 <replaceable>...</replaceable>
 acrobat-reader-7.0 - ?      <lineannotation>(package is not available at all)</lineannotation>
 autoconf-2.59      = 2.59   <lineannotation>(same version)</lineannotation>
 firefox-1.0.4      &lt; 1.0.7  <lineannotation>(a more recent version is available)</lineannotation>
 <replaceable>...</replaceable>
+</screen>
 
-$ nix-env -qa '.*zip.*' <lineannotation>(show all packages with “zip” in the name)</lineannotation>
+</para>
+
+<para>To show all packages with “<literal>zip</literal>” in the name:
+
+<screen>
+$ nix-env -qa '.*zip.*'
 bzip2-1.0.6
 gzip-1.6
 zip-3.0
-<replaceable>...</replaceable>
+<replaceable>…</replaceable>
+</screen>
 
-$ nix-env -qa '.*(firefox|chromium).*' <lineannotation>(show all packages with “firefox” or “chromium” in the name)</lineannotation>
+</para>
+
+<para>To show all packages with “<literal>firefox</literal>” or
+“<literal>chromium</literal>” in the name:
+
+<screen>
+$ nix-env -qa '.*(firefox|chromium).*'
 chromium-37.0.2062.94
 chromium-beta-38.0.2125.24
 firefox-32.0.3
 firefox-with-plugins-13.0.1
-<replaceable>...</replaceable>
+<replaceable>…</replaceable>
 </screen>
 
+</para>
+
+<para>To show all packages in the latest revision of the Nixpkgs
+repository:
+
+<screen>
+$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa
+</screen>
+
+</para>
+
 </refsection>
 
 </refsection>

doc/manual/command-ref/nix-push.xml

@@ -27,6 +27,7 @@
     <arg><option>--manifest</option></arg>
     <arg><option>--manifest-path</option> <replaceable>filename</replaceable></arg>
     <arg><option>--url-prefix</option> <replaceable>url</replaceable></arg>
+    <arg><option>--key-file</option> <replaceable>path</replaceable></arg>
     <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
   </cmdsynopsis>
 </refsynopsisdiv>
@@ -43,7 +44,7 @@ source and instead download binaries from the cache
 automatically.</para>
 
 <para><command>nix-push</command> performs the following actions.
-      
+
 <orderedlist>
 
   <listitem><para>Each path in <replaceable>paths</replaceable> is
@@ -101,9 +102,9 @@ automatically.</para>
   <varlistentry><term><option>--bzip2</option></term>
 
     <listitem><para>Compress NARs using <command>bzip2</command>
-    instead of <command>xz -9</command>.  The latter compresses about
-    30% better on typical archives, decompresses about twice as fast,
-    but compresses a lot slower and is not supported by Nix prior to
+    instead of <command>xz</command>.  The latter compresses about 30%
+    better on typical archives, decompresses about twice as fast, but
+    compresses a lot slower and is not supported by Nix prior to
     version 1.2.</para></listitem>
 
   </varlistentry>
@@ -155,6 +156,19 @@ automatically.</para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--key-file</option> <replaceable>path</replaceable></term>
+
+    <listitem><para>Sign the binary cache using the secret key stored
+    in <replaceable>path</replaceable>. This secret key must have been
+    created using <command
+    linkend="rsec-nix-store-generate-binary-cache-key">nix-store
+    --generate-binary-cache-key</command>. Users of this binary cache
+    should add the corresponding public key to the option
+    <option>binary-cache-public-keys</option> in
+    <filename>nix.conf</filename>.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </refsection>
@@ -203,6 +217,40 @@ $ nix-pull http://example.org/cache
 
 to cause the binaries to be used by subsequent Nix operations.</para>
 
+<para>To generate a signed binary cache, you must first generate a key
+pair, in this example called <literal>cache.example.org-1</literal>,
+storing the secret key in <filename>./sk</filename> and the public key
+in <filename>./pk</filename>:
+
+<screen>
+$ nix-store --generate-binary-cache-key cache.example.org-1 sk pk
+
+$ cat sk
+cache.example.org-1:jcMRQYFo8pQKzTtimpQLIPeHkMYZjfhB24hGfwF+u9PuX8H8FO7q564+X3G/JDlqqIqGar3OXRRwS9N3Wh3vbw==
+
+$ cat pk
+cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728=
+</screen>
+
+You can then generate a binary cache signed with the secret key:
+
+<screen>
+$ nix-push --dest /tmp/cache --key-file ./sk $(type -p firefox)
+</screen>
+
+Users who wish to verify the integrity of binaries downloaded from
+your cache would add the following to their
+<filename>nix.conf</filename>:
+
+<programlisting>
+binary-caches = http://cache.example.org
+signed-binary-caches = *
+binary-cache-public-keys = cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728=
+</programlisting>
+
+Nix will then ignore any binary that has a missing, incorrect or
+unrecognised signature.</para>
+
 </refsection>
 
 
@@ -224,7 +272,7 @@ Priority: 10
 The properties that are currently supported are:
 
 <variablelist>
-  
+
   <varlistentry><term><literal>StoreDir</literal></term>
 
     <listitem><para>The path of the Nix store to which this binary
@@ -303,12 +351,13 @@ NarHash: sha256:0s491y1h9hxj5ghiizlxk7ax6jwbha00zwn7lpyd5xg5bhf60vzg
 NarSize: 109521136
 References: 2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16 ...
 Deriver: 7akyyc87ka32xwmqza9dvyg5pwx3j212-glibc-2.7.drv
+Sig: cache.example.org-1:WepnSp2UT0odDpR3NRjPVhJBHmdBgSBSTbHpdh4SCz92nGXwFY82bkPEmISoC0hGqBXDXEmB6y3Ohgna3mMgDg==
 </screen>
 
 The fields are as follows:
 
 <variablelist>
-  
+
   <varlistentry><term><literal>StorePath</literal></term>
 
     <listitem><para>The full store path, including the name part
@@ -381,6 +430,22 @@ The fields are as follows:
 
   </varlistentry>
 
+  <varlistentry><term><literal>Sig</literal></term>
+
+    <listitem><para>A signature of the the form
+    <literal><replaceable>key-name</replaceable>:<replaceable>sig</replaceable></literal>,
+    where <replaceable>key-name</replaceable> is the symbolic name of
+    the key pair used to sign and verify the cache
+    (e.g. <literal>cache.example.org-1</literal>), and
+    <replaceable>sig</replaceable> is the actual signature, computed
+    over the <varname>StorePath</varname>, <varname>NarHash</varname>,
+    <varname>NarSize</varname> and <varname>References</varname>
+    fields using the <link
+    xlink:href="http://ed25519.cr.yp.to/">Ed25519 public-key signature
+    system</link>.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </para>

doc/manual/command-ref/nix-shell.xml

@@ -29,6 +29,7 @@
       <replaceable>attrPath</replaceable>
     </arg>
     <arg><option>--command</option> <replaceable>cmd</replaceable></arg>
+    <arg><option>--run</option> <replaceable>cmd</replaceable></arg>
     <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg>
     <arg><option>--pure</option></arg>
     <group choice='req'>
@@ -60,6 +61,13 @@ derivation for development.</para>
 <filename>shell.nix</filename> if it exists, and
 <filename>default.nix</filename> otherwise.</para>
 
+<para>If <replaceable>path</replaceable> starts with
+<literal>http://</literal> or <literal>https://</literal>, it is
+interpreted as the URL of a tarball that will be downloaded and
+unpacked to a temporary location. The tarball must include a single
+top-level directory containing at least a file named
+<filename>default.nix</filename>.</para>
+
 <para>If the derivation defines the variable
 <varname>shellHook</varname>, it will be evaluated after
 <literal>$stdenv/setup</literal> has been sourced.  Since this hook is
@@ -92,11 +100,24 @@ also <xref linkend="sec-common-options" />.</phrase></para>
   <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term>
 
     <listitem><para>In the environment of the derivation, run the
-    shell command <replaceable>cmd</replaceable> instead of starting
-    an interactive shell.  However, if you end the shell command with
-    <literal>return</literal>, you still get an interactive shell.
-    This can be useful for doing any additional
-    initialisation.</para></listitem>
+    shell command <replaceable>cmd</replaceable>. This command is
+    executed in an interactive shell. (Use <option>--run</option> to
+    use a non-interactive shell instead.) However, a call to
+    <literal>exit</literal> is implicitly added to the command, so the
+    shell will exit after running the command. To prevent this, add
+    <literal>return</literal> at the end; e.g. <literal>--command
+    "echo Hello; return"</literal> will print <literal>Hello</literal>
+    and then drop you into the interactive shell. This can be useful
+    for doing any additional initialisation.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--run</option> <replaceable>cmd</replaceable></term>
+
+    <listitem><para>Like <option>--command</option>, but executes the
+    command in a non-interactive shell. This means (among other
+    things) that if you hit Ctrl-C while the command is running, the
+    shell exits.</para></listitem>
 
   </varlistentry>
 
@@ -135,6 +156,15 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
+  <varlistentry><term><option>-i</option> <replaceable>interpreter</replaceable></term>
+
+    <listitem><para>The chained script interpreter to be invoked by
+    <command>nix-shell</command>. Only applicable in
+    <literal>#!</literal>-scripts (described <link
+    linkend="ssec-nix-shell-shebang">below</link>).</para>
+
+    </listitem></varlistentry>
+
 </variablelist>
 
 <para>The following common options are supported:</para>
@@ -184,6 +214,133 @@ $ nix-shell -p sqlite xorg.libX11
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
 </screen>
 
+The <command>-p</command> flag looks up Nixpkgs in the Nix search
+path. You can override it by passing <option>-I</option> or setting
+<envar>NIX_PATH</envar>. For example, the following gives you a shell
+containing the Pan package from a specific revision of Nixpkgs:
+
+<screen>
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+
+[nix-shell:~]$ pan --version
+Pan 0.139
+</screen>
+
+</para>
+
+</refsection>
+
+
+<refsection xml:id="ssec-nix-shell-shebang"><title>Use as a <literal>#!</literal>-interpreter</title>
+
+<para>You can use <command>nix-shell</command> as a script interpreter
+to allow scripts written in arbitrary languages to obtain their own
+dependencies via Nix. This is done by starting the script with the
+following lines:
+
+<programlisting>
+#! /usr/bin/env nix-shell
+#! nix-shell -i <replaceable>real-interpreter</replaceable> -p <replaceable>packages</replaceable>
+</programlisting>
+
+where <replaceable>real-interpreter</replaceable> is the “real” script
+interpreter that will be invoked by <command>nix-shell</command> after
+it has obtained the dependencies and initialised the environment, and
+<replaceable>packages</replaceable> are the attribute names of the
+dependencies in Nixpkgs.</para>
+
+<para>The lines starting with <literal>#! nix-shell</literal> specify
+<command>nix-shell</command> options (see above). Note that you cannot
+write <literal>#1 /usr/bin/env nix-shell -i ...</literal> because
+<command>/usr/bin/env</command> does not support passing options to
+the interpreter.</para>
+
+<para>For example, here is a Python script that depends on Python and
+the <literal>prettytable</literal> package:
+
+<programlisting>
+#! /usr/bin/env nix-shell
+#! nix-shell -i python -p python pythonPackages.prettytable
+
+import prettytable
+
+# Print a simple table.
+t = prettytable.PrettyTable(["N", "N^2"])
+for n in range(1, 10): t.add_row([n, n * n])
+print t
+</programlisting>
+
+</para>
+
+<para>Similarly, the following is a Perl script that specifies that it
+requires Perl and the <literal>HTML::TokeParser::Simple</literal> and
+<literal>LWP</literal> packages:
+
+<programlisting>
+#! /usr/bin/env nix-shell
+#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP
+
+use HTML::TokeParser::Simple;
+
+# Fetch nixos.org and print all hrefs.
+my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/');
+
+while (my $token = $p->get_tag("a")) {
+    my $href = $token->get_attr("href");
+    print "$href\n" if $href;
+}
+</programlisting>
+
+</para>
+
+<para>Finally, the following Haskell script uses a specific branch of
+Nixpkgs/NixOS (the 14.12 stable branch):
+
+<programlisting><![CDATA[
+#! /usr/bin/env nix-shell
+#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP haskellPackages.tagsoup
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz
+
+import Network.HTTP
+import Text.HTML.TagSoup
+
+-- Fetch nixos.org and print all hrefs.
+main = do
+  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+  body <- getResponseBody resp
+  let tags = filter (isTagOpenName "a") $ parseTags body
+  let tags' = map (fromAttrib "href") tags
+  mapM_ putStrLn $ filter (/= "") tags'
+]]></programlisting>
+
+If you want to be even more precise, you can specify a specific
+revision of Nixpkgs:
+
+<programlisting>
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
+</programlisting>
+
+</para>
+
+<para>The examples above all used <option>-p</option> to get
+dependencies from Nixpkgs. You can also use a Nix expression to build
+your own dependencies. For example, the Python example could have been
+written as:
+
+<programlisting>
+#! /usr/bin/env nix-shell
+#! nix-shell deps.nix -i python
+</programlisting>
+
+where the file <filename>deps.nix</filename> in the same directory
+as the <literal>#!</literal>-script contains:
+
+<programlisting>
+with import &lt;nixpkgs> {};
+
+runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } ""
+</programlisting>
+
 </para>
 
 </refsection>

doc/manual/command-ref/nix-store.xml

@@ -1338,6 +1338,56 @@ $ nix-store --clear-failed-paths *
 </refsection>
 
 
+<!--######################################################################-->
+
+<refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'>
+      <option>--generate-binary-cache-key</option>
+      <option>key-name</option>
+      <option>secret-key-file</option>
+      <option>public-key-file</option>
+    </arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+
+<para>This command generates an <link
+xlink:href="http://ed25519.cr.yp.to/">Ed25519 key pair</link> that can
+be used to create a signed binary cache. It takes three mandatory
+parameters:
+
+<orderedlist>
+
+  <listitem><para>A key name, such as
+  <literal>cache.example.org-1</literal>, that is used to look up keys
+  on the client when it verifies signatures. It can be anything, but
+  it’s suggested to use the host name of your cache
+  (e.g. <literal>cache.example.org</literal>) with a suffix denoting
+  the number of the key (to be incremented every time you need to
+  revoke a key).</para></listitem>
+
+  <listitem><para>The file name where the secret key is to be
+  stored.</para></listitem>
+
+  <listitem><para>The file name where the public key is to be
+  stored.</para></listitem>
+
+</orderedlist>
+
+For an example, see the manual page for <command
+linkend="sec-nix-push">nix-push</command>.</para>
+
+</refsection>
+
+</refsection>
+
+
 <!--######################################################################-->
 
 <refsection condition="manpage"><title>Environment variables</title>

doc/manual/expressions/advanced-attributes.xml

@@ -40,7 +40,7 @@ allowedReferences = [];
     recursively.  For example,
 
 <programlisting>
-allowedReferences = [ foobar ];
+allowedRequisites = [ foobar ];
 </programlisting>
 
     enforces that the output of a derivation cannot have any other
@@ -90,6 +90,33 @@ derivation {
   </varlistentry>
 
 
+  <varlistentry><term><varname>impureEnvVars</varname></term>
+
+    <listitem><para>This attribute allows you to specify a list of
+    environment variables that should be passed from the environment
+    of the calling user to the builder.  Usually, the environment is
+    cleared completely when the builder is executed, but with this
+    attribute you can allow specific environment variables to be
+    passed unmodified.  For example, <function>fetchurl</function> in
+    Nixpkgs has the line
+
+<programlisting>
+impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
+</programlisting>
+
+    to make it use the proxy server configuration specified by the
+    user in the environment variables <envar>http_proxy</envar> and
+    friends.</para>
+
+    <para>This attribute is only allowed in <link
+    linkend="fixed-output-drvs">fixed-output derivations</link>, where
+    impurities such as these are okay since (the hash of) the output
+    is known in advance.  It is ignored for all other
+    derivations.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="fixed-output-drvs">
     <term><varname>outputHash</varname></term>
     <term><varname>outputHashAlgo</varname></term>
@@ -215,29 +242,29 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>impureEnvVars</varname></term>
+  <varlistentry><term><varname>passAsFile</varname></term>
 
-    <listitem><para>This attribute allows you to specify a list of
-    environment variables that should be passed from the environment
-    of the calling user to the builder.  Usually, the environment is
-    cleared completely when the builder is executed, but with this
-    attribute you can allow specific environment variables to be
-    passed unmodified.  For example, <function>fetchurl</function> in
-    Nixpkgs has the line
+    <listitem><para>A list of names of attributes that should be
+    passed via files rather than environment variables.  For example,
+    if you have
 
-<programlisting>
-impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
-</programlisting>
+    <programlisting>
+passAsFile = ["big"];
+big = "a very long string";
+    </programlisting>
 
-    to make it use the proxy server configuration specified by the
-    user in the environment variables <envar>http_proxy</envar> and
-    friends.</para>
-
-    <para>This attribute is only allowed in <link
-    linkend="fixed-output-drvs">fixed-output derivations</link>, where
-    impurities such as these are okay since (the hash of) the output
-    is known in advance.  It is ignored for all other
-    derivations.</para></listitem>
+    then when the builder runs, the environment variable
+    <envar>bigPath</envar> will contain the absolute path to a
+    temporary file containing <literal>a very long
+    string</literal>. That is, for any attribute
+    <replaceable>x</replaceable> listed in
+    <varname>passAsFile</varname>, Nix will pass an environment
+    variable <envar><replaceable>x</replaceable>Path</envar> holding
+    the path of the file containing the value of attribute
+    <replaceable>x</replaceable>. This is useful when you need to pass
+    large strings to a builder, since most operating systems impose a
+    limit on the size of the environment (typically, a few hundred
+    kilobyte).</para></listitem>
 
   </varlistentry>
 

doc/manual/expressions/builtins.xml

@@ -211,6 +211,45 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
+  <varlistentry><term><function>builtins.fetchurl</function>
+  <replaceable>url</replaceable></term>
+
+    <listitem><para>Download the specified URL and return the path of
+    the downloaded file. This function is not available if <link
+    linkend="conf-restrict-eval">restricted evaluation mode</link> is
+    enabled.</para></listitem>
+
+  </varlistentry>
+
+
+  <varlistentry><term><function>fetchTarball</function>
+  <replaceable>url</replaceable></term>
+
+    <listitem><para>Download the specified URL, unpack it and return
+    the path of the unpacked tree. The file must be a tape archive
+    (<filename>.tar</filename>) compressed with
+    <literal>gzip</literal>, <literal>bzip2</literal> or
+    <literal>xz</literal>. The top-level path component of the files
+    in the tarball is removed, so it is best if the tarball contains a
+    single directory at top level. The typical use of the function is
+    to obtain external Nix expression dependencies, such as a
+    particular version of Nixpkgs, e.g.
+
+<programlisting>
+with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {};
+
+stdenv.mkDerivation { … }
+</programlisting>
+
+    </para>
+
+    <para>This function is not available if <link
+    linkend="conf-restrict-eval">restricted evaluation mode</link> is
+    enabled.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry><term><function>builtins.filter</function>
   <replaceable>f</replaceable> <replaceable>xs</replaceable></term>
 

doc/manual/expressions/language-values.xml

@@ -155,7 +155,14 @@ stdenv.mkDerivation {
   expression that contained it.  For instance, if a Nix expression in
   <filename>/foo/bar/bla.nix</filename> refers to
   <filename>../xyzzy/fnord.nix</filename>, the absolute path is
-  <filename>/foo/xyzzy/fnord.nix</filename>.</para></listitem>
+  <filename>/foo/xyzzy/fnord.nix</filename>.</para>
+
+  <para>If the first component of a path is a <literal>~</literal>,
+  it is interpreted as if the rest of the path were relative to the
+  user's home directory. e.g. <filename>~/foo</filename> would be
+  equivalent to <filename>/home/edolstra/foo</filename> for a user
+  whose home directory is <filename>/home/edolstra</filename>.
+  </para></listitem>
 
   <listitem><para><emphasis>Booleans</emphasis> with values
   <literal>true</literal> and

doc/manual/installation/multi-user.xml

@@ -52,6 +52,34 @@ This creates 10 build users. There can never be more concurrent builds
 than the number of build users, so you may want to increase this if
 you expect to do many builds at the same time.</para>
 
+<para>On Mac OS X, you can create the required group and users by
+running the following script:
+
+<programlisting>
+#! /bin/bash -e
+
+dseditgroup -o create nixbld -q
+
+gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }')
+
+echo "created nixbld group with gid $gid"
+
+for i in $(seq 1 10); do
+    user=/Users/nixbld$i
+    uid="$((30000 + $i))"
+    dscl . create $user
+    dscl . create $user RealName "Nix build user $i"
+    dscl . create $user PrimaryGroupID "$gid"
+    dscl . create $user UserShell /usr/bin/false
+    dscl . create $user NFSHomeDirectory /var/empty
+    dscl . create $user UniqueID "$uid"
+    dseditgroup -o edit -a nixbld$i -t user nixbld
+    echo "created nixbld$i user with uid $uid"
+done
+</programlisting>
+
+</para>
+
 </simplesect>
 
 

doc/manual/introduction/about-nix.xml

@@ -197,7 +197,7 @@ collection</emphasis> (Nixpkgs).</para>
 <simplesect><title>Managing build environments</title>
 
 <para>Nix is extremely useful for developers as it makes it easy to
-automatically set up the the build environment for a package. Given a
+automatically set up the build environment for a package. Given a
 Nix expression that describes the dependencies of your package, the
 command <command>nix-shell</command> will build or download those
 dependencies if they’re not already in your Nix store, and then start

doc/manual/manual.xml

@@ -4,10 +4,8 @@
       version="5.0">
 
   <info>
-
     <title>Nix Package Manager Guide</title>
-
-    <edition>Version <xi:include href="version.txt" parse="text" /></edition>
+    <subtitle>Version <xi:include href="version.txt" parse="text" /></subtitle>
 
     <author>
       <personname>

doc/manual/packages/basic-package-mgmt.xml

@@ -36,7 +36,7 @@ it’s much more convenient to use the Nixpkgs
 <emphasis>channel</emphasis>, since it makes it easy to stay up to
 date with new versions of Nixpkgs. (Channels are described in more
 detail in <xref linkend="sec-channels"/>.) Nixpkgs is automatically
-added to your list of “subscribed” channels when when you install
+added to your list of “subscribed” channels when you install
 Nix. If this is not the case for some reason, you can add it as
 follows:
 

doc/manual/packages/profiles.xml

@@ -120,8 +120,7 @@ can also see all available generations:
 <screen>
 $ nix-env --list-generations</screen></para>
 
-<para>Actually, there is another level of indirection not shown in the
-figure above.  You generally wouldn’t have
+<para>You generally wouldn’t have
 <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
 in your <envar>PATH</envar>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current

doc/manual/release-notes/release-notes.xml

@@ -12,6 +12,7 @@
 </partintro>
 -->
 
+<xi:include href="rl-1.9.xml" />
 <xi:include href="rl-1.8.xml" />
 <xi:include href="rl-1.7.xml" />
 <xi:include href="rl-1.6.1.xml" />

doc/manual/release-notes/rl-1.8.xml

@@ -113,4 +113,11 @@ $ nix-store -l $(which xterm)
 
 </itemizedlist>
 
+<para>This release has contributions from Adam Szkoda, Aristid
+Breitkreuz, Bob van der Linden, Charles Strahan, darealshinji, Eelco
+Dolstra, Gergely Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic,
+Mikey Ariel, Paul Colomiets, Ricardo M.  Correia, Ricky Elrod, Robert
+Helgesson, Rob Vermaas, Russell O'Connor, Shea Levy, Shell Turner,
+Sönke Hahn, Steve Purcell, Vladimír Čunát and Wout Mertens.</para>
+
 </section>

doc/manual/release-notes/rl-1.9.xml

@@ -0,0 +1,216 @@
+<section xmlns="http://docbook.org/ns/docbook"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      xmlns:xi="http://www.w3.org/2001/XInclude"
+      version="5.0"
+      xml:id="ssec-relnotes-1.9">
+
+<title>Release 1.9 (June 12, 2015)</title>
+
+<para>In addition to the usual bug fixes, this release has the
+following new features:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Signed binary cache support. You can enable signature
+    checking by adding the following to <filename>nix.conf</filename>:
+
+<programlisting>
+signed-binary-caches = *
+binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+</programlisting>
+
+    This will prevent Nix from downloading any binary from the cache
+    that is not signed by one of the keys listed in
+    <option>binary-cache-public-keys</option>.</para>
+
+    <para>Signature checking is only supported if you built Nix with
+    the <literal>libsodium</literal> package.</para>
+
+    <para>Note that while Nix has had experimental support for signed
+    binary caches since version 1.7, this release changes the
+    signature format in a backwards-incompatible way.</para>
+
+  </listitem>
+
+  <listitem>
+
+    <para>Automatic downloading of Nix expression tarballs. In various
+    places, you can now specify the URL of a tarball containing Nix
+    expressions (such as Nixpkgs), which will be downloaded and
+    unpacked automatically. For example:</para>
+
+    <itemizedlist>
+
+      <listitem><para>In <command>nix-env</command>:
+
+<screen>
+$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox
+</screen>
+
+      This installs Firefox from the latest tested and built revision
+      of the NixOS 14.12 channel.</para></listitem>
+
+      <listitem><para>In <command>nix-build</command> and
+      <command>nix-shell</command>:
+
+<screen>
+$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello
+</screen>
+
+      This builds GNU Hello from the latest revision of the Nixpkgs
+      master branch.</para></listitem>
+
+      <listitem><para>In the Nix search path (as specified via
+      <envar>NIX_PATH</envar> or <option>-I</option>). For example, to
+      start a shell containing the Pan package from a specific version
+      of Nixpkgs:
+
+<screen>
+$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
+</screen>
+
+      </para></listitem>
+
+      <listitem><para>In <command>nixos-rebuild</command> (on NixOS):
+
+<screen>
+$ nixos-rebuild test -I nixpkgs=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz
+</screen>
+
+      </para></listitem>
+
+      <listitem><para>In Nix expressions, via the new builtin function <function>fetchTarball</function>:
+
+<programlisting>
+with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; …
+</programlisting>
+
+      (This is not allowed in restricted mode.)</para></listitem>
+
+    </itemizedlist>
+
+  </listitem>
+
+  <listitem>
+
+    <para><command>nix-shell</command> improvements:</para>
+
+    <itemizedlist>
+
+      <listitem><para><command>nix-shell</command> now has a flag
+      <option>--run</option> to execute a command in the
+      <command>nix-shell</command> environment,
+      e.g. <literal>nix-shell --run make</literal>. This is like
+      the existing <option>--command</option> flag, except that it
+      uses a non-interactive shell (ensuring that hitting Ctrl-C won’t
+      drop you into the child shell).</para></listitem>
+
+      <listitem><para><command>nix-shell</command> can now be used as
+      a <literal>#!</literal>-interpreter. This allows you to write
+      scripts that dynamically fetch their own dependencies. For
+      example, here is a Haskell script that, when invoked, first
+      downloads GHC and the Haskell packages on which it depends:
+
+<programlisting>
+#! /usr/bin/env nix-shell
+#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP
+
+import Network.HTTP
+
+main = do
+  resp &lt;- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
+  body &lt;- getResponseBody resp
+  print (take 100 body)
+</programlisting>
+
+      Of course, the dependencies are cached in the Nix store, so the
+      second invocation of this script will be much
+      faster.</para></listitem>
+
+    </itemizedlist>
+
+  </listitem>
+
+  <listitem>
+
+    <para>Chroot improvements:</para>
+
+    <itemizedlist>
+
+      <listitem><para>Chroot builds are now supported on Mac OS X
+      (using its sandbox mechanism).</para></listitem>
+
+      <listitem><para>If chroots are enabled, they are now used for
+      all derivations, including fixed-output derivations (such as
+      <function>fetchurl</function>). The latter do have network
+      access, but can no longer access the host filesystem. If you
+      need the old behaviour, you can set the option
+      <option>build-use-chroot</option> to
+      <literal>relaxed</literal>.</para></listitem>
+
+      <listitem><para>On Linux, if chroots are enabled, builds are
+      performed in a private PID namespace once again. (This
+      functionality was lost in Nix 1.8.)</para></listitem>
+
+      <listitem><para>Store paths listed in
+      <option>build-chroot-dirs</option> are now automatically
+      expanded to their closure. For instance, if you want
+      <filename>/nix/store/…-bash/bin/sh</filename> mounted in your
+      chroot as <filename>/bin/sh</filename>, you only need to say
+      <literal>build-chroot-dirs =
+      /bin/sh=/nix/store/…-bash/bin/sh</literal>; it is no longer
+      necessary to specify the dependencies of Bash.</para></listitem>
+
+    </itemizedlist>
+
+  </listitem>
+
+  <listitem><para>The new derivation attribute
+  <varname>passAsFile</varname> allows you to specify that the
+  contents of derivation attributes should be passed via files rather
+  than environment variables. This is useful if you need to pass very
+  long strings that exceed the size limit of the environment. The
+  Nixpkgs function <function>writeTextFile</function> uses
+  this.</para></listitem>
+
+  <listitem><para>You can now use <literal>~</literal> in Nix file
+  names to refer to your home directory, e.g. <literal>import
+  ~/.nixpkgs/config.nix</literal>.</para></listitem>
+
+  <listitem><para>Nix has a new option <option>restrict-eval</option>
+  that allows limiting what paths the Nix evaluator has access to. By
+  passing <literal>--option restrict-eval true</literal> to Nix, the
+  evaluator will throw an exception if an attempt is made to access
+  any file outside of the Nix search path. This is primarily intended
+  for Hydra to ensure that a Hydra jobset only refers to its declared
+  inputs (and is therefore reproducible).</para></listitem>
+
+  <listitem><para><command>nix-env</command> now only creates a new
+  “generation” symlink in <filename>/nix/var/nix/profiles</filename>
+  if something actually changed.</para></listitem>
+
+  <listitem><para>The environment variable <envar>NIX_PAGER</envar>
+  can now be set to override <envar>PAGER</envar>. You can set it to
+  <literal>cat</literal> to disable paging for Nix commands
+  only.</para></listitem>
+
+  <listitem><para>Failing <literal>&lt;...></literal>
+  lookups now show position information.</para></listitem>
+
+  <listitem><para>Improved Boehm GC use: we disabled scanning for
+  interior pointers, which should reduce the “<literal>Repeated
+  allocation of very large block</literal>” warnings and associated
+  retention of memory.</para></listitem>
+
+</itemizedlist>
+
+<para>This release has contributions from aszlig, Benjamin Staffin,
+Charles Strahan, Christian Theune, Daniel Hahler, Danylo Hlynskyi
+Daniel Peebles, Dan Peebles, Domen Kožar, Eelco Dolstra, Harald van
+Dijk, Hoang Xuan Phu, Jaka Hudoklin, Jeff Ramnani, j-keck, Linquize,
+Luca Bruno, Michael Merickel, Oliver Dunkl, Rob Vermaas, Rok Garbas,
+Shea Levy, Tobias Geerinckx-Rice and William A. Kennington III.</para>
+
+</section>
+

local.mk

@@ -8,6 +8,7 @@ clean-files += Makefile.config
 
 GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
 
-$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+$(foreach i, config.h $(call rwildcard, src/lib*, *.hh) src/nix-store/serve-protocol.hh, \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644)))

misc/vim/syntax/nix.vim

@@ -1,37 +0,0 @@
-" Vim syntax file
-" Language:	nix
-" Maintainer:	Marc Weber <marco-oweber@gmx.de>
-"               Modify and commit if you feel that way
-" Last Change:	2007 Dec
-
-" Quit when a (custom) syntax file was already loaded
-if exists("b:current_syntax")
-  finish
-endif
-
-syn keyword	nixKeyword	let throw inherit import true false null with
-syn keyword	nixConditional	if else then
-syn keyword     nixBrace        ( ) { } =
-syn keyword     nixBuiltin         __currentSystem __currentTime __isFunction __getEnv __trace __toPath __pathExists 
-  \ __readFile __toXML __toFile __filterSource __attrNames __getAttr __hasAttr __isAttrs __listToAttrs __isList 
-  \ __head __tail __add __sub __lessThan __substring __stringLength
-
-syn match nixAttr "\w\+\ze\s*="
-syn match nixFuncArg "\zs\w\+\ze\s*:"
-syn region nixStringParam start=+\${+ end=+}+
-syn region nixMultiLineComment start=+/\*+ skip=+\\"+ end=+\*/+
-syn match  nixEndOfLineComment "#.*$"
-syn region nixStringIndented start=+''+ skip=+'''\|''${\|"+ end=+''+ contains=nixStringParam
-syn region nixString         start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam
-
-hi def link nixKeyword       Keyword
-hi def link nixConditional   Conditional
-hi def link nixBrace         Special
-hi def link nixString        String
-hi def link nixStringIndented String
-hi def link nixBuiltin       Special
-hi def link nixStringParam   Macro
-hi def link nixMultiLineComment Comment
-hi def link nixEndOfLineComment Comment
-hi def link nixAttr        Identifier
-hi def link nixFuncArg     Identifier

perl/lib/Nix/Config.pm.in

@@ -1,5 +1,7 @@
 package Nix::Config;
 
+use MIME::Base64;
+
 $version = "@PACKAGE_VERSION@";
 
 $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
@@ -19,24 +21,33 @@ $useBindings = "@perlbindings@" eq "yes";
 
 %config = ();
 
+%binaryCachePublicKeys = ();
+
+$defaultPublicKeys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
+
 sub readConfig {
     if (defined $ENV{'_NIX_OPTIONS'}) {
         foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) {
             my ($n, $v) = split '=', $s, 2;
             $config{$n} = $v;
         }
-        return;
+    } else {
+        my $config = "$confDir/nix.conf";
+        return unless -f $config;
+
+        open CONFIG, "<$config" or die "cannot open ‘$config’";
+        while (<CONFIG>) {
+            /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next;
+            $config{$1} = $2;
+        }
+        close CONFIG;
     }
 
-    my $config = "$confDir/nix.conf";
-    return unless -f $config;
-
-    open CONFIG, "<$config" or die "cannot open ‘$config’";
-    while (<CONFIG>) {
-        /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next;
-        $config{$1} = $2;
+    foreach my $s (split(/ /, $config{"binary-cache-public-keys"} // $defaultPublicKeys)) {
+        my ($keyName, $publicKey) = split ":", $s;
+        next unless defined $keyName && defined $publicKey;
+        $binaryCachePublicKeys{$keyName} = decode_base64($publicKey);
     }
-    close CONFIG;
 }
 
 return 1;

perl/lib/Nix/Crypto.pm

@@ -1,42 +0,0 @@
-package Nix::Crypto;
-
-use strict;
-use MIME::Base64;
-use Nix::Store;
-use Nix::Config;
-use IPC::Open2;
-
-our @ISA = qw(Exporter);
-our @EXPORT = qw(signString isValidSignature);
-
-sub signString {
-    my ($privateKeyFile, $s) = @_;
-    my $hash = hashString("sha256", 0, $s);
-    my ($from, $to);
-    my $pid = open2($from, $to, $Nix::Config::openssl, "rsautl", "-sign", "-inkey", $privateKeyFile);
-    print $to $hash;
-    close $to;
-    local $/ = undef;
-    my $sig = <$from>;
-    close $from;
-    waitpid($pid, 0);
-    die "$0: OpenSSL returned exit code $? while signing hash\n" if $? != 0;
-    my $sig64 = encode_base64($sig, "");
-    return $sig64;
-}
-
-sub isValidSignature {
-    my ($publicKeyFile, $sig64, $s) = @_;
-    my ($from, $to);
-    my $pid = open2($from, $to, $Nix::Config::openssl, "rsautl", "-verify", "-inkey", $publicKeyFile, "-pubin");
-    print $to decode_base64($sig64);
-    close $to;
-    my $decoded = <$from>;
-    close $from;
-    waitpid($pid, 0);
-    return 0 if $? != 0;
-    my $hash = hashString("sha256", 0, $s);
-    return $decoded eq $hash;
-}
-
-1;

perl/lib/Nix/Manifest.pm

@@ -8,11 +8,12 @@ use Cwd;
 use File::stat;
 use File::Path;
 use Fcntl ':flock';
+use MIME::Base64;
 use Nix::Config;
-use Nix::Crypto;
+use Nix::Store;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo);
+our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo fingerprintPath);
 
 
 sub addNAR {
@@ -376,7 +377,6 @@ EOF
 }
 
 
-
 # Delete all old manifests downloaded from a given URL.
 sub deleteOldManifests {
     my ($url, $curUrlFile) = @_;
@@ -394,12 +394,29 @@ sub deleteOldManifests {
 }
 
 
+# Return a fingerprint of a store path to be used in binary cache
+# signatures. It contains the store path, the base-32 SHA-256 hash of
+# the contents of the path, and the references.
+sub fingerprintPath {
+    my ($storePath, $narHash, $narSize, $references) = @_;
+    die if substr($storePath, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir;
+    die if substr($narHash, 0, 7) ne "sha256:";
+    # Convert hash from base-16 to base-32, if necessary.
+    $narHash = "sha256:" . convertHash("sha256", substr($narHash, 7), 1)
+        if length($narHash) == 71;
+    die if length($narHash) != 59;
+    foreach my $ref (@{$references}) {
+        die if substr($ref, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir;
+    }
+    return "1;" . $storePath . ";" . $narHash . ";" . $narSize . ";" . join(",", @{$references});
+}
+
+
 # Parse a NAR info file.
 sub parseNARInfo {
     my ($storePath, $content, $requireValidSig, $location) = @_;
 
     my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system, $sig);
-    my $signedData = "";
     my $compression = "bzip2";
     my @refs;
 
@@ -415,8 +432,7 @@ sub parseNARInfo {
         elsif ($1 eq "References") { @refs = split / /, $2; }
         elsif ($1 eq "Deriver") { $deriver = $2; }
         elsif ($1 eq "System") { $system = $2; }
-        elsif ($1 eq "Signature") { $sig = $2; last; }
-        $signedData .= "$line\n";
+        elsif ($1 eq "Sig") { $sig = $2; }
     }
 
     return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash;
@@ -434,30 +450,37 @@ sub parseNARInfo {
         };
 
     if ($requireValidSig) {
+        # FIXME: might be useful to support multiple signatures per .narinfo.
+
         if (!defined $sig) {
             warn "NAR info file ‘$location’ lacks a signature; ignoring\n";
             return undef;
         }
-        my ($sigVersion, $keyName, $sig64) = split ";", $sig;
-        $sigVersion //= 0;
-        if ($sigVersion != 1) {
-            warn "NAR info file ‘$location’ has unsupported version $sigVersion; ignoring\n";
-            return undef;
-        }
+        my ($keyName, $sig64) = split ":", $sig;
         return undef unless defined $keyName && defined $sig64;
-        my $publicKeyFile = $Nix::Config::config{"binary-cache-public-key-$keyName"};
-        if (!defined $publicKeyFile) {
+
+        my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName};
+        if (!defined $publicKey) {
             warn "NAR info file ‘$location’ is signed by unknown key ‘$keyName’; ignoring\n";
             return undef;
         }
-        if (! -f $publicKeyFile) {
-            die "binary cache public key file ‘$publicKeyFile’ does not exist\n";
+
+        my $fingerprint;
+        eval {
+            $fingerprint = fingerprintPath(
+                $storePath, $narHash, $narSize,
+                [ map { "$Nix::Config::storeDir/$_" } @refs ]);
+        };
+        if ($@) {
+            warn "cannot compute fingerprint of ‘$location’; ignoring\n";
             return undef;
         }
-        if (!isValidSignature($publicKeyFile, $sig64, $signedData)) {
-            warn "NAR info file ‘$location’ has an invalid signature; ignoring\n";
+
+        if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) {
+            warn "NAR info file ‘$location’ has an incorrect signature; ignoring\n";
             return undef;
         }
+
         $res->{signedBy} = $keyName;
     }
 

perl/lib/Nix/Store.pm

@@ -13,10 +13,12 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] );
 our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
 
 our @EXPORT = qw(
+    setVerbosity
     isValidPath queryReferences queryPathInfo queryDeriver queryPathHash
     queryPathFromHashPart
     topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths
-    hashPath hashFile hashString
+    hashPath hashFile hashString convertHash
+    signString checkSignature
     addToStore makeFixedOutputPath
     derivationFromPath
 );

perl/lib/Nix/Store.xs

@@ -11,6 +11,10 @@
 #include <misc.hh>
 #include <util.hh>
 
+#if HAVE_SODIUM
+#include <sodium.h>
+#endif
+
 
 using namespace nix;
 
@@ -44,6 +48,11 @@ void init()
         doInit();
 
 
+void setVerbosity(int level)
+    CODE:
+        verbosity = (Verbosity) level;
+
+
 int isValidPath(char * path)
     CODE:
         try {
@@ -223,11 +232,68 @@ SV * hashString(char * algo, int base32, char * s)
         }
 
 
+SV * convertHash(char * algo, char * s, int toBase32)
+    PPCODE:
+        try {
+            Hash h = parseHash16or32(parseHashType(algo), s);
+            string s = toBase32 ? printHash32(h) : printHash(h);
+            XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
+        } catch (Error & e) {
+            croak(e.what());
+        }
+
+
+SV * signString(SV * secretKey_, char * msg)
+    PPCODE:
+        try {
+#if HAVE_SODIUM
+            STRLEN secretKeyLen;
+            unsigned char * secretKey = (unsigned char *) SvPV(secretKey_, secretKeyLen);
+            if (secretKeyLen != crypto_sign_SECRETKEYBYTES)
+                throw Error("secret key is not valid");
+
+            unsigned char sig[crypto_sign_BYTES];
+            unsigned long long sigLen;
+            crypto_sign_detached(sig, &sigLen, (unsigned char *) msg, strlen(msg), secretKey);
+            XPUSHs(sv_2mortal(newSVpv((char *) sig, sigLen)));
+#else
+            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
+#endif
+        } catch (Error & e) {
+            croak(e.what());
+        }
+
+
+int checkSignature(SV * publicKey_, SV * sig_, char * msg)
+    CODE:
+        try {
+#if HAVE_SODIUM
+            STRLEN publicKeyLen;
+            unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen);
+            if (publicKeyLen != crypto_sign_PUBLICKEYBYTES)
+                throw Error("public key is not valid");
+
+            STRLEN sigLen;
+            unsigned char * sig = (unsigned char *) SvPV(sig_, sigLen);
+            if (sigLen != crypto_sign_BYTES)
+                throw Error("signature is not valid");
+
+            RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0;
+#else
+            throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
+#endif
+        } catch (Error & e) {
+            croak(e.what());
+        }
+    OUTPUT:
+        RETVAL
+
+
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             doInit();
-            Path path = store->addToStore(srcPath, recursive, parseHashType(algo));
+            Path path = store->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo));
             XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
         } catch (Error & e) {
             croak(e.what());

perl/local.mk

@@ -5,8 +5,7 @@ nix_perl_sources := \
   $(d)/lib/Nix/SSH.pm \
   $(d)/lib/Nix/CopyClosure.pm \
   $(d)/lib/Nix/Config.pm.in \
-  $(d)/lib/Nix/Utils.pm \
-  $(d)/lib/Nix/Crypto.pm
+  $(d)/lib/Nix/Utils.pm
 
 nix_perl_modules := $(nix_perl_sources:.in=)
 
@@ -23,16 +22,18 @@ ifeq ($(perlbindings), yes)
 
   Store_SOURCES := $(Store_DIR)/Store.cc
 
-  Store_LIBS = libstore libutil
-
   Store_CXXFLAGS = \
     -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \
     -D_FILE_OFFSET_BITS=64 -Wno-unused-variable -Wno-literal-suffix -Wno-reserved-user-defined-literal
 
+  Store_LIBS = libstore libutil
+
+  Store_LDFLAGS := $(SODIUM_LIBS)
+
   ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
     archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
     libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
-    Store_LDFLAGS = $(shell find ${archlib} -name ${libperl})
+    Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})
   endif
 
   Store_ALLOW_UNDEFINED = 1

release.nix

@@ -24,7 +24,7 @@ let
 
         buildInputs =
           [ curl bison flex perl libxml2 libxslt bzip2
-            tetex dblatex nukeReferences pkgconfig sqlite
+            tetex dblatex nukeReferences pkgconfig sqlite libsodium
             docbook5 docbook5_xsl
           ] ++ lib.optional (!lib.inNixShell) git;
 
@@ -80,7 +80,9 @@ let
         name = "nix";
         src = tarball;
 
-        buildInputs = [ curl perl bzip2 openssl pkgconfig sqlite boehmgc ];
+        buildInputs =
+          [ curl perl bzip2 openssl pkgconfig sqlite boehmgc ]
+          ++ lib.optional stdenv.isLinux libsodium;
 
         configureFlags = ''
           --disable-init-state
@@ -91,14 +93,6 @@ let
           --sysconfdir=/etc
         '';
 
-        # Provide a default value for the ‘build-chroot-dirs’ setting
-        # that includes /bin/sh pointing to bash.
-        preHook = lib.optionalString stdenv.isLinux (
-          let sh = stdenv.shell; in
-          ''
-            export DEFAULT_CHROOT_DIRS="/bin/sh=${sh} $(tr '\n' ' ' < ${writeReferencesToFile sh})"
-          '');
-
         enableParallelBuilding = true;
 
         makeFlags = "profiledir=$(out)/etc/profile.d";
@@ -181,23 +175,33 @@ let
       };
 
 
-    rpm_fedora20i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora20i386);
-    rpm_fedora20x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora20x86_64);
+    rpm_fedora18i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora18i386) [];
+    rpm_fedora18x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora18x86_64) [];
+    rpm_fedora19i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora19i386) [];
+    rpm_fedora19x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora19x86_64) [];
+    rpm_fedora20i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora20i386) [];
+    rpm_fedora20x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora20x86_64) [];
+    rpm_fedora21i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora21i386) [ "libsodium-devel" ];
+    rpm_fedora21x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora21x86_64) [ "libsodium-devel" ];
 
 
-    deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386);
-    deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64);
+    deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386) [];
+    deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64) [];
+    deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ];
+    deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ];
 
-    deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386);
-    deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64);
-    deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386);
-    deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64);
-    deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386);
-    deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64);
-    deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386);
-    deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64);
-    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386);
-    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64);
+    deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386) [];
+    deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64) [];
+    deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386) [];
+    deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64) [];
+    deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) [];
+    deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) [];
+    deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) [];
+    deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) [];
+    deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [];
+    deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [];
+    deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ];
+    deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ];
 
 
     # System tests.
@@ -245,12 +249,14 @@ let
           binaryTarball.x86_64-linux
           deb_debian7i386
           deb_debian7x86_64
-          deb_ubuntu1404i386
-          deb_ubuntu1404x86_64
-          deb_ubuntu1410i386
-          deb_ubuntu1410x86_64
+          deb_ubuntu1404i386 # LTS
+          deb_ubuntu1404x86_64 # LTS
+          deb_ubuntu1504i386
+          deb_ubuntu1504x86_64
           rpm_fedora20i386
           rpm_fedora20x86_64
+          rpm_fedora21i386
+          rpm_fedora21x86_64
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
@@ -264,7 +270,7 @@ let
   makeRPM_x86_64 = makeRPM "x86_64-linux";
 
   makeRPM =
-    system: diskImageFun:
+    system: diskImageFun: extraPackages:
 
     with import <nixpkgs> { inherit system; };
 
@@ -272,7 +278,9 @@ let
       name = "nix-rpm";
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" ]; };
+        { extraPackages =
+            [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" "libcurl-devel" ]
+            ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
@@ -283,7 +291,7 @@ let
   makeDeb_x86_64 = makeDeb "x86_64-linux";
 
   makeDeb =
-    system: diskImageFun:
+    system: diskImageFun: extraPackages:
 
     with import <nixpkgs> { inherit system; };
 
@@ -291,7 +299,9 @@ let
       name = "nix-deb";
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" ]; };
+        { extraPackages =
+            [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" ]
+            ++ extraPackages; };
       memSize = 1024;
       meta.schedulingPriority = 50;
       configureFlags = "--sysconfdir=/etc";

scripts/build-remote.pl.in

@@ -16,6 +16,8 @@ binmode STDERR, ":encoding(utf8)";
 
 my $debug = defined $ENV{NIX_DEBUG_HOOK};
 
+setVerbosity(0); # make exportPath() less verbose
+
 
 # General operation:
 #
@@ -82,7 +84,7 @@ if (defined $conf && -e $conf) {
         push @machines,
             { hostName => $tokens[0]
             , systemTypes => [ split(/,/, $tokens[1]) ]
-            , sshKeys => $tokens[2]
+            , sshKey => $tokens[2]
             , maxJobs => int($tokens[3])
             , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1)
             , supportedFeatures => [ @supportedFeatures, @mandatoryFeatures ]
@@ -201,7 +203,7 @@ REQ: while (1) {
 
 
         # Connect to the selected machine.
-        my @sshOpts = ("-i", $machine->{sshKeys});
+        my @sshOpts = ("-i", $machine->{sshKey});
         $hostName = $machine->{hostName};
         eval {
             ($from, $to) = connectToRemoteNix($hostName, \@sshOpts, "2>&4");

scripts/copy-from-other-stores.pl.in

@@ -9,6 +9,7 @@ my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 
 STDOUT->autoflush(1);
+binmode STDERR, ":encoding(utf8)";
 
 my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
 
@@ -93,7 +94,7 @@ elsif ($ARGV[0] eq "--substitute") {
     my ($store, $sourcePath) = findStorePath $storePath;
     die unless $store;
     print STDERR "\n*** Copying ‘$storePath’ from ‘$sourcePath’\n\n";
-    system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $destPath") == 0
+    system("@coreutils@/cp", "-rpd", $sourcePath, $destPath) == 0
         or die "cannot copy ‘$sourcePath’ to ‘$storePath’";
     print "\n"; # no hash to verify
 }

scripts/download-from-binary-cache.pl.in

@@ -47,11 +47,17 @@ $caBundle = "/etc/ssl/certs/ca-certificates.crt" if !$caBundle && -f "/etc/ssl/c
 
 my $userName = getpwuid($<) || $ENV{"USER"} or die "cannot figure out user name";
 
+my $userAgent = "Nix/$Nix::Config::version";
+
 sub isTrue {
     my ($x) = @_;
     return $x eq "true" || $x eq "1";
 }
 
+# FIXME: this should be cache URLs required to have valid signatures,
+# or "*" to require signatures on all binary caches.
+# FIXME: should binary caches using a key in
+# ‘binary-cache-public-keys’ be trusted by default?
 my $requireSignedBinaryCaches = ($Nix::Config::config{"signed-binary-caches"} // "0") ne "0";
 
 my $curlConnectTimeout = int(
@@ -75,7 +81,7 @@ sub addRequest {
     $curl->setopt(CURLOPT_FOLLOWLOCATION, 1);
     $curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle;
     $curl->setopt(CURLOPT_SSL_VERIFYPEER, 0) unless isTrue($Nix::Config::config{"verify-https-binary-caches"} // "1");
-    $curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version");
+    $curl->setopt(CURLOPT_USERAGENT, $userAgent);
     $curl->setopt(CURLOPT_NOBODY, 1) if $head;
     $curl->setopt(CURLOPT_FAILONERROR, 1);
     $curl->setopt(CURLOPT_CONNECTTIMEOUT, $curlConnectTimeout);
@@ -555,7 +561,7 @@ sub downloadBinary {
         die if $requireSignedBinaryCaches && !defined $info->{signedBy};
         print STDERR "\n*** Downloading ‘$url’ ", ($requireSignedBinaryCaches ? "(signed by ‘$info->{signedBy}’) " : ""), "to ‘$storePath’...\n";
         checkURL $url;
-        if (system("$Nix::Config::curl --fail --location --insecure --connect-timeout $curlConnectTimeout '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
+        if (system("$Nix::Config::curl --fail --location --insecure --connect-timeout $curlConnectTimeout -A '$userAgent' '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
             warn "download of ‘$url’ failed" . ($! ? ": $!" : "") . "\n";
             next;
         }

scripts/install-nix-from-closure.sh

@@ -41,7 +41,7 @@ mkdir -p $dest/store
 
 echo -n "copying Nix to $dest/store..." >&2
 
-for i in $(cd $self/store && echo *); do
+for i in $(cd $self/store >/dev/null && echo *); do
     echo -n "." >&2
     i_tmp="$dest/store/$i.$$"
     if [ -e "$i_tmp" ]; then
@@ -75,7 +75,7 @@ fi
 # Install an SSL certificate bundle.
 if [ -z "$SSL_CERT_FILE" -o ! -f "$SSL_CERT_FILE" ]; then
     $nix/bin/nix-env -i "$cacert"
-    export SSL_CERT_FILE="$HOME/.nix-profile/etc/ca-bundle.crt"
+    export SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt"
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.

scripts/local.mk

@@ -1,7 +1,6 @@
 nix_bin_scripts := \
   $(d)/nix-build \
   $(d)/nix-channel \
-  $(d)/nix-collect-garbage \
   $(d)/nix-copy-closure \
   $(d)/nix-generate-patches \
   $(d)/nix-install-package \

scripts/nix-build.in

@@ -5,6 +5,8 @@ use strict;
 use Nix::Config;
 use Nix::Store;
 use Nix::Utils;
+use File::Basename;
+use Cwd;
 
 binmode STDERR, ":encoding(utf8)";
 
@@ -14,6 +16,7 @@ my $runEnv = $0 =~ /nix-shell$/;
 my $pure = 0;
 my $fromArgs = 0;
 my $packages = 0;
+my $interactive = 1;
 
 my @instArgs = ();
 my @buildArgs = ();
@@ -25,6 +28,9 @@ my @envExclude = ();
 
 my $myName = $runEnv ? "nix-shell" : "nix-build";
 
+my $inShebang = 0;
+my $script;
+my @savedArgs;
 
 my $tmpDir = mkTempDir($myName);
 
@@ -35,6 +41,30 @@ my $drvLink = "$tmpDir/derivation";
 $SIG{'INT'} = sub { exit 1 };
 
 
+# Heuristic to see if we're invoked as a shebang script, namely, if we
+# have a single argument, it's the name of an executable file, and it
+# starts with "#!".
+if ($runEnv && defined $ARGV[0] && $ARGV[0] !~ /nix-shell/) {
+    $script = $ARGV[0];
+    if (-f $script && -x $script) {
+        open SCRIPT, "<$script" or die "$0: cannot open ‘$script’: $!\n";
+        my $first = <SCRIPT>;
+        if ($first =~ /^\#\!/) {
+            $inShebang = 1;
+            @savedArgs = @ARGV; shift @savedArgs;
+            @ARGV = ();
+            while (<SCRIPT>) {
+                chomp;
+                if (/^\#\!\s*nix-shell (.*)$/) {
+                    push @ARGV, split(/ /, $1);
+                }
+            }
+        }
+        close SCRIPT;
+    }
+}
+
+
 for (my $n = 0; $n < scalar @ARGV; $n++) {
     my $arg = $ARGV[$n];
 
@@ -131,10 +161,11 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         $runEnv = 1;
     }
 
-    elsif ($arg eq "--command") {
+    elsif ($arg eq "--command" || $arg eq "--run") {
         $n++;
         die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
-        $envCommand = "$ARGV[$n]\nexit $!";
+        $envCommand = "$ARGV[$n]\nexit";
+        $interactive = 0 if $arg eq "--run";
     }
 
     elsif ($arg eq "--exclude") {
@@ -155,15 +186,39 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         $packages = 1;
     }
 
+    elsif ($inShebang && $arg eq "-i") {
+        $n++;
+        die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
+        my $interpreter = $ARGV[$n];
+        # Überhack to support Perl. Perl examines the shebang and
+        # executes it unless it contains the string "perl" or "indir",
+        # or (undocumented) argv[0] does not contain "perl". Exploit
+        # the latter by doing "exec -a".
+        my $execArgs = $interpreter =~ /perl/ ? "-a PERL" : "";
+        sub shellEscape {
+            my $s = $_;
+            $s =~ s/'/'\\''/g;
+            return "'" . $s . "'";
+        }
+        $envCommand = "exec $execArgs $interpreter $script ${\(join ' ', (map shellEscape, @savedArgs))}";
+    }
+
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
 
+    elsif ($arg eq "-Q" || $arg eq "--no-build-output") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+    }
+
     else {
         push @exprs, $arg;
     }
 }
 
+die "$0: ‘-p’ and ‘-E’ are mutually exclusive\n" if $packages && $fromArgs;
+
 if ($packages) {
     push @instArgs, "--expr";
     @exprs = (
@@ -182,6 +237,11 @@ foreach my $expr (@exprs) {
     # Instantiate.
     my @drvPaths;
     if ($expr !~ /^\/.*\.drv$/) {
+        # If we're in a #! script, interpret filenames relative to the
+        # script.
+        $expr = dirname(Cwd::abs_path($script)) . "/" . $expr
+            if $inShebang && !$packages && $expr !~ /^\//;
+
         # !!! would prefer the perl 5.8.0 pipe open feature here.
         my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
         while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
@@ -232,17 +292,20 @@ foreach my $expr (@exprs) {
             ($pure ? '' : 'p=$PATH; ' ) .
             'dontAddDisableDepTrack=1; ' .
             '[ -e $stdenv/setup ] && source $stdenv/setup; ' .
-            'if [ "$(type -t runHook)" = function ]; then runHook shellHook; fi; ' .
             ($pure ? '' : 'PATH=$PATH:$p; unset p; ') .
             'set +e; ' .
             '[ -n "$PS1" ] && PS1="\n\[\033[1;32m\][nix-shell:\w]$\[\033[0m\] "; ' .
+            'if [ "$(type -t runHook)" = function ]; then runHook shellHook; fi; ' .
             'unset NIX_ENFORCE_PURITY; ' .
             'unset NIX_INDENT_MAKE; ' .
             'shopt -u nullglob; ' .
             'unset TZ; ' . (defined $ENV{'TZ'} ? "export TZ='${ENV{'TZ'}}'; " : '') .
             $envCommand);
         $ENV{BASH_ENV} = $rcfile;
-        exec($ENV{NIX_BUILD_SHELL} // "bash", "--rcfile", $rcfile);
+        my @args = ($ENV{NIX_BUILD_SHELL} // "bash");
+        push @args, "--rcfile" if $interactive;
+        push @args, $rcfile;
+        exec @args;
         die;
     }
 

scripts/nix-channel.in

@@ -6,6 +6,7 @@ use File::Basename;
 use File::Path qw(mkpath);
 use Nix::Config;
 use Nix::Manifest;
+use File::Temp qw(tempdir);
 
 binmode STDERR, ":encoding(utf8)";
 
@@ -98,42 +99,14 @@ sub update {
         my $url = $channels{$name};
         my $origUrl = "$url/MANIFEST";
 
-        # Check if $url is a redirect.  If so, follow it now to ensure
-        # consistency if the redirection is changed between
-        # downloading the manifest and the tarball.
-        my $headers = `$Nix::Config::curl --silent --head '$url'`;
+        # We want to download the url to a file to see if it's a tarball while also checking if we
+        # got redirected in the process, so that we can grab the various parts of a nix channel
+        # definition from a consistent location if the redirect changes mid-download.
+        my $tmpdir = tempdir( CLEANUP => 1 );
+        my $filename;
+        ($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`;
         die "$0: unable to check ‘$url’\n" if $? != 0;
-        $headers =~ s/\r//g;
-        $url = $1 if $headers =~ /^Location:\s*(.*)\s*$/m;
-
-        # Check if the channel advertises a binary cache.
-        my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
-        my $extraAttrs = "";
-        my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true";
-        if ($? == 0 && $binaryCacheURL ne "") {
-            $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
-            deleteOldManifests($origUrl, undef);
-        } else {
-            $getManifest = 1;
-        }
-
-        if ($getManifest) {
-            # No binary cache, so pull the channel manifest.
-            mkdir $manifestDir, 0755 unless -e $manifestDir;
-            die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir;
-            $ENV{'NIX_ORIG_URL'} = $origUrl;
-            system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
-                or die "cannot pull manifest from ‘$url’\n";
-        }
-
-        # Download the channel tarball.
-        my $fullURL = "$url/nixexprs.tar.xz";
-        system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or
-            $fullURL = "$url/nixexprs.tar.bz2";
-        print STDERR "downloading Nix expressions from ‘$fullURL’...\n";
-        my ($hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`;
-        die "cannot fetch ‘$fullURL’\n" if $? != 0;
-        chomp $path;
+        chomp $url;
 
         # If the URL contains a version number, append it to the name
         # attribute (so that "nix-env -q" on the channels profile
@@ -141,6 +114,52 @@ sub update {
         my $cname = $name;
         $cname .= $1 if basename($url) =~ /(-\d.*)$/;
 
+        my $path;
+        my $ret = -1;
+        if (-e "$tmpdir/$filename" && $filename =~ /\.tar\.(gz|bz2|xz)$/) {
+            # Get our temporary download into the store.
+            (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url 'file://$tmpdir/$filename'`;
+            chomp $path;
+
+            # Try unpacking the expressions to see if they'll be valid for us to process later.
+            # Like anything in nix, this will cache the result so we don't do it again outside of the loop below.
+            $ret = system("$Nix::Config::binDir/nix-build --no-out-link -E 'import <nix/unpack-channel.nix> " .
+                          "{ name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; }'");
+        }
+
+        # The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it
+        my $extraAttrs = "";
+        if ($ret != 0) {
+            # Check if the channel advertises a binary cache.
+            my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
+            my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true";
+            if ($? == 0 && $binaryCacheURL ne "") {
+                $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
+                deleteOldManifests($origUrl, undef);
+            } else {
+                $getManifest = 1;
+            }
+
+            if ($getManifest) {
+                # No binary cache, so pull the channel manifest.
+                mkdir $manifestDir, 0755 unless -e $manifestDir;
+                die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir;
+                $ENV{'NIX_ORIG_URL'} = $origUrl;
+                system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
+                    or die "cannot pull manifest from ‘$url’\n";
+            }
+
+            # Download the channel tarball.
+            my $fullURL = "$url/nixexprs.tar.xz";
+            system("$Nix::Config::curl --fail --silent --head '$fullURL' > /dev/null") == 0 or
+                $fullURL = "$url/nixexprs.tar.bz2";
+            print STDERR "downloading Nix expressions from ‘$fullURL’...\n";
+            (my $hash, $path) = `PRINT_PATH=1 QUIET=1 $Nix::Config::binDir/nix-prefetch-url '$fullURL'`;
+            die "cannot fetch ‘$fullURL’\n" if $? != 0;
+            chomp $path;
+        }
+
+        # Regardless of where it came from, add the expression representing this channel to accumulated expression
         $exprs .= "'f: f { name = \"$cname\"; channelName = \"$name\"; src = builtins.storePath \"$path\"; $extraAttrs }' ";
     }
 

scripts/nix-collect-garbage.in

@@ -1,65 +0,0 @@
-#! @perl@ -w @perlFlags@
-
-use strict;
-use Nix::Config;
-
-my $profilesDir = "@localstatedir@/nix/profiles";
-
-
-# Process the command line arguments.
-my @args = ();
-my $arg;
-
-my $removeOld = 0;
-my $gen;
-my $dryRun = 0;
-
-while ($arg = shift) {
-    if ($arg eq "--delete-old" || $arg eq "-d") {
-        $removeOld = 1;
-        $gen = "old";
-    } elsif ($arg eq "--delete-older-than") {
-        $removeOld = 1;
-        $gen = shift;
-    } elsif ($arg eq "--dry-run") {
-        $dryRun = 1;
-    } elsif ($arg eq "--help") {
-        exec "man nix-collect-garbage" or die;
-    } else {
-        push @args, $arg;
-    }
-}
-
-
-# If `-d' was specified, remove all old generations of all profiles.
-# Of course, this makes rollbacks to before this point in time
-# impossible.
-
-sub removeOldGenerations;
-sub removeOldGenerations {
-    my $dir = shift;
-
-    my $dh;
-    opendir $dh, $dir or die;
-
-    foreach my $name (sort (readdir $dh)) {
-        next if $name eq "." || $name eq "..";
-        $name = $dir . "/" . $name;
-        if (-l $name && (readlink($name) =~ /link/)) {
-            print STDERR "removing old generations of profile $name\n";
-
-            system("$Nix::Config::binDir/nix-env", "-p", $name, "--delete-generations", $gen, $dryRun ? "--dry-run" : ());
-        }
-        elsif (! -l $name && -d $name) {
-            removeOldGenerations $name;
-        }
-    }
-
-    closedir $dh or die;
-}
-
-removeOldGenerations $profilesDir if $removeOld;
-
-
-# Run the actual garbage collector.
-exec "$Nix::Config::binDir/nix-store", "--gc", @args unless $dryRun;

scripts/nix-install-package.in

@@ -71,7 +71,7 @@ sub barf {
 my $pkgFile = $source;
 if ($fromURL) {
     $pkgFile = "$tmpDir/tmp.nixpkg";
-    system("@curl@", "--silent", $source, "-o", $pkgFile) == 0
+    system("@curl@", "-L", "--silent", $source, "-o", $pkgFile) == 0
         or barf "curl failed: $?";
 }
 

scripts/nix-profile.sh.in

@@ -9,6 +9,7 @@ if [ -n "$HOME" ]; then
     fi
 
     export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH
+    export MANPATH=$NIX_LINK/share/man:$MANPATH
 
     # Subscribe the user to the Nixpkgs channel by default.
     if [ ! -e $HOME/.nix-channels ]; then
@@ -21,11 +22,15 @@ if [ -n "$HOME" ]; then
     export NIX_PATH=${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs
 
     # Set $SSL_CERT_FILE so that Nixpkgs applications like curl work.
-    if [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Fedora, NixOS
-        export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
-    elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # Ubuntu, Debian
+    if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
         export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-    elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # fall back to Nix profile
+    elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS
+        export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt
+    elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
+        export SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt
+    elif [ -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in Nix profile
+        export SSL_CERT_FILE="$NIX_LINK/etc/ssl/certs/ca-bundle.crt"
+    elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile
         export SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 fi

scripts/nix-push.in

@@ -6,11 +6,11 @@ use File::Basename;
 use File::Path qw(mkpath);
 use File::stat;
 use File::Copy;
+use MIME::Base64;
 use Nix::Config;
 use Nix::Store;
 use Nix::Manifest;
 use Nix::Utils;
-use Nix::Crypto;
 
 binmode STDERR, ":encoding(utf8)";
 
@@ -27,9 +27,9 @@ my $writeManifest = 0;
 my $manifestPath;
 my $archivesURL;
 my $link = 0;
-my $privateKeyFile;
-my $keyName;
+my $secretKeyFile;
 my @roots;
+my @buildArgs;
 
 for (my $n = 0; $n < scalar @ARGV; $n++) {
     my $arg = $ARGV[$n];
@@ -61,14 +61,14 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         $archivesURL = $ARGV[$n];
     } elsif ($arg eq "--link") {
         $link = 1;
-    } elsif ($arg eq "--key") {
+    } elsif ($arg eq "--key-file") {
         $n++;
         die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
-        $privateKeyFile = $ARGV[$n];
-    } elsif ($arg eq "--key-name") {
+        $secretKeyFile = $ARGV[$n];
+    } elsif ($arg eq "--max-jobs" || $arg eq "-j") {
         $n++;
         die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
-        $keyName = $ARGV[$n];
+        push @buildArgs, ($arg, $ARGV[$n]);
     } elsif (substr($arg, 0, 1) eq "-") {
         die "$0: unknown flag ‘$arg’\n";
     } else {
@@ -110,7 +110,7 @@ my %narFiles;
 foreach my $storePath (@storePaths) {
     my $pathHash = substr(basename($storePath), 0, 32);
     my $narInfoFile = "$destDir/$pathHash.narinfo";
-    if (-e $narInfoFile) {
+    if (!$force && -e $narInfoFile) {
         my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read ‘$narInfoFile’\n";
         my $narFile = "$destDir/$narInfo->{url}";
         if (-e $narFile) {
@@ -158,7 +158,7 @@ close NIX;
 # Build the Nix expression.
 print STDERR "building compressed archives...\n";
 my @narPaths;
-my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |")
+my $pid = open(READ, "-|", "$Nix::Config::binDir/nix-build", $nixExpr, "-o", "$tmpDir/result", @buildArgs)
     or die "cannot run nix-build";
 while (<READ>) {
     chomp;
@@ -257,9 +257,14 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) {
         }
     }
 
-    if (defined $privateKeyFile && defined $keyName) {
-        my $sig = signString($privateKeyFile, $info);
-        $info .= "Signature: 1;$keyName;$sig\n";
+    if (defined $secretKeyFile) {
+        my $s = readFile $secretKeyFile;
+        chomp $s;
+        my ($keyName, $secretKey) = split ":", $s;
+        die "invalid secret key file ‘$secretKeyFile’\n" unless defined $keyName && defined $secretKey;
+        my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
+        my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), "");
+        $info .= "Sig: $keyName:$sig\n";
     }
 
     my $pathHash = substr(basename($storePath), 0, 32);

src/libexpr/common-opts.cc

@@ -1,5 +1,6 @@
 #include "common-opts.hh"
-#include "../libmain/shared.hh"
+#include "shared.hh"
+#include "download.hh"
 #include "util.hh"
 
 
@@ -53,7 +54,9 @@ bool parseSearchPathArg(Strings::iterator & i,
 
 Path lookupFileArg(EvalState & state, string s)
 {
-    if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
+    if (isUri(s))
+        return downloadFileCached(s, true);
+    else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
         Path p = s.substr(1, s.size() - 2);
         return state.findFile(p);
     } else

src/libexpr/download.cc

@@ -0,0 +1,236 @@
+#include "download.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "hash.hh"
+#include "store-api.hh"
+
+#include <curl/curl.h>
+
+namespace nix {
+
+struct Curl
+{
+    CURL * curl;
+    string data;
+    string etag, status, expectedETag;
+
+    struct curl_slist * requestHeaders;
+
+    static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp)
+    {
+        Curl & c(* (Curl *) userp);
+        size_t realSize = size * nmemb;
+        c.data.append((char *) contents, realSize);
+        return realSize;
+    }
+
+    static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp)
+    {
+        Curl & c(* (Curl *) userp);
+        size_t realSize = size * nmemb;
+        string line = string((char *) contents, realSize);
+        printMsg(lvlVomit, format("got header: %1%") % trim(line));
+        if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
+            c.etag = "";
+            auto ss = tokenizeString<vector<string>>(line, " ");
+            c.status = ss.size() >= 2 ? ss[1] : "";
+        } else {
+            auto i = line.find(':');
+            if (i != string::npos) {
+                string name = trim(string(line, 0, i));
+                if (name == "ETag") { // FIXME: case
+                    c.etag = trim(string(line, i + 1));
+                    /* Hack to work around a GitHub bug: it sends
+                       ETags, but ignores If-None-Match. So if we get
+                       the expected ETag on a 200 response, then shut
+                       down the connection because we already have the
+                       data. */
+                    printMsg(lvlDebug, format("got ETag: %1%") % c.etag);
+                    if (c.etag == c.expectedETag && c.status == "200") {
+                        printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag"));
+                        return 0;
+                    }
+                }
+            }
+        }
+        return realSize;
+    }
+
+    static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+    {
+        return _isInterrupted;
+    }
+
+    Curl()
+    {
+        requestHeaders = 0;
+
+        curl = curl_easy_init();
+        if (!curl) throw Error("unable to initialize curl");
+
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+        curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str());
+        curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str());
+        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl);
+
+        curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback);
+        curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl);
+
+        curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+        curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+    }
+
+    ~Curl()
+    {
+        if (curl) curl_easy_cleanup(curl);
+        if (requestHeaders) curl_slist_free_all(requestHeaders);
+    }
+
+    bool fetch(const string & url, const string & expectedETag = "")
+    {
+        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+
+        data.clear();
+
+        if (requestHeaders) {
+            curl_slist_free_all(requestHeaders);
+            requestHeaders = 0;
+        }
+
+        if (!expectedETag.empty()) {
+            this->expectedETag = expectedETag;
+            requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str());
+        }
+
+        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders);
+
+        CURLcode res = curl_easy_perform(curl);
+        checkInterrupt();
+        if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false;
+        if (res != CURLE_OK)
+            throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)")
+                % url % curl_easy_strerror(res) % res);
+
+        long httpStatus = 0;
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus);
+        if (httpStatus == 304) return false;
+
+        return true;
+    }
+};
+
+
+DownloadResult downloadFile(string url, string expectedETag)
+{
+    DownloadResult res;
+    Curl curl;
+    if (curl.fetch(url, expectedETag)) {
+        res.cached = false;
+        res.data = curl.data;
+    } else
+        res.cached = true;
+    res.etag = curl.etag;
+    return res;
+}
+
+
+Path downloadFileCached(const string & url, bool unpack)
+{
+    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    createDirs(cacheDir);
+
+    string urlHash = printHash32(hashString(htSHA256, url));
+
+    Path dataFile = cacheDir + "/" + urlHash + ".info";
+    Path fileLink = cacheDir + "/" + urlHash + "-file";
+
+    Path storePath;
+
+    string expectedETag;
+
+    int ttl = settings.get("tarball-ttl", 60 * 60);
+    bool skip = false;
+
+    if (pathExists(fileLink) && pathExists(dataFile)) {
+        storePath = readLink(fileLink);
+        store->addTempRoot(storePath);
+        if (store->isValidPath(storePath)) {
+            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
+            if (ss.size() >= 3 && ss[0] == url) {
+                time_t lastChecked;
+                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
+                    skip = true;
+                else if (!ss[1].empty()) {
+                    printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
+                    expectedETag = ss[1];
+                }
+            }
+        } else
+            storePath = "";
+    }
+
+    string name;
+    auto p = url.rfind('/');
+    if (p != string::npos) name = string(url, p + 1);
+
+    if (!skip) {
+
+        if (storePath.empty())
+            printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
+        else
+            printMsg(lvlInfo, format("checking ‘%1%’...") % url);
+
+        try {
+            auto res = downloadFile(url, expectedETag);
+
+            if (!res.cached)
+                storePath = store->addTextToStore(name, res.data, PathSet(), false);
+
+            assert(!storePath.empty());
+            replaceSymlink(storePath, fileLink);
+
+            writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n");
+        } catch (DownloadError & e) {
+            if (storePath.empty()) throw;
+            printMsg(lvlError, format("warning: %1%; using cached result") % e.msg());
+        }
+    }
+
+    if (unpack) {
+        Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
+        Path unpackedStorePath;
+        if (pathExists(unpackedLink)) {
+            unpackedStorePath = readLink(unpackedLink);
+            store->addTempRoot(unpackedStorePath);
+            if (!store->isValidPath(unpackedStorePath))
+                unpackedStorePath = "";
+        }
+        if (unpackedStorePath.empty()) {
+            printMsg(lvlInfo, format("unpacking ‘%1%’...") % url);
+            Path tmpDir = createTempDir();
+            AutoDelete autoDelete(tmpDir, true);
+            // FIXME: this requires GNU tar for decompression.
+            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
+            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false);
+        }
+        replaceSymlink(unpackedStorePath, unpackedLink);
+        return unpackedStorePath;
+    }
+
+    return storePath;
+}
+
+
+bool isUri(const string & s)
+{
+    size_t pos = s.find("://");
+    if (pos == string::npos) return false;
+    string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https" || scheme == "file";
+}
+
+
+}

src/libexpr/download.hh

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "types.hh"
+#include <string>
+
+namespace nix {
+
+struct DownloadResult
+{
+    bool cached;
+    string data, etag;
+};
+
+DownloadResult downloadFile(string url, string expectedETag = "");
+
+Path downloadFileCached(const string & url, bool unpack);
+
+MakeError(DownloadError, Error)
+
+bool isUri(const string & s);
+
+}

src/libexpr/eval.cc

@@ -21,9 +21,6 @@
 
 #else
 
-#define GC_STRDUP strdup
-#define GC_MALLOC malloc
-
 #define NEW new
 
 #endif
@@ -32,6 +29,32 @@
 namespace nix {
 
 
+static char * dupString(const char * s)
+{
+    char * t;
+#if HAVE_BOEHMGC
+    t = GC_strdup(s);
+#else
+    t = strdup(s);
+#endif
+    if (!t) throw std::bad_alloc();
+    return t;
+}
+
+
+static void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_malloc(n);
+#else
+    p = malloc(n);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());
@@ -168,6 +191,49 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
 }
 
 
+static bool gcInitialised = false;
+
+void initGC()
+{
+    if (gcInitialised) return;
+
+#if HAVE_BOEHMGC
+    /* Initialise the Boehm garbage collector. */
+    GC_set_all_interior_pointers(0);
+
+    GC_INIT();
+
+    GC_oom_fn = oomHandler;
+
+    /* Set the initial heap size to something fairly big (25% of
+       physical RAM, up to a maximum of 384 MiB) so that in most cases
+       we don't need to garbage collect at all.  (Collection has a
+       fairly significant overhead.)  The heap size can be overridden
+       through libgc's GC_INITIAL_HEAP_SIZE environment variable.  We
+       should probably also provide a nix.conf setting for this.  Note
+       that GC_expand_hp() causes a lot of virtual, but not physical
+       (resident) memory to be allocated.  This might be a problem on
+       systems that don't overcommit. */
+    if (!getenv("GC_INITIAL_HEAP_SIZE")) {
+        size_t size = 32 * 1024 * 1024;
+#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
+        size_t maxSize = 384 * 1024 * 1024;
+        long pageSize = sysconf(_SC_PAGESIZE);
+        long pages = sysconf(_SC_PHYS_PAGES);
+        if (pageSize != -1)
+            size = (pageSize * pages) / 4; // 25% of RAM
+        if (size > maxSize) size = maxSize;
+#endif
+        debug(format("setting initial heap size to %1% bytes") % size);
+        GC_expand_hp(size);
+    }
+
+#endif
+
+    gcInitialised = true;
+}
+
+
 EvalState::EvalState(const Strings & _searchPath)
     : sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
@@ -195,44 +261,9 @@ EvalState::EvalState(const Strings & _searchPath)
     nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0;
     countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
 
-#if HAVE_BOEHMGC
-    static bool gcInitialised = false;
-    if (!gcInitialised) {
+    restricted = settings.get("restrict-eval", false);
 
-        /* Initialise the Boehm garbage collector.  This isn't
-           necessary on most platforms, but for portability we do it
-           anyway. */
-        GC_INIT();
-
-        GC_oom_fn = oomHandler;
-
-        /* Set the initial heap size to something fairly big (25% of
-           physical RAM, up to a maximum of 384 MiB) so that in most
-           cases we don't need to garbage collect at all.  (Collection
-           has a fairly significant overhead.)  The heap size can be
-           overridden through libgc's GC_INITIAL_HEAP_SIZE environment
-           variable.  We should probably also provide a nix.conf
-           setting for this.  Note that GC_expand_hp() causes a lot of
-           virtual, but not physical (resident) memory to be
-           allocated.  This might be a problem on systems that don't
-           overcommit. */
-        if (!getenv("GC_INITIAL_HEAP_SIZE")) {
-            size_t size = 32 * 1024 * 1024;
-#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
-            size_t maxSize = 384 * 1024 * 1024;
-            long pageSize = sysconf(_SC_PAGESIZE);
-            long pages = sysconf(_SC_PHYS_PAGES);
-            if (pageSize != -1)
-                size = (pageSize * pages) / 4; // 25% of RAM
-            if (size > maxSize) size = maxSize;
-#endif
-            debug(format("setting initial heap size to %1% bytes") % size);
-            GC_expand_hp(size);
-        }
-
-        gcInitialised = true;
-    }
-#endif
+    assert(gcInitialised);
 
     /* Initialise the Nix expression search path. */
     Strings paths = tokenizeString<Strings>(getEnv("NIX_PATH", ""), ":");
@@ -250,6 +281,33 @@ EvalState::~EvalState()
 }
 
 
+Path EvalState::checkSourcePath(const Path & path_)
+{
+    if (!restricted) return path_;
+
+    /* Resolve symlinks. */
+    Path path = canonPath(path_, true);
+
+    for (auto & i : searchPath)
+        if (path == i.second || isInDir(path, i.second))
+            return path;
+
+    /* To support import-from-derivation, allow access to anything in
+       the store. FIXME: only allow access to paths that have been
+       constructed by this evaluation. */
+    if (isInStore(path)) return path;
+
+#if 0
+    /* Hack to support the chroot dependencies of corepkgs (see
+       corepkgs/config.nix.in). */
+    if (path == settings.nixPrefix && isStorePath(settings.nixPrefix))
+        return path;
+#endif
+
+    throw RestrictedPathError(format("access to path ‘%1%’ is forbidden in restricted mode") % path_);
+}
+
+
 void EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
@@ -359,7 +417,7 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 
 void mkString(Value & v, const char * s)
 {
-    mkStringNoCopy(v, GC_STRDUP(s));
+    mkStringNoCopy(v, dupString(s));
 }
 
 
@@ -369,9 +427,9 @@ void mkString(Value & v, const string & s, const PathSet & context)
     if (!context.empty()) {
         unsigned int n = 0;
         v.string.context = (const char * *)
-            GC_MALLOC((context.size() + 1) * sizeof(char *));
+            allocBytes((context.size() + 1) * sizeof(char *));
         foreach (PathSet::const_iterator, i, context)
-            v.string.context[n++] = GC_STRDUP(i->c_str());
+            v.string.context[n++] = dupString(i->c_str());
         v.string.context[n] = 0;
     }
 }
@@ -379,7 +437,7 @@ void mkString(Value & v, const string & s, const PathSet & context)
 
 void mkPath(Value & v, const char * s)
 {
-    mkPathNoCopy(v, GC_STRDUP(s));
+    mkPathNoCopy(v, dupString(s));
 }
 
 
@@ -412,7 +470,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 Value * EvalState::allocValue()
 {
     nrValues++;
-    return (Value *) GC_MALLOC(sizeof(Value));
+    return (Value *) allocBytes(sizeof(Value));
 }
 
 
@@ -422,10 +480,10 @@ Env & EvalState::allocEnv(unsigned int size)
 
     nrEnvs++;
     nrValuesInEnvs += size;
-    Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *));
+    Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
     env->size = size;
 
-    /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */
+    /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */
     for (unsigned i = 0; i < size; ++i)
         env->values[i] = 0;
 
@@ -443,7 +501,7 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 
 Bindings * EvalState::allocBindings(Bindings::size_t capacity)
 {
-    return new (GC_MALLOC(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity);
 }
 
 
@@ -452,7 +510,7 @@ void EvalState::mkList(Value & v, unsigned int length)
     clearValue(v);
     v.type = tList;
     v.list.length = length;
-    v.list.elems = length ? (Value * *) GC_MALLOC(length * sizeof(Value *)) : 0;
+    v.list.elems = length ? (Value * *) allocBytes(length * sizeof(Value *)) : 0;
     nrListElems += length;
 }
 
@@ -555,7 +613,7 @@ void EvalState::evalFile(const Path & path, Value & v)
     }
 
     startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path2);
-    Expr * e = parseExprFromFile(path2);
+    Expr * e = parseExprFromFile(checkSourcePath(path2));
     try {
         eval(e, v);
     } catch (Error & e) {
@@ -748,6 +806,23 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
+{
+    std::ostringstream out;
+    bool first = true;
+    for (auto & i : attrPath) {
+        if (!first) out << '.'; else first = false;
+        try {
+            out << getName(i, state, env);
+        } catch (Error & e) {
+            assert(!i.symbol.set());
+            out << "\"${" << *i.expr << "}\"";
+        }
+    }
+    return out.str();
+}
+
+
 unsigned long nrLookups = 0;
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
@@ -774,16 +849,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 }
             } else {
                 state.forceAttrs(*vAttrs, pos);
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                    AttrPath staticPath;
-                    AttrPath::const_iterator j;
-                    for (j = attrPath.begin(); j != i; ++j)
-                        staticPath.push_back(AttrName(getName(*j, state, env)));
-                    staticPath.push_back(AttrName(getName(*j, state, env)));
-                    for (j = j + 1; j != attrPath.end(); ++j)
-                        staticPath.push_back(*j);
-                    throwEvalError("attribute ‘%1%’ missing, at %2%", showAttrPath(staticPath), pos);
-                }
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    throwEvalError("attribute ‘%1%’ missing, at %2%", name, pos);
             }
             vAttrs = j->value;
             pos2 = j->pos;
@@ -795,7 +862,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
             addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n",
-                showAttrPath(attrPath), *pos2);
+                showAttrPath(state, env, attrPath), *pos2);
         throw;
     }
 
@@ -1174,7 +1241,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == tPath) {
         if (!context.empty())
             throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos);
-        mkPath(v, s.str().c_str());
+        auto path = canonPath(s.str());
+        mkPath(v, path.c_str());
     } else
         mkString(v, s.str(), context);
 }
@@ -1199,8 +1267,13 @@ void EvalState::forceValueDeep(Value & v)
         forceValue(v);
 
         if (v.type == tAttrs) {
-            foreach (Bindings::iterator, i, *v.attrs)
-                recurse(*i->value);
+            for (auto & i : *v.attrs)
+                try {
+                    recurse(*i.value);
+                } catch (Error & e) {
+                    addErrorPrefix(e, "while evaluating the attribute ‘%1%’ at %2%:\n", i.name, *i.pos);
+                    throw;
+                }
         }
 
         else if (v.type == tList) {
@@ -1357,8 +1430,8 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         dstPath = srcToStore[path];
     else {
         dstPath = settings.readOnlyMode
-            ? computeStorePathForPath(path).first
-            : store->addToStore(path, true, htSHA256, defaultPathFilter, repair);
+            ? computeStorePathForPath(checkSourcePath(path)).first
+            : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         srcToStore[path] = dstPath;
         printMsg(lvlChatty, format("copied source ‘%1%’ -> ‘%2%’")
             % path % dstPath);
@@ -1466,6 +1539,7 @@ void EvalState::printStats()
 
     printMsg(v, format("  time elapsed: %1%") % cpuTime);
     printMsg(v, format("  size of a value: %1%") % sizeof(Value));
+    printMsg(v, format("  size of an attr: %1%") % sizeof(Attr));
     printMsg(v, format("  environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs);
     printMsg(v, format("  list elements: %1% (%2% bytes)") % nrListElems % bLists);
     printMsg(v, format("  list concatenations: %1%") % nrListConcats);
@@ -1482,6 +1556,13 @@ void EvalState::printStats()
     printMsg(v, format("  number of function calls: %1%") % nrFunctionCalls);
     printMsg(v, format("  total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets));
 
+#if HAVE_BOEHMGC
+    GC_word heapSize, totalBytes;
+    GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
+    printMsg(v, format("  current Boehm heap size: %1% bytes") % heapSize);
+    printMsg(v, format("  total Boehm heap allocations: %1% bytes") % totalBytes);
+#endif
+
     if (countCalls) {
         v = lvlInfo;
 
@@ -1543,12 +1624,20 @@ size_t valueSize(Value & v)
             sz += doString(v.path);
             break;
         case tAttrs:
-            for (auto & i : *v.attrs)
-                sz += doValue(*i.value);
+            if (seen.find(v.attrs) == seen.end()) {
+                seen.insert(v.attrs);
+                sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity();
+                for (auto & i : *v.attrs)
+                    sz += doValue(*i.value);
+            }
             break;
         case tList:
-            for (unsigned int n = 0; n < v.list.length; ++n)
-                sz += doValue(*v.list.elems[n]);
+            if (seen.find(v.list.elems) == seen.end()) {
+                seen.insert(v.list.elems);
+                sz += v.list.length * sizeof(Value *);
+                for (unsigned int n = 0; n < v.list.length; ++n)
+                    sz += doValue(*v.list.elems[n]);
+            }
             break;
         case tThunk:
             sz += doEnv(*v.thunk.env);
@@ -1580,7 +1669,7 @@ size_t valueSize(Value & v)
         if (seen.find(&env) != seen.end()) return 0;
         seen.insert(&env);
 
-        size_t sz = sizeof(Env);
+        size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
 
         for (unsigned int i = 0; i < env.size; ++i)
             if (env.values[i])

src/libexpr/eval.hh

@@ -39,10 +39,10 @@ public:
     typedef uint32_t size_t;
 
 private:
-    size_t size_, capacity;
+    size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(uint32_t capacity) : size_(0), capacity(capacity) { }
+    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:
@@ -54,7 +54,7 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity);
+        assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
@@ -76,6 +76,8 @@ public:
 
     void sort();
 
+    size_t capacity() { return capacity_; }
+
     friend class EvalState;
 };
 
@@ -119,6 +121,10 @@ std::ostream & operator << (std::ostream & str, const Value & v);
 typedef list<std::pair<string, Path> > SearchPath;
 
 
+/* Initialise the Boehm GC, if applicable. */
+void initGC();
+
+
 class EvalState
 {
 public:
@@ -133,6 +139,10 @@ public:
        already exist there. */
     bool repair;
 
+    /* If set, don't allow access to files outside of the Nix search
+       path or to environment variables. */
+    bool restricted;
+
 private:
     SrcToStore srcToStore;
 
@@ -153,6 +163,8 @@ public:
 
     void addToSearchPath(const string & s, bool warn = false);
 
+    Path checkSourcePath(const Path & path);
+
     /* Parse a Nix expression from the specified file. */
     Expr * parseExprFromFile(const Path & path);
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
@@ -169,7 +181,7 @@ public:
 
     /* Look up a file in the search path. */
     Path findFile(const string & path);
-    Path findFile(SearchPath & searchPath, const string & path);
+    Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos);
 
     /* Evaluate an expression to normal form, storing the result in
        value `v'. */

src/libexpr/lexer.l

@@ -80,6 +80,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s)
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+
+HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+
 SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
 URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
@@ -159,6 +160,7 @@ or          { return OR_KW; }
 <IND_STRING>.    return yytext[0]; /* just in case: shouldn't be reached */
 
 {PATH}      { yylval->path = strdup(yytext); return PATH; }
+{HPATH}     { yylval->path = strdup(yytext); return HPATH; }
 {SPATH}     { yylval->path = strdup(yytext); return SPATH; }
 {URI}       { yylval->uri = strdup(yytext); return URI; }
 

src/libexpr/local.mk

@@ -8,7 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
 libexpr_LIBS = libutil libstore libformat
 
-libexpr_LDFLAGS = -ldl
+libexpr_LDFLAGS = -ldl -lcurl
 
 # The dependency on libgc must be propagated (i.e. meaning that
 # programs/libraries that use libexpr must explicitly pass -lgc),

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % pos.file % pos.line % pos.column).str();
+        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
     return str;
 }
 
@@ -200,15 +200,12 @@ string showAttrPath(const AttrPath & attrPath)
 {
     std::ostringstream out;
     bool first = true;
-    foreach (AttrPath::const_iterator, i, attrPath) {
-        if (!first)
-            out << '.';
+    for (auto & i : attrPath) {
+        if (!first) out << '.'; else first = false;
+        if (i.symbol.set())
+            out << i.symbol;
         else
-            first = false;
-        if (i->symbol.set())
-            out << i->symbol;
-        else
-            out << "\"${" << *i->expr << "}\"";
+            out << "\"${" << *i.expr << "}\"";
     }
     return out.str();
 }

src/libexpr/nixexpr.hh

@@ -16,6 +16,7 @@ MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
 MakeError(UndefinedVarError, Error)
+MakeError(RestrictedPathError, Error)
 
 
 /* Position objects. */

src/libexpr/parser.y

@@ -268,7 +268,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %token <id> ID ATTRPATH
 %token <e> STR IND_STR
 %token <n> INT
-%token <path> PATH SPATH
+%token <path> PATH HPATH SPATH
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
@@ -286,7 +286,6 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %left '*' '/'
 %right CONCAT
 %nonassoc '?'
-%nonassoc '~'
 %nonassoc NEGATE
 
 %%
@@ -376,6 +375,7 @@ expr_simple
       $$ = stripIndentation(CUR_POS, data->symbols, *$2);
   }
   | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
+  | HPATH { $$ = new ExprPath(getEnv("HOME", "") + string{$1 + 1}); }
   | SPATH {
       string path($1 + 1, strlen($1) - 2);
       $$ = new ExprApp(CUR_POS,
@@ -527,6 +527,8 @@ formal
 #include <unistd.h>
 
 #include <eval.hh>
+#include <download.hh>
+#include <store-api.hh>
 
 
 namespace nix {
@@ -611,10 +613,14 @@ void EvalState::addToSearchPath(const string & s, bool warn)
         path = string(s, pos + 1);
     }
 
+    if (isUri(path))
+        path = downloadFileCached(path, true);
+
     path = absPath(path);
     if (pathExists(path)) {
         debug(format("adding path ‘%1%’ to the search path") % path);
-        searchPath.push_back(std::pair<string, Path>(prefix, path));
+        /* Resolve symlinks in the path to support restricted mode. */
+        searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true)));
     } else if (warn)
         printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path);
 }
@@ -626,22 +632,27 @@ Path EvalState::findFile(const string & path)
 }
 
 
-Path EvalState::findFile(SearchPath & searchPath, const string & path)
+Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos)
 {
-    foreach (SearchPath::iterator, i, searchPath) {
+    for (auto & i : searchPath) {
+        assert(!isUri(i.second));
         Path res;
-        if (i->first.empty())
-            res = i->second + "/" + path;
+        if (i.first.empty())
+            res = i.second + "/" + path;
         else {
-            if (path.compare(0, i->first.size(), i->first) != 0 ||
-                (path.size() > i->first.size() && path[i->first.size()] != '/'))
+            if (path.compare(0, i.first.size(), i.first) != 0 ||
+                (path.size() > i.first.size() && path[i.first.size()] != '/'))
                 continue;
-            res = i->second +
-                (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size()));
+            res = i.second +
+                (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size()));
         }
         if (pathExists(res)) return canonPath(res);
     }
-    throw ThrownError(format("file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)") % path);
+    format f = format(
+        "file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)"
+        + string(pos ? ", at %2%" : ""));
+    f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    throw ThrownError(f % path % pos);
 }
 
 

src/libexpr/primops.cc

@@ -9,6 +9,7 @@
 #include "json-to-value.hh"
 #include "names.hh"
 #include "eval-inline.hh"
+#include "download.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -80,6 +81,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             % path % e.path % pos);
     }
 
+    path = state.checkSourcePath(path);
+
     if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) {
         Derivation drv = readDerivation(path);
         Value & w = *state.allocValue();
@@ -100,7 +103,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         }
         w.attrs->sort();
         Value fun;
-        state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun);
+        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
         state.forceFunction(fun, pos);
         mkApp(v, fun, w);
         state.forceAttrs(v, pos);
@@ -133,7 +136,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 /* !!! Should we pass the Pos or the file name too? */
 extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
 
-/* Load a ValueInitializer from a dso and return whatever it initializes */
+/* Load a ValueInitializer from a DSO and return whatever it initializes */
 static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
@@ -146,6 +149,8 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args
             % path % e.path % pos);
     }
 
+    path = state.checkSourcePath(path);
+
     string sym = state.forceStringNoCtx(*args[1], pos);
 
     void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL);
@@ -187,7 +192,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
         case tPrimOpApp:
             t = "lambda";
             break;
-	case tExternal:
+        case tExternal:
             t = args[0]->external->typeOf();
             break;
         default: abort();
@@ -380,7 +385,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, getEnv(name));
+    mkString(v, state.restricted ? "" : getEnv(name));
 }
 
 
@@ -680,7 +685,7 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu
 static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    Path path = state.coerceToPath(pos, *args[0], context);
+    Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context));
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
@@ -701,7 +706,15 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
     Path path = state.coerceToPath(pos, *args[0], context);
     if (!context.empty())
         throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos);
-    mkBool(v, pathExists(path));
+    try {
+        mkBool(v, pathExists(state.checkSourcePath(path)));
+    } catch (SysError & e) {
+        /* Don't give away info from errors while canonicalising
+           ‘path’ in restricted mode. */
+        mkBool(v, false);
+    } catch (RestrictedPathError & e) {
+        mkBool(v, false);
+    }
 }
 
 
@@ -710,7 +723,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
 static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context)), context);
+    mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context);
 }
 
 
@@ -736,7 +749,10 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
         throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%")
             % path % e.path % pos);
     }
-    mkString(v, readFile(path).c_str());
+    string s = readFile(state.checkSourcePath(path));
+    if (s.find((char) 0) != string::npos)
+        throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path);
+    mkString(v, s.c_str());
 }
 
 
@@ -763,7 +779,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
             throw EvalError(format("attribute ‘path’ missing, at %1%") % pos);
         string path = state.coerceToPath(pos, *i->value, context);
 
-        searchPath.push_back(std::pair<string, Path>(prefix, path));
+        searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path)));
     }
 
     string path = state.forceStringNoCtx(*args[1], pos);
@@ -775,7 +791,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
             % path % e.path % pos);
     }
 
-    mkPath(v, state.findFile(searchPath, path).c_str());
+    mkPath(v, state.findFile(searchPath, path, pos).c_str());
 }
 
 /* Read a directory (without . or ..) */
@@ -790,13 +806,13 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
             % path % e.path % pos);
     }
 
-    DirEntries entries = readDirectory(path);
+    DirEntries entries = readDirectory(state.checkSourcePath(path));
     state.mkAttrs(v, entries.size());
 
     for (auto & ent : entries) {
         Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
-            ent.type = getFileType(path);
+            ent.type = getFileType(path + "/" + ent.name);
         mkStringNoCopy(*ent_val,
             ent.type == DT_REG ? "regular" :
             ent.type == DT_DIR ? "directory" :
@@ -927,9 +943,11 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
 
     FilterFromExpr filter(state, *args[0]);
 
+    path = state.checkSourcePath(path);
+
     Path dstPath = settings.readOnlyMode
         ? computeStorePathForPath(path, true, htSHA256, filter).first
-        : store->addToStore(path, true, htSHA256, filter, state.repair);
+        : store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair);
 
     mkString(v, dstPath, singleton<PathSet>(dstPath));
 }
@@ -1199,7 +1217,7 @@ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value
 }
 
 
-/* Return a list consisting of everything but the the first element of
+/* Return a list consisting of everything but the first element of
    a list.  Warning: this function takes O(n) time, so you probably
    don't want to use it!  */
 static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1465,6 +1483,55 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a
 }
 
 
+/*************************************************************
+ * Networking
+ *************************************************************/
+
+
+void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
+    const string & who, bool unpack)
+{
+    if (state.restricted) throw Error(format("‘%1%’ is not allowed in restricted mode") % who);
+
+    string url;
+
+    state.forceValue(*args[0]);
+
+    if (args[0]->type == tAttrs) {
+
+        state.forceAttrs(*args[0], pos);
+
+        for (auto & attr : *args[0]->attrs) {
+            string name(attr.name);
+            if (name == "url")
+                url = state.forceStringNoCtx(*attr.value, *attr.pos);
+            else
+                throw EvalError(format("unsupported argument ‘%1%’ to ‘%2%’, at %3%") % attr.name % who % attr.pos);
+        }
+
+        if (url.empty())
+            throw EvalError(format("‘url’ argument required, at %1%") % pos);
+
+    } else
+        url = state.forceStringNoCtx(*args[0], pos);
+
+    Path res = downloadFileCached(url, unpack);
+    mkString(v, res, PathSet({res}));
+}
+
+
+static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchurl", false);
+}
+
+
+static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    fetch(state, pos, args, v, "fetchTarball", true);
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -1493,12 +1560,15 @@ void EvalState::createBaseEnv()
     mkInt(v, time(0));
     addConstant("__currentTime", v);
 
-    mkString(v, settings.thisSystem.c_str());
+    mkString(v, settings.thisSystem);
     addConstant("__currentSystem", v);
 
-    mkString(v, nixVersion.c_str());
+    mkString(v, nixVersion);
     addConstant("__nixVersion", v);
 
+    mkString(v, settings.nixStore);
+    addConstant("__storeDir", v);
+
     /* Language version.  This should be increased every time a new
        language feature gets added.  It's not necessary to increase it
        when primops get added, because you can just use `builtins ?
@@ -1600,10 +1670,13 @@ void EvalState::createBaseEnv()
     // Derivations
     addPrimOp("derivationStrict", 1, prim_derivationStrict);
 
+    // Networking
+    addPrimOp("__fetchurl", 1, prim_fetchurl);
+    addPrimOp("fetchTarball", 1, prim_fetchTarball);
+
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = findFile("nix/derivation.nix");
-    assert(!path.empty());
+    string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix";
     sDerivationNix = symbols.create(path);
     evalFile(path, v);
     addConstant("derivation", v);

src/libmain/shared.cc

@@ -293,8 +293,9 @@ int handleExceptions(const string & programName, std::function<void()> fun)
 RunPager::RunPager()
 {
     if (!isatty(STDOUT_FILENO)) return;
-    string pager = getEnv("PAGER", "default");
-    if (pager == "" || pager == "cat") return;
+    char * pager = getenv("NIX_PAGER");
+    if (!pager) pager = getenv("PAGER");
+    if (pager && ((string) pager == "" || (string) pager == "cat")) return;
 
     /* Ignore SIGINT. The pager will handle it (and we'll get
        SIGPIPE). */
@@ -314,8 +315,8 @@ RunPager::RunPager()
             throw SysError("dupping stdin");
         if (!getenv("LESS"))
             setenv("LESS", "FRSXMK", 1);
-        if (pager != "default")
-            execl("/bin/sh", "sh", "-c", pager.c_str(), NULL);
+        if (pager)
+            execl("/bin/sh", "sh", "-c", pager, NULL);
         execlp("pager", "pager", NULL);
         execlp("less", "less", NULL);
         execlp("more", "more", NULL);
@@ -341,4 +342,19 @@ RunPager::~RunPager()
 }
 
 
+string showBytes(unsigned long long bytes)
+{
+    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
+}
+
+
+PrintFreed::~PrintFreed()
+{
+    if (show)
+        std::cout << format("%1% store paths deleted, %2% freed\n")
+            % results.paths.size()
+            % showBytes(results.bytesFreed);
+}
+
+
 }

src/libmain/shared.hh

@@ -83,4 +83,21 @@ private:
 
 extern volatile ::sig_atomic_t blockInt;
 
+
+/* GC helpers. */
+
+string showBytes(unsigned long long bytes);
+
+class GCResults;
+
+struct PrintFreed
+{
+    bool show;
+    const GCResults & results;
+    PrintFreed(bool show, const GCResults & results)
+        : show(show), results(results) { }
+    ~PrintFreed();
+};
+
+
 }

src/libstore/build.cc

@@ -20,6 +20,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/utsname.h>
+#include <sys/select.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
@@ -38,6 +39,9 @@
 #if HAVE_SYS_MOUNT_H
 #include <sys/mount.h>
 #endif
+#if HAVE_SYS_SYSCALL_H
+#include <sys/syscall.h>
+#endif
 #if HAVE_SCHED_H
 #include <sched.h>
 #endif
@@ -48,7 +52,16 @@
 #include <linux/fs.h>
 #endif
 
-#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS)
+#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root)
+
+/* chroot-like behavior from Apple's sandbox */
+#if __APPLE__
+    #define SANDBOX_ENABLED 1
+    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
+#else
+    #define SANDBOX_ENABLED 0
+    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin"
+#endif
 
 #if CHROOT_ENABLED
 #include <sys/socket.h>
@@ -986,7 +999,7 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes && !willBuildLocally(drv))
+    if (settings.useSubstitutes && substitutesAllowed(drv))
         foreach (PathSet::iterator, i, invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair));
 
@@ -1174,22 +1187,6 @@ void DerivationGoal::inputsRealised()
 }
 
 
-PathSet outputPaths(const DerivationOutputs & outputs)
-{
-    PathSet paths;
-    foreach (DerivationOutputs::const_iterator, i, outputs)
-        paths.insert(i->second.path);
-    return paths;
-}
-
-
-static string get(const StringPairs & map, const string & key)
-{
-    StringPairs::const_iterator i = map.find(key);
-    return i == map.end() ? (string) "" : i->second;
-}
-
-
 static bool canBuildLocally(const string & platform)
 {
     return platform == settings.thisSystem
@@ -1200,12 +1197,25 @@ static bool canBuildLocally(const string & platform)
 }
 
 
+static string get(const StringPairs & map, const string & key, const string & def = "")
+{
+    StringPairs::const_iterator i = map.find(key);
+    return i == map.end() ? def : i->second;
+}
+
+
 bool willBuildLocally(const Derivation & drv)
 {
     return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
 }
 
 
+bool substitutesAllowed(const Derivation & drv)
+{
+    return get(drv.env, "allowSubstitutes", "1") == "1";
+}
+
+
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
@@ -1228,7 +1238,7 @@ void DerivationGoal::tryToBuild()
        can't acquire the lock, then continue; hopefully some other
        goal can start a build, and if not, the main loop will sleep a
        few seconds and then retry this goal. */
-    if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) {
+    if (!outputLocks.lockPaths(outputPaths(drv), "", false)) {
         worker.waitForAWhile(shared_from_this());
         return;
     }
@@ -1249,7 +1259,7 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
-    missingPaths = outputPaths(drv.outputs);
+    missingPaths = outputPaths(drv);
     if (buildMode != bmCheck)
         foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i);
 
@@ -1586,6 +1596,13 @@ void chmod_(const Path & path, mode_t mode)
 }
 
 
+int childEntry(void * arg)
+{
+    ((DerivationGoal *) arg)->runChild();
+    return 1;
+}
+
+
 void DerivationGoal::startBuilder()
 {
     startNest(nest, lvlInfo, format(
@@ -1627,14 +1644,28 @@ void DerivationGoal::startBuilder()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
 
-    /* Add all bindings specified in the derivation. */
-    foreach (StringPairs::iterator, i, drv.env)
-        env[i->first] = i->second;
-
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700);
 
+    /* Add all bindings specified in the derivation via the
+       environments, except those listed in the passAsFile
+       attribute. Those are passed as file names pointing to
+       temporary files containing the contents. */
+    PathSet filesToChown;
+    StringSet passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile"));
+    int fileNr = 0;
+    for (auto & i : drv.env) {
+        if (passAsFile.find(i.first) == passAsFile.end()) {
+            env[i.first] = i.second;
+        } else {
+            Path p = tmpDir + "/.attr-" + int2String(fileNr++);
+            writeFile(p, i.second);
+            filesToChown.insert(p);
+            env[i.first + "Path"] = p;
+        }
+    }
+
     /* For convenience, set an environment pointing to the top build
        directory. */
     env["NIX_BUILD_TOP"] = tmpDir;
@@ -1727,23 +1758,11 @@ void DerivationGoal::startBuilder()
         buildUser.kill();
 
         /* Change ownership of the temporary build directory. */
-        if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
-            throw SysError(format("cannot change ownership of ‘%1%’") % tmpDir);
+        filesToChown.insert(tmpDir);
 
-        /* Check that the Nix store has the appropriate permissions,
-           i.e., owned by root and mode 1775 (sticky bit on so that
-           the builder can create its output but not mess with the
-           outputs of other processes). */
-        struct stat st;
-        if (stat(settings.nixStore.c_str(), &st) == -1)
-            throw SysError(format("cannot stat ‘%1%’") % settings.nixStore);
-        if (!(st.st_mode & S_ISVTX) ||
-            ((st.st_mode & S_IRWXG) != S_IRWXG) ||
-            (st.st_gid != buildUser.getGID()))
-            throw Error(format(
-                "builder does not have write permission to ‘%2%’; "
-                "try ‘chgrp %1% %2%; chmod 1775 %2%’")
-                % buildUser.getGID() % settings.nixStore);
+        for (auto & p : filesToChown)
+            if (chown(p.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
+                throw SysError(format("cannot change ownership of ‘%1%’") % p);
     }
 
 
@@ -1752,14 +1771,78 @@ void DerivationGoal::startBuilder()
        functions like fetchurl (which needs a proper /etc/resolv.conf)
        work properly.  Purity checking for fixed-output derivations
        is somewhat pointless anyway. */
-    useChroot = settings.useChroot;
-
-    if (fixedOutput) useChroot = false;
-
-    /* Hack to allow derivations to disable chroot builds. */
-    if (get(drv.env, "__noChroot") == "1") useChroot = false;
+    {
+        string x = settings.get("build-use-chroot", string("false"));
+        if (x != "true" && x != "false" && x != "relaxed")
+            throw Error("option ‘build-use-chroot’ must be set to one of ‘true’, ‘false’ or ‘relaxed’");
+        if (x == "true") {
+            if (get(drv.env, "__noChroot") == "1")
+                throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-chroot’ is ‘true’") % drvPath);
+            useChroot = true;
+        }
+        else if (x == "false")
+            useChroot = false;
+        else if (x == "relaxed")
+            useChroot = !fixedOutput && get(drv.env, "__noChroot") != "1";
+    }
 
     if (useChroot) {
+
+        string defaultChrootDirs;
+#if CHROOT_ENABLED
+        if (isInStore(BASH_PATH))
+            defaultChrootDirs = "/bin/sh=" BASH_PATH;
+#endif
+
+        /* Allow a user-configurable set of directories from the
+           host file system. */
+        PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", defaultChrootDirs));
+        PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string("")));
+        dirs.insert(dirs2.begin(), dirs2.end());
+
+        for (auto & i : dirs) {
+            size_t p = i.find('=');
+            if (p == string::npos)
+                dirsInChroot[i] = i;
+            else
+                dirsInChroot[string(i, 0, p)] = string(i, p + 1);
+        }
+        dirsInChroot[tmpDir] = tmpDir;
+
+        /* Add the closure of store paths to the chroot. */
+        PathSet closure;
+        for (auto & i : dirsInChroot)
+            if (isInStore(i.second))
+                computeFSClosure(worker.store, toStorePath(i.second), closure);
+        for (auto & i : closure)
+            dirsInChroot[i] = i;
+
+        string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES));
+        PathSet allowedPaths = tokenizeString<StringSet>(allowed);
+
+        /* This works like the above, except on a per-derivation level */
+        Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps"));
+
+        for (auto & i : impurePaths) {
+            bool found = false;
+            /* Note: we're not resolving symlinks here to prevent
+               giving a non-root user info about inaccessible
+               files. */
+            Path canonI = canonPath(i);
+            /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
+            for (auto & a : allowedPaths) {
+                Path canonA = canonPath(a);
+                if (canonI == canonA || isInDir(canonI, canonA)) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+                throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed);
+
+            dirsInChroot[i] = i;
+        }
+
 #if CHROOT_ENABLED
         /* Create a temporary directory in which we set up the chroot
            environment using bind-mounts.  We put it in the Nix store
@@ -1773,6 +1856,12 @@ void DerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir);
 
+        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+            throw SysError(format("cannot create ‘%1%’") % chrootRootDir);
+
+        if (chown(chrootRootDir.c_str(), 0, buildUser.getGID()) == -1)
+            throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir);
+
         /* Create a writable /tmp in the chroot.  Many builders need
            this.  (Of course they should really respect $TMPDIR
            instead.) */
@@ -1799,21 +1888,8 @@ void DerivationGoal::startBuilder()
                 % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
 
         /* Create /etc/hosts with localhost entry. */
-        writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
-
-        /* Bind-mount a user-configurable set of directories from the
-           host file system. */
-        PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS)));
-        PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string("")));
-        dirs.insert(dirs2.begin(), dirs2.end());
-        for (auto & i : dirs) {
-            size_t p = i.find('=');
-            if (p == string::npos)
-                dirsInChroot[i] = i;
-            else
-                dirsInChroot[string(i, 0, p)] = string(i, p + 1);
-        }
-        dirsInChroot[tmpDir] = tmpDir;
+        if (!fixedOutput)
+            writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
            rather than the whole Nix store.  This prevents any access
@@ -1822,8 +1898,12 @@ void DerivationGoal::startBuilder()
            can be bind-mounted).  !!! As an extra security
            precaution, make the fake Nix store only writable by the
            build user. */
-        createDirs(chrootRootDir + settings.nixStore);
-        chmod_(chrootRootDir + settings.nixStore, 01777);
+        Path chrootStoreDir = chrootRootDir + settings.nixStore;
+        createDirs(chrootStoreDir);
+        chmod_(chrootStoreDir, 01775);
+
+        if (chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1)
+            throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir);
 
         foreach (PathSet::iterator, i, inputPaths) {
             struct stat st;
@@ -1858,6 +1938,9 @@ void DerivationGoal::startBuilder()
             foreach (DerivationOutputs::iterator, i, drv.outputs)
                 dirsInChroot.erase(i->second.path);
 
+#elif SANDBOX_ENABLED
+        /* We don't really have any parent prep work to do (yet?)
+           All work happens in the child, instead. */
 #else
         throw Error("chroot builds are not supported on this platform");
 #endif
@@ -1891,6 +1974,42 @@ void DerivationGoal::startBuilder()
                 }
     }
 
+    if (settings.preBuildHook != "") {
+        printMsg(lvlChatty, format("executing pre-build hook ‘%1%’")
+            % settings.preBuildHook);
+        auto args = useChroot ? Strings({drvPath, chrootRootDir}) :
+            Strings({ drvPath });
+        enum BuildHookState {
+            stBegin,
+            stExtraChrootDirs
+        };
+        auto state = stBegin;
+        auto lines = runProgram(settings.preBuildHook, false, args);
+        auto lastPos = std::string::size_type{0};
+        for (auto nlPos = lines.find('\n'); nlPos != string::npos;
+                nlPos = lines.find('\n', lastPos)) {
+            auto line = std::string{lines, lastPos, nlPos};
+            lastPos = nlPos + 1;
+            if (state == stBegin) {
+                if (line == "extra-chroot-dirs") {
+                    state = stExtraChrootDirs;
+                } else {
+                    throw Error(format("unknown pre-build hook command ‘%1%’")
+                        % line);
+                }
+            } else if (state == stExtraChrootDirs) {
+                if (line == "") {
+                    state = stBegin;
+                } else {
+                    auto p = line.find('=');
+                    if (p == string::npos)
+                        dirsInChroot[line] = line;
+                    else
+                        dirsInChroot[string(line, 0, p)] = string(line, p + 1);
+                }
+            }
+        }
+    }
 
     /* Run the builder. */
     printMsg(lvlChatty, format("executing builder ‘%1%’") % drv.builder);
@@ -1902,11 +2021,70 @@ void DerivationGoal::startBuilder()
     builderOut.create();
 
     /* Fork a child to build the package. */
-    ProcessOptions options;
-    options.allowVfork = !buildUser.enabled();
-    pid = startProcess([&]() {
-        runChild();
-    }, options);
+#if CHROOT_ENABLED
+    if (useChroot) {
+        /* Set up private namespaces for the build:
+
+           - The PID namespace causes the build to start as PID 1.
+             Processes outside of the chroot are not visible to those
+             on the inside, but processes inside the chroot are
+             visible from the outside (though with different PIDs).
+
+           - The private mount namespace ensures that all the bind
+             mounts we do will only show up in this process and its
+             children, and will disappear automatically when we're
+             done.
+
+           - The private network namespace ensures that the builder
+             cannot talk to the outside world (or vice versa).  It
+             only has a private loopback interface. (Fixed-output
+             derivations are not run in a private network namespace
+             to allow functions like fetchurl to work.)
+
+           - The IPC namespace prevents the builder from communicating
+             with outside processes using SysV IPC mechanisms (shared
+             memory, message queues, semaphores).  It also ensures
+             that all IPC objects are destroyed when the builder
+             exits.
+
+           - The UTS namespace ensures that builders see a hostname of
+             localhost rather than the actual hostname.
+
+           We use a helper process to do the clone() to work around
+           clone() being broken in multi-threaded programs due to
+           at-fork handlers not being run. Note that we use
+           CLONE_PARENT to ensure that the real builder is parented to
+           us.
+        */
+        ProcessOptions options;
+        options.allowVfork = false;
+        Pid helper = startProcess([&]() {
+            char stack[32 * 1024];
+            int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
+            if (!fixedOutput) flags |= CLONE_NEWNET;
+            pid_t child = clone(childEntry, stack + sizeof(stack) - 8, flags, this);
+            if (child == -1 && errno == EINVAL)
+                /* Fallback for Linux < 2.13 where CLONE_NEWPID and
+                   CLONE_PARENT are not allowed together. */
+                child = clone(childEntry, stack + sizeof(stack) - 8, flags & ~CLONE_NEWPID, this);
+            if (child == -1) throw SysError("cloning builder process");
+            writeFull(builderOut.writeSide, int2String(child) + "\n");
+            _exit(0);
+        }, options);
+        if (helper.wait(true) != 0)
+            throw Error("unable to start build process");
+        pid_t tmp;
+        if (!string2Int<pid_t>(readLine(builderOut.readSide), tmp)) abort();
+        pid = tmp;
+    } else
+#endif
+    {
+        ProcessOptions options;
+        options.allowVfork = !buildUser.enabled();
+        pid = startProcess([&]() {
+            runChild();
+        }, options);
+    }
 
     /* parent */
     pid.setSeparatePG(true);
@@ -1922,7 +2100,6 @@ void DerivationGoal::startBuilder()
         printMsg(lvlError, format("@ build-started %1% - %2% %3%")
             % drvPath % drv.platform % logFile);
     }
-
 }
 
 
@@ -1938,30 +2115,6 @@ void DerivationGoal::runChild()
 #if CHROOT_ENABLED
         if (useChroot) {
 
-            /* Set up private namespaces for the build:
-
-               - The private mount namespace ensures that all the bind
-                 mounts we do will only show up in this process and
-                 its children, and will disappear automatically when
-                 we're done.
-
-               - The private network namespace ensures that the
-                 builder cannot talk to the outside world (or vice
-                 versa).  It only has a private loopback interface.
-
-               - The IPC namespace prevents the builder from
-                 communicating with outside processes using SysV IPC
-                 mechanisms (shared memory, message queues,
-                 semaphores).  It also ensures that all IPC objects
-                 are destroyed when the builder exits.
-
-               - The UTS namespace ensures that builders see a
-                 hostname of localhost rather than the actual
-                 hostname.
-            */
-            if (unshare(CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS) == -1)
-                throw SysError("setting up private namespaces");
-
             /* Initialise the loopback interface. */
             AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
             if (fd == -1) throw SysError("cannot open IP socket");
@@ -1998,12 +2151,17 @@ void DerivationGoal::runChild()
                     throw SysError(format("unable to make filesystem ‘%1%’ private") % fs);
             }
 
+            /* Bind-mount chroot directory to itself, to treat it as a
+               different filesystem from /, as needed for pivot_root. */
+            if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
+                throw SysError(format("unable to bind mount ‘%1%’") % chrootRootDir);
+
             /* Set up a nearly empty /dev, unless the user asked to
                bind-mount the host /dev. */
+            Strings ss;
             if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
                 createDirs(chrootRootDir + "/dev/shm");
                 createDirs(chrootRootDir + "/dev/pts");
-                Strings ss;
                 ss.push_back("/dev/full");
 #ifdef __linux__
                 if (pathExists("/dev/kvm"))
@@ -2014,13 +2172,24 @@ void DerivationGoal::runChild()
                 ss.push_back("/dev/tty");
                 ss.push_back("/dev/urandom");
                 ss.push_back("/dev/zero");
-                foreach (Strings::iterator, i, ss) dirsInChroot[*i] = *i;
                 createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
                 createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
                 createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
                 createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
             }
 
+            /* Fixed-output derivations typically need to access the
+               network, so give them access to /etc/resolv.conf and so
+               on. */
+            if (fixedOutput) {
+                ss.push_back("/etc/resolv.conf");
+                ss.push_back("/etc/nsswitch.conf");
+                ss.push_back("/etc/services");
+                ss.push_back("/etc/hosts");
+            }
+
+            for (auto & i : ss) dirsInChroot[i] = i;
+
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot
                environment. */
@@ -2069,13 +2238,26 @@ void DerivationGoal::runChild()
                 chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
             }
 
-            /* Do the chroot().  Below we do a chdir() to the
-               temporary build directory to make sure the current
-               directory is in the chroot.  (Actually the order
-               doesn't matter, since due to the bind mount tmpDir and
-               tmpRootDit/tmpDir are the same directories.) */
-            if (chroot(chrootRootDir.c_str()) == -1)
+            /* Do the chroot(). */
+            if (chdir(chrootRootDir.c_str()) == -1)
+                throw SysError(format("cannot change directory to ‘%1%’") % chrootRootDir);
+
+            if (mkdir("real-root", 0) == -1)
+                throw SysError("cannot create real-root directory");
+
+#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+            if (pivot_root(".", "real-root") == -1)
+                throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root"));
+#undef pivot_root
+
+            if (chroot(".") == -1)
                 throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir);
+
+            if (umount2("real-root", MNT_DETACH) == -1)
+                throw SysError("cannot unmount real root filesystem");
+
+            if (rmdir("real-root") == -1)
+                throw SysError("cannot remove real-root directory");
         }
 #endif
 
@@ -2114,7 +2296,6 @@ void DerivationGoal::runChild()
         Strings envStrs;
         foreach (Environment::const_iterator, i, env)
             envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp));
-        auto envArr = stringsToCharPtrs(envStrs);
 
         /* If we are running in `build-users' mode, then switch to the
            user we allocated above.  Make sure that we drop all root
@@ -2139,19 +2320,142 @@ void DerivationGoal::runChild()
 
         /* Fill in the arguments. */
         Strings args;
-        string builderBasename = baseNameOf(drv.builder);
-        args.push_back(builderBasename);
+
+        const char *builder = "invalid";
+
+        string sandboxProfile;
+        if (useChroot && SANDBOX_ENABLED) {
+            /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
+            PathSet ancestry;
+
+            /* We build the ancestry before adding all inputPaths to the store because we know they'll
+               all have the same parents (the store), and there might be lots of inputs. This isn't
+               particularly efficient... I doubt it'll be a bottleneck in practice */
+            for (auto & i : dirsInChroot) {
+                Path cur = i.first;
+                while (cur.compare("/") != 0) {
+                    cur = dirOf(cur);
+                    ancestry.insert(cur);
+                }
+            }
+
+            /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
+               path component this time, since it's typically /nix/store and we care about that. */
+            Path cur = settings.nixStore;
+            while (cur.compare("/") != 0) {
+                ancestry.insert(cur);
+                cur = dirOf(cur);
+            }
+
+            /* Add all our input paths to the chroot */
+            for (auto & i : inputPaths)
+                dirsInChroot[i] = i;
+
+
+            /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */
+            sandboxProfile += "(version 1)\n";
+
+            /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
+            if (settings.get("darwin-log-sandbox-violations", false)) {
+                sandboxProfile += "(deny default)\n";
+            } else {
+                sandboxProfile += "(deny default (with no-log))\n";
+            }
+
+            sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n";
+
+            sandboxProfile += "(allow file-read-metadata\n"
+                "\t(literal \"/var\")\n"
+                "\t(literal \"/tmp\")\n"
+                "\t(literal \"/etc\")\n"
+                "\t(literal \"/etc/nix\")\n"
+                "\t(literal \"/etc/nix/nix.conf\"))\n";
+
+            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
+               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+            Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
+
+            /* They don't like trailing slashes on subpath directives */
+            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
+
+            /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */
+            sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str();
+
+            sandboxProfile += "(allow process-fork)\n";
+            sandboxProfile += "(allow sysctl-read)\n";
+            sandboxProfile += "(allow signal (target same-sandbox))\n";
+
+            /* Enables getpwuid (used by git and others) */
+            sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n";
+
+            /* Allow local networking operations, mostly because lots of test suites use it and it seems mostly harmless */
+            sandboxProfile += "(allow network* (local ip) (remote unix-socket))";
+
+
+            /* Our rwx outputs */
+            sandboxProfile += "(allow file-read* file-write* process-exec\n";
+            for (auto & i : missingPaths) {
+                sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
+            }
+            sandboxProfile += ")\n";
+
+            /* Our inputs (transitive dependencies and any impurities computed above)
+               Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning
+               mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that
+               denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since
+               it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of
+               these is the entire GHC Haskell ecosystem. */
+            sandboxProfile += "(allow file-read* file-write* process-exec\n";
+            for (auto & i : dirsInChroot) {
+                if (i.first != i.second)
+                    throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin"));
+
+                string path = i.first;
+                struct stat st;
+                if (lstat(path.c_str(), &st))
+                    throw SysError(format("getting attributes of path ‘%1%’") % path);
+                if (S_ISDIR(st.st_mode))
+                    sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
+                else
+                    sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str();
+            }
+            sandboxProfile += ")\n";
+
+            /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that,
+               you open up the entire filesystem because you end up with (subpath "/") */
+            sandboxProfile += "(allow file-read-metadata\n";
+            for (auto & i : ancestry) {
+                sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str();
+            }
+            sandboxProfile += ")\n";
+
+            builder = "/usr/bin/sandbox-exec";
+            args.push_back("sandbox-exec");
+            args.push_back("-p");
+            args.push_back(sandboxProfile);
+            args.push_back(drv.builder);
+        } else {
+            builder = drv.builder.c_str();
+            string builderBasename = baseNameOf(drv.builder);
+            args.push_back(builderBasename);
+        }
+
         foreach (Strings::iterator, i, drv.args)
             args.push_back(rewriteHashes(*i, rewritesToTmp));
-        auto argArr = stringsToCharPtrs(args);
 
         restoreSIGPIPE();
 
         /* Indicate that we managed to set up the build environment. */
         writeFull(STDERR_FILENO, "\n");
 
+        /* This needs to be after that fateful '\n', and I didn't want to duplicate code */
+        if (useChroot && SANDBOX_ENABLED) {
+            printMsg(lvlDebug, "Generated sandbox profile:");
+            printMsg(lvlDebug, sandboxProfile);
+        }
+
         /* Execute the program.  This should not return. */
-        execve(drv.builder.c_str(), (char * *) &argArr[0], (char * *) &envArr[0]);
+        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 
         throw SysError(format("executing ‘%1%’") % drv.builder);
 
@@ -2779,7 +3083,6 @@ void SubstitutionGoal::tryToRun()
     args.push_back("--substitute");
     args.push_back(storePath);
     args.push_back(destPath);
-    auto argArr = stringsToCharPtrs(args);
 
     /* Fork the substitute program. */
     pid = startProcess([&]() {
@@ -2789,7 +3092,7 @@ void SubstitutionGoal::tryToRun()
         if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
             throw SysError("cannot dup output pipe into stdout");
 
-        execv(sub.c_str(), (char * *) &argArr[0]);
+        execv(sub.c_str(), stringsToCharPtrs(args).data());
 
         throw SysError(format("executing ‘%1%’") % sub);
     });

src/libstore/derivations.cc

@@ -285,4 +285,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
+PathSet outputPaths(const Derivation & drv)
+{
+    PathSet paths;
+    for (auto & i : drv.outputs)
+        paths.insert(i.second.path);
+    return paths;
+}
+
+
 }

src/libstore/derivations.hh

@@ -89,5 +89,6 @@ Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outpu
 
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
+PathSet outputPaths(const Derivation & drv);
 
 }

src/libstore/gc.cc

@@ -96,7 +96,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath,
                 "(are you running nix-build inside the store?)") % gcRoot);
 
     if (indirect) {
-        /* Don't clobber the the link if it already exists and doesn't
+        /* Don't clobber the link if it already exists and doesn't
            point to the Nix store. */
         if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
             throw Error(format("cannot create symlink ‘%1%’; already exists") % gcRoot);

src/libstore/globals.cc

@@ -47,7 +47,6 @@ Settings::Settings()
     syncBeforeRegistering = false;
     useSubstitutes = true;
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
-    useChroot = false;
     useSshSubstituter = true;
     impersonateLinux26 = false;
     keepLog = true;
@@ -68,6 +67,7 @@ Settings::Settings()
 
 void Settings::processEnvironment()
 {
+    nixPrefix = NIX_PREFIX;
     nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
     nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
     nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
@@ -144,6 +144,14 @@ bool Settings::get(const string & name, bool def)
 }
 
 
+int Settings::get(const string & name, int def)
+{
+    int res = def;
+    _get(res, name);
+    return res;
+}
+
+
 void Settings::update()
 {
     _get(tryFallback, "build-fallback");
@@ -158,7 +166,6 @@ void Settings::update()
     _get(syncBeforeRegistering, "sync-before-registering");
     _get(useSubstitutes, "build-use-substitutes");
     _get(buildUsersGroup, "build-users-group");
-    _get(useChroot, "build-use-chroot");
     _get(impersonateLinux26, "build-impersonate-linux-26");
     _get(keepLog, "build-keep-log");
     _get(compressLog, "build-compress-log");
@@ -175,6 +182,7 @@ void Settings::update()
     _get(logServers, "log-servers");
     _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
     _get(useCaseHack, "use-case-hack");
+    _get(preBuildHook, "pre-build-hook");
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {

src/libstore/globals.hh

@@ -27,6 +27,8 @@ struct Settings {
 
     bool get(const string & name, bool def);
 
+    int get(const string & name, int def);
+
     void update();
 
     string pack();
@@ -40,6 +42,8 @@ struct Settings {
 
     Path nixDataDir; /* !!! fix */
 
+    Path nixPrefix;
+
     /* The directory where we log various operations. */
     Path nixLogDir;
 
@@ -145,9 +149,6 @@ struct Settings {
     /* The Unix group that contains the build users. */
     string buildUsersGroup;
 
-    /* Whether to build in chroot. */
-    bool useChroot;
-
     /* Set of ssh connection strings for the ssh substituter */
     Strings sshSubstituterHosts;
 
@@ -205,6 +206,10 @@ struct Settings {
     /* Whether the importNative primop should be enabled */
     bool enableImportNative;
 
+    /* The hook to run just before a build to set derivation-specific
+       build settings */
+    Path preBuildHook;
+
 private:
     SettingsMap settings, overrides;
 

src/libstore/local-store.cc

@@ -13,6 +13,7 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/select.h>
 #include <sys/time.h>
 #include <unistd.h>
 #include <utime.h>
@@ -256,20 +257,23 @@ LocalStore::LocalStore(bool reserveSpace)
         if (chmod(perUserDir.c_str(), 01777) == -1)
             throw SysError(format("could not set permissions on ‘%1%’ to 1777") % perUserDir);
 
+        mode_t perm = 01775;
+
         struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
         if (!gr)
-            throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist")
+            printMsg(lvlError, format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist")
                 % settings.buildUsersGroup);
+        else {
+            struct stat st;
+            if (stat(settings.nixStore.c_str(), &st))
+                throw SysError(format("getting attributes of path ‘%1%’") % settings.nixStore);
 
-        struct stat st;
-        if (stat(settings.nixStore.c_str(), &st))
-            throw SysError(format("getting attributes of path ‘%1%’") % settings.nixStore);
-
-        if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) {
-            if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
-                throw SysError(format("changing ownership of path ‘%1%’") % settings.nixStore);
-            if (chmod(settings.nixStore.c_str(), 01775) == -1)
-                throw SysError(format("changing permissions on path ‘%1%’") % settings.nixStore);
+            if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
+                if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
+                    throw SysError(format("changing ownership of path ‘%1%’") % settings.nixStore);
+                if (chmod(settings.nixStore.c_str(), perm) == -1)
+                    throw SysError(format("changing permissions on path ‘%1%’") % settings.nixStore);
+            }
         }
     }
 
@@ -1402,7 +1406,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
 }
 
 
-Path LocalStore::addToStore(const Path & _srcPath,
+Path LocalStore::addToStore(const string & name, const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
 {
     Path srcPath(absPath(_srcPath));
@@ -1417,7 +1421,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
     else
         sink.s = readFile(srcPath);
 
-    return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair);
+    return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair);
 }
 
 

src/libstore/local-store.hh

@@ -130,7 +130,7 @@ public:
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos);
 
-    Path addToStore(const Path & srcPath,
+    Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, bool repair = false);
 

src/libstore/local.mk

@@ -15,6 +15,7 @@ ifeq ($(OS), SunOS)
 endif
 
 libstore_CXXFLAGS = \
+ -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \
  -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
@@ -22,7 +23,7 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
- -DDEFAULT_CHROOT_DIRS="\"$(DEFAULT_CHROOT_DIRS)\""
+ -DBASH_PATH="\"$(bash)\""
 
 $(d)/local-store.cc: $(d)/schema.sql.hh
 

src/libstore/misc.cc

@@ -120,7 +120,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
                 if (invalid.empty()) continue;
 
                 todoDrv.insert(*i);
-                if (settings.useSubstitutes && !willBuildLocally(drv))
+                if (settings.useSubstitutes && substitutesAllowed(drv))
                     query.insert(invalid.begin(), invalid.end());
             }
 
@@ -144,7 +144,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
             PathSet outputs;
             bool mustBuild = false;
-            if (settings.useSubstitutes && !willBuildLocally(drv)) {
+            if (settings.useSubstitutes && substitutesAllowed(drv)) {
                 foreach (DerivationOutputs::iterator, j, drv.outputs) {
                     if (!wantOutput(j->first, i2.second)) continue;
                     if (!store.isValidPath(j->second.path)) {

src/libstore/misc.hh

@@ -34,5 +34,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
 bool willBuildLocally(const Derivation & drv);
 
+bool substitutesAllowed(const Derivation & drv);
+
 
 }

src/libstore/profiles.cc

@@ -80,7 +80,25 @@ Path createGeneration(Path profile, Path outPath)
        previous ones. */
     int dummy;
     Generations gens = findGenerations(profile, dummy);
-    unsigned int num = gens.size() > 0 ? gens.back().number : 0;
+
+    unsigned int num;
+    if (gens.size() > 0) {
+        Generation last = gens.back();
+
+        if (readLink(last.path) == outPath) {
+            /* We only create a new generation symlink if it differs
+               from the last one.
+
+               This helps keeping gratuitous installs/rebuilds from piling
+               up uncontrolled numbers of generations, cluttering up the
+               UI like grub. */
+            return last.path;
+        }
+
+        num = gens.back().number;
+    } else {
+        num = 0;
+    }
 
     /* Create the new generation.  Note that addPermRoot() blocks if
        the garbage collector is running to prevent the stuff we've
@@ -111,21 +129,94 @@ void deleteGeneration(const Path & profile, unsigned int gen)
 }
 
 
+static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
+{
+    if (dryRun)
+        printMsg(lvlInfo, format("would remove generation %1%") % gen);
+    else {
+        printMsg(lvlInfo, format("removing generation %1%") % gen);
+        deleteGeneration(profile, gen);
+    }
+}
+
+
+void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
+{
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    int curGen;
+    Generations gens = findGenerations(profile, curGen);
+
+    if (gensToDelete.find(curGen) != gensToDelete.end())
+        throw Error(format("cannot delete current generation of profile %1%’") % profile);
+
+    for (auto & i : gens) {
+        if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
+        deleteGeneration2(profile, i.number, dryRun);
+    }
+}
+
+
+void deleteOldGenerations(const Path & profile, bool dryRun)
+{
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    int curGen;
+    Generations gens = findGenerations(profile, curGen);
+
+    for (auto & i : gens)
+        if (i.number != curGen)
+            deleteGeneration2(profile, i.number, dryRun);
+}
+
+
+void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
+{
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    int curGen;
+    Generations gens = findGenerations(profile, curGen);
+
+    bool canDelete = false;
+    for (auto i = gens.rbegin(); i != gens.rend(); ++i)
+        if (canDelete) {
+            assert(i->creationTime < t);
+            if (i->number != curGen)
+                deleteGeneration2(profile, i->number, dryRun);
+        } else if (i->creationTime < t) {
+            /* We may now start deleting generations, but we don't
+               delete this generation yet, because this generation was
+               still the one that was active at the requested point in
+               time. */
+            canDelete = true;
+        }
+}
+
+
+void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
+{
+    time_t curTime = time(0);
+    string strDays = string(timeSpec, 0, timeSpec.size() - 1);
+    int days;
+
+    if (!string2Int(strDays, days) || days < 1)
+        throw Error(format("invalid number of days specifier ‘%1%’") % timeSpec);
+
+    time_t oldTime = curTime - days * 24 * 3600;
+
+    deleteGenerationsOlderThan(profile, oldTime, dryRun);
+}
+
+
 void switchLink(Path link, Path target)
 {
     /* Hacky. */
     if (dirOf(target) == dirOf(link)) target = baseNameOf(target);
 
-    Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link));
-    createSymlink(target, tmp);
-    /* The rename() system call is supposed to be essentially atomic
-       on Unix.  That is, if we have links `current -> X' and
-       `new_current -> Y', and we rename new_current to current, a
-       process accessing current will see X or Y, but never a
-       file-not-found or other error condition.  This is sufficient to
-       atomically switch user environments. */
-    if (rename(tmp.c_str(), link.c_str()) != 0)
-        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link);
+    replaceSymlink(target, link);
 }
 
 

src/libstore/profiles.hh

@@ -30,11 +30,19 @@ typedef list<Generation> Generations;
 /* Returns the list of currently present generations for the specified
    profile, sorted by generation number. */
 Generations findGenerations(Path profile, int & curGen);
-    
+
 Path createGeneration(Path profile, Path outPath);
 
 void deleteGeneration(const Path & profile, unsigned int gen);
 
+void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
+
+void deleteOldGenerations(const Path & profile, bool dryRun);
+
+void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
+
+void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun);
+
 void switchLink(Path link, Path target);
 
 /* Ensure exclusive access to a profile.  Any command that modifies

src/libstore/remote-store.cc

@@ -387,7 +387,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 }
 
 
-Path RemoteStore::addToStore(const Path & _srcPath,
+Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@@ -397,7 +397,7 @@ Path RemoteStore::addToStore(const Path & _srcPath,
     Path srcPath(absPath(_srcPath));
 
     writeInt(wopAddToStore, to);
-    writeString(baseNameOf(srcPath), to);
+    writeString(name, to);
     /* backwards compatibility hack */
     writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
     writeInt(recursive ? 1 : 0, to);
@@ -587,6 +587,16 @@ void RemoteStore::optimiseStore()
     readInt(from);
 }
 
+bool RemoteStore::verifyStore(bool checkContents, bool repair)
+{
+    openConnection();
+    writeInt(wopVerifyStore, to);
+    writeInt(checkContents, to);
+    writeInt(repair, to);
+    processStderr();
+    return readInt(from) != 0;
+}
+
 void RemoteStore::processStderr(Sink * sink, Source * source)
 {
     to.flush();

src/libstore/remote-store.hh

@@ -21,15 +21,15 @@ public:
     RemoteStore();
 
     ~RemoteStore();
-    
+
     /* Implementations of abstract store API methods. */
-    
+
     bool isValidPath(const Path & path);
 
     PathSet queryValidPaths(const PathSet & paths);
-    
+
     PathSet queryAllValidPaths();
-    
+
     ValidPathInfo queryPathInfo(const Path & path);
 
     Hash queryPathHash(const Path & path);
@@ -39,21 +39,21 @@ public:
     void queryReferrers(const Path & path, PathSet & referrers);
 
     Path queryDeriver(const Path & path);
-    
+
     PathSet queryValidDerivers(const Path & path);
 
     PathSet queryDerivationOutputs(const Path & path);
-    
+
     StringSet queryDerivationOutputNames(const Path & path);
 
     Path queryPathFromHashPart(const string & hashPart);
-    
+
     PathSet querySubstitutablePaths(const PathSet & paths);
-    
+
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos);
-    
-    Path addToStore(const Path & srcPath,
+
+    Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, bool repair = false);
 
@@ -64,7 +64,7 @@ public:
         Sink & sink);
 
     Paths importPaths(bool requireSignature, Source & source);
-    
+
     void buildPaths(const PathSet & paths, BuildMode buildMode);
 
     void ensurePath(const Path & path);
@@ -72,19 +72,20 @@ public:
     void addTempRoot(const Path & path);
 
     void addIndirectRoot(const Path & path);
-    
+
     void syncWithGC();
-    
+
     Roots findRoots();
 
     void collectGarbage(const GCOptions & options, GCResults & results);
-    
+
     PathSet queryFailedPaths();
 
     void clearFailedPaths(const PathSet & paths);
 
     void optimiseStore();
 
+    bool verifyStore(bool checkContents, bool repair);
 private:
     AutoCloseFD fdSocket;
     FdSink to;

src/libstore/store-api.hh

@@ -54,7 +54,7 @@ struct GCOptions
 };
 
 
-struct GCResults 
+struct GCResults
 {
     /* Depending on the action, the GC roots, or the paths that would
        be or have been deleted. */
@@ -82,7 +82,7 @@ struct SubstitutablePathInfo
 typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
 
 
-struct ValidPathInfo 
+struct ValidPathInfo
 {
     Path path;
     Path deriver;
@@ -100,13 +100,13 @@ typedef list<ValidPathInfo> ValidPathInfos;
 enum BuildMode { bmNormal, bmRepair, bmCheck };
 
 
-class StoreAPI 
+class StoreAPI
 {
 public:
 
     virtual ~StoreAPI() { }
 
-    /* Check whether a path is valid. */ 
+    /* Check whether a path is valid. */
     virtual bool isValidPath(const Path & path) = 0;
 
     /* Query which of the given paths is valid. */
@@ -118,7 +118,7 @@ public:
     /* Query information about a valid path. */
     virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
 
-    /* Query the hash of a valid path. */ 
+    /* Query the hash of a valid path. */
     virtual Hash queryPathHash(const Path & path) = 0;
 
     /* Query the set of outgoing FS references for a store path.  The
@@ -150,7 +150,7 @@ public:
     /* Query the full store path given the hash part of a valid store
        path, or "" if the path doesn't exist. */
     virtual Path queryPathFromHashPart(const string & hashPart) = 0;
-    
+
     /* Query which of the given paths have substitutes. */
     virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
 
@@ -159,12 +159,12 @@ public:
        info, it's omitted from the resulting ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) = 0;
-    
+
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        The function object `filter' can be used to exclude files (see
        libutil/archive.hh). */
-    virtual Path addToStore(const Path & srcPath,
+    virtual Path addToStore(const string & name, const Path & srcPath,
         bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, bool repair = false) = 0;
 
@@ -254,6 +254,10 @@ public:
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
     virtual void optimiseStore() = 0;
+
+    /* Check the integrity of the Nix store.  Returns true if errors
+       remain. */
+    virtual bool verifyStore(bool checkContents, bool repair) = 0;
 };
 
 
@@ -267,7 +271,7 @@ bool isStorePath(const Path & path);
 
 /* Extract the name part of the given store path. */
 string storePathToName(const Path & path);
-    
+
 void checkStoreName(const string & name);
 
 
@@ -288,7 +292,7 @@ Path followLinksToStorePath(const Path & path);
 /* Constructs a unique store path name. */
 Path makeStorePath(const string & type,
     const Hash & hash, const string & name);
-    
+
 Path makeOutputPath(const string & id,
     const Hash & hash, const string & name);
 

src/libstore/worker-protocol.hh

@@ -42,7 +42,8 @@ typedef enum {
     wopQueryValidPaths = 31,
     wopQuerySubstitutablePaths = 32,
     wopQueryValidDerivers = 33,
-    wopOptimiseStore = 34
+    wopOptimiseStore = 34,
+    wopVerifyStore = 35
 } WorkerOp;
 
 

src/libutil/hash.cc

@@ -84,7 +84,7 @@ string printHash(const Hash & hash)
     return string(buf, hash.hashSize * 2);
 }
 
-    
+
 Hash parseHash(HashType ht, const string & s)
 {
     Hash hash(ht);
@@ -92,7 +92,7 @@ Hash parseHash(HashType ht, const string & s)
         throw Error(format("invalid hash ‘%1%’") % s);
     for (unsigned int i = 0; i < hash.hashSize; i++) {
         string s2(s, i * 2, 2);
-        if (!isxdigit(s2[0]) || !isxdigit(s2[1])) 
+        if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
             throw Error(format("invalid hash ‘%1%’") % s);
         std::istringstream str(s2);
         int n;
@@ -103,24 +103,6 @@ Hash parseHash(HashType ht, const string & s)
 }
 
 
-static unsigned char divMod(unsigned char * bytes, unsigned char y)
-{
-    unsigned int borrow = 0;
-
-    int pos = Hash::maxHashSize - 1;
-    while (pos >= 0 && !bytes[pos]) --pos;
-
-    for ( ; pos >= 0; --pos) {
-        unsigned int s = bytes[pos] + (borrow << 8);
-        unsigned int d = s / y;
-        borrow = s % y;
-        bytes[pos] = d;
-    }
-
-    return borrow;
-}
-
-
 unsigned int hashLength32(const Hash & hash)
 {
     return (hash.hashSize * 8 - 1) / 5 + 1;
@@ -136,19 +118,19 @@ string printHash32(const Hash & hash)
     Hash hash2(hash);
     unsigned int len = hashLength32(hash);
 
-    const char * chars = base32Chars.data();
-    
-    string s(len, '0');
+    string s;
+    s.reserve(len);
 
-    int pos = len - 1;
-    while (pos >= 0) {
-        unsigned char digit = divMod(hash2.hash, 32);
-        s[pos--] = chars[digit];
+    for (int n = len - 1; n >= 0; n--) {
+        unsigned int b = n * 5;
+        unsigned int i = b / 8;
+        unsigned int j = b % 8;
+        unsigned char c =
+            (hash.hash[i] >> j)
+            | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
+        s.push_back(base32Chars[c & 0x1f]);
     }
 
-    for (unsigned int i = 0; i < hash2.maxHashSize; ++i)
-        assert(hash2.hash[i] == 0);
-
     return s;
 }
 
@@ -159,51 +141,24 @@ string printHash16or32(const Hash & hash)
 }
 
 
-static bool mul(unsigned char * bytes, unsigned char y, int maxSize)
-{
-    unsigned char carry = 0;
-
-    for (int pos = 0; pos < maxSize; ++pos) {
-        unsigned int m = bytes[pos] * y + carry;
-        bytes[pos] = m & 0xff;
-        carry = m >> 8;
-    }
-
-    return carry;
-}
-
-
-static bool add(unsigned char * bytes, unsigned char y, int maxSize)
-{
-    unsigned char carry = y;
-
-    for (int pos = 0; pos < maxSize; ++pos) {
-        unsigned int m = bytes[pos] + carry;
-        bytes[pos] = m & 0xff;
-        carry = m >> 8;
-        if (carry == 0) break;
-    }
-
-    return carry;
-}
-
-
 Hash parseHash32(HashType ht, const string & s)
 {
     Hash hash(ht);
+    unsigned int len = hashLength32(ht);
+    assert(s.size() == len);
 
-    const char * chars = base32Chars.data();
-
-    for (unsigned int i = 0; i < s.length(); ++i) {
-        char c = s[i];
+    for (unsigned int n = 0; n < len; ++n) {
+        char c = s[len - n - 1];
         unsigned char digit;
         for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
-            if (chars[digit] == c) break;
+            if (base32Chars[digit] == c) break;
         if (digit >= 32)
             throw Error(format("invalid base-32 hash ‘%1%’") % s);
-        if (mul(hash.hash, 32, hash.hashSize) ||
-            add(hash.hash, digit, hash.hashSize))
-            throw Error(format("base-32 hash ‘%1%’ is too large") % s);
+        unsigned int b = n * 5;
+        unsigned int i = b / 8;
+        unsigned int j = b % 8;
+        hash.hash[i] |= digit << j;
+        if (i < hash.hashSize - 1) hash.hash[i + 1] |= digit >> (8 - j);
     }
 
     return hash;
@@ -299,7 +254,7 @@ Hash hashFile(HashType ht, const Path & path)
         if (n == -1) throw SysError(format("reading file ‘%1%’") % path);
         update(ht, ctx, buf, n);
     }
-    
+
     finish(ht, ctx, hash.hash);
     return hash;
 }
@@ -311,7 +266,7 @@ HashSink::HashSink(HashType ht) : ht(ht)
     bytes = 0;
     start(ht, *ctx);
 }
-    
+
 HashSink::~HashSink()
 {
     bufPos = 0;
@@ -369,7 +324,7 @@ HashType parseHashType(const string & s)
     else return htUnknown;
 }
 
- 
+
 string printHashType(HashType ht)
 {
     if (ht == htMD5) return "md5";
@@ -378,5 +333,5 @@ string printHashType(HashType ht)
     else throw Error("cannot print unknown hash type");
 }
 
- 
+
 }

src/libutil/util.cc

@@ -193,8 +193,12 @@ Path readLink(const Path & path)
     if (!S_ISLNK(st.st_mode))
         throw Error(format("‘%1%’ is not a symlink") % path);
     char buf[st.st_size];
-    if (readlink(path.c_str(), buf, st.st_size) != st.st_size)
+    ssize_t rlsize = readlink(path.c_str(), buf, st.st_size);
+    if (rlsize == -1)
         throw SysError(format("reading symbolic link ‘%1%’") % path);
+    else if (rlsize > st.st_size)
+        throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%")
+            % path % rlsize % st.st_size);
     return string(buf, st.st_size);
 }
 
@@ -409,6 +413,17 @@ void createSymlink(const Path & target, const Path & link)
 }
 
 
+void replaceSymlink(const Path & target, const Path & link)
+{
+    Path tmp = canonPath(dirOf(link) + "/.new_" + baseNameOf(link));
+
+    createSymlink(target, tmp);
+
+    if (rename(tmp.c_str(), link.c_str()) != 0)
+        throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % link);
+}
+
+
 LogType logType = ltPretty;
 Verbosity verbosity = lvlInfo;
 
@@ -912,43 +927,55 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 }
 
 
-std::vector<const char *> stringsToCharPtrs(const Strings & ss)
+std::vector<char *> stringsToCharPtrs(const Strings & ss)
 {
-    std::vector<const char *> res;
-    for (auto & s : ss) res.push_back(s.c_str());
+    std::vector<char *> res;
+    for (auto & s : ss) res.push_back((char *) s.c_str());
     res.push_back(0);
     return res;
 }
 
 
-string runProgram(Path program, bool searchPath, const Strings & args)
+string runProgram(Path program, bool searchPath, const Strings & args,
+    const string & input)
 {
     checkInterrupt();
 
     /* Create a pipe. */
-    Pipe pipe;
-    pipe.create();
+    Pipe out, in;
+    out.create();
+    if (!input.empty()) in.create();
 
     /* Fork. */
     Pid pid = startProcess([&]() {
-        if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
+        if (dup2(out.writeSide, STDOUT_FILENO) == -1)
             throw SysError("dupping stdout");
+        if (!input.empty()) {
+            if (dup2(in.readSide, STDIN_FILENO) == -1)
+                throw SysError("dupping stdin");
+        }
 
         Strings args_(args);
         args_.push_front(program);
-        auto cargs = stringsToCharPtrs(args_);
 
         if (searchPath)
-            execvp(program.c_str(), (char * *) &cargs[0]);
+            execvp(program.c_str(), stringsToCharPtrs(args_).data());
         else
-            execv(program.c_str(), (char * *) &cargs[0]);
+            execv(program.c_str(), stringsToCharPtrs(args_).data());
 
         throw SysError(format("executing ‘%1%’") % program);
     });
 
-    pipe.writeSide.close();
+    out.writeSide.close();
 
-    string result = drainFD(pipe.readSide);
+    /* FIXME: This can deadlock if the input is too long. */
+    if (!input.empty()) {
+        in.readSide.close();
+        writeFull(in.writeSide, input);
+        in.writeSide.close();
+    }
+
+    string result = drainFD(out.readSide);
 
     /* Wait for the child to finish. */
     int status = pid.wait(true);
@@ -1059,6 +1086,15 @@ string chomp(const string & s)
 }
 
 
+string trim(const string & s, const string & whitespace)
+{
+    auto i = s.find_first_not_of(whitespace);
+    if (i == string::npos) return "";
+    auto j = s.find_last_not_of(whitespace);
+    return string(s, i, j == string::npos ? j : j - i + 1);
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
@@ -1191,4 +1227,63 @@ string filterANSIEscapes(const string & s, bool nixOnly)
 }
 
 
+static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+
+string base64Encode(const string & s)
+{
+    string res;
+    int data = 0, nbits = 0;
+
+    for (char c : s) {
+        data = data << 8 | (unsigned char) c;
+        nbits += 8;
+        while (nbits >= 6) {
+            nbits -= 6;
+            res.push_back(base64Chars[data >> nbits & 0x3f]);
+        }
+    }
+
+    if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]);
+    while (res.size() % 4) res.push_back('=');
+
+    return res;
+}
+
+
+string base64Decode(const string & s)
+{
+    bool init = false;
+    char decode[256];
+    if (!init) {
+        // FIXME: not thread-safe.
+        memset(decode, -1, sizeof(decode));
+        for (int i = 0; i < 64; i++)
+            decode[(int) base64Chars[i]] = i;
+        init = true;
+    }
+
+    string res;
+    unsigned int d = 0, bits = 0;
+
+    for (char c : s) {
+        if (c == '=') break;
+        if (c == '\n') continue;
+
+        char digit = decode[(unsigned char) c];
+        if (digit == -1)
+            throw Error("invalid character in Base64 string");
+
+        bits += 6;
+        d = d << 6 | digit;
+        if (bits >= 8) {
+            res.push_back(d >> (bits - 8) & 0xff);
+            bits -= 8;
+        }
+    }
+
+    return res;
+}
+
+
 }

src/libutil/util.hh

@@ -110,6 +110,9 @@ Paths createDirs(const Path & path);
 /* Create a symlink. */
 void createSymlink(const Path & target, const Path & link);
 
+/* Atomically create or replace a symlink. */
+void replaceSymlink(const Path & target, const Path & link);
+
 
 template<class T, class A>
 T singleton(const A & a)
@@ -285,14 +288,14 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P
 /* Run a program and return its stdout in a string (i.e., like the
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
-    const Strings & args = Strings());
+    const Strings & args = Strings(), const string & input = "");
 
 MakeError(ExecError, Error)
 
 /* Convert a list of strings to a null-terminated vector of char
    *'s. The result must not be accessed beyond the lifetime of the
    list of strings. */
-std::vector<const char *> stringsToCharPtrs(const Strings & ss);
+std::vector<char *> stringsToCharPtrs(const Strings & ss);
 
 /* Close all file descriptors except stdin, stdout, stderr, and those
    listed in the given set.  Good practice in child processes. */
@@ -334,6 +337,10 @@ string concatStringsSep(const string & sep, const StringSet & ss);
 string chomp(const string & s);
 
 
+/* Remove whitespace from the start and end of a string. */
+string trim(const string & s, const string & whitespace = " \n\r\t");
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);
@@ -398,4 +405,9 @@ void ignoreException();
 string filterANSIEscapes(const string & s, bool nixOnly = false);
 
 
+/* Base64 encoding/decoding. */
+string base64Encode(const string & s);
+string base64Decode(const string & s);
+
+
 }

src/nix-collect-garbage/local.mk

@@ -0,0 +1,7 @@
+programs += nix-collect-garbage
+
+nix-collect-garbage_DIR := $(d)
+
+nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc
+
+nix-collect-garbage_LIBS = libmain libstore libutil libformat

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -0,0 +1,82 @@
+#include "store-api.hh"
+#include "profiles.hh"
+#include "shared.hh"
+#include "globals.hh"
+
+#include <iostream>
+
+using namespace nix;
+
+std::string deleteOlderThan;
+bool dryRun = false;
+
+
+/* If `-d' was specified, remove all old generations of all profiles.
+ * Of course, this makes rollbacks to before this point in time
+ * impossible. */
+
+void removeOldGenerations(std::string dir)
+{
+    if (access(dir.c_str(), R_OK) != 0) return;
+
+    bool canWrite = access(dir.c_str(), W_OK) == 0;
+
+    for (auto & i : readDirectory(dir)) {
+        checkInterrupt();
+
+        auto path = dir + "/" + i.name;
+        auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type;
+
+        if (type == DT_LNK && canWrite) {
+            auto link = readLink(path);
+            if (link.find("link") != string::npos) {
+                printMsg(lvlInfo, format("removing old generations of profile %1%") % path);
+                if (deleteOlderThan != "")
+                    deleteGenerationsOlderThan(path, deleteOlderThan, dryRun);
+                else
+                    deleteOldGenerations(path, dryRun);
+            }
+        } else if (type == DT_DIR) {
+            removeOldGenerations(path);
+        }
+    }
+}
+
+int main(int argc, char * * argv)
+{
+    bool removeOld = false;
+    Strings extraArgs;
+
+    return handleExceptions(argv[0], [&]() {
+        initNix();
+
+        parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
+            if (*arg == "--help")
+                showManPage("nix-collect-garbage");
+            else if (*arg == "--version")
+                printVersion("nix-collect-garbage");
+            else if (*arg == "--delete-old" || *arg == "-d") removeOld = true;
+            else if (*arg == "--delete-older-than") {
+                removeOld = true;
+                deleteOlderThan = getArg(*arg, arg, end);
+            }
+            else if (*arg == "--dry-run") dryRun = true;
+            else
+                extraArgs.push_back(*arg);
+            return true;
+        });
+
+        auto profilesDir = settings.nixStateDir + "/profiles";
+        if (removeOld) removeOldGenerations(profilesDir);
+
+        // Run the actual garbage collector.
+        if (!dryRun) {
+            store = openStore(false);
+            GCOptions options;
+            options.action = GCOptions::gcDeleteDead;
+            GCResults results;
+            PrintFreed freed(true, results);
+            store->collectGarbage(options, results);
+        }
+    });
+}

src/nix-daemon/local.mk

@@ -11,5 +11,3 @@ nix-daemon_LDFLAGS = -pthread
 ifeq ($(OS), SunOS)
         nix-daemon_LDFLAGS += -lsocket
 endif
-
-$(eval $(call install-symlink, nix-daemon, $(bindir)/nix-worker))

src/nix-daemon/nix-daemon.cc

@@ -519,6 +519,18 @@ static void performOp(bool trusted, unsigned int clientVersion,
         writeInt(1, to);
         break;
 
+    case wopVerifyStore: {
+        bool checkContents = readInt(from) != 0;
+        bool repair = readInt(from) != 0;
+        startWork();
+        if (repair && !trusted)
+            throw Error("you are not privileged to repair paths");
+        bool errors = store->verifyStore(checkContents, repair);
+        stopWork();
+        writeInt(errors, to);
+        break;
+    }
+
     default:
         throw Error(format("invalid operation %1%") % op);
     }

src/nix-env/nix-env.cc

@@ -132,6 +132,8 @@ static void getAllExprs(EvalState & state,
             Value & vArg(*state.allocValue());
             state.getBuiltin("import", vFun);
             mkString(vArg, path2);
+            if (v.attrs->size() == v.attrs->capacity())
+                throw Error(format("too many Nix expressions in directory ‘%1%’") % path);
             mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg);
         }
         else if (S_ISDIR(st.st_mode))
@@ -160,7 +162,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v)
        ~/.nix-defexpr directory that includes some system-wide
        directory). */
     if (S_ISDIR(st.st_mode)) {
-        state.mkAttrs(v, 16);
+        state.mkAttrs(v, 1024);
         state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0);
         StringSet attrs;
         getAllExprs(state, path, attrs, v);
@@ -1260,73 +1262,24 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs
 }
 
 
-static void deleteGeneration2(Globals & globals, unsigned int gen)
-{
-    if (globals.dryRun)
-        printMsg(lvlInfo, format("would remove generation %1%") % gen);
-    else {
-        printMsg(lvlInfo, format("removing generation %1%") % gen);
-        deleteGeneration(globals.profile, gen);
-    }
-
-}
-
-
 static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag ‘%1%’") % opFlags.front());
 
-    PathLocks lock;
-    lockProfile(lock, globals.profile);
-
-    int curGen;
-    Generations gens = findGenerations(globals.profile, curGen);
-
-    for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) {
-
-        if (*i == "old") {
-            for (Generations::iterator j = gens.begin(); j != gens.end(); ++j)
-                if (j->number != curGen)
-                    deleteGeneration2(globals, j->number);
-        } else if (i->size() >= 2 && tolower(*i->rbegin()) == 'd') {
-            time_t curTime = time(NULL);
-            time_t oldTime;
-            string strDays = string(*i, 0, i->size() - 1);
-            int days;
-
-            if (!string2Int(strDays, days) || days < 1)
-                throw UsageError(format("invalid number of days specifier ‘%1%’") % *i);
-
-            oldTime = curTime - days * 24 * 3600;
-
-            bool canDelete = false;
-            for (Generations::reverse_iterator j = gens.rbegin(); j != gens.rend(); ++j) {
-                if (canDelete) {
-                    assert(j->creationTime < oldTime);
-                    deleteGeneration2(globals, j->number);
-                } else if (j->creationTime < oldTime) {
-                    /* We may now start deleting generations, but we don't delete
-                       this generation yet, because this generation was still the
-                       one that was active at the requested point in time. */
-                    canDelete = true;
-                }
-            }
-        } else {
-            int n;
-            if (!string2Int(*i, n) || n < 0)
-                throw UsageError(format("invalid generation specifier ‘%1%’")  % *i);
-            bool found = false;
-            for (Generations::iterator j = gens.begin(); j != gens.end(); ++j) {
-                if (j->number == n) {
-                    deleteGeneration2(globals, j->number);
-                    found = true;
-                    break;
-                }
-            }
-            if (!found)
-                printMsg(lvlError, format("generation %1% does not exist") % n);
+    if (opArgs.size() == 1 && opArgs.front() == "old") {
+        deleteOldGenerations(globals.profile, globals.dryRun);
+    } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) {
+        deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun);
+    } else {
+        std::set<unsigned int> gens;
+        for (auto & i : opArgs) {
+            unsigned int n;
+            if (!string2Int(i, n) || n < 0)
+                throw UsageError(format("invalid generation number ‘%1%’") % i);
+            gens.insert(n);
         }
+        deleteGenerations(globals.profile, gens, globals.dryRun);
     }
 }
 
@@ -1335,6 +1288,7 @@ int main(int argc, char * * argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+        initGC();
 
         Strings opFlags, opArgs, searchPath;
         std::map<string, string> autoArgs_;
@@ -1420,6 +1374,8 @@ int main(int argc, char * * argv)
 
         if (!op) throw UsageError("no operation specified");
 
+        store = openStore();
+
         globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath));
         globals.state->repair = repair;
 
@@ -1438,8 +1394,6 @@ int main(int argc, char * * argv)
                 : canonPath(settings.nixStateDir + "/profiles/default");
         }
 
-        store = openStore();
-
         op(globals, opFlags, opArgs);
 
         globals.state->printStats();

src/nix-instantiate/nix-instantiate.cc

@@ -93,6 +93,7 @@ int main(int argc, char * * argv)
 {
     return handleExceptions(argv[0], [&]() {
         initNix();
+        initGC();
 
         Strings files, searchPath;
         bool readStdin = false;
@@ -154,14 +155,16 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        if (evalOnly && !wantsReadWrite)
+            settings.readOnlyMode = true;
+
+        store = openStore();
+
         EvalState state(searchPath);
         state.repair = repair;
 
         Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
 
-        if (evalOnly && !wantsReadWrite)
-            settings.readOnlyMode = true;
-
         if (attrPaths.empty()) attrPaths.push_back("");
 
         if (findFile) {
@@ -173,8 +176,6 @@ int main(int argc, char * * argv)
             return;
         }
 
-        store = openStore();
-
         if (readStdin) {
             Expr * e = parseStdin(state);
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,
@@ -182,10 +183,10 @@ int main(int argc, char * * argv)
         } else if (files.empty() && !fromArgs)
             files.push_back("./default.nix");
 
-        foreach (Strings::iterator, i, files) {
+        for (auto & i : files) {
             Expr * e = fromArgs
-                ? state.parseExprFromString(*i, absPath("."))
-                : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i)));
+                ? state.parseExprFromString(i, absPath("."))
+                : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, i)));
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }

src/nix-store/local.mk

@@ -6,6 +6,6 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc)
 
 nix-store_LIBS = libmain libstore libutil libformat
 
-nix-store_LDFLAGS = -lbz2 -pthread
+nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS)
 
 nix-store_CXXFLAGS = -DCURL=\"$(curl)\"

src/nix-store/nix-store.cc

@@ -20,6 +20,10 @@
 
 #include <bzlib.h>
 
+#if HAVE_SODIUM
+#include <sodium.h>
+#endif
+
 
 using namespace nix;
 using std::cin;
@@ -45,12 +49,11 @@ LocalStore & ensureLocalStore()
 
 static Path useDeriver(Path path)
 {
-    if (!isDerivation(path)) {
-        path = store->queryDeriver(path);
-        if (path == "")
-            throw Error(format("deriver of path ‘%1%’ is not known") % path);
-    }
-    return path;
+    if (isDerivation(path)) return path;
+    Path drvPath = store->queryDeriver(path);
+    if (drvPath == "")
+        throw Error(format("deriver of path ‘%1%’ is not known") % path);
+    return drvPath;
 }
 
 
@@ -136,7 +139,8 @@ static void opRealise(Strings opFlags, Strings opArgs)
         unknown = PathSet();
     }
 
-    printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize);
+    if (settings.get("print-missing", true))
+        printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (dryRun) return;
 
@@ -158,8 +162,8 @@ static void opAdd(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
-    for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i)
-        cout << format("%1%\n") % store->addToStore(*i);
+    for (auto & i : opArgs)
+        cout << format("%1%\n") % store->addToStore(baseNameOf(i), i);
 }
 
 
@@ -180,8 +184,8 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     HashType hashAlgo = parseHashType(opArgs.front());
     opArgs.pop_front();
 
-    for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i)
-        cout << format("%1%\n") % store->addToStore(*i, recursive, hashAlgo);
+    for (auto & i : opArgs)
+        cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo);
 }
 
 
@@ -621,28 +625,6 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 }
 
 
-static string showBytes(unsigned long long bytes)
-{
-    return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
-}
-
-
-struct PrintFreed
-{
-    bool show;
-    const GCResults & results;
-    PrintFreed(bool show, const GCResults & results)
-        : show(show), results(results) { }
-    ~PrintFreed()
-    {
-        if (show)
-            cout << format("%1% store paths deleted, %2% freed\n")
-                % results.paths.size()
-                % showBytes(results.bytesFreed);
-    }
-};
-
-
 static void opGC(Strings opFlags, Strings opArgs)
 {
     bool printRoots = false;
@@ -786,7 +768,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
         else if (*i == "--repair") repair = true;
         else throw UsageError(format("unknown flag ‘%1%’") % *i);
 
-    if (ensureLocalStore().verifyStore(checkContents, repair)) {
+    if (store->verifyStore(checkContents, repair)) {
         printMsg(lvlError, "warning: not all errors were fixed");
         throw Exit(1);
     }
@@ -1006,6 +988,34 @@ static void opServe(Strings opFlags, Strings opArgs)
 }
 
 
+static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
+{
+    foreach (Strings::iterator, i, opFlags)
+        throw UsageError(format("unknown flag ‘%1%’") % *i);
+
+    if (opArgs.size() != 3) throw UsageError("three arguments expected");
+    auto i = opArgs.begin();
+    string keyName = *i++;
+    string secretKeyFile = *i++;
+    string publicKeyFile = *i++;
+
+#if HAVE_SODIUM
+    sodium_init();
+
+    unsigned char pk[crypto_sign_PUBLICKEYBYTES];
+    unsigned char sk[crypto_sign_SECRETKEYBYTES];
+    if (crypto_sign_keypair(pk, sk) != 0)
+        throw Error("key generation failed");
+
+    writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES)));
+    umask(0077);
+    writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES)));
+#else
+    throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
+#endif
+}
+
+
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
@@ -1072,14 +1082,16 @@ int main(int argc, char * * argv)
                 op = opQueryFailedPaths;
             else if (*arg == "--clear-failed-paths")
                 op = opClearFailedPaths;
+            else if (*arg == "--serve")
+                op = opServe;
+            else if (*arg == "--generate-binary-cache-key")
+                op = opGenerateBinaryCacheKey;
             else if (*arg == "--add-root")
                 gcRoot = absPath(getArg(*arg, arg, end));
             else if (*arg == "--indirect")
                 indirectRoot = true;
             else if (*arg == "--no-output")
                 noOutput = true;
-            else if (*arg == "--serve")
-                op = opServe;
             else if (*arg != "" && arg->at(0) == '-') {
                 opFlags.push_back(*arg);
                 if (*arg == "--max-freed" || *arg == "--max-links" || *arg == "--max-atime") /* !!! hack */

tests/binary-cache.sh

@@ -87,3 +87,55 @@ rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
 grep -q "Downloading" $TEST_ROOT/log
+
+
+if [ -n "$HAVE_SODIUM" ]; then
+
+# Create a signed binary cache.
+clearCache
+
+declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 ))
+publicKey="$(cat $TEST_ROOT/pk1)"
+
+res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2))
+badKey="$(cat $TEST_ROOT/pk2)"
+
+res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
+otherKey="$(cat $TEST_ROOT/pk3)"
+
+nix-push --dest $cacheDir --key-file $TEST_ROOT/sk1 $outPath
+
+
+# Downloading should fail if we don't provide a key.
+clearStore
+
+rm -f $NIX_STATE_DIR/binary-cache*
+
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' )
+
+
+# And it should fail if we provide an incorrect key.
+clearStore
+
+rm -f $NIX_STATE_DIR/binary-cache*
+
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$badKey")
+
+
+# It should succeed if we provide the correct key.
+nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$otherKey $publicKey"
+
+
+# It should fail if we corrupt the .narinfo.
+clearStore
+
+for i in $cacheDir/*.narinfo; do
+    grep -v References $i > $i.tmp
+    mv $i.tmp $i
+done
+
+rm -f $NIX_STATE_DIR/binary-cache*
+
+(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey")
+
+fi # HAVE_LIBSODIUM

tests/common.sh.in

@@ -25,6 +25,7 @@ export dot=@dot@
 export xmllint="@xmllint@"
 export SHELL="@bash@"
 export PAGER=cat
+export HAVE_SODIUM="@HAVE_SODIUM@"
 
 export version=@PACKAGE_VERSION@
 export system=@system@

tests/lang/eval-okay-context.nix

@@ -1,4 +1,4 @@
-let s = "foo ${builtins.substring 33 100 (baseNameOf ./eval-okay-context.nix)} bar";
+let s = "foo ${builtins.substring 33 100 (baseNameOf "${./eval-okay-context.nix}")} bar";
 in
   if s != "foo eval-okay-context.nix bar"
   then abort "context not discarded"

tests/local.mk

@@ -11,7 +11,7 @@ nix_tests = \
   binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
-  check-reqs.sh
+  check-reqs.sh pass-as-file.sh tarball.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))

tests/nix-channel.sh

@@ -41,3 +41,26 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
 # Do an install.
 nix-env -i dependencies
 [ -e $TEST_ROOT/var/nix/profiles/default/foobar ]
+
+
+
+clearProfiles
+clearManifests
+rm -f $TEST_ROOT/.nix-channels
+
+# Test updating from a tarball
+nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 foo
+nix-channel --update
+
+# Do a query.
+nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml
+if [ "$xmllint" != false ]; then
+    $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
+fi
+grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
+grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+
+# Do an install.
+nix-env -i dependencies
+[ -e $TEST_ROOT/var/nix/profiles/default/foobar ]
+

tests/pass-as-file.sh

@@ -0,0 +1,17 @@
+source common.sh
+
+clearStore
+
+outPath=$(nix-build --no-out-link -E "
+with import ./config.nix;
+
+mkDerivation {
+  name = \"pass-as-file\";
+  passAsFile = [ \"foo\" ];
+  foo = [ \"xyzzy\" ];
+  builder = builtins.toFile \"builder.sh\" ''
+    [ \"\$(cat \$fooPath)\" = xyzzy ]
+    touch \$out
+  '';
+}
+")

tests/remote-builds.nix

@@ -14,6 +14,7 @@ let
     { services.openssh.enable = true;
       virtualisation.writableStore = true;
       nix.package = nix;
+      nix.useChroot = true;
     };
 
   # Trivial Nix expression to build remotely.

tests/tarball.sh

@@ -0,0 +1,23 @@
+source common.sh
+
+clearStore
+
+export HOME=$TEST_ROOT/home
+rm -rf $TEST_ROOT/home
+
+tarroot=$TEST_ROOT/tarball
+rm -rf $tarroot
+mkdir -p $tarroot
+cp dependencies.nix $tarroot/default.nix
+cp config.nix dependencies.builder*.sh $tarroot/
+
+tarball=$TEST_ROOT/tarball.tar.xz
+(cd $TEST_ROOT && tar c tarball) | xz > $tarball
+
+nix-env -f file://$tarball -qa --out-path | grep -q dependencies
+
+nix-build file://$tarball
+
+nix-build '<foo>' -I foo=file://$tarball
+
+nix-build -E "import (fetchTarball file://$tarball)"

tests/user-envs.sh

@@ -99,6 +99,16 @@ if nix-env -q '*' | grep -q bar; then false; fi
 nix-env --list-generations
 test "$(nix-env --list-generations | wc -l)" -eq 7
 
+# Doing the same operation twice results in the same generation, which triggers
+# "lazy" behaviour and does not create a new symlink.
+
+nix-env -i foo
+nix-env -i foo
+
+# Count generations.
+nix-env --list-generations
+test "$(nix-env --list-generations | wc -l)" -eq 8
+
 # Switch to a specified generation.
 nix-env --switch-generation 7
 [ "$(nix-store -q --resolve $profiles/test)" = "$oldGen" ]

version

@@ -1 +1 @@
-1.8
+1.9