diff --git a/flake.nix b/flake.nix
index 2ae547e31..db0aceb31 100644
--- a/flake.nix
+++ b/flake.nix
@@ -293,6 +293,8 @@
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
+          nix-eval-jobs = forAllSystems (system: self.packages.${system}.nix-eval-jobs.tests.nix-eval-jobs);
+
           # This is x86_64-linux only, just because we have significantly
           # cheaper x86_64-linux compute in CI.
           # It is clangStdenv because clang's sanitizers are nicer.
diff --git a/subprojects/nix-eval-jobs/default.nix b/subprojects/nix-eval-jobs/default.nix
index 5eaeb5e26..a0ec6369b 100644
--- a/subprojects/nix-eval-jobs/default.nix
+++ b/subprojects/nix-eval-jobs/default.nix
@@ -3,49 +3,90 @@
   lib,
   nix,
   pkgs,
-  srcDir ? null,
+  srcDir ? ./.,
+  callPackage,
 }:
 
 let
-  filterMesonBuild = builtins.filterSource (
-    path: type: type != "directory" || baseNameOf path != "build"
-  );
-in
-stdenv.mkDerivation {
-  pname = "nix-eval-jobs";
-  version = "2.93.0-dev";
-  src = if srcDir == null then filterMesonBuild ./. else srcDir;
-  buildInputs = with pkgs; [
-    nlohmann_json
-    nix
-    boost
-  ];
-  nativeBuildInputs =
-    with pkgs;
-    [
-      meson
-      pkg-config
-      ninja
-      # nlohmann_json can be only discovered via cmake files
-      cmake
-      # XXX: ew
-      nix.passthru.capnproto-lix
-    ]
-    ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]);
-
-  # nix-fast-build wants the nix attr on nix-eval-jobs
-  passthru = {
-    inherit nix;
-  };
-
-  meta = {
-    description = "Hydra's builtin hydra-eval-jobs as a standalone";
-    homepage = "https://github.com/nix-community/nix-eval-jobs";
-    license = lib.licenses.gpl3;
-    maintainers = with lib.maintainers; [
-      adisbladis
-      mic92
+  sourceBase = if srcDir == null then ./. else srcDir;
+  package = stdenv.mkDerivation (finalAttrs: {
+    pname = "nix-eval-jobs";
+    version = "2.93.0-dev";
+    src = lib.fileset.toSource {
+      root = sourceBase;
+      fileset = lib.fileset.unions [
+        ./meson.build
+        ./src
+      ];
+    };
+    buildInputs = with pkgs; [
+      nlohmann_json
+      nix
+      boost
     ];
-    platforms = lib.platforms.unix;
-  };
-}
+    nativeBuildInputs =
+      with pkgs;
+      [
+        meson
+        pkg-config
+        ninja
+        # nlohmann_json can be only discovered via cmake files
+        cmake
+        # XXX: ew
+        nix.passthru.capnproto-lix
+      ]
+      ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]);
+
+    passthru = {
+      inherit nix;
+      tests.nix-eval-jobs = callPackage (
+        {
+          stdenv,
+          python3Packages,
+          path,
+        }:
+        let
+          nejAttrs = finalAttrs;
+        in
+        stdenv.mkDerivation (finalAttrs: {
+          pname = "nix-eval-jobs-tests";
+          inherit (nejAttrs) version;
+          src = lib.fileset.toSource {
+            root = sourceBase;
+            fileset = lib.fileset.unions [
+              ./pyproject.toml
+              ./tests
+            ];
+          };
+
+          nativeBuildInputs = [
+            python3Packages.pytest
+            package
+          ];
+
+          env.NEJ_NIXPKGS_PATH = "${path}";
+
+          dontConfigure = true;
+          buildPhase = ''
+            export NIX_REMOTE="$NIX_BUILD_TOP"
+            export HOME="$(mktemp -d)"
+            pytest
+          '';
+          installPhase = "touch $out";
+        })
+      ) { };
+    };
+
+    meta = {
+      description = "Hydra's builtin hydra-eval-jobs as a standalone";
+      homepage = "https://github.com/nix-community/nix-eval-jobs";
+      license = lib.licenses.gpl3;
+      maintainers = with lib.maintainers; [
+        adisbladis
+        mic92
+      ];
+      platforms = lib.platforms.unix;
+    };
+  });
+in
+package
diff --git a/subprojects/nix-eval-jobs/src/worker.cc b/subprojects/nix-eval-jobs/src/worker.cc
index 934119d7e..35c176407 100644
--- a/subprojects/nix-eval-jobs/src/worker.cc
+++ b/subprojects/nix-eval-jobs/src/worker.cc
@@ -144,7 +144,7 @@ void worker(nix::ref<nix::eval_cache::CachingEvaluator> evaluator,
                                     .dynamic_pointer_cast<nix::LocalFSStore>();
                             auto storePath =
                                 localStore->parseStorePath(drv.drvPath);
-                            localStore->addPermRoot(storePath, root);
+                            aio.blockOn(localStore->addPermRoot(storePath, root));
                         }
                     }
                 } else {
diff --git a/subprojects/nix-eval-jobs/tests/assets/ci.nix b/subprojects/nix-eval-jobs/tests/assets/ci.nix
index e16369219..55bed43ca 100644
--- a/subprojects/nix-eval-jobs/tests/assets/ci.nix
+++ b/subprojects/nix-eval-jobs/tests/assets/ci.nix
@@ -1,6 +1,6 @@
 {
   pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { },
-  system ? pkgs.system,
+  system ? pkgs.stdenv.hostPlatform.system,
 }:
 
 {
diff --git a/subprojects/nix-eval-jobs/tests/assets/flake.nix b/subprojects/nix-eval-jobs/tests/assets/flake.nix
index 7f6cc61fa..abd5faa08 100644
--- a/subprojects/nix-eval-jobs/tests/assets/flake.nix
+++ b/subprojects/nix-eval-jobs/tests/assets/flake.nix
@@ -19,7 +19,7 @@
               recursion = [ recursion ];
             in
             derivation {
-              inherit (pkgs) system;
+              inherit (pkgs.stdenv.hostPlatform) system;
               name = "drvB";
               recursiveAttr = recursion;
               builder = ":";
diff --git a/subprojects/nix-eval-jobs/tests/test_eval.py b/subprojects/nix-eval-jobs/tests/test_eval.py
index 8fa41cb72..166dc8029 100755
--- a/subprojects/nix-eval-jobs/tests/test_eval.py
+++ b/subprojects/nix-eval-jobs/tests/test_eval.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import json
+import os
 import subprocess
 import pytest
 from pathlib import Path
@@ -8,22 +9,57 @@ from tempfile import TemporaryDirectory
 from typing import Any, Dict, List
 
 TEST_ROOT = Path(__file__).parent.resolve()
-PROJECT_ROOT = TEST_ROOT.parent
-BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs")
+# subprojects/nix-eval-jobs/tests
+# PROJECT_ROOT = TEST_ROOT.parent.parent.parent
+# BIN = PROJECT_ROOT.joinpath("outputs", "out", "bin", "nix-eval-jobs")
+BIN = "nix-eval-jobs"
+
+
+def check_gc_root(gcRootDir: str, drvPath: str):
+    """
+    Make sure the expected GC root exists in the given dir
+    """
+    link_name = os.path.basename(drvPath)
+    symlink_path = os.path.join(gcRootDir, link_name)
+    assert os.path.islink(symlink_path) and drvPath == os.readlink(symlink_path)
+
+
+def evaluate(
+    tempdir: TemporaryDirectory,
+    expected_statuscode: int = 0,
+    extra_args: List[str] = [],
+) -> tuple[Dict[str, Dict[str, Any]], str]:
+    if nixpkgs_path := os.getenv("NEJ_NIXPKGS_PATH"):
+        if "--flake" in extra_args:
+            extra_args.extend(["--override-input", "nixpkgs", f"path:{nixpkgs_path}"])
+        else:
+            extra_args.extend(["--arg", "pkgs", f"import {nixpkgs_path} {{}}"])
+
+    cmd = [
+        str(BIN),
+        "--gc-roots-dir",
+        tempdir,
+        "--meta",
+        "--extra-experimental-features",
+        "flakes",
+    ] + extra_args
+    res = subprocess.run(
+        cmd,
+        cwd=TEST_ROOT.joinpath("assets"),
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    assert res.returncode == expected_statuscode
+    print(res.stdout)
+    print(res.stderr)
+    return [json.loads(r) for r in res.stdout.split("\n") if r], res.stderr
 
 
 def common_test(extra_args: List[str]) -> List[Dict[str, Any]]:
     with TemporaryDirectory() as tempdir:
-        cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args
-        res = subprocess.run(
-            cmd,
-            cwd=TEST_ROOT.joinpath("assets"),
-            text=True,
-            check=True,
-            stdout=subprocess.PIPE,
-        )
-
-        results = [json.loads(r) for r in res.stdout.split("\n") if r]
+        results, _ = evaluate(tempdir, 0, extra_args)
         assert len(results) == 4
 
         built_job = results[0]
@@ -32,14 +68,17 @@ def common_test(extra_args: List[str]) -> List[Dict[str, Any]]:
         assert built_job["outputs"]["out"].startswith("/nix/store")
         assert built_job["drvPath"].endswith(".drv")
         assert built_job["meta"]["broken"] is False
+        check_gc_root(tempdir, built_job['drvPath'])
 
         dotted_job = results[1]
         assert dotted_job["attr"] == '"dotted.attr"'
         assert dotted_job["attrPath"] == ["dotted.attr"]
+        check_gc_root(tempdir, dotted_job['drvPath'])
 
         recurse_drv = results[2]
         assert recurse_drv["attr"] == "recurse.drvB"
         assert recurse_drv["name"] == "drvB"
+        check_gc_root(tempdir, recurse_drv['drvPath'])
 
         substituted_job = results[3]
         assert substituted_job["attr"] == "substitutedJob"
@@ -73,48 +112,31 @@ def test_expression() -> None:
 
 def test_eval_error() -> None:
     with TemporaryDirectory() as tempdir:
-        cmd = [
-            str(BIN),
-            "--gc-roots-dir",
+        results, _ = evaluate(
             tempdir,
-            "--meta",
-            "--workers",
-            "1",
-            "--flake",
-            ".#legacyPackages.x86_64-linux.brokenPkgs",
-        ]
-        res = subprocess.run(
-            cmd,
-            cwd=TEST_ROOT.joinpath("assets"),
-            text=True,
-            stdout=subprocess.PIPE,
+            0,
+            ["--workers", "1", "--flake", ".#legacyPackages.x86_64-linux.brokenPkgs"],
         )
-        print(res.stdout)
-        attrs = json.loads(res.stdout)
-        assert attrs["attr"] == "brokenPackage"
-        assert "this is an evaluation error" in attrs["error"]
+        assert len(results) == 1
+
+        attr = results[0]
+        assert attr["attr"] == "brokenPackage"
+        assert "this is an evaluation error" in attr["error"]
 
 
 @pytest.mark.infiniterecursion
 def test_recursion_error() -> None:
     with TemporaryDirectory() as tempdir:
-        cmd = [
-            str(BIN),
-            "--gc-roots-dir",
+        results, stderr = evaluate(
             tempdir,
-            "--meta",
-            "--workers",
-            "1",
-            "--flake",
-            ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs",
-        ]
-        res = subprocess.run(
-            cmd,
-            cwd=TEST_ROOT.joinpath("assets"),
-            text=True,
-            stderr=subprocess.PIPE,
+            1,
+            [
+                "--workers",
+                "1",
+                "--flake",
+                ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs",
+            ],
         )
-        assert res.returncode == 1
-        print(res.stderr)
-        assert "packageWithInfiniteRecursion" in res.stderr
-        assert "possible infinite recursion" in res.stderr
+        print(stderr)
+        assert "packageWithInfiniteRecursion" in stderr
+        assert "possible infinite recursion" in stderr