diff --git a/package.nix b/package.nix index 2a5a7143c..d9e7678c2 100644 --- a/package.nix +++ b/package.nix @@ -10,6 +10,7 @@ # anyway, but also allows easily overriding the patch logic. boehmgc-nix ? __forDefaults.boehmgc-nix, boehmgc, + buildPackages, nlohmann_json, build-release-notes ? __forDefaults.build-release-notes, boost, @@ -178,6 +179,14 @@ let ./tests/unit (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ]; + + # python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858 + lixPythonForBuild = python3.pythonOnBuildForHost.withPackages (p: [ + p.pytest + p.pytest-xdist + p.python-frontmatter + p.aiohttp + ]); in assert (lintInsteadOfBuild -> lix-clang-tidy != null); stdenv.mkDerivation (finalAttrs: { @@ -247,12 +256,7 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ - # python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858 - (python3.pythonOnBuildForHost.withPackages (p: [ - p.pytest - p.pytest-xdist - p.python-frontmatter - ])) + lixPythonForBuild meson ninja cmake @@ -328,7 +332,11 @@ stdenv.mkDerivation (finalAttrs: { propagatedBuildInputs = lib.optionals (!finalAttrs.dontBuild) maybePropagatedInputs; - disallowedReferences = [ boost ]; + disallowedReferences = [ + boost + buildPackages.python3 + lixPythonForBuild + ]; # Needed for Meson to find Boost. # https://github.com/NixOS/nixpkgs/issues/86131. @@ -505,9 +513,10 @@ stdenv.mkDerivation (finalAttrs: { # wrapped python instead of build inputs for its python inputs p.pytest p.pytest-xdist + p.aiohttp + p.python-frontmatter p.yapf - p.python-frontmatter p.requests p.xdg-base-dirs p.packaging diff --git a/tests/functional2/store/test_evil_nars.py b/tests/functional2/store/test_evil_nars.py index 937eed0ce..e1df94d84 100644 --- a/tests/functional2/store/test_evil_nars.py +++ b/tests/functional2/store/test_evil_nars.py @@ -1,5 +1,7 @@ import pytest import os +import unicodedata +from pathlib import Path from ..testlib.nar import * from ..testlib.fixtures import Nix from io import BytesIO @@ -12,59 +14,59 @@ meow_nfd = meow_nfd_.encode('utf-8') assert meow_nfc != meow_nfd EVIL_NARS: list[tuple[str, NarItem]] = [ - ('valid-dir-1', Directory([ - (b'a-nested', Directory([ + ('valid-dir-1', DirectoryUnordered([ + (b'a-nested', DirectoryUnordered([ (b'loopy', Symlink(b'../abc-nested')) ])), (b'b-file', Regular(False, b'meow kbity')), (b'c-exe', Regular(True, b'#!/usr/bin/env cat\nmeow kbity')), ])), - ('invalid-slashes-1', Directory([ + ('invalid-slashes-1', DirectoryUnordered([ (b'meow', Symlink(b'meowmeow')), (b'meow/nya', Regular(False, b'eepy')), ])), - ('invalid-dot-1', Directory([ + ('invalid-dot-1', DirectoryUnordered([ (b'.', Symlink(b'meowmeow')), ])), - ('invalid-dot-2', Directory([ + ('invalid-dot-2', DirectoryUnordered([ (b'..', Symlink(b'meowmeow')), ])), - ('invalid-nul-1', Directory([ + ('invalid-nul-1', DirectoryUnordered([ (b'meow\0nya', Symlink(b'meowmeow')), ])), - ('invalid-misorder-1', Directory([ + ('invalid-misorder-1', DirectoryUnordered([ (b'zzz', Regular(False, b'eepy')), (b'kbity', Regular(False, b'meow')), ])), - ('invalid-dupe-1', Directory([ + ('invalid-dupe-1', DirectoryUnordered([ (b'zzz', Regular(False, b'eepy')), (b'zzz', Regular(False, b'meow')), ])), - ('invalid-dupe-2', Directory([ - (b'zzz', Directory([ + ('invalid-dupe-2', DirectoryUnordered([ + (b'zzz', DirectoryUnordered([ (b'meow', Regular(False, b'kbity')) ])), (b'zzz', Regular(False, b'meow')), ])), - ('invalid-dupe-3', Directory([ - (b'zzz', Directory([ + ('invalid-dupe-3', DirectoryUnordered([ + (b'zzz', DirectoryUnordered([ (b'meow', Regular(False, b'kbity')) ])), - (b'zzz', Directory([ + (b'zzz', DirectoryUnordered([ (b'meow', Regular(False, b'kbityy')) ])), ])), - ('invalid-dupe-4', Directory([ + ('invalid-dupe-4', DirectoryUnordered([ (b'zzz', Symlink(b'../kbity')), - (b'zzz', Directory([ + (b'zzz', DirectoryUnordered([ (b'meow', Regular(False, b'kbityy')) ])), ])), - ('invalid-casehack-1', Directory([ + ('invalid-casehack-1', DirectoryUnordered([ (b'ZZZ~nix~case~hack~2', Regular(False, b'meow')), (b'zzz~nix~case~hack~1', Regular(False, b'eepy')), ])), - ('invalid-casehack-2', Directory([ + ('invalid-casehack-2', DirectoryUnordered([ (b'ZZZ~nix~case~hack~1', Regular(False, b'meow')), (b'zzz~nix~case~hack~1', Regular(False, b'eepy')), ])), @@ -106,12 +108,12 @@ def test_unicode_evil_nar(nix: Nix, tmp_path: Path): # normalization is not applied to this system pytest.skip('filesystem does not use unicode normalization') - test_evil_nar(nix, 'invalid-unicode-normalization-1', Directory([ + test_evil_nar(nix, 'invalid-unicode-normalization-1', DirectoryUnordered([ # méow (meow_nfd, Regular(False, b'eepy')), (meow_nfc, Symlink(b'meowmeow')), ])) - test_evil_nar(nix, 'invalid-unicode-normalization-2', Directory([ + test_evil_nar(nix, 'invalid-unicode-normalization-2', DirectoryUnordered([ # méow (meow_nfd, Symlink(b'meowmeow')), (meow_nfc, Regular(False, b'eepy')), diff --git a/tests/functional2/testlib/fixtures.py b/tests/functional2/testlib/fixtures/__init__.py similarity index 97% rename from tests/functional2/testlib/fixtures.py rename to tests/functional2/testlib/fixtures/__init__.py index 259132c14..49d686772 100644 --- a/tests/functional2/testlib/fixtures.py +++ b/tests/functional2/testlib/fixtures/__init__.py @@ -20,6 +20,8 @@ class CommandResult: def ok(self): if self.rc != 0: + print('stdout:', self.stdout_s) + print('stderr:', self.stderr_s) raise subprocess.CalledProcessError(returncode=self.rc, cmd=self.cmd, stderr=self.stderr, @@ -28,6 +30,8 @@ class CommandResult: def expect(self, rc: int): if self.rc != rc: + print('stdout:', self.stdout_s) + print('stderr:', self.stderr_s) raise subprocess.CalledProcessError(returncode=self.rc, cmd=self.cmd, stderr=self.stderr, diff --git a/tests/functional2/testlib/fixtures/http_server.py b/tests/functional2/testlib/fixtures/http_server.py new file mode 100644 index 000000000..857772975 --- /dev/null +++ b/tests/functional2/testlib/fixtures/http_server.py @@ -0,0 +1,124 @@ +""" +HTTP server fixture for tests which binds to an auto-assigned port on localhost. +""" +import asyncio +import contextlib +import dataclasses +import time +import socket +import threading +from typing import Tuple +from queue import Queue +import aiohttp.web as web + + +@dataclasses.dataclass +class HttpServer: + app: web.Application + port: int + + +class Event_ts(asyncio.Event): + """ + A thread safe version of the asyncio Event + + NOTE: clear() is not thread safe + + Taken from https://stackoverflow.com/a/33006667 + """ + + def __init__(self, + *args, + loop: asyncio.AbstractEventLoop | None = None, + **kwargs): + """ + Creates a thread-safe event for the given loop (or the loop of the current thread). + """ + super().__init__(*args, **kwargs) + self.target_loop = loop or asyncio.get_running_loop() + + def set(self): + self.target_loop.call_soon_threadsafe(super().set) + + +def _make_localhost_socket() -> Tuple[socket.socket, int]: + """Creates a localhost-bound socket with an auto-assigned port.""" + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind(('::1', 0)) + # Shouldn't matter because we dynamically allocate ports, but this is generally preferred. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + _, port = sock.getsockname()[:2] + + return (sock, port) + + +def _server_thread(app: web.Application, sock: socket.socket, + shutdown_ev_q: Queue): + + async def async_main(): + nonlocal app, sock + # Due to Reasons(tm) of event loop lifecycles and stuff of the sort, + # it's far easier to just send the event object to the other thread + # from inside the loop where it already knows which loop it is. + shutdown_ev = Event_ts() + shutdown_ev_q.put(shutdown_ev) + + runner = web.AppRunner(app, handle_signals=False) + await runner.setup() + site = web.SockSite(runner, sock) + await site.start() + await shutdown_ev.wait() + await runner.cleanup() + + asyncio.run(async_main()) + + +@contextlib.contextmanager +def http_server(app: web.Application): + """ + Creates an http server on an automatically chosen port on the host + running the given web.Application, gives you the port for it. + + The server is run on a separate thread. + """ + # n.b. pytest doesn't directly support asyncio. There's a bunch of + # complexity that we could go through to do this or we could just throw the + # async on a thread which was what we would do to the web server anyway if + # it was blocking. + shutdown_ev_q = Queue() + thr = None + sock = None + shutdown_ev = None + try: + sock, port = _make_localhost_socket() + thr = threading.Thread(target=_server_thread, + args=(app, sock, shutdown_ev_q), + name=f'functional2 httpd [::1]:{port}') + thr.start() + shutdown_ev = shutdown_ev_q.get() + yield HttpServer(app=app, port=port) + finally: + if shutdown_ev: + shutdown_ev.set() + if thr: + thr.join() + if sock: + sock.close() + + +def dev_main(): + """A little test server for poking at this manually""" + + async def root(_req: web.Request): + return web.Response(body='hello world') + + app = web.Application() + app.add_routes([web.get('/', root)]) + + with http_server(app) as httpd: + print(f'Listening on http://[::1]:{httpd.port}') + time.sleep(3600) + + +if __name__ == '__main__': + dev_main() diff --git a/tests/functional2/testlib/nar.py b/tests/functional2/testlib/nar.py index ffeb5a809..927e3feab 100644 --- a/tests/functional2/testlib/nar.py +++ b/tests/functional2/testlib/nar.py @@ -7,14 +7,14 @@ See "The Purely Functional Software Deployment Model", fig. 5.2 [1]. from abc import ABCMeta, abstractmethod import dataclasses import struct -from pathlib import Path from typing import Protocol -import unicodedata class Writable(Protocol): """Realistically could just be IOBase but this is more constrained""" - def write(self, data: bytes, /) -> int: ... + + def write(self, data: bytes, /) -> int: + ... @dataclasses.dataclass @@ -68,7 +68,7 @@ class Regular(NarItem): @dataclasses.dataclass -class Directory(NarItem): +class DirectoryUnordered(NarItem): entries: list[tuple[bytes, NarItem]] """Entries in the directory, not required to be in order because this nar is evil""" type_ = b'directory' @@ -91,6 +91,15 @@ class Directory(NarItem): self.entry(out, name, entry) +@dataclasses.dataclass +class Directory(NarItem): + entries: dict[bytes, NarItem] + + def serialize_type(self, out: NarListener): + for name, item in sorted(self.entries.items(), key=lambda v: v[0]): + DirectoryUnordered.entry(out, name, item) + + @dataclasses.dataclass class Symlink(NarItem): target: bytes