guiwf/lix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge changes Iaa096f7d,I48b7c1be into main

Browse Source 
* changes:
  functional2: tidy up some of the NAR handling
  functional2: add http server fixture
This commit is contained in:
jade
2025-01-26 02:05:38 +00:00
committed by Gerrit Code Review
parent 0ad79775b6 aa1aacb3e0
commit 0262879b0d
5 changed files with 179 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.nix
+17 -8
View File
@@ -10,6 +10,7 @@
# anyway, but also allows easily overriding the patch logic.
boehmgc-nix ? __forDefaults.boehmgc-nix,
boehmgc,
buildPackages,
nlohmann_json,
build-release-notes ? __forDefaults.build-release-notes,
boost,
@@ -178,6 +179,14 @@ let
./tests/unit
(fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
];
# python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858
lixPythonForBuild = python3.pythonOnBuildForHost.withPackages (p: [
p.pytest
p.pytest-xdist
p.python-frontmatter
p.aiohttp
]);
in
assert (lintInsteadOfBuild -> lix-clang-tidy != null);
stdenv.mkDerivation (finalAttrs: {
@@ -247,12 +256,7 @@ stdenv.mkDerivation (finalAttrs: {
nativeBuildInputs =
[
# python3.withPackages does not splice properly, see https://github.com/NixOS/nixpkgs/issues/305858
(python3.pythonOnBuildForHost.withPackages (p: [
p.pytest
p.pytest-xdist
p.python-frontmatter
]))
lixPythonForBuild
meson
ninja
cmake
@@ -328,7 +332,11 @@ stdenv.mkDerivation (finalAttrs: {
propagatedBuildInputs = lib.optionals (!finalAttrs.dontBuild) maybePropagatedInputs;
disallowedReferences = [ boost ];
disallowedReferences = [
boost
buildPackages.python3
lixPythonForBuild
];
# Needed for Meson to find Boost.
# https://github.com/NixOS/nixpkgs/issues/86131.
@@ -505,9 +513,10 @@ stdenv.mkDerivation (finalAttrs: {
# wrapped python instead of build inputs for its python inputs
p.pytest
p.pytest-xdist
p.aiohttp
p.python-frontmatter
p.yapf
p.python-frontmatter
p.requests
p.xdg-base-dirs
p.packaging
tests/functional2/store/test_evil_nars.py
+21 -19
View File
@@ -1,5 +1,7 @@
import pytest
import os
import unicodedata
from pathlib import Path
from ..testlib.nar import *
from ..testlib.fixtures import Nix
from io import BytesIO
@@ -12,59 +14,59 @@ meow_nfd = meow_nfd_.encode('utf-8')
assert meow_nfc != meow_nfd
EVIL_NARS: list[tuple[str, NarItem]] = [
('valid-dir-1', Directory([
(b'a-nested', Directory([
('valid-dir-1', DirectoryUnordered([
(b'a-nested', DirectoryUnordered([
(b'loopy', Symlink(b'../abc-nested'))
])),
(b'b-file', Regular(False, b'meow kbity')),
(b'c-exe', Regular(True, b'#!/usr/bin/env cat\nmeow kbity')),
])),
('invalid-slashes-1', Directory([
('invalid-slashes-1', DirectoryUnordered([
(b'meow', Symlink(b'meowmeow')),
(b'meow/nya', Regular(False, b'eepy')),
])),
('invalid-dot-1', Directory([
('invalid-dot-1', DirectoryUnordered([
(b'.', Symlink(b'meowmeow')),
])),
('invalid-dot-2', Directory([
('invalid-dot-2', DirectoryUnordered([
(b'..', Symlink(b'meowmeow')),
])),
('invalid-nul-1', Directory([
('invalid-nul-1', DirectoryUnordered([
(b'meow\0nya', Symlink(b'meowmeow')),
])),
('invalid-misorder-1', Directory([
('invalid-misorder-1', DirectoryUnordered([
(b'zzz', Regular(False, b'eepy')),
(b'kbity', Regular(False, b'meow')),
])),
('invalid-dupe-1', Directory([
('invalid-dupe-1', DirectoryUnordered([
(b'zzz', Regular(False, b'eepy')),
(b'zzz', Regular(False, b'meow')),
])),
('invalid-dupe-2', Directory([
(b'zzz', Directory([
('invalid-dupe-2', DirectoryUnordered([
(b'zzz', DirectoryUnordered([
(b'meow', Regular(False, b'kbity'))
])),
(b'zzz', Regular(False, b'meow')),
])),
('invalid-dupe-3', Directory([
(b'zzz', Directory([
('invalid-dupe-3', DirectoryUnordered([
(b'zzz', DirectoryUnordered([
(b'meow', Regular(False, b'kbity'))
])),
(b'zzz', Directory([
(b'zzz', DirectoryUnordered([
(b'meow', Regular(False, b'kbityy'))
])),
])),
('invalid-dupe-4', Directory([
('invalid-dupe-4', DirectoryUnordered([
(b'zzz', Symlink(b'../kbity')),
(b'zzz', Directory([
(b'zzz', DirectoryUnordered([
(b'meow', Regular(False, b'kbityy'))
])),
])),
('invalid-casehack-1', Directory([
('invalid-casehack-1', DirectoryUnordered([
(b'ZZZ~nix~case~hack~2', Regular(False, b'meow')),
(b'zzz~nix~case~hack~1', Regular(False, b'eepy')),
])),
('invalid-casehack-2', Directory([
('invalid-casehack-2', DirectoryUnordered([
(b'ZZZ~nix~case~hack~1', Regular(False, b'meow')),
(b'zzz~nix~case~hack~1', Regular(False, b'eepy')),
])),
@@ -106,12 +108,12 @@ def test_unicode_evil_nar(nix: Nix, tmp_path: Path):
# normalization is not applied to this system
pytest.skip('filesystem does not use unicode normalization')
test_evil_nar(nix, 'invalid-unicode-normalization-1', Directory([
test_evil_nar(nix, 'invalid-unicode-normalization-1', DirectoryUnordered([
# méow
(meow_nfd, Regular(False, b'eepy')),
(meow_nfc, Symlink(b'meowmeow')),
]))
test_evil_nar(nix, 'invalid-unicode-normalization-2', Directory([
test_evil_nar(nix, 'invalid-unicode-normalization-2', DirectoryUnordered([
# méow
(meow_nfd, Symlink(b'meowmeow')),
(meow_nfc, Regular(False, b'eepy')),
tests/functional2/testlib/fixtures.py → tests/functional2/testlib/fixtures/__init__.py
+4
View File
@@ -20,6 +20,8 @@ class CommandResult:
def ok(self):
if self.rc != 0:
print('stdout:', self.stdout_s)
print('stderr:', self.stderr_s)
raise subprocess.CalledProcessError(returncode=self.rc,
cmd=self.cmd,
stderr=self.stderr,
@@ -28,6 +30,8 @@ class CommandResult:
def expect(self, rc: int):
if self.rc != rc:
print('stdout:', self.stdout_s)
print('stderr:', self.stderr_s)
raise subprocess.CalledProcessError(returncode=self.rc,
cmd=self.cmd,
stderr=self.stderr,
tests/functional2/testlib/fixtures/http_server.py
+124
View File
@@ -0,0 +1,124 @@
"""
HTTP server fixture for tests which binds to an auto-assigned port on localhost.
"""
import asyncio
import contextlib
import dataclasses
import time
import socket
import threading
from typing import Tuple
from queue import Queue
import aiohttp.web as web
@dataclasses.dataclass
class HttpServer:
app: web.Application
port: int
class Event_ts(asyncio.Event):
"""
A thread safe version of the asyncio Event
NOTE: clear() is not thread safe
Taken from https://stackoverflow.com/a/33006667
"""
def __init__(self,
*args,
loop: asyncio.AbstractEventLoop | None = None,
**kwargs):
"""
Creates a thread-safe event for the given loop (or the loop of the current thread).
"""
super().__init__(*args, **kwargs)
self.target_loop = loop or asyncio.get_running_loop()
def set(self):
self.target_loop.call_soon_threadsafe(super().set)
def _make_localhost_socket() -> Tuple[socket.socket, int]:
"""Creates a localhost-bound socket with an auto-assigned port."""
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
sock.bind(('::1', 0))
# Shouldn't matter because we dynamically allocate ports, but this is generally preferred.
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
_, port = sock.getsockname()[:2]
return (sock, port)
def _server_thread(app: web.Application, sock: socket.socket,
shutdown_ev_q: Queue):
async def async_main():
nonlocal app, sock
# Due to Reasons(tm) of event loop lifecycles and stuff of the sort,
# it's far easier to just send the event object to the other thread
# from inside the loop where it already knows which loop it is.
shutdown_ev = Event_ts()
shutdown_ev_q.put(shutdown_ev)
runner = web.AppRunner(app, handle_signals=False)
await runner.setup()
site = web.SockSite(runner, sock)
await site.start()
await shutdown_ev.wait()
await runner.cleanup()
asyncio.run(async_main())
@contextlib.contextmanager
def http_server(app: web.Application):
"""
Creates an http server on an automatically chosen port on the host
running the given web.Application, gives you the port for it.
The server is run on a separate thread.
"""
# n.b. pytest doesn't directly support asyncio. There's a bunch of
# complexity that we could go through to do this or we could just throw the
# async on a thread which was what we would do to the web server anyway if
# it was blocking.
shutdown_ev_q = Queue()
thr = None
sock = None
shutdown_ev = None
try:
sock, port = _make_localhost_socket()
thr = threading.Thread(target=_server_thread,
args=(app, sock, shutdown_ev_q),
name=f'functional2 httpd [::1]:{port}')
thr.start()
shutdown_ev = shutdown_ev_q.get()
yield HttpServer(app=app, port=port)
finally:
if shutdown_ev:
shutdown_ev.set()
if thr:
thr.join()
if sock:
sock.close()
def dev_main():
"""A little test server for poking at this manually"""
async def root(_req: web.Request):
return web.Response(body='hello world')
app = web.Application()
app.add_routes([web.get('/', root)])
with http_server(app) as httpd:
print(f'Listening on http://[::1]:{httpd.port}')
time.sleep(3600)
if __name__ == '__main__':
dev_main()
tests/functional2/testlib/nar.py
+13 -4
View File
@@ -7,14 +7,14 @@ See "The Purely Functional Software Deployment Model", fig. 5.2 [1].
from abc import ABCMeta, abstractmethod
import dataclasses
import struct
from pathlib import Path
from typing import Protocol
import unicodedata
class Writable(Protocol):
"""Realistically could just be IOBase but this is more constrained"""
def write(self, data: bytes, /) -> int: ...
def write(self, data: bytes, /) -> int:
...
@dataclasses.dataclass
@@ -68,7 +68,7 @@ class Regular(NarItem):
@dataclasses.dataclass
class Directory(NarItem):
class DirectoryUnordered(NarItem):
entries: list[tuple[bytes, NarItem]]
"""Entries in the directory, not required to be in order because this nar is evil"""
type_ = b'directory'
@@ -91,6 +91,15 @@ class Directory(NarItem):
self.entry(out, name, entry)
@dataclasses.dataclass
class Directory(NarItem):
entries: dict[bytes, NarItem]
def serialize_type(self, out: NarListener):
for name, item in sorted(self.entries.items(), key=lambda v: v[0]):
DirectoryUnordered.entry(out, name, item)
@dataclasses.dataclass
class Symlink(NarItem):
target: bytes