diff --git a/subprojects/nix-eval-jobs/.clang-format b/subprojects/nix-eval-jobs/.clang-format new file mode 100644 index 000000000..b389fd02b --- /dev/null +++ b/subprojects/nix-eval-jobs/.clang-format @@ -0,0 +1,3 @@ +BasedOnStyle: llvm +IndentWidth: 4 +SortIncludes: false diff --git a/subprojects/nix-eval-jobs/.github/dependabot.yml b/subprojects/nix-eval-jobs/.github/dependabot.yml new file mode 100644 index 000000000..5ace4600a --- /dev/null +++ b/subprojects/nix-eval-jobs/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/subprojects/nix-eval-jobs/.github/workflows/nix-github-actions.yml b/subprojects/nix-eval-jobs/.github/workflows/nix-github-actions.yml new file mode 100644 index 000000000..cafc26ccc --- /dev/null +++ b/subprojects/nix-eval-jobs/.github/workflows/nix-github-actions.yml @@ -0,0 +1,66 @@ +name: Nix actions + +on: + pull_request: + push: + branches: + - main + - staging + - release-* + +jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + echo "matrix=$(nix eval --json '.#githubActions.matrix')" >> "$GITHUB_OUTPUT" + + nix-build: + needs: nix-matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + - run: nix build -L ".#${{ matrix.attr }}" + tests: + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v25 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + extra_nix_config: | + accept-flake-config = true + - name: Build + run: nix develop -c bash -c 'meson setup -Db_sanitize=address,undefined build && ninja -C build' + - name: Run tests + run: nix develop -c pytest ./tests -m 'not infiniterecursion' + + # address sanitizer will lead to out-of-memory in the infinite recursion case + - name: Build without sanitizer + run: nix develop -c bash -c 'rm -rf build && meson setup build && ninja -C build' + - name: Run tests + run: nix develop -c pytest ./tests -m 'infiniterecursion' + + collect: + runs-on: ubuntu-latest + needs: + - nix-build + - tests + steps: + - run: "true" diff --git a/subprojects/nix-eval-jobs/.github/workflows/publish.yml b/subprojects/nix-eval-jobs/.github/workflows/publish.yml new file mode 100644 index 000000000..58660afa5 --- /dev/null +++ b/subprojects/nix-eval-jobs/.github/workflows/publish.yml @@ -0,0 +1,22 @@ +name: "Publish a flake to flakestry" +on: + push: + tags: + - "v?[0-9]+.[0-9]+.[0-9]+" + - "v?[0-9]+.[0-9]+" + workflow_dispatch: + inputs: + tag: + description: "The existing tag to publish" + type: "string" + required: true +jobs: + publish-flake: + runs-on: ubuntu-latest + permissions: + id-token: "write" + contents: "read" + steps: + - uses: flakestry/flakestry-publish@main + with: + version: "${{ inputs.tag || github.ref_name }}" diff --git a/subprojects/nix-eval-jobs/.gitignore b/subprojects/nix-eval-jobs/.gitignore new file mode 100644 index 000000000..c6f9d8c6f --- /dev/null +++ b/subprojects/nix-eval-jobs/.gitignore @@ -0,0 +1,57 @@ +.DS_Store +.idea +*.log + +tmp/ + + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# build directory +/build +# nix-build +/result + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# nix-direnv +.direnv diff --git a/subprojects/nix-eval-jobs/.mergify.yml b/subprojects/nix-eval-jobs/.mergify.yml new file mode 100644 index 000000000..21dea9be7 --- /dev/null +++ b/subprojects/nix-eval-jobs/.mergify.yml @@ -0,0 +1,18 @@ +queue_rules: + - name: default + merge_conditions: + - check-success=collect + +defaults: + actions: + queue: + allow_merging_configuration_change: true + method: rebase + +pull_request_rules: + - name: merge using the merge queue + conditions: + - base=main + - label~=merge-queue|dependencies + actions: + queue: {} diff --git a/subprojects/nix-eval-jobs/.prettierrc.js b/subprojects/nix-eval-jobs/.prettierrc.js new file mode 100644 index 000000000..dc1e020bf --- /dev/null +++ b/subprojects/nix-eval-jobs/.prettierrc.js @@ -0,0 +1,3 @@ +module.exports = { + "plugins": ["prettier-plugin-toml"], +}; diff --git a/subprojects/nix-eval-jobs/LICENSE.md b/subprojects/nix-eval-jobs/LICENSE.md new file mode 100644 index 000000000..1bbb815ce --- /dev/null +++ b/subprojects/nix-eval-jobs/LICENSE.md @@ -0,0 +1,614 @@ +# GNU General Public License + +_Version 3, 29 June 2007_ _Copyright © 2007 Free Software Foundation, Inc. +<>_ + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +## Preamble + +The GNU General Public License is a free, copyleft license for software and +other kinds of works. + +The licenses for most software and other practical works are designed to take +away your freedom to share and change the works. By contrast, the GNU General +Public License is intended to guarantee your freedom to share and change all +versions of a program--to make sure it remains free software for all its users. +We, the Free Software Foundation, use the GNU General Public License for most of +our software; it applies also to any other work released this way by its +authors. You can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for them if you wish), that you +receive source code or can get it if you want it, that you can change the +software or use pieces of it in new free programs, and that you know you can do +these things. + +To protect your rights, we need to prevent others from denying you these rights +or asking you to surrender the rights. Therefore, you have certain +responsibilities if you distribute copies of the software, or if you modify it: +responsibilities to respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a +fee, you must pass on to the recipients the same freedoms that you received. You +must make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: **(1)** +assert copyright on the software, and **(2)** offer you this License giving you +legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there +is no warranty for this free software. For both users' and authors' sake, the +GPL requires that modified versions be marked as changed, so that their problems +will not be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified +versions of the software inside them, although the manufacturer can do so. This +is fundamentally incompatible with the aim of protecting users' freedom to +change the software. The systematic pattern of such abuse occurs in the area of +products for individuals to use, which is precisely where it is most +unacceptable. Therefore, we have designed this version of the GPL to prohibit +the practice for those products. If such problems arise substantially in other +domains, we stand ready to extend this provision to those domains in future +versions of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. States +should not allow patents to restrict development and use of software on +general-purpose computers, but in those that do, we wish to avoid the special +danger that patents applied to a free program could make it effectively +proprietary. To prevent this, the GPL assures that patents cannot be used to +render the program non-free. + +The precise terms and conditions for copying, distribution and modification +follow. + +## TERMS AND CONDITIONS + +### 0. Definitions + +“This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of works, +such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this License. Each +licensee is addressed as “you”. “Licensees” and “recipients” may be individuals +or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in a +fashion requiring copyright permission, other than the making of an exact copy. +The resulting work is called a “modified version” of the earlier work or a work +“based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on the +Program. + +To “propagate” a work means to do anything with it that, without permission, +would make you directly or secondarily liable for infringement under applicable +copyright law, except executing it on a computer or modifying a private copy. +Propagation includes copying, distribution (with or without modification), +making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other parties to +make or receive copies. Mere interaction with a user through a computer network, +with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the extent +that it includes a convenient and prominently visible feature that **(1)** +displays an appropriate copyright notice, and **(2)** tells the user that there +is no warranty for the work (except to the extent that warranties are provided), +that licensees may convey the work under this License, and how to view a copy of +this License. If the interface presents a list of user commands or options, such +as a menu, a prominent item in the list meets this criterion. + +### 1. Source Code + +The “source code” for a work means the preferred form of the work for making +modifications to it. “Object code” means any non-source form of a work. + +A “Standard Interface” means an interface that either is an official standard +defined by a recognized standards body, or, in the case of interfaces specified +for a particular programming language, one that is widely used among developers +working in that language. + +The “System Libraries” of an executable work include anything, other than the +work as a whole, that **(a)** is included in the normal form of packaging a +Major Component, but which is not part of that Major Component, and **(b)** +serves only to enable use of the work with that Major Component, or to implement +a Standard Interface for which an implementation is available to the public in +source code form. A “Major Component”, in this context, means a major essential +component (kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to produce the +work, or an object code interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the source +code needed to generate, install, and (for an executable work) run the object +code and to modify the work, including scripts to control those activities. +However, it does not include the work's System Libraries, or general-purpose +tools or generally available free programs which are used unmodified in +performing those activities but which are not part of the work. For example, +Corresponding Source includes interface definition files associated with source +files for the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, such as by +intimate data communication or control flow between those subprograms and other +parts of the work. + +The Corresponding Source need not include anything that users can regenerate +automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +### 2. Basic Permissions + +All rights granted under this License are granted for the term of copyright on +the Program, and are irrevocable provided the stated conditions are met. This +License explicitly affirms your unlimited permission to run the unmodified +Program. The output from running a covered work is covered by this License only +if the output, given its content, constitutes a covered work. This License +acknowledges your rights of fair use or other equivalent, as provided by +copyright law. + +You may make, run and propagate covered works that you do not convey, without +conditions so long as your license otherwise remains in force. You may convey +covered works to others for the sole purpose of having them make modifications +exclusively for you, or provide you with facilities for running those works, +provided that you comply with the terms of this License in conveying all +material for which you do not control copyright. Those thus making or running +the covered works for you must do so exclusively on your behalf, under your +direction and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions +stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law + +No covered work shall be deemed part of an effective technological measure under +any applicable law fulfilling obligations under article 11 of the WIPO copyright +treaty adopted on 20 December 1996, or similar laws prohibiting or restricting +circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention is +effected by exercising rights under this License with respect to the covered +work, and you disclaim any intention to limit operation or modification of the +work as a means of enforcing, against the work's users, your or third parties' +legal rights to forbid circumvention of technological measures. + +### 4. Conveying Verbatim Copies + +You may convey verbatim copies of the Program's source code as you receive it, +in any medium, provided that you conspicuously and appropriately publish on each +copy an appropriate copyright notice; keep intact all notices stating that this +License and any non-permissive terms added in accord with section 7 apply to the +code; keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may +offer support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions + +You may convey a work based on the Program, or the modifications to produce it +from the Program, in the form of source code under the terms of section 4, +provided that you also meet all of these conditions: + +- **a)** The work must carry prominent notices stating that you modified it, and + giving a relevant date. +- **b)** The work must carry prominent notices stating that it is released under + this License and any conditions added under section 7. This requirement + modifies the requirement in section 4 to “keep intact all notices”. +- **c)** You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore apply, + along with any applicable section 7 additional terms, to the whole of the + work, and all its parts, regardless of how they are packaged. This License + gives no permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. +- **d)** If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive interfaces + that do not display Appropriate Legal Notices, your work need not make them do + so. + +A compilation of a covered work with other separate and independent works, which +are not by their nature extensions of the covered work, and which are not +combined with it such as to form a larger program, in or on a volume of a +storage or distribution medium, is called an “aggregate” if the compilation and +its resulting copyright are not used to limit the access or legal rights of the +compilation's users beyond what the individual works permit. Inclusion of a +covered work in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +### 6. Conveying Non-Source Forms + +You may convey a covered work in object code form under the terms of sections 4 +and 5, provided that you also convey the machine-readable Corresponding Source +under the terms of this License, in one of these ways: + +- **a)** Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the Corresponding + Source fixed on a durable physical medium customarily used for software + interchange. +- **b)** Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written offer, + valid for at least three years and valid for as long as you offer spare parts + or customer support for that product model, to give anyone who possesses the + object code either **(1)** a copy of the Corresponding Source for all the + software in the product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no more than + your reasonable cost of physically performing this conveying of source, or + **(2)** access to copy the Corresponding Source from a network server at no + charge. +- **c)** Convey individual copies of the object code with a copy of the written + offer to provide the Corresponding Source. This alternative is allowed only + occasionally and noncommercially, and only if you received the object code + with such an offer, in accord with subsection 6b. +- **d)** Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the Corresponding + Source in the same way through the same place at no further charge. You need + not require recipients to copy the Corresponding Source along with the object + code. If the place to copy the object code is a network server, the + Corresponding Source may be on a different server (operated by you or a third + party) that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the Corresponding + Source, you remain obligated to ensure that it is available for as long as + needed to satisfy these requirements. +- **e)** Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of the work + are being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the +Corresponding Source as a System Library, need not be included in conveying the +object code work. + +A “User Product” is either **(1)** a “consumer product”, which means any +tangible personal property which is normally used for personal, family, or +household purposes, or **(2)** anything designed or sold for incorporation into +a dwelling. In determining whether a product is a consumer product, doubtful +cases shall be resolved in favor of coverage. For a particular product received +by a particular user, “normally used” refers to a typical or common use of that +class of product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected to use, +the product. A product is a consumer product regardless of whether the product +has substantial commercial, industrial or non-consumer uses, unless such uses +represent the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, procedures, +authorization keys, or other information required to install and execute +modified versions of a covered work in that User Product from a modified version +of its Corresponding Source. The information must suffice to ensure that the +continued functioning of the modified object code is in no case prevented or +interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as part of a +transaction in which the right of possession and use of the User Product is +transferred to the recipient in perpetuity or for a fixed term (regardless of +how the transaction is characterized), the Corresponding Source conveyed under +this section must be accompanied by the Installation Information. But this +requirement does not apply if neither you nor any third party retains the +ability to install modified object code on the User Product (for example, the +work has been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates for a +work that has been modified or installed by the recipient, or for the User +Product in which it has been modified or installed. Access to a network may be +denied when the modification itself materially and adversely affects the +operation of the network or violates the rules and protocols for communication +across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord +with this section must be in a format that is publicly documented (and with an +implementation available to the public in source code form), and must require no +special password or key for unpacking, reading or copying. + +### 7. Additional Terms + +“Additional permissions” are terms that supplement the terms of this License by +making exceptions from one or more of its conditions. Additional permissions +that are applicable to the entire Program shall be treated as though they were +included in this License, to the extent that they are valid under applicable +law. If additional permissions apply only to part of the Program, that part may +be used separately under those permissions, but the entire Program remains +governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any +additional permissions from that copy, or from any part of it. (Additional +permissions may be written to require their own removal in certain cases when +you modify the work.) You may place additional permissions on material, added by +you to a covered work, for which you have or can give appropriate copyright +permission. + +Notwithstanding any other provision of this License, for material you add to a +covered work, you may (if authorized by the copyright holders of that material) +supplement the terms of this License with terms: + +- **a)** Disclaiming warranty or limiting liability differently from the terms + of sections 15 and 16 of this License; or +- **b)** Requiring preservation of specified reasonable legal notices or author + attributions in that material or in the Appropriate Legal Notices displayed by + works containing it; or +- **c)** Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in reasonable ways + as different from the original version; or +- **d)** Limiting the use for publicity purposes of names of licensors or + authors of the material; or +- **e)** Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or +- **f)** Requiring indemnification of licensors and authors of that material by + anyone who conveys the material (or modified versions of it) with contractual + assumptions of liability to the recipient, for any liability that these + contractual assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered “further restrictions” +within the meaning of section 10. If the Program as you received it, or any part +of it, contains a notice stating that it is governed by this License along with +a term that is a further restriction, you may remove that term. If a license +document contains a further restriction but permits relicensing or conveying +under this License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does not survive +such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, +in the relevant source files, a statement of the additional terms that apply to +those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a +separately written license, or stated as exceptions; the above requirements +apply either way. + +### 8. Termination + +You may not propagate or modify a covered work except as expressly provided +under this License. Any attempt otherwise to propagate or modify it is void, and +will automatically terminate your rights under this License (including any +patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a +particular copyright holder is reinstated **(a)** provisionally, unless and +until the copyright holder explicitly and finally terminates your license, and +**(b)** permanently, if the copyright holder fails to notify you of the +violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated +permanently if the copyright holder notifies you of the violation by some +reasonable means, this is the first time you have received notice of violation +of this License (for any work) from that copyright holder, and you cure the +violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of +parties who have received copies or rights from you under this License. If your +rights have been terminated and not permanently reinstated, you do not qualify +to receive new licenses for the same material under section 10. + +### 9. Acceptance Not Required for Having Copies + +You are not required to accept this License in order to receive or run a copy of +the Program. Ancillary propagation of a covered work occurring solely as a +consequence of using peer-to-peer transmission to receive a copy likewise does +not require acceptance. However, nothing other than this License grants you +permission to propagate or modify any covered work. These actions infringe +copyright if you do not accept this License. Therefore, by modifying or +propagating a covered work, you indicate your acceptance of this License to do +so. + +### 10. Automatic Licensing of Downstream Recipients + +Each time you convey a covered work, the recipient automatically receives a +license from the original licensors, to run, modify and propagate that work, +subject to this License. You are not responsible for enforcing compliance by +third parties with this License. + +An “entity transaction” is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered work results +from an entity transaction, each party to that transaction who receives a copy +of the work also receives whatever licenses to the work the party's predecessor +in interest had or could give under the previous paragraph, plus a right to +possession of the Corresponding Source of the work from the predecessor in +interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights +granted or affirmed under this License. For example, you may not impose a +license fee, royalty, or other charge for exercise of rights granted under this +License, and you may not initiate litigation (including a cross-claim or +counterclaim in a lawsuit) alleging that any patent claim is infringed by +making, using, selling, offering for sale, or importing the Program or any +portion of it. + +### 11. Patents + +A “contributor” is a copyright holder who authorizes use under this License of +the Program or a work on which the Program is based. The work thus licensed is +called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or +controlled by the contributor, whether already acquired or hereafter acquired, +that would be infringed by some manner, permitted by this License, of making, +using, or selling its contributor version, but do not include claims that would +be infringed only as a consequence of further modification of the contributor +version. For purposes of this definition, “control” includes the right to grant +patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent +license under the contributor's essential patent claims, to make, use, sell, +offer for sale, import and otherwise run, modify and propagate the contents of +its contributor version. + +In the following three paragraphs, a “patent license” is any express agreement +or commitment, however denominated, not to enforce a patent (such as an express +permission to practice a patent or covenant not to sue for patent infringement). +To “grant” such a patent license to a party means to make such an agreement or +commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the +Corresponding Source of the work is not available for anyone to copy, free of +charge and under the terms of this License, through a publicly available network +server or other readily accessible means, then you must either **(1)** cause the +Corresponding Source to be so available, or **(2)** arrange to deprive yourself +of the benefit of the patent license for this particular work, or **(3)** +arrange, in a manner consistent with the requirements of this License, to extend +the patent license to downstream recipients. “Knowingly relying” means you have +actual knowledge that, but for the patent license, your conveying the covered +work in a country, or your recipient's use of the covered work in a country, +would infringe one or more identifiable patents in that country that you have +reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you +convey, or propagate by procuring conveyance of, a covered work, and grant a +patent license to some of the parties receiving the covered work authorizing +them to use, propagate, modify or convey a specific copy of the covered work, +then the patent license you grant is automatically extended to all recipients of +the covered work and works based on it. + +A patent license is “discriminatory” if it does not include within the scope of +its coverage, prohibits the exercise of, or is conditioned on the non-exercise +of one or more of the rights that are specifically granted under this License. +You may not convey a covered work if you are a party to an arrangement with a +third party that is in the business of distributing software, under which you +make payment to the third party based on the extent of your activity of +conveying the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory patent +license **(a)** in connection with copies of the covered work conveyed by you +(or copies made from those copies), or **(b)** primarily for and in connection +with specific products or compilations that contain the covered work, unless you +entered into that arrangement, or that patent license was granted, prior to 28 +March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied +license or other defenses to infringement that may otherwise be available to you +under applicable patent law. + +### 12. No Surrender of Others' Freedom + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not excuse +you from the conditions of this License. If you cannot convey a covered work so +as to satisfy simultaneously your obligations under this License and any other +pertinent obligations, then as a consequence you may not convey it at all. For +example, if you agree to terms that obligate you to collect a royalty for +further conveying from those to whom you convey the Program, the only way you +could satisfy both those terms and this License would be to refrain entirely +from conveying the Program. + +### 13. Use with the GNU Affero General Public License + +Notwithstanding any other provision of this License, you have permission to link +or combine any covered work with a work licensed under version 3 of the GNU +Affero General Public License into a single combined work, and to convey the +resulting work. The terms of this License will continue to apply to the part +which is the covered work, but the special requirements of the GNU Affero +General Public License, section 13, concerning interaction through a network +will apply to the combination as such. + +### 14. Revised Versions of this License + +The Free Software Foundation may publish revised and/or new versions of the GNU +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program specifies +that a certain numbered version of the GNU General Public License “or any later +version” applies to it, you have the option of following the terms and +conditions either of that numbered version or of any later version published by +the Free Software Foundation. If the Program does not specify a version number +of the GNU General Public License, you may choose any version ever published by +the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the +GNU General Public License can be used, that proxy's public statement of +acceptance of a version permanently authorizes you to choose that version for +the Program. + +Later license versions may give you additional or different permissions. +However, no additional obligations are imposed on any author or copyright holder +as a result of your choosing to follow a later version. + +### 15. Disclaimer of Warranty + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER +PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE +QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +### 16. Limitation of Liability + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY +COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, +INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE +THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE +PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY +HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16 + +If the disclaimer of warranty and limitation of liability provided above cannot +be given local legal effect according to their terms, reviewing courts shall +apply local law that most closely approximates an absolute waiver of all civil +liability in connection with the Program, unless a warranty or assumption of +liability accompanies a copy of the Program in return for a fee. + +_END OF TERMS AND CONDITIONS_ + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use +to the public, the best way to achieve this is to make it free software which +everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively state the exclusion of +warranty; and each file should have at least the “copyright” line and a pointer +to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like +this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type 'show c' for details. + +The hypothetical commands `show w` and `show c` should show the appropriate +parts of the General Public License. Of course, your program's commands might be +different; for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if +any, to sign a “copyright disclaimer” for the program, if necessary. For more +information on this, and how to apply and follow the GNU GPL, see +<>. + +The GNU General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may consider +it more useful to permit linking proprietary applications with the library. If +this is what you want to do, use the GNU Lesser General Public License instead +of this License. But first, please read +<>. diff --git a/subprojects/nix-eval-jobs/README.md b/subprojects/nix-eval-jobs/README.md new file mode 100644 index 000000000..e2c8bcc18 --- /dev/null +++ b/subprojects/nix-eval-jobs/README.md @@ -0,0 +1,156 @@ +# nix-eval-jobs + +> [!NOTE] +> This is a fork of nix-eval-jobs that works with Lix. + +This project evaluates nix attribute sets in parallel with streamable json +output. This is useful for time and memory intensive evaluations such as NixOS +machines, i.e. in a CI context. The evaluation is done with a controllable +number of threads that are restarted when their memory consumption exceeds a +certain threshold. + +To facilitate integration, nix-eval-jobs creates garbage collection roots for +each evaluated derivation (drv file, not the build) within the provided +attribute. This prevents race conditions between the nix garbage collection +service and user-started nix builds processes. + +## Why using nix-eval-jobs? + +- Faster evaluation by using threads +- Memory used for evaluation is reclaimed after nix-eval-jobs finish, so that + the build can use it. +- Evaluation of jobs can fail individually + +## Example + +In the following example we evaluate the hydraJobs attribute of the +[patchelf](https://github.com/NixOS/patchelf) flake: + +```console +$ nix-eval-jobs --gc-roots-dir gcroot --flake 'github:NixOS/patchelf#hydraJobs' +{"attr":"coverage","attrPath":["coverage"],"drvPath":"/nix/store/fmbqzaq8mim1423879lhn9whs6imx5w4-patchelf-coverage-0.18.0.drv","inputDrvs":{"/nix/store/23632hx2c98lbbjld279dx0w08lxn6kp-hook.drv":["out"],"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/vslywm6kbazi37q1vbq8y7bi884yc6yx-lcov-1.16.drv":["out"],"/nix/store/y964yq4vz1gsn7azd44vyg65gnr4gpvi-hook.drv":["out"]},"name":"patchelf-coverage-0.18.0","outputs":{"out":"/nix/store/gfni9sbhhwhxxfqziq1fs3n82bvw962l-patchelf-coverage-0.18.0"},"system":"x86_64-linux"} +{"attr":"patchelf-win32","attrPath":["patchelf-win32"],"drvPath":"/nix/store/s38l0fg5ja6j8qpws7slw2ws0c6v0qcf-patchelf-i686-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"],"/nix/store/fbnhh18m4jh6cwa92am2sv3aqzjnzpdd-stdenv-linux.drv":["out"]},"name":"patchelf-i686-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/w8r4h1xk71fryb99df8aszp83kfhw3bc-patchelf-i686-w64-mingw32-0.18.0"},"system":"x86_64-linux"} +{"attr":"patchelf-win64","attrPath":["patchelf-win64"],"drvPath":"/nix/store/wxpym6d3dxr1w9syhinp7f058gwxfmd3-patchelf-x86_64-w64-mingw32-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/71lv5lsr1y59bv1b91jc9gg0n85kf1sq-stdenv-linux.drv":["out"],"/nix/store/b2p151ilwqpd47fbmzz50a5cmj12ixbf-hook.drv":["out"]},"name":"patchelf-x86_64-w64-mingw32-0.18.0","outputs":{"out":"/nix/store/fkq5428l2xsb84yj0cc6q1lkvsrga7sv-patchelf-x86_64-w64-mingw32-0.18.0"},"system":"x86_64-linux"} +{"attr":"release","attrPath":["release"],"drvPath":"/nix/store/3xpwg8f623dpkh6cblv2fzcq5n99xl0j-patchelf-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9rmihrl9ys0sap6827xyns0y73vqafjx-patchelf-0.18.0.drv":["out"],"/nix/store/am2zqx3pyc1i14f888jna785h0f841sg-patchelf-0.18.0.drv":["out"],"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv":["out"],"/nix/store/csjiccxbwpfv55m8kqs2xwrkkha14dnq-patchelf-0.18.0.drv":["out"],"/nix/store/jsrnpxdx5vmpnakd9bkb3sk3lgh0k8hm-patchelf-0.18.0.drv":["out"],"/nix/store/k8a51ax83554c67g98xf3y751vjgjs7m-patchelf-0.18.0.drv":["out"],"/nix/store/wq3ncl207isqqkqmsa5ql4fg19jbrhxg-stdenv-linux.drv":["out"]},"name":"patchelf-0.18.0","outputs":{"out":"/nix/store/d0mzprvv3vhasj23r1a6qn8qip0srbc4-patchelf-0.18.0"},"system":"x86_64-linux"} +{"attr":"tarball","attrPath":["tarball"],"drvPath":"/nix/store/c0gg7lj101xhd8v2b3cjl5dwwkpxfc0q-patchelf-tarball-0.18.0.drv","inputDrvs":{"/nix/store/6z1jfnqqgyqr221zgbpm30v91yfj3r45-bash-5.1-p16.drv":["out"],"/nix/store/9d754glmsvpjm5kxvgsjslvgv356kbmn-libtool-2.4.7.drv":["out"],"/nix/store/ap9g09fxbicj836zm88d56dn3ff4clxl-stdenv-linux.drv":["out"],"/nix/store/f1ksgsyplvb0sli4pls6k6vsfvmv519d-autoconf-2.71.drv":["out"],"/nix/store/jf58lcnch1bmpbi2188c59w5zr1cqrx2-automake-1.16.5.drv":["out"]},"name":"patchelf-tarball-0.18.0","outputs":{"out":"/nix/store/72pz5awc7gpwdqxrdsy8j0bvg2n7z78q-patchelf-tarball-0.18.0"},"system":"x86_64-linux"} +``` + +The output here is newline-seperated json according to https://jsonlines.org. + +The code is derived from [hydra's](https://github.com/nixos/hydra) eval-jobs +executable. + +## Further options + +```console +$ nix-eval-jobs --help +USAGE: nix-eval-jobs [options] expr + + --arg Pass the value *expr* as the argument *name* to Nix functions. + --argstr Pass the string *string* as the argument *name* to Nix functions. + --check-cache-status Check if the derivations are present locally or in any configured substituters (i.e. binary cache). The information will be exposed in the `isCached` field of the JSON output. + --debug Set the logging verbosity level to 'debug'. + --eval-store + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + + --expr treat the argument as a Nix expression + --flake build a flake + --force-recurse force recursion (don't respect recurseIntoAttrs) + --gc-roots-dir garbage collector roots directory + --help show usage information + --impure allow impure expressions + + --log-format Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`. + --max-memory-size maximum evaluation memory size in megabyte (4GiB per worker by default) + --meta include derivation meta field in output + --option Set the Nix configuration setting *name* to *value* (overriding `nix.conf`). + --override-flake Override the flake registries, redirecting *original-ref* to *resolved-ref*. + --override-input Override a specific flake input (e.g. `dwarffs/nixpkgs`). + --quiet Decrease the logging verbosity level. + --repair During evaluation, rewrite missing or corrupted files in the Nix store. During building, rebuild missing or corrupted store paths. + --show-trace print out a stack trace in case of evaluation errors + --verbose Increase the logging verbosity level. + --workers number of evaluate workers +``` + +## Potential use-cases for the tool + +**Faster evaluator in deployment tools.** When evaluating NixOS machines, +evaluation can take several minutes when run on a single core. This limits +scalability for large deployments with deployment tools such as +[NixOps](https://github.com/NixOS/nixops). + +**Faster evaluator in CIs.** In addition to evaluation speed for CIs, it is also +useful if evaluation of individual jobs in CIs can fail, as opposed to failing +the entire jobset. For CIs that allow dynamic build steps to be created, one can +also take advantage of the fact that nix-eval-jobs outputs the derivation path +separately. This allows separate logs and success status per job instead of a +single large log file. In the +[wiki](https://github.com/nix-community/nix-eval-jobs/wiki#ci-example-configurations) +we collect example ci configuration for various CIs. + +## Organisation of this repository + +`main` follows Lix HEAD, and is updated alongside the Lix NixOS module. When we +release we will make a `release-2.90` etc branch, which receives backports. + +The version of nix-eval-jobs follows the major version of Lix and minor +versions of nix-eval-jobs are released as necessary when changes are made in +n-e-j itself. + +## Projects using nix-eval-jobs + +- [nix-fast-build](https://github.com/Mic92/nix-fast-build) - Combine the power + of nix-eval-jobs with nix-output-monitor to speed-up your evaluation and + building process +- [buildbot-nix](https://github.com/Mic92/buildbot-nix) - A nixos module to make + buildbot a proper Nix-CI +- [colmena](https://github.com/zhaofengli/colmena) - A simple, stateless NixOS + deployment tool +- [robotnix](https://github.com/danielfullmer/robotnix) - Build Android (AOSP) + using Nix, used in their + [CI](https://github.com/danielfullmer/robotnix/blob/38b80700ee4265c306dcfdcce45056e32ab2973f/.github/workflows/instantiate.yml#L18) + +## FAQ + +### How can I check if my package already have been uploaded in the binary cache? + +If you provide the `--check-cache-status`, the json will contain a `"isCached"` +key in its json, that is true or false based on the status. + +### How can I evaluate nixpkgs? + +If you want to evaluate nixpkgs in the same way +[hydra](https://hydra.nixos.org/) does it, use this snippet: + +```console +$ nix-eval-jobs --force-recurse pkgs/top-level/release.nix +``` + +### nix-eval-jobs consumes too much memory / is too slow + +By default, nix-eval-jobs spawns as many worker processes as there are hardware +threads in the system and limits the memory usage for each worker to 4GB. + +However, keep in mind that each worker process may need to re-evaluate shared +dependencies of the attributes, which can introduce some overhead for each +evaluation or cause workers to exceed their memory limit. If you encounter these +situations, you can tune the following options: + +`--workers`: This option allows you to set the number of evaluation workers that +nix-eval-jobs should spawn. You can increase or decrease this number to optimize +the evaluation speed and memory usage. For example, if you have a system with +many CPU cores but limited memory, you may want to reduce the number of workers +to avoid exceeding the memory limit. + +`--max-memory-size`: This option allows you to adjust the memory limit for each +worker process. By default, it's set to 4GiB, but you can increase or decrease +this value as needed. For example, if you have a system with a lot of memory and +want to speed up the evaluation, you may want to increase the memory limit to +allow workers to cache more data in memory before getting restarted by +nix-eval-jobs. Note that this is not a hard limit and memory usage may rise +above the limit momentarily before the worker process exits. + +Overall, tuning these options can help you optimize the performance and memory +usage of nix-eval-jobs to better fit your system and evaluation needs. diff --git a/subprojects/nix-eval-jobs/default.nix b/subprojects/nix-eval-jobs/default.nix new file mode 100644 index 000000000..9b398d61d --- /dev/null +++ b/subprojects/nix-eval-jobs/default.nix @@ -0,0 +1,38 @@ +{ stdenv +, lib +, nix +, pkgs +, srcDir ? null +}: + +let + filterMesonBuild = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != "build"); +in +stdenv.mkDerivation { + pname = "nix-eval-jobs"; + version = "2.93.0-dev"; + src = if srcDir == null then filterMesonBuild ./. else srcDir; + buildInputs = with pkgs; [ + nlohmann_json + nix + boost + ]; + nativeBuildInputs = with pkgs; [ + meson + pkg-config + ninja + # nlohmann_json can be only discovered via cmake files + cmake + # XXX: ew + nix.passthru.capnproto-lix + ] ++ (lib.optional stdenv.cc.isClang [ pkgs.clang-tools ]); + + meta = { + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + homepage = "https://github.com/nix-community/nix-eval-jobs"; + license = lib.licenses.gpl3; + maintainers = with lib.maintainers; [ adisbladis mic92 ]; + platforms = lib.platforms.unix; + }; +} diff --git a/subprojects/nix-eval-jobs/dev/treefmt.nix b/subprojects/nix-eval-jobs/dev/treefmt.nix new file mode 100644 index 000000000..f505413f2 --- /dev/null +++ b/subprojects/nix-eval-jobs/dev/treefmt.nix @@ -0,0 +1,44 @@ +{ pkgs, ... }: { + # Used to find the project root + projectRootFile = "flake.lock"; + + programs.deno.enable = true; + + programs.clang-format.enable = true; + + settings.formatter = { + nix = { + command = "sh"; + options = [ + "-eucx" + '' + ${pkgs.deadnix}/bin/deadnix --edit "$@" + + for i in "$@"; do + ${pkgs.statix}/bin/statix fix "$i" + done + + ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt "$@" + '' + "--" + ]; + includes = [ "*.nix" ]; + excludes = [ ]; + }; + + clang-format = { }; + + python = { + command = "sh"; + options = [ + "-eucx" + '' + ${pkgs.python3.pkgs.black}/bin/black "$@" + ${pkgs.ruff}/bin/ruff --fix "$@" + '' + "--" # this argument is ignored by bash + ]; + includes = [ "*.py" ]; + }; + }; +} diff --git a/subprojects/nix-eval-jobs/flake.lock b/subprojects/nix-eval-jobs/flake.lock new file mode 100644 index 000000000..656b2cfc5 --- /dev/null +++ b/subprojects/nix-eval-jobs/flake.lock @@ -0,0 +1,177 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1730504689, + "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "506278e768c2a08bec68eb62932193e341f55c90", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "lix": { + "inputs": { + "flake-compat": "flake-compat", + "nix2container": "nix2container", + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1737857294, + "narHash": "sha256-bzC+anLF/NlgolaMoB4uTFgSejLJlTzPcNF1Kbq/BP0=", + "rev": "4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1", + "type": "tarball", + "url": "https://git.lix.systems/api/v1/repos/lix-project/lix/archive/4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1.tar.gz?rev=4af6b5ed9f8f2412bef5331b8e3b93f3ad305ea1" + }, + "original": { + "type": "tarball", + "url": "https://git.lix.systems/lix-project/lix/archive/main.tar.gz" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1731952509, + "narHash": "sha256-p4gB3Rhw8R6Ak4eMl8pqjCPOLCZRqaehZxdZ/mbFClM=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "7b5f051df789b6b20d259924d349a9ba3319b226", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nix2container": { + "flake": false, + "locked": { + "lastModified": 1724996935, + "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1737672001, + "narHash": "sha256-YnHJJ19wqmibLQdUeq9xzE6CjrMA568KN/lFPuSVs4I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "035f8c0853c2977b24ffc4d0a42c74f00b182cd8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11-small", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "pre-commit-hooks": { + "flake": false, + "locked": { + "lastModified": 1733318908, + "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "6f4e2a2112050951a314d2733a994fbab94864c6", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "lix": "lix", + "nix-github-actions": "nix-github-actions", + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1732292307, + "narHash": "sha256-5WSng844vXt8uytT5djmqBCkopyle6ciFgteuA9bJpw=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "705df92694af7093dfbb27109ce16d828a79155f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/subprojects/nix-eval-jobs/flake.nix b/subprojects/nix-eval-jobs/flake.nix new file mode 100644 index 000000000..cc805f22d --- /dev/null +++ b/subprojects/nix-eval-jobs/flake.nix @@ -0,0 +1,52 @@ +{ + description = "Hydra's builtin hydra-eval-jobs as a standalone"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small"; + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; + inputs.treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nix-github-actions.url = "github:nix-community/nix-github-actions"; + inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; + inputs.lix = { + url = "https://git.lix.systems/lix-project/lix/archive/main.tar.gz"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = inputs @ { flake-parts, ... }: + let + inherit (inputs.nixpkgs) lib; + inherit (inputs) self; + in + flake-parts.lib.mkFlake { inherit inputs; } + { + systems = inputs.nixpkgs.lib.systems.flakeExposed; + imports = [ inputs.treefmt-nix.flakeModule ]; + + flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = { + inherit (self.checks) x86_64-linux; + x86_64-darwin = builtins.removeAttrs self.checks.x86_64-darwin [ "treefmt" ]; + }; + }; + + perSystem = { pkgs, self', inputs', ... }: + let + drvArgs = { + srcDir = self; + nix = inputs'.lix.packages.default; + }; + in + { + treefmt.imports = [ ./dev/treefmt.nix ]; + packages.nix-eval-jobs = pkgs.callPackage ./default.nix drvArgs; + packages.clangStdenv-nix-eval-jobs = pkgs.callPackage ./default.nix (drvArgs // { stdenv = pkgs.clangStdenv; }); + packages.default = self'.packages.nix-eval-jobs; + devShells.default = pkgs.callPackage ./shell.nix drvArgs; + + checks = builtins.removeAttrs self'.packages [ "default" ] // { + shell = self'.devShells.default; + }; + }; + }; +} diff --git a/subprojects/nix-eval-jobs/meson.build b/subprojects/nix-eval-jobs/meson.build new file mode 100644 index 000000000..a4eede748 --- /dev/null +++ b/subprojects/nix-eval-jobs/meson.build @@ -0,0 +1,20 @@ +project('nix-eval-jobs', 'cpp', + version : '2.93.0-dev', + license : 'GPL-3.0', + default_options : [ + 'debug=true', + 'optimization=2', + 'cpp_std=c++20', + ], +) + +nix_main_dep = dependency('lix-main', required: true) +nix_store_dep = dependency('lix-store', required: true) +nix_expr_dep = dependency('lix-expr', required: true) +nix_cmd_dep = dependency('lix-cmd', required: true) +threads_dep = dependency('threads', required: true) +nlohmann_json_dep = dependency('nlohmann_json', required: true) +boost_dep = dependency('boost', required: true) +kj_async_dep = dependency('kj-async', required: true) + +subdir('src') diff --git a/subprojects/nix-eval-jobs/pyproject.toml b/subprojects/nix-eval-jobs/pyproject.toml new file mode 100644 index 000000000..0b0d0d11a --- /dev/null +++ b/subprojects/nix-eval-jobs/pyproject.toml @@ -0,0 +1,4 @@ +[tool.pytest.ini_options] +markers = [ + "infiniterecursion: mark test as infinite recursion", +] diff --git a/subprojects/nix-eval-jobs/renovate.json b/subprojects/nix-eval-jobs/renovate.json new file mode 100644 index 000000000..da5dfa5ed --- /dev/null +++ b/subprojects/nix-eval-jobs/renovate.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ], + "lockFileMaintenance": { + "enabled": true, + "extends": [ + "schedule:weekly" + ] + }, + "labels": ["dependencies"], + "nix": { + "enabled": true + } +} diff --git a/subprojects/nix-eval-jobs/shell.nix b/subprojects/nix-eval-jobs/shell.nix new file mode 100644 index 000000000..7035ed9e1 --- /dev/null +++ b/subprojects/nix-eval-jobs/shell.nix @@ -0,0 +1,42 @@ +{ pkgs ? ( + let + inherit (builtins) fromJSON readFile; + flakeLock = fromJSON (readFile ./flake.lock); + inherit (flakeLock.nodes.nixpkgs) locked; + nixpkgs = assert locked.type == "github"; builtins.fetchTarball { + url = "https://github.com/${locked.owner}/${locked.repo}/archive/${locked.rev}.tar.gz"; + sha256 = locked.narHash; + }; + in + import nixpkgs { } + ) +, srcDir ? null +, nix +}: + +let + inherit (pkgs) lib stdenv; + nix-eval-jobs = pkgs.callPackage ./default.nix { + inherit srcDir nix; + }; +in +pkgs.mkShell { + inherit (nix-eval-jobs) buildInputs; + nativeBuildInputs = nix-eval-jobs.nativeBuildInputs ++ [ + pkgs.clang-tools + (pkgs.python3.withPackages (ps: [ + ps.pytest + ])) + ] ++ lib.optional stdenv.isLinux # broken on darwin + (pkgs.writeShellScriptBin "update-include-what-you-use" '' + #!${pkgs.stdenv.shell} + export PATH=${pkgs.include-what-you-use}/bin:$PATH + find src -type f -name '*.cpp' -o -name '*.hh' -print0 | \ + xargs -n1 --null include-what-you-use -std=c++20 -isystem ${lib.getDev nix}/include/nix 2>&1 | \ + fix_includes.py + ''); + + shellHook = lib.optionalString stdenv.isLinux '' + export NIX_DEBUG_INFO_DIRS="${pkgs.curl.debug}/lib/debug:${nix.debug}/lib/debug''${NIX_DEBUG_INFO_DIRS:+:$NIX_DEBUG_INFO_DIRS}" + ''; +} diff --git a/subprojects/nix-eval-jobs/src/autotools-config.h.in b/subprojects/nix-eval-jobs/src/autotools-config.h.in new file mode 100644 index 000000000..4cee198b7 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/autotools-config.h.in @@ -0,0 +1 @@ +#mesondefine HAVE_STRUCT_DIRENT_D_TYPE diff --git a/subprojects/nix-eval-jobs/src/buffered-io.cc b/subprojects/nix-eval-jobs/src/buffered-io.cc new file mode 100644 index 000000000..469dbe23f --- /dev/null +++ b/subprojects/nix-eval-jobs/src/buffered-io.cc @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include + +#include "buffered-io.hh" + +[[nodiscard]] int tryWriteLine(int fd, std::string s) { + s += "\n"; + std::string_view sv{s}; + while (!sv.empty()) { + nix::checkInterrupt(); + ssize_t res = write(fd, sv.data(), sv.size()); + if (res == -1 && errno != EINTR) { + return -errno; + } + if (res > 0) { + sv.remove_prefix(res); + } + } + return 0; +} + +LineReader::LineReader(int fd) { + stream = fdopen(fd, "r"); + if (!stream) { + throw nix::Error("fdopen failed: %s", strerror(errno)); + } +} + +LineReader::~LineReader() { + fclose(stream); + free(buffer); +} + +LineReader::LineReader(LineReader &&other) { + stream = other.stream; + other.stream = nullptr; + buffer = other.buffer; + other.buffer = nullptr; + len = other.len; + other.len = 0; +} + +[[nodiscard]] std::string_view LineReader::readLine() { + ssize_t read = getline(&buffer, &len, stream); + + if (read == -1) { + return {}; // Return an empty string_view in case of error + } + + nix::checkInterrupt(); + + // Remove trailing newline + return std::string_view(buffer, read - 1); +} diff --git a/subprojects/nix-eval-jobs/src/buffered-io.hh b/subprojects/nix-eval-jobs/src/buffered-io.hh new file mode 100644 index 000000000..e1f068500 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/buffered-io.hh @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include + +[[nodiscard]] int tryWriteLine(int fd, std::string s); + +class LineReader { + public: + LineReader(int fd); + ~LineReader(); + + LineReader(LineReader &&other); + [[nodiscard]] std::string_view readLine(); + + private: + FILE *stream = nullptr; + char *buffer = nullptr; + size_t len = 0; +}; diff --git a/subprojects/nix-eval-jobs/src/drv.cc b/subprojects/nix-eval-jobs/src/drv.cc new file mode 100644 index 000000000..bacbf2a1c --- /dev/null +++ b/subprojects/nix-eval-jobs/src/drv.cc @@ -0,0 +1,134 @@ +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "drv.hh" +#include "eval-args.hh" + +static bool +queryIsCached(nix::Store &store, + std::map> &outputs) { + uint64_t downloadSize, narSize; + nix::StorePathSet willBuild, willSubstitute, unknown; + + std::vector paths; + for (auto const &[key, val] : outputs) { + if (val) { + paths.push_back(followLinksToStorePathWithOutputs(store, *val)); + } + } + + store.queryMissing(toDerivedPaths(paths), willBuild, willSubstitute, + unknown, downloadSize, narSize); + return willBuild.empty() && unknown.empty(); +} + +/* The fields of a derivation that are printed in json form */ +Drv::Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, + MyArgs &args) { + + auto localStore = state.ctx.store.dynamic_pointer_cast(); + + try { + // CA derivations do not have static output paths, so we have to + // defensively not query output paths in case we encounter one. + for (auto &[outputName, optOutputPath] : + drvInfo.queryOutputs(state, !nix::experimentalFeatureSettings.isEnabled( + nix::Xp::CaDerivations))) { + if (optOutputPath) { + outputs[outputName] = + localStore->printStorePath(*optOutputPath); + } else { + assert(nix::experimentalFeatureSettings.isEnabled( + nix::Xp::CaDerivations)); + outputs[outputName] = std::nullopt; + } + } + } catch (const std::exception &e) { + state.ctx.errors.make( + "derivation '%s' does not have valid outputs: %s", + attrPath, e.what() + ).debugThrow(); + } + + if (args.meta) { + nlohmann::json meta_; + for (auto &metaName : drvInfo.queryMetaNames(state)) { + nix::NixStringContext context; + std::stringstream ss; + + auto metaValue = drvInfo.queryMeta(state, metaName); + // Skip non-serialisable types + // TODO: Fix serialisation of derivations to store paths + if (metaValue == 0) { + continue; + } + + nix::printValueAsJSON(state, true, *metaValue, nix::noPos, ss, + context); + + meta_[metaName] = nlohmann::json::parse(ss.str()); + } + meta = meta_; + } + if (args.checkCacheStatus) { + cacheStatus = queryIsCached(*localStore, outputs) + ? Drv::CacheStatus::Cached + : Drv::CacheStatus::Uncached; + } else { + cacheStatus = Drv::CacheStatus::Unknown; + } + + drvPath = localStore->printStorePath(drvInfo.requireDrvPath(state)); + + auto drv = localStore->readDerivation(drvInfo.requireDrvPath(state)); + for (const auto &[inputDrvPath, inputNode] : drv.inputDrvs.map) { + std::set inputDrvOutputs; + for (auto &outputName : inputNode.value) { + inputDrvOutputs.insert(outputName); + } + inputDrvs[localStore->printStorePath(inputDrvPath)] = inputDrvOutputs; + } + name = drvInfo.queryName(state); + system = drv.platform; +} + +void to_json(nlohmann::json &json, const Drv &drv) { + std::map outputsJson; + for (auto &[name, optPath] : drv.outputs) { + outputsJson[name] = + optPath ? nlohmann::json(*optPath) : nlohmann::json(nullptr); + } + + json = nlohmann::json{{"name", drv.name}, + {"system", drv.system}, + {"drvPath", drv.drvPath}, + {"outputs", outputsJson}, + {"inputDrvs", drv.inputDrvs}}; + + if (drv.meta.has_value()) { + json["meta"] = drv.meta.value(); + } + + if (drv.cacheStatus != Drv::CacheStatus::Unknown) { + json["isCached"] = drv.cacheStatus == Drv::CacheStatus::Cached; + } +} diff --git a/subprojects/nix-eval-jobs/src/drv.hh b/subprojects/nix-eval-jobs/src/drv.hh new file mode 100644 index 000000000..4cfc6a0bb --- /dev/null +++ b/subprojects/nix-eval-jobs/src/drv.hh @@ -0,0 +1,33 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eval-args.hh" + +class MyArgs; + +namespace nix { +class EvalState; +struct DrvInfo; +} // namespace nix + +/* The fields of a derivation that are printed in json form */ +struct Drv { + std::string name; + std::string system; + std::string drvPath; + + enum class CacheStatus { Cached, Uncached, Unknown } cacheStatus; + std::map> outputs; + std::map> inputDrvs; + std::optional meta; + + Drv(std::string &attrPath, nix::EvalState &state, nix::DrvInfo &drvInfo, MyArgs &args); +}; +void to_json(nlohmann::json &json, const Drv &drv); diff --git a/subprojects/nix-eval-jobs/src/eval-args.cc b/subprojects/nix-eval-jobs/src/eval-args.cc new file mode 100644 index 000000000..57b839c28 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/eval-args.cc @@ -0,0 +1,105 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eval-args.hh" + +MyArgs::MyArgs(nix::AsyncIoRoot & aio) : MixCommonArgs("nix-eval-jobs"), aio_(aio) { + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { + printf("USAGE: nix-eval-jobs [options] expr\n\n"); + for (const auto &[name, flag] : longFlags) { + if (hiddenCategories.count(flag->category)) { + continue; + } + printf(" --%-20s %s\n", name.c_str(), + flag->description.c_str()); + } + ::exit(0); + }}, + }); + + addFlag({.longName = "impure", + .description = "allow impure expressions", + .handler = {&impure, true}}); + + addFlag({.longName = "force-recurse", + .description = "force recursion (don't respect recurseIntoAttrs)", + .handler = {&forceRecurse, true}}); + + addFlag({.longName = "gc-roots-dir", + .description = "garbage collector roots directory", + .labels = {"path"}, + .handler = {&gcRootsDir}}); + + addFlag( + {.longName = "workers", + .description = "number of evaluate workers", + .labels = {"workers"}, + .handler = {[=, this](std::string s) { nrWorkers = std::stoi(s); }}}); + + addFlag({.longName = "max-memory-size", + .description = "maximum evaluation memory size in megabyte " + "(4GiB per worker by default)", + .labels = {"size"}, + .handler = { + [=, this](std::string s) { maxMemorySize = std::stoi(s); }}}); + + addFlag({.longName = "flake", + .description = "build a flake", + .handler = {&flake, true}}); + + addFlag({.longName = "meta", + .description = "include derivation meta field in output", + .handler = {&meta, true}}); + + addFlag({.longName = "check-cache-status", + .description = + "Check if the derivations are present locally or in " + "any configured substituters (i.e. binary cache). The " + "information " + "will be exposed in the `isCached` field of the JSON output.", + .handler = {&checkCacheStatus, true}}); + + addFlag( + {.longName = "show-trace", + .description = "print out a stack trace in case of evaluation errors", + .handler = {&showTrace, true}}); + + addFlag({.longName = "expr", + .shortName = 'E', + .description = "treat the argument as a Nix expression", + .handler = {&fromArgs, true}}); + + // usually in MixFlakeOptions + addFlag({ + .longName = "override-input", + .description = + "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", + .category = category, + .labels = {"input-path", "flake-url"}, + .handler = {[&](std::string inputPath, std::string flakeRef) { + // overriden inputs are unlocked + lockFlags.allowUnlocked = true; + lockFlags.inputOverrides.insert_or_assign( + nix::flake::parseInputPath(inputPath), + nix::parseFlakeRef(flakeRef, nix::absPath("."), true)); + }}, + }); + + expectArg("expr", &releaseExpr); +} + +void MyArgs::parseArgs(char **argv, int argc) { + parseCmdline(nix::Strings(argv + 1, argv + argc)); +} diff --git a/subprojects/nix-eval-jobs/src/eval-args.hh b/subprojects/nix-eval-jobs/src/eval-args.hh new file mode 100644 index 000000000..1316c154f --- /dev/null +++ b/subprojects/nix-eval-jobs/src/eval-args.hh @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class MyArgs : virtual public nix::MixEvalArgs, + virtual public nix::MixCommonArgs, + virtual public nix::RootArgs { + // intentionally hidden in this subclass because it's mondo dangerous + // in n-e-j due to all the forking we do for worker process creation. + nix::AsyncIoRoot & aio_; + nix::AsyncIoRoot & aio() override { return aio_; } + public: + std::string releaseExpr; + nix::Path gcRootsDir; + bool flake = false; + bool fromArgs = false; + bool meta = false; + bool showTrace = false; + bool impure = false; + bool forceRecurse = false; + bool checkCacheStatus = false; + size_t nrWorkers = 1; + size_t maxMemorySize = 4096; + + // usually in MixFlakeOptions + nix::flake::LockFlags lockFlags = {.updateLockFile = false, + .writeLockFile = false, + .useRegistries = false, + .allowUnlocked = false}; + + MyArgs(nix::AsyncIoRoot & aio); + MyArgs(const MyArgs&) = delete; + + void parseArgs(char** argv, int argc); +}; diff --git a/subprojects/nix-eval-jobs/src/meson.build b/subprojects/nix-eval-jobs/src/meson.build new file mode 100644 index 000000000..4ece5923b --- /dev/null +++ b/subprojects/nix-eval-jobs/src/meson.build @@ -0,0 +1,34 @@ +src = files( + 'nix-eval-jobs.cc', + 'eval-args.cc', + 'drv.cc', + 'buffered-io.cc', + 'worker.cc', +) + +cc = meson.get_compiler('cpp') + +autotool_config = configuration_data() +# nix defines this with autotools +if cc.has_member('struct dirent', 'd_type', prefix: '#include ') + autotool_config.set('HAVE_STRUCT_DIRENT_D_TYPE', 1) +endif + +configure_file( + input: 'autotools-config.h.in', + output: 'autotools-config.h', + configuration: autotool_config +) + +executable('nix-eval-jobs', src, + dependencies : [ + nix_main_dep, + nix_store_dep, + nix_expr_dep, + nix_cmd_dep, + boost_dep, + nlohmann_json_dep, + threads_dep + ], + install: true, + cpp_args: ['--include', 'autotools-config.h']) diff --git a/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc new file mode 100644 index 000000000..8990398f5 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/nix-eval-jobs.cc @@ -0,0 +1,396 @@ +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eval-args.hh" +#include "buffered-io.hh" +#include "worker.hh" + +using namespace nix; +using namespace nlohmann; + +using Processor = std::function state, Bindings &autoArgs, + AutoCloseFD &to, AutoCloseFD &from, MyArgs &args, AsyncIoRoot &aio)>; + +/* Auto-cleanup of fork's process and fds. */ +struct Proc { + AutoCloseFD to, from; + Pid pid; + + Proc(MyArgs &myArgs, const Processor &proc) { + Pipe toPipe, fromPipe; + toPipe.create(); + fromPipe.create(); + auto p = startProcess( + [&, + to{std::make_shared(std::move(fromPipe.writeSide))}, + from{ + std::make_shared(std::move(toPipe.readSide))}]() { + debug("created worker process %d", getpid()); + try { + AsyncIoRoot aio; + auto evalStore = myArgs.evalStoreUrl + ? openStore(*myArgs.evalStoreUrl) + : openStore(); + auto evaluator = + nix::make_ref( + aio, myArgs.searchPath, evalStore); + Bindings &autoArgs = *myArgs.getAutoArgs(*evaluator); + proc(evaluator, autoArgs, *to, *from, myArgs, aio); + } catch (Error &e) { + nlohmann::json err; + auto msg = e.msg(); + err["error"] = nix::filterANSIEscapes(msg, true); + printError(msg); + if (tryWriteLine(to->get(), err.dump()) < 0) { + return; // main process died + }; + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + if (tryWriteLine(to->get(), "restart") < 0) { + return; // main process died + } + } + }, + ProcessOptions{}); + + to = std::move(toPipe.writeSide); + from = std::move(fromPipe.readSide); + pid = std::move(p); + } + + ~Proc() {} +}; + +// We'd highly prefer using std::thread here; but this won't let us configure the stack +// size. macOS uses 512KiB size stacks for non-main threads, and musl defaults to 128k. +// While Nix configures a 64MiB size for the main thread, this doesn't propagate to the +// threads we launch here. It turns out, running the evaluator under an anemic stack of +// 0.5MiB has it overflow way too quickly. Hence, we have our own custom Thread struct. +struct Thread { + pthread_t thread; + + Thread(const Thread &) = delete; + Thread(Thread &&) noexcept = default; + + Thread(std::function f) { + int s; + pthread_attr_t attr; + + auto func = std::make_unique>(std::move(f)); + + if ((s = pthread_attr_init(&attr)) != 0) { + throw SysError(s, "calling pthread_attr_init"); + } + if ((s = pthread_attr_setstacksize(&attr, 64 * 1024 * 1024)) != 0) { + throw SysError(s, "calling pthread_attr_setstacksize"); + } + if ((s = pthread_create(&thread, &attr, Thread::init, + func.release())) != 0) { + throw SysError(s, "calling pthread_launch"); + } + if ((s = pthread_attr_destroy(&attr)) != 0) { + throw SysError(s, "calling pthread_attr_destroy"); + } + } + + void join() { + int s; + s = pthread_join(thread, nullptr); + if (s != 0) { + throw SysError(s, "calling pthread_join"); + } + } + + private: + static void *init(void *ptr) { + std::unique_ptr> func; + func.reset(static_cast *>(ptr)); + + (*func)(); + return 0; + } +}; + +struct State { + std::set todo = json::array({json::array()}); + std::set active; + std::exception_ptr exc; +}; + +void handleBrokenWorkerPipe(Proc &proc, std::string_view msg) { + // we already took the process status from Proc, no + // need to wait for it again to avoid error messages + pid_t pid = proc.pid.release(); + while (1) { + int status; + int rc = waitpid(pid, &status, WNOHANG); + if (rc == 0) { + kill(pid, SIGKILL); + throw Error("BUG: while %s, worker pipe got closed but evaluation " + "worker still running?", + msg); + } else if (rc == -1) { + kill(pid, SIGKILL); + throw Error( + "BUG: while %s, waitpid for evaluation worker failed: %s", msg, + strerror(errno)); + } else { + if (WIFEXITED(status)) { + if (WEXITSTATUS(status) == 1) { + throw Error( + "while %s, evaluation worker exited with exit code 1, " + "(possible infinite recursion)", + msg); + } + throw Error("while %s, evaluation worker exited with %d", msg, + WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + switch (WTERMSIG(status)) { + case SIGKILL: + throw Error( + "while %s, evaluation worker got killed by SIGKILL, " + "maybe " + "memory limit reached?", + msg); + break; +#ifdef __APPLE__ + case SIGBUS: + throw Error( + "while %s, evaluation worker got killed by SIGBUS, " + "(possible infinite recursion)", + msg); + break; +#else + case SIGSEGV: + throw Error( + "while %s, evaluation worker got killed by SIGSEGV, " + "(possible infinite recursion)", + msg); +#endif + } + throw Error( + "while %s, evaluation worker got killed by signal %d (%s)", + msg, WTERMSIG(status), strsignal(WTERMSIG(status))); + } // else ignore WIFSTOPPED and WIFCONTINUED + } + } +} + +std::string joinAttrPath(json &attrPath) { + std::string joined; + for (auto &element : attrPath) { + if (!joined.empty()) { + joined += '.'; + } + joined += element.get(); + } + return joined; +} + +void collector(MyArgs &myArgs, Sync &state_, + std::condition_variable &wakeup) { + try { + std::optional> proc_; + std::optional> fromReader_; + + while (true) { + if (!proc_.has_value()) { + proc_ = std::make_unique(myArgs, worker); + fromReader_ = + std::make_unique(proc_.value()->from.release()); + } + auto proc = std::move(proc_.value()); + auto fromReader = std::move(fromReader_.value()); + + /* Check whether the existing worker process is still there. */ + auto s = fromReader->readLine(); + if (s.empty()) { + handleBrokenWorkerPipe(*proc.get(), "checking worker process"); + } else if (s == "restart") { + proc_ = std::nullopt; + fromReader_ = std::nullopt; + continue; + } else if (s != "next") { + try { + auto json = json::parse(s); + throw Error("worker error: %s", (std::string)json["error"]); + } catch (const json::exception &e) { + throw Error( + "Received invalid JSON from worker: %s\n json: '%s'", + e.what(), s); + } + } + + /* Wait for a job name to become available. */ + json attrPath; + + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || + state->exc) { + if (tryWriteLine(proc->to.get(), "exit") < 0) { + handleBrokenWorkerPipe(*proc.get(), "sending exit"); + } + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } + + /* Tell the worker to evaluate it. */ + if (tryWriteLine(proc->to.get(), "do " + attrPath.dump()) < 0) { + auto msg = "sending attrPath '" + joinAttrPath(attrPath) + "'"; + handleBrokenWorkerPipe(*proc.get(), msg); + } + + /* Wait for the response. */ + auto respString = fromReader->readLine(); + if (respString.empty()) { + auto msg = "reading result for attrPath '" + + joinAttrPath(attrPath) + "'"; + handleBrokenWorkerPipe(*proc.get(), msg); + } + json response; + try { + response = json::parse(respString); + } catch (const json::exception &e) { + throw Error( + "Received invalid JSON from worker: %s\n json: '%s'", + e.what(), respString); + } + + /* Handle the response. */ + std::vector newAttrs; + if (response.find("attrs") != response.end()) { + for (auto &i : response["attrs"]) { + json newAttr = json(response["attrPath"]); + newAttr.emplace_back(i); + newAttrs.push_back(newAttr); + } + } else { + auto state(state_.lock()); + std::cout << respString << "\n" << std::flush; + } + + proc_ = std::move(proc); + fromReader_ = std::move(fromReader); + + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto p : newAttrs) { + state->todo.insert(p); + } + wakeup.notify_all(); + } + } + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } +} + +int main(int argc, char **argv) { + + /* Prevent undeclared dependencies in the evaluation via + $NIX_PATH. */ + unsetenv("NIX_PATH"); + + /* We are doing the garbage collection by killing forks */ + setenv("GC_DONT_GC", "1", 1); + + return handleExceptions(argv[0], [&]() { + initNix(); + initLibExpr(); + + nix::AsyncIoRoot aio; + MyArgs myArgs(aio); + + myArgs.parseArgs(argv, argc); + + /* When building a flake, use pure evaluation (no access to + 'getEnv', 'currentSystem' etc. */ + if (myArgs.impure) { + evalSettings.pureEval.override(false); + } else if (myArgs.flake) { + evalSettings.pureEval.override(true); + } + + if (myArgs.releaseExpr == "") + throw UsageError("no expression specified"); + + if (myArgs.gcRootsDir == "") { + printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + } else { + myArgs.gcRootsDir = std::filesystem::absolute(myArgs.gcRootsDir); + } + + if (myArgs.showTrace) { + loggerSettings.showTrace.override(true); + } + + Sync state_; + + /* Start a collector thread per worker process. */ + std::vector threads; + std::condition_variable wakeup; + for (size_t i = 0; i < myArgs.nrWorkers; i++) { + threads.emplace_back(std::bind(collector, std::ref(myArgs), + std::ref(state_), std::ref(wakeup))); + } + + for (auto &thread : threads) + thread.join(); + + auto state(state_.lock()); + + if (state->exc) + std::rethrow_exception(state->exc); + }); +} diff --git a/subprojects/nix-eval-jobs/src/worker.cc b/subprojects/nix-eval-jobs/src/worker.cc new file mode 100644 index 000000000..a11696809 --- /dev/null +++ b/subprojects/nix-eval-jobs/src/worker.cc @@ -0,0 +1,215 @@ +#include // IWYU pragma: keep + +// doesn't exist on macOS +// IWYU pragma: no_include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "worker.hh" +#include "drv.hh" +#include "buffered-io.hh" +#include "eval-args.hh" + +static nix::Value *releaseExprTopLevelValue(nix::EvalState &state, + nix::Bindings &autoArgs, + MyArgs &args) { + nix::Value vTop; + + if (args.fromArgs) { + nix::Expr &e = state.ctx.parseExprFromString(args.releaseExpr, + nix::CanonPath::fromCwd()); + state.eval(e, vTop); + } else { + state.evalFile(nix::lookupFileArg(state.ctx, args.releaseExpr), vTop); + } + + auto vRoot = state.ctx.mem.allocValue(); + + state.autoCallFunction(autoArgs, vTop, *vRoot); + + return vRoot; +} + +static std::string attrPathJoin(nlohmann::json input) { + return std::accumulate(input.begin(), input.end(), std::string(), + [](std::string ss, std::string s) { + // Escape token if containing dots + if (s.find(".") != std::string::npos) { + s = "\"" + s + "\""; + } + return ss.empty() ? s : ss + "." + s; + }); +} + +void worker(nix::ref evaluator, + nix::Bindings &autoArgs, nix::AutoCloseFD &to, + nix::AutoCloseFD &from, MyArgs &args, nix::AsyncIoRoot &aio) { + + nix::Value *vRoot = [&]() { + auto state = evaluator->begin(aio); + if (args.flake) { + auto [flakeRef, fragment, outputSpec] = + nix::parseFlakeRefWithFragmentAndExtendedOutputsSpec( + args.releaseExpr, nix::absPath(".")); + nix::InstallableFlake flake{ + {}, evaluator, std::move(flakeRef), fragment, outputSpec, + {}, {}, args.lockFlags}; + + return flake.toValue(*state).first; + } else { + return releaseExprTopLevelValue(*state, autoArgs, args); + } + }(); + + LineReader fromReader(from.release()); + auto state = evaluator->begin(aio); + + while (true) { + /* Wait for the collector to send us a job name. */ + if (tryWriteLine(to.get(), "next") < 0) { + return; // main process died + } + + auto s = fromReader.readLine(); + if (s == "exit") { + break; + } + if (!s.starts_with("do ")) { + fprintf(stderr, "worker error: received invalid command '%s'\n", + s.data()); + abort(); + } + auto path = nlohmann::json::parse(s.substr(3)); + auto attrPathS = attrPathJoin(path); + + /* Evaluate it and send info back to the collector. */ + nlohmann::json reply = + nlohmann::json{{"attr", attrPathS}, {"attrPath", path}}; + try { + auto vTmp = + nix::findAlongAttrPath(*state, attrPathS, autoArgs, *vRoot) + .first; + + auto v = evaluator->mem.allocValue(); + state->autoCallFunction(autoArgs, *vTmp, *v); + + if (v->type() == nix::nAttrs) { + if (auto drvInfo = nix::getDerivation(*state, *v, false)) { + auto drv = Drv(attrPathS, *state, *drvInfo, args); + reply.update(drv); + + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + if (args.gcRootsDir != "") { + nix::Path root = + args.gcRootsDir + "/" + + std::string(nix::baseNameOf(drv.drvPath)); + if (!nix::pathExists(root)) { + auto localStore = + evaluator->store + .dynamic_pointer_cast(); + auto storePath = + localStore->parseStorePath(drv.drvPath); + localStore->addPermRoot(storePath, root); + } + } + } else { + auto attrs = nlohmann::json::array(); + bool recurse = + args.forceRecurse || + path.size() == 0; // Dont require `recurseForDerivations + // = true;` for top-level attrset + + for (auto &i : + v->attrs->lexicographicOrder(evaluator->symbols)) { + const std::string &name = evaluator->symbols[i->name]; + attrs.push_back(name); + + if (name == "recurseForDerivations" && + !args.forceRecurse) { + auto attrv = v->attrs->get( + evaluator->s.recurseForDerivations); + recurse = state->forceBool( + *attrv->value, attrv->pos, + "while evaluating recurseForDerivations"); + } + } + if (recurse) + reply["attrs"] = std::move(attrs); + else + reply["attrs"] = nlohmann::json::array(); + } + } else { + // We ignore everything that cannot be build + reply["attrs"] = nlohmann::json::array(); + } + } catch (nix::EvalError &e) { + auto err = e.info(); + std::ostringstream oss; + nix::showErrorInfo(oss, err, nix::loggerSettings.showTrace.get()); + auto msg = oss.str(); + + // Transmits the error we got from the previous evaluation + // in the JSON output. + reply["error"] = nix::filterANSIEscapes(msg, true); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + fprintf(stderr, "%s\n", msg.c_str()); + } catch ( + const std::exception &e) { // FIXME: for some reason the catch block + // above, doesn't trigger on macOS (?) + auto msg = e.what(); + reply["error"] = nix::filterANSIEscapes(msg, true); + fprintf(stderr, "%s\n", msg); + } + + if (tryWriteLine(to.get(), reply.dump()) < 0) { + return; // main process died + } + + /* If our RSS exceeds the maximum, exit. The collector will + start a new process. */ + struct rusage r; + getrusage(RUSAGE_SELF, &r); + if ((size_t)r.ru_maxrss > args.maxMemorySize * 1024) + break; + } + + if (tryWriteLine(to.get(), "restart") < 0) { + return; // main process died + }; +} diff --git a/subprojects/nix-eval-jobs/src/worker.hh b/subprojects/nix-eval-jobs/src/worker.hh new file mode 100644 index 000000000..d4c41caff --- /dev/null +++ b/subprojects/nix-eval-jobs/src/worker.hh @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include +#include + +#include "eval-args.hh" + +class MyArgs; + +namespace nix { +class AutoCloseFD; +class Bindings; +class EvalState; +template class ref; +} // namespace nix + +void worker(nix::ref evaluator, + nix::Bindings &autoArgs, nix::AutoCloseFD &to, + nix::AutoCloseFD &from, MyArgs &args, nix::AsyncIoRoot &aio); diff --git a/subprojects/nix-eval-jobs/tests/assets/ci.nix b/subprojects/nix-eval-jobs/tests/assets/ci.nix new file mode 100644 index 000000000..5354be14b --- /dev/null +++ b/subprojects/nix-eval-jobs/tests/assets/ci.nix @@ -0,0 +1,35 @@ +{ pkgs ? import (builtins.getFlake (toString ./.)).inputs.nixpkgs { } +, system ? pkgs.system +}: + +{ + builtJob = pkgs.writeText "job1" "job1"; + substitutedJob = pkgs.hello; + + dontRecurse = { + # This shouldn't build as `recurseForDerivations = true;` is not set + # recurseForDerivations = true; + + # This should not build + drvB = derivation { + inherit system; + name = "drvA"; + builder = ":"; + }; + }; + + recurse = { + # This should build + recurseForDerivations = true; + + # This should not build + drvB = derivation { + inherit system; + name = "drvB"; + builder = ":"; + }; + }; + + "dotted.attr" = pkgs.hello; + +} diff --git a/subprojects/nix-eval-jobs/tests/assets/flake.lock b/subprojects/nix-eval-jobs/tests/assets/flake.lock new file mode 100644 index 000000000..551565bcb --- /dev/null +++ b/subprojects/nix-eval-jobs/tests/assets/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1702172177, + "narHash": "sha256-2T1DjuXz0bVxy5g8oF9FYioHOLWkXw5EdW687NDQakE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2873a73123077953f3e6f34964466018876d87c4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/subprojects/nix-eval-jobs/tests/assets/flake.nix b/subprojects/nix-eval-jobs/tests/assets/flake.nix new file mode 100644 index 000000000..5a322776b --- /dev/null +++ b/subprojects/nix-eval-jobs/tests/assets/flake.nix @@ -0,0 +1,29 @@ +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + + outputs = { nixpkgs, ... }: + let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in + { + hydraJobs = import ./ci.nix { inherit pkgs; }; + + legacyPackages.x86_64-linux = { + brokenPkgs = { + brokenPackage = throw "this is an evaluation error"; + }; + infiniteRecursionPkgs = { + packageWithInfiniteRecursion = + let + recursion = [ recursion ]; + in + derivation { + inherit (pkgs) system; + name = "drvB"; + recursiveAttr = recursion; + builder = ":"; + }; + }; + }; + }; +} diff --git a/subprojects/nix-eval-jobs/tests/test_eval.py b/subprojects/nix-eval-jobs/tests/test_eval.py new file mode 100644 index 000000000..8fa41cb72 --- /dev/null +++ b/subprojects/nix-eval-jobs/tests/test_eval.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import pytest +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, List + +TEST_ROOT = Path(__file__).parent.resolve() +PROJECT_ROOT = TEST_ROOT.parent +BIN = PROJECT_ROOT.joinpath("build", "src", "nix-eval-jobs") + + +def common_test(extra_args: List[str]) -> List[Dict[str, Any]]: + with TemporaryDirectory() as tempdir: + cmd = [str(BIN), "--gc-roots-dir", tempdir, "--meta"] + extra_args + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + check=True, + stdout=subprocess.PIPE, + ) + + results = [json.loads(r) for r in res.stdout.split("\n") if r] + assert len(results) == 4 + + built_job = results[0] + assert built_job["attr"] == "builtJob" + assert built_job["name"] == "job1" + assert built_job["outputs"]["out"].startswith("/nix/store") + assert built_job["drvPath"].endswith(".drv") + assert built_job["meta"]["broken"] is False + + dotted_job = results[1] + assert dotted_job["attr"] == '"dotted.attr"' + assert dotted_job["attrPath"] == ["dotted.attr"] + + recurse_drv = results[2] + assert recurse_drv["attr"] == "recurse.drvB" + assert recurse_drv["name"] == "drvB" + + substituted_job = results[3] + assert substituted_job["attr"] == "substitutedJob" + assert substituted_job["name"].startswith("hello-") + assert substituted_job["meta"]["broken"] is False + return results + + +def test_flake() -> None: + results = common_test(["--flake", ".#hydraJobs"]) + for result in results: + assert "isCached" not in result + + +def test_query_cache_status() -> None: + results = common_test(["--flake", ".#hydraJobs", "--check-cache-status"]) + # FIXME in the nix sandbox we cannot query binary caches + # this would need some local one + for result in results: + assert "isCached" in result + + +def test_expression() -> None: + results = common_test(["ci.nix"]) + for result in results: + assert "isCached" not in result + + with open(TEST_ROOT.joinpath("assets/ci.nix"), "r") as ci_nix: + common_test(["-E", ci_nix.read()]) + + +def test_eval_error() -> None: + with TemporaryDirectory() as tempdir: + cmd = [ + str(BIN), + "--gc-roots-dir", + tempdir, + "--meta", + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.brokenPkgs", + ] + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + stdout=subprocess.PIPE, + ) + print(res.stdout) + attrs = json.loads(res.stdout) + assert attrs["attr"] == "brokenPackage" + assert "this is an evaluation error" in attrs["error"] + + +@pytest.mark.infiniterecursion +def test_recursion_error() -> None: + with TemporaryDirectory() as tempdir: + cmd = [ + str(BIN), + "--gc-roots-dir", + tempdir, + "--meta", + "--workers", + "1", + "--flake", + ".#legacyPackages.x86_64-linux.infiniteRecursionPkgs", + ] + res = subprocess.run( + cmd, + cwd=TEST_ROOT.joinpath("assets"), + text=True, + stderr=subprocess.PIPE, + ) + assert res.returncode == 1 + print(res.stderr) + assert "packageWithInfiniteRecursion" in res.stderr + assert "possible infinite recursion" in res.stderr