From b7254760ccedc8fc81a21b4e707c628556df1f6c Mon Sep 17 00:00:00 2001 From: seth Date: Tue, 7 Nov 2023 21:39:13 -0500 Subject: initial commit --- .github/dependabot.yml | 8 +++ .github/workflows/ci.yaml | 99 ++++++++++++++++++++++++++++++++++++ .github/workflows/example.yaml | 55 ++++++++++++++++++++ LICENSE | 21 ++++++++ README.md | 90 ++++++++++++++++++++++++++++++++ flake.lock | 26 ++++++++++ flake.nix | 14 +++++ lib.nix | 80 +++++++++++++++++++++++++++++ module.nix | 113 +++++++++++++++++++++++++++++++++++++++++ test/lib/flake.lock | 42 +++++++++++++++ test/lib/flake.nix | 48 +++++++++++++++++ test/module/flake.lock | 63 +++++++++++++++++++++++ test/module/flake.nix | 39 ++++++++++++++ 13 files changed, 698 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/example.yaml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 lib.nix create mode 100644 module.nix create mode 100644 test/lib/flake.lock create mode 100644 test/lib/flake.nix create mode 100644 test/module/flake.lock create mode 100644 test/module/flake.nix diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1d662ce --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + commit-message: + prefix: "actions" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a542c01 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,99 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + eval: + strategy: + matrix: + test: [lib, module] + + runs-on: ubuntu-latest + + outputs: + lib-matrix: ${{ steps.lib-generate.outputs.matrix }} + module-matrix: ${{ steps.module-generate.outputs.matrix }} + + steps: + - uses: actions/checkout@v4 + - uses: nixbuild/nix-quick-install-action@v26 + + - name: generate matrix + id: ${{ matrix.test }}-generate + run: | + cd test/${{ matrix.test }} + set -e + echo "matrix=$(nix eval --show-trace --json .#githubWorkflow.matrix)" >> "$GITHUB_OUTPUT" + + # would be nice to have another matrix for these but meh + test-lib: + needs: eval + + strategy: + matrix: ${{ fromJSON(needs.eval.outputs.lib-matrix) }} + + runs-on: ${{ matrix.os }} + + name: build (${{ matrix.attr }}) + + steps: + - uses: actions/checkout@v4 + + - name: setup qemu + if: matrix.arch == 'aarch64' + run: | + sudo apt update -y + sudo apt install -y qemu-user-static + + - name: install nix + if: matrix.arch != 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + + - name: install nix (with aarch64) + if: matrix.arch == 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + with: + nix_conf: "extra-platforms = aarch64-linux arm-linux" + + - name: build ${{ matrix.attr }} + run: | + cd test/lib + nix build -L .#${{ matrix.attr }} + + test-module: + needs: eval + + strategy: + matrix: ${{ fromJSON(needs.eval.outputs.module-matrix) }} + + runs-on: ${{ matrix.os }} + + name: build (${{ matrix.attr }}) + + steps: + - uses: actions/checkout@v4 + + - name: setup qemu + if: matrix.arch == 'aarch64' + run: | + sudo apt update -y + sudo apt install -y qemu-user-static + + - name: install nix + if: matrix.arch != 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + + - name: install nix (with aarch64) + if: matrix.arch == 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + with: + nix_conf: "extra-platforms = aarch64-linux arm-linux" + + - name: build ${{ matrix.attr }} + run: | + cd test/module + nix build -L .#${{ matrix.attr }} diff --git a/.github/workflows/example.yaml b/.github/workflows/example.yaml new file mode 100644 index 0000000..5c376c1 --- /dev/null +++ b/.github/workflows/example.yaml @@ -0,0 +1,55 @@ +name: Example + +on: + workflow_dispatch: + +jobs: + eval: + runs-on: ubuntu-latest + + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + + steps: + - uses: actions/checkout@v4 + + - name: install nix + uses: nixbuild/nix-quick-install-action@v26 + + - name: generate matrix + id: generate + run: | + set -Eeu + echo "matrix=$(nix eval --show-trace --json .#githubWorkflow.matrix)" >> "$GITHUB_OUTPUT" + + build: + needs: eval + + strategy: + matrix: ${{ fromJSON(needs.eval.outputs.matrix) }} + + runs-on: ${{ matrix.os }} + + name: build (${{matrix.attr}}) + + steps: + - uses: actions/checkout@v4 + + - name: setup qemu + if: matrix.arch == 'aarch64' + run: | + sudo apt update -y + sudo apt install -y qemu-user-static + + - name: install nix + if: matrix.arch != 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + + - name: install nix (with aarch64) + if: matrix.arch == 'aarch64' + uses: nixbuild/nix-quick-install-action@v26 + with: + nix_conf: "extra-platforms = aarch64-linux arm-linux" + + - name: build ${{ matrix.attr }} + run: nix build -L --fallback .#${{ matrix.attr }} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..171b89c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 seth + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8f97230 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# nix2workflow + +![test status](https://github.com/getchoo/nix2workflow/actions/workflows/ci.yaml/badge.svg) + +nix2workflow is a library for generating github matrices from regular nix flake outputs. + +## usage + +we offer both a standard library for use in any flake, along with +a [flake-parts](https://flake.parts/) module for easier integration. + +you can find an example workflow for use in your own project in +[./.github/workflows/example.yaml](./.github/workflows/example.yaml). + +### flake module + +a basic setup might look like this. please see the [module](./module.nix) +for all options + +```nix +{ + imports = [nix2workflow.flakeModule]; + + githubWorkflowGenerator = { + outputs = [ + "checks" + "devShells" + "nixosConfigurations" + "packages" + ]; + + overrides = { + checks.systems = ["x86_64-linux"]; + }; + }; +} +``` + +a full example can be found in [./test/module/flake.nix](./test/module/flake.nix) + +### library + +the regular library will have a more complicated setup, though +it also allows using lower level functions and has no restrictions on +what flake outputs are used. + +```nix +{ + githubworkflow = let + workflow = nix2workflow.lib {inherit self;}; + outputs = [ + "checks" + "devShells" + "nixosConfigurations" + "packages" + ]; + in { + matrix.include = lib.concatLists ( + map ( + output: + workflow.mkMatrix { + inherit output; + # you can also specify what systems to build each output for + systems = ["x86_64-linux" "aarch64-darwin"]; + } + ) + outputs + ); + }; +} +``` + +you can see a full example in [./test/lib/flake.nix](./test/lib/flake.nix) + +### in workflows + +when the matrix is imported, a few variables with added to the `matrix` context. +these can allow you to customize your workflow based on what packages are building +- such as enabling QEMU when building for aarch64 + +| name | use | +| --- | --- | +| `os` | the operating system of the current output. usually `ubuntu-latest` or `macos-latest` | +| `arch` | the architecture of the current output. will be `aarch64` or `x64` | +| `attr` | the flake attribute of the current output (can really be anything) | + +## related projects + - [nix-community/nix-github-actions](https://github.com/nix-community/nix-github-actions/) + - this is the primary inspiration for this project - and i believe also one of the first + projects to attempt this, so kudos! i just wanted a more opionated and expandable approach :) diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..e9a69a9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs-lib": { + "locked": { + "lastModified": 1699145078, + "narHash": "sha256-OO1b3jiMUGjafD2ErkbTPVgUlhmyWo2Z5i0k2kD1ViU=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "174d7dc67189bc4a53f1bffb4fb9d0f13b79cd3c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..f6d4cd3 --- /dev/null +++ b/flake.nix @@ -0,0 +1,14 @@ +{ + description = "generate github matrices with nix!"; + + inputs.nixpkgs-lib.url = "github:nix-community/nixpkgs.lib"; + + outputs = { + self, + nixpkgs-lib, + ... + }: { + lib = import ./lib.nix nixpkgs-lib.lib; + flakeModule = import ./module.nix self.lib; + }; +} diff --git a/lib.nix b/lib.nix new file mode 100644 index 0000000..f7f07f3 --- /dev/null +++ b/lib.nix @@ -0,0 +1,80 @@ +lib: { + self, + platforms ? { + "x86_64-linux" = { + os = "ubuntu-latest"; + arch = "x64"; + }; + + "aarch64-linux" = { + os = "ubuntu-latest"; + arch = "aarch64"; + }; + + "x86_64-darwin" = { + os = "macos-latest"; + arch = "x64"; + }; + }, + ... +}: let + platforms' = + platforms + // { + fallback = lib.warn "an output in the job matrix is not supported!" { + os = null; + arch = null; + }; + }; + + mkMatrixMulti = systems: output: + lib.flatten ( + lib.mapAttrsToList ( + system: + lib.mapAttrsToList ( + attr: _: { + inherit (platforms'.${system} or platforms'.fallback) arch os; + attr = "${output}.${system}.${attr}"; + } + ) + ) + (lib.getAttrs systems self.${output}) + ); + + mkMatrixFlat = { + output, + suffix ? "", + }: + lib.mapAttrsToList ( + attr: deriv: { + inherit (platforms'.${deriv.pkgs.system} or platforms'.fallback) os arch; + attr = "${output}.${attr}${suffix}"; + } + ) + self.${output}; +in { + inherit + mkMatrixMulti + mkMatrixFlat + ; + + mkMatrix = { + output, + systems ? (builtins.attrNames platforms), + }: let + systemMatrix = mkMatrixFlat { + inherit output; + suffix = ".config.system.build.toplevel"; + }; + in + { + "nixosConfigurations" = systemMatrix; + "darwinConfigurations" = systemMatrix; + "homeConfigurations" = mkMatrixFlat { + inherit output; + suffix = ".activationPackage"; + }; + } + .${output} + or (mkMatrixMulti systems output); +} diff --git a/module.nix b/module.nix new file mode 100644 index 0000000..189dc69 --- /dev/null +++ b/module.nix @@ -0,0 +1,113 @@ +workflowLib': { + config, + lib, + self, + ... +}: let + cfg = config.githubWorkflowGenerator; + + inherit + (builtins) + attrNames + elem + ; + + inherit + (lib) + filter + getAttrs + mapAttrsToList + mdDoc + mkOption + literalExpression + types + ; + + workflowLib = workflowLib' ( + {inherit self;} + // lib.mkIf (cfg.platforms != {}) { + inherit (cfg) platforms; + } + ); + + supportedOutputs = [ + "apps" + "checks" + "devShells" + "darwinConfigurations" + "homeConfigurations" + "nixosConfigurations" + "packages" + ]; + + platformMap = { + options = { + arch = mkOption { + description = mdDoc "the architecture of a system"; + type = types.str; + default = null; + example = literalExpression "x86_64"; + }; + + os = mkOption { + description = mdDoc "the name of an os supported by github runners"; + type = types.str; + default = null; + example = literalExpression "ubuntu-latest"; + }; + }; + }; + + overrides = { + options = { + systems = mkOption { + description = mdDoc "list of systems to build an output for"; + type = types.listOf types.str; + default = builtins.attrNames cfg.platforms; + }; + }; + }; + + jobs = lib.concatLists ( + mapAttrsToList ( + output: _: + workflowLib.mkMatrix ({inherit output;} // cfg.overrides.${output} or {}) + ) + (getAttrs cfg.outputs self) + ); +in { + options = { + githubWorkflowGenerator = { + outputs = mkOption { + description = mdDoc "outputs to include in workflow"; + type = types.listOf types.str; + default = filter (output: elem output supportedOutputs) (attrNames self); + }; + + platforms = mkOption { + description = mdDoc '' + an attrset that can map a nix system to an architecture and os supported by github + ''; + type = types.attrsOf (types.submodule platformMap); + default = {}; + }; + + overrides = mkOption { + description = mdDoc "overrides for mkMatrix args"; + type = types.attrsOf (types.submodule overrides); + default = {}; + example = literalExpression '' + { + githubWorkflowGenerator.overrides = { + checks.systems = [ "x86_64-linux" ]; + }; + } + ''; + }; + }; + }; + + config.flake.githubWorkflow = { + matrix.include = jobs; + }; +} diff --git a/test/lib/flake.lock b/test/lib/flake.lock new file mode 100644 index 0000000..a71eb72 --- /dev/null +++ b/test/lib/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "call-flake": { + "locked": { + "lastModified": 1699147280, + "narHash": "sha256-bmhE1TmrJG4ba93l9WQTLuYM53kwGQAjYHRvHOeuxWU=", + "owner": "divnix", + "repo": "call-flake", + "rev": "7d993ee93bd35a9e6574ec76d54b7d44087495ad", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "call-flake", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1699099776, + "narHash": "sha256-X09iKJ27mGsGambGfkKzqvw5esP1L/Rf8H3u3fCqIiU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "85f1ba3e51676fa8cc604a3d863d729026a6b8eb", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "call-flake": "call-flake", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/test/lib/flake.nix b/test/lib/flake.nix new file mode 100644 index 0000000..c4607dc --- /dev/null +++ b/test/lib/flake.nix @@ -0,0 +1,48 @@ +{ + inputs = { + nixpkgs.url = "nixpkgs/nixos-unstable"; + call-flake.url = "github:divnix/call-flake"; + }; + + outputs = { + self, + nixpkgs, + call-flake, + ... + }: let + inherit (nixpkgs) lib; + + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + forAllSystems = fn: lib.genAttrs systems (sys: fn nixpkgs.legacyPackages.${sys}); + workflow = (call-flake ../../.).lib {inherit self;}; + in { + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + packages = [pkgs.hello]; + }; + }); + + packages = forAllSystems (pkgs: { + inherit (pkgs) hello; + default = pkgs.hello; + }); + + githubWorkflow = let + outputs = ["packages" "devShells"]; + jobs = lib.concatLists ( + map ( + output: workflow.mkMatrix {inherit output;} + ) + outputs + ); + in { + matrix.include = jobs; + }; + }; +} diff --git a/test/module/flake.lock b/test/module/flake.lock new file mode 100644 index 0000000..c2979aa --- /dev/null +++ b/test/module/flake.lock @@ -0,0 +1,63 @@ +{ + "nodes": { + "call-flake": { + "locked": { + "lastModified": 1699147280, + "narHash": "sha256-bmhE1TmrJG4ba93l9WQTLuYM53kwGQAjYHRvHOeuxWU=", + "owner": "divnix", + "repo": "call-flake", + "rev": "7d993ee93bd35a9e6574ec76d54b7d44087495ad", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "call-flake", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1699099776, + "narHash": "sha256-X09iKJ27mGsGambGfkKzqvw5esP1L/Rf8H3u3fCqIiU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "85f1ba3e51676fa8cc604a3d863d729026a6b8eb", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698882062, + "narHash": "sha256-HkhafUayIqxXyHH1X8d9RDl1M2CkFgZLjKD3MzabiEo=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8c9fa2545007b49a5db5f650ae91f227672c3877", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "root": { + "inputs": { + "call-flake": "call-flake", + "nixpkgs": "nixpkgs", + "parts": "parts" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/test/module/flake.nix b/test/module/flake.nix new file mode 100644 index 0000000..69aa4a2 --- /dev/null +++ b/test/module/flake.nix @@ -0,0 +1,39 @@ +{ + inputs = { + nixpkgs.url = "nixpkgs/nixos-unstable"; + + call-flake.url = "github:divnix/call-flake"; + + parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + }; + + outputs = { + parts, + call-flake, + ... + } @ inputs: + parts.lib.mkFlake {inherit inputs;} { + imports = [(call-flake ../../.).flakeModule]; + + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + perSystem = {pkgs, ...}: { + devShells.default = pkgs.mkShell { + packages = [pkgs.hello]; + }; + + packages = { + inherit (pkgs) hello; + default = pkgs.hello; + }; + }; + }; +} -- cgit v1.2.3