From 75361f9a8a0e353c73c9f4b537085d2e46eef639 Mon Sep 17 00:00:00 2001 From: "Collin J. Doering" Date: Sat, 15 Jul 2023 16:37:59 -0400 Subject: [PATCH] Initial commit --- .envrc | 10 ++++ .gitignore | 1 + Dockerfile | 7 +++ Makefile | 33 +++++++++++ README.org | 142 +++++++++++++++++++++++++++++++++++++++++++++++ channels.scm | 11 ++++ manifest-dev.scm | 3 + manifest.scm | 8 +++ shell.sh | 13 +++++ 9 files changed, 228 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.org create mode 100644 channels.scm create mode 100644 manifest-dev.scm create mode 100644 manifest.scm create mode 100755 shell.sh diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..fbb332f --- /dev/null +++ b/.envrc @@ -0,0 +1,10 @@ +use_guix-shell() { + CHANNEL_FILE=channels.scm + if [ -f $CHANNEL_FILE ]; then + eval "$(guix time-machine -C $CHANNEL_FILE -- shell "$@" --search-paths)" + else + eval "$(guix shell "$@" --search-paths)" + fi +} + +use guix-shell -m manifest-dev.scm diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..335ec95 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.tar.gz diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..640af1c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM scratch +ADD guix-tarball-pack.tar.gz /guix-builder + +ENV PATH=/guix-builder/bin + +VOLUME /var/guix/daemon-socket/socket /gnu/store /etc/ssl +ENTRYPOINT ["/guix-builder/bin/bash"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..973e56e --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +.DEFAULT_GOAL := build + +GUIX_MANIFEST := manifest.scm +GUIX_MANIFEST_DEV := manifest-dev.scm + +TARBALL ?= guix-tarball-pack.tar.gz +IMAGE_TAG ?= guix-builder + +$(TARBALL): + @cp $$(guix pack -R -S /bin=bin -S /etc/ssl=etc/ssl -m $(GUIX_MANIFEST)) $@ + @chmod +w $@ + +.PHONY: build +build: $(TARBALL) + @docker build -t $(IMAGE_TAG) . + +.PHONY: run +run: build + @docker run --rm -it \ + -v /var/guix/daemon-socket/socket:/var/guix/daemon-socket/socket \ + -v /gnu/store:/gnu/store:ro \ + -v /etc/ssl:/etc/ssl:ro \ + -e HOME=/tmp \ + -w /tmp \ + $(IMAGE_TAG):latest + +.PHONY: shell +shell: + @./shell.sh $(GUIX_MANIFEST) $(GUIX_MANIFEST_DEV) + +.PHONY: +clean: + rm $(TARBALL) diff --git a/README.org b/README.org new file mode 100644 index 0000000..051452e --- /dev/null +++ b/README.org @@ -0,0 +1,142 @@ +#+TITLE: Guix Builder +#+AUTHOR: Collin J. Doering + +#+begin_abstract +This repository defines a minimal OCI container containing guix and a few other tools useful +for CI/CD jobs. The container is expected to be run from a system that already has the +~guix-daemon~ installed and running, and is most useful in the context of existing CI/CD +tools that already use an existing OCI container runtime (eg. docker, podman, etc..). +#+end_abstract + +* Prerequisites + +- ~guix-daemon~ installed and running on the host +- ~dockerd~ to build and run the produced image +- ~direnv~ (optional) + +* Repository Structure + +- ~channels.scm~ :: [[*Guix Channel File][Guix Channel File]]. +- ~Dockerfile~ :: From scratch image that uses guix pack tar.gz output. +- ~Makefile~ :: Various make targets to build and run the guix-builder OCI image. +- ~manifest-dev.scm~ :: Packages required for development. +- ~manifest.scm~ :: Packages to be installed in the OCI image. +- ~.gitignore~ :: Files ignored by git. +- ~README.org~ :: Org-mode[fn:1] documentation. +- ~shell.sh~ :: Launch a containerized development shell. + +** Guix Channel File + +Guix channels[fn:2] allow for Guix to be customized and extended. They are also critical for +replicating a Guix system[fn:3]. To ensure reproducibility, a ~channels.scm~ file is provided +in this repository that is expected to be used during deployment. It pins external guix +channels to specific versions. + +* Setup development environment + +~direnv~ is the preferred and optimal way to enter a development environment. Use ~direnv allow~. + +Alternatively, ~./shell.sh~ can be invoked directly + +* Building the image + +#+begin_src shell + make [build] +#+end_src + +~build~ is optional, as its the default target. + +* Using the image + +Now that the OCI image has been produced, some care must be taken when running it. Namely, +the following volumes are expected to be mounted. + +- ~/var/guix/daemon-socket/socket~ :: the ~guix-daemon~ socket +- ~/gnu/store~ :: The hosts guix store +- ~/etc/ssl~ :: The hosts ssl certificates (required for ~guix time-machine ...~ commands) + +Additionally, for ~guix~ to function appropriately, the ~HOME~ environment variable must be +set. This allows guix to store its cache (mandatory for many operations, eg ~guix pull~). +Depending on use-case, it is sometimes useful to retain the guix cache between container +invocations. To do so, mount a volume to ~$HOME/.cache/guix~ (replacing ~$HOME~ with what it +will be set to within the container). + +A complete example is given below, where the aforementioned volumes are mounted, ~HOME~ is +set to ~/tmp~, and a volume for retaining the guix cache between invocations is put in place. +The working directory within the container is also set with the ~-w~ option, however this is +not required. + +#+begin_src shell + docker run --rm -it \ + -v /var/guix/daemon-socket/socket:/var/guix/daemon-socket/socket \ + -v /gnu/store:/gnu/store:ro \ + -v /etc/ssl:/etc/ssl:ro \ + -v /var/lib/ci/guix-cache:/tmp/.cache/guix \ + -e HOME=/tmp \ + -w /tmp \ + guix-builder:latest +#+end_src + +** Using the image with DroneCI / WoodpeckerCI + +The primary use-case for the image produced by this repository is for ci/cd jobs (mandatorily +on a host that is running guix-daemon itself). This allows for the hosts guix store to be +reused, allowing for a node-local caching layer, and enabling optimal ci/cd build/deploy +times. + +Any ci/cd tool that can run OCI containers should be compatible with this image, though the +optimal setup assumes it provides a mechanism to mount volumes and set environment variables +(by an administrator). Both DroneCI and WoodpeckerCI allow for this, using +~DRONE_RUNNER_VOLUMES~ and ~WOODPECKER_BACKEND_DOCKER_VOLUMES~ respectively. + +Note, in WoodpeckerCI, this functionality has not yet made it into a release (see [[https://github.com/woodpecker-ci/woodpecker/pull/1203][PR]]). + +Below is an example of running ~drone-runner-docker~, setup to spawn ~guix~ images produced +by this repository: + +#+begin_src shell + docker run -d \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e DRONE_RPC_PROTO="https" \ + -e DRONE_RPC_HOST="" \ + -e DRONE_RPC_SECRET="" \ + -e DRONE_RUNNER_CAPACITY=4 \ + -e DRONE_RUNNER_NAME="" \ + -e DRONE_RUNNER_LABELS=guix:on \ + -e DRONE_RUNNER_VOLUMES=/var/guix/daemon-socket/socket:/var/guix/daemon-socket/socket,/gnu/store:/gnu/store,/var/lib/ci/guix-cache:/.cache/guix \ + -p 3001:3000 \ + --restart unless-stopped \ + --name runner-guix \ + drone/drone-runner-docker:1.6.3 +#+end_src + +* FAQ + +** Why not produce this image with ~guix pack -f docker ...~ directly? + +One may ask, why not simply produce a OCI container using ~guix pack~ directly? Eg. + +#+begin_src shell + guix pack -f docker -S /bin=bin --entry-point=bin/guix -m manifest.scm +#+end_src + +Well, I'm glad you asked! This produces a container that has the guix store in its default +location ~/gnu/store~. Using the same method as described in [[*Using the image with DroneCI / WoodpeckerCI][Using the image with DroneCI / +WoodpeckerCI]], this container image works, however it has an implicit constraint: all store +items used within the pack must also exist in the hosts guix store. This is because the guix +store is volume mounted into the container at runtime, shadowing the already existing +~/gnu/store~ directory put in place via ~guix pack~. There are ways one could work around +this constraint, however the best solution would be to have a guix container that is +independent of the host (as described in this repository). + +** What about running a ~guix-daemon~ within a container (so that builds can be completely isolated)? + +This currently is not possible ... TBD (more detail) + +* Footnotes + +[fn:1] https://orgmode.org/ + +[fn:2] https://guix.gnu.org/manual/en/html_node/Channels.html + +[fn:3] https://guix.gnu.org/manual/en/html_node/Replicating-Guix.html diff --git a/channels.scm b/channels.scm new file mode 100644 index 0000000..66f92d1 --- /dev/null +++ b/channels.scm @@ -0,0 +1,11 @@ +(list (channel + (name 'guix) + (url "https://git.savannah.gnu.org/git/guix.git") + (branch "master") + (commit + "1b07f397dc17e31ad55b80a4efd34fdcb5b3c690") + (introduction + (make-channel-introduction + "9edb3f66fd807b096b48283debdcddccfea34bad" + (openpgp-fingerprint + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA"))))) diff --git a/manifest-dev.scm b/manifest-dev.scm new file mode 100644 index 0000000..4fa1847 --- /dev/null +++ b/manifest-dev.scm @@ -0,0 +1,3 @@ +(specifications->manifest + (list "make" + "docker-cli")) diff --git a/manifest.scm b/manifest.scm new file mode 100644 index 0000000..d9080a5 --- /dev/null +++ b/manifest.scm @@ -0,0 +1,8 @@ +(specifications->manifest + (list "guix" + "coreutils" + "bash" + "nss-certs" + "grep" + "gawk" + "sed")) diff --git a/shell.sh b/shell.sh new file mode 100755 index 0000000..e729e89 --- /dev/null +++ b/shell.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Usage: ./shell.sh [manifest] [dev-manifest] + +GUIX_MANIFEST="${1:-manifest.scm}" +GUIX_MANIFEST_DEV="${2:-manifest-dev.scm}" + +export PS1="\W [env]\$ " + +exec guix time-machine -C channels.scm -- shell -m $GUIX_MANIFEST -m $GUIX_MANIFEST_DEV -E '^PS1$' -C \ + --expose=/var/run/docker.sock=/var/run/docker.sock \ + --expose=/var/guix/daemon-socket/socket=/var/guix/daemon-socket/socket \ + --expose=/gnu/store=/gnu/store