build-system/go: Build with a filesystem union of Go dependencies.

This basically changes (guix build-system go) so that instead of looking
up its dependencies in a list of directories in $GOPATH, all the
Go dependencies are symlinked into a single directory.

Fixes <https://bugs.gnu.org/33620>.

* guix/build/go-build-system.scm (setup-go-environment): New variable.
(setup-environment, install-source): Remove variables.
(unpack): Unpack the source relative to $GOPATH.
(install): Do not install the compiled objects in the 'pkg' directory.
Install the source code in this phase, and only install the source of
the package named by IMPORT-PATH.
* doc/guix.texi (Build Systems): Adjust accordingly.
* gnu/packages/docker.scm (docker): Import (guix build union) on the build side
and adjust to build phase name changes in (guix build-system go).
* gnu/packages/shellutils.scm (direnv): Likewise.
* gnu/packages/databases.scm (mongo-tools)[arguments]:
Set '#:install-source #f'.
* gnu/packages/music.scm (demlo)[arguments]: Move the 'install-scripts'
phase after the 'install' phase.
This commit is contained in:
Leo Famulari 2018-01-06 15:47:47 -05:00
parent 5a14b913ad
commit e3900a4d64
No known key found for this signature in database
GPG Key ID: 2646FA30BACA7F08
7 changed files with 90 additions and 72 deletions

View File

@ -5791,8 +5791,8 @@ some cases, you will need to unpack the package's source code to a
different directory structure than the one indicated by the import path, different directory structure than the one indicated by the import path,
and @code{#:unpack-path} should be used in such cases. and @code{#:unpack-path} should be used in such cases.
Packages that provide Go libraries should be installed along with their Packages that provide Go libraries should install their source code into
source code. The key @code{#:install-source?}, which defaults to the built output. The key @code{#:install-source?}, which defaults to
@code{#t}, controls whether or not the source code is installed. It can @code{#t}, controls whether or not the source code is installed. It can
be set to @code{#f} for packages that only provide executable files. be set to @code{#f} for packages that only provide executable files.
@end defvr @end defvr

View File

@ -2610,6 +2610,7 @@ transforms idiomatic python function calls to well-formed SQL queries.")
#:modules ((srfi srfi-1) #:modules ((srfi srfi-1)
(guix build go-build-system) (guix build go-build-system)
(guix build utils)) (guix build utils))
#:install-source? #f
#:phases #:phases
(let ((all-tools (let ((all-tools
'("bsondump" "mongodump" "mongoexport" "mongofiles" '("bsondump" "mongodump" "mongoexport" "mongofiles"
@ -2629,8 +2630,6 @@ transforms idiomatic python function calls to well-formed SQL queries.")
(("skipping restore of system.profile collection\", db)") (("skipping restore of system.profile collection\", db)")
"skipping restore of system.profile collection\")")) "skipping restore of system.profile collection\")"))
#t)) #t))
;; We don't need to install the source code for end-user applications
(delete 'install-source)
(replace 'build (replace 'build
(lambda _ (lambda _
(for-each (lambda (tool) (for-each (lambda (tool)

View File

@ -249,9 +249,11 @@ network attachments.")
`(#:modules `(#:modules
((guix build gnu-build-system) ((guix build gnu-build-system)
((guix build go-build-system) #:prefix go:) ((guix build go-build-system) #:prefix go:)
(guix build union)
(guix build utils)) (guix build utils))
#:imported-modules #:imported-modules
(,@%gnu-build-system-modules (,@%gnu-build-system-modules
(guix build union)
(guix build go-build-system)) (guix build go-build-system))
#:phases #:phases
(modify-phases %standard-phases (modify-phases %standard-phases
@ -412,8 +414,8 @@ network attachments.")
;; Make build faster ;; Make build faster
(setenv "GOCACHE" "/tmp") (setenv "GOCACHE" "/tmp")
#t)) #t))
(add-before 'build 'setup-environment (add-before 'build 'setup-go-environment
(assoc-ref go:%standard-phases 'setup-environment)) (assoc-ref go:%standard-phases 'setup-go-environment))
(replace 'build (replace 'build
(lambda _ (lambda _
;; Our LD doesn't like the statically linked relocatable things ;; Our LD doesn't like the statically linked relocatable things

View File

@ -4361,7 +4361,7 @@ console music players.")
dir "/sbin")) dir "/sbin"))
(list ffmpeg chromaprint)))) (list ffmpeg chromaprint))))
#t))) #t)))
(add-after 'install-source 'install-scripts (add-after 'install 'install-scripts
(lambda* (#:key outputs #:allow-other-keys) (lambda* (#:key outputs #:allow-other-keys)
(let* ((out (assoc-ref outputs "out")) (let* ((out (assoc-ref outputs "out"))
(root (string-append out "/src/gitlab.com/ambrevar/demlo")) (root (string-append out "/src/gitlab.com/ambrevar/demlo"))

View File

@ -118,15 +118,17 @@ are already there.")
#:make-flags (list (string-append "DESTDIR=" (assoc-ref %outputs "out"))) #:make-flags (list (string-append "DESTDIR=" (assoc-ref %outputs "out")))
#:modules ((guix build gnu-build-system) #:modules ((guix build gnu-build-system)
((guix build go-build-system) #:prefix go:) ((guix build go-build-system) #:prefix go:)
(guix build union)
(guix build utils)) (guix build utils))
#:imported-modules (,@%gnu-build-system-modules #:imported-modules (,@%gnu-build-system-modules
(guix build union)
(guix build go-build-system)) (guix build go-build-system))
#:phases #:phases
(modify-phases %standard-phases (modify-phases %standard-phases
(delete 'configure) (delete 'configure)
;; Help the build scripts find the Go language dependencies. ;; Help the build scripts find the Go language dependencies.
(add-after 'unpack 'setup-go-environment (add-before 'unpack 'setup-go-environment
(assoc-ref go:%standard-phases 'setup-environment))))) (assoc-ref go:%standard-phases 'setup-go-environment)))))
(inputs (inputs
`(("go" ,go) `(("go" ,go)
("go-github-com-burntsushi-toml" ,go-github-com-burntsushi-toml) ("go-github-com-burntsushi-toml" ,go-github-com-burntsushi-toml)

View File

@ -39,6 +39,7 @@
(define %go-build-system-modules (define %go-build-system-modules
;; Build-side modules imported and used by default. ;; Build-side modules imported and used by default.
`((guix build go-build-system) `((guix build go-build-system)
(guix build union)
,@%gnu-build-system-modules)) ,@%gnu-build-system-modules))
(define (default-go) (define (default-go)
@ -87,6 +88,7 @@
(guile #f) (guile #f)
(imported-modules %go-build-system-modules) (imported-modules %go-build-system-modules)
(modules '((guix build go-build-system) (modules '((guix build go-build-system)
(guix build union)
(guix build utils)))) (guix build utils))))
(define builder (define builder
`(begin `(begin

View File

@ -1,6 +1,6 @@
;;; GNU Guix --- Functional package management for GNU ;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2016 Petter <petter@mykolab.ch> ;;; Copyright © 2016 Petter <petter@mykolab.ch>
;;; Copyright © 2017 Leo Famulari <leo@famulari.name> ;;; Copyright © 2017, 2019 Leo Famulari <leo@famulari.name>
;;; ;;;
;;; This file is part of GNU Guix. ;;; This file is part of GNU Guix.
;;; ;;;
@ -19,6 +19,7 @@
(define-module (guix build go-build-system) (define-module (guix build go-build-system)
#:use-module ((guix build gnu-build-system) #:prefix gnu:) #:use-module ((guix build gnu-build-system) #:prefix gnu:)
#:use-module (guix build union)
#:use-module (guix build utils) #:use-module (guix build utils)
#:use-module (ice-9 match) #:use-module (ice-9 match)
#:use-module (srfi srfi-1) #:use-module (srfi srfi-1)
@ -38,24 +39,26 @@
;; results. [0] ;; results. [0]
;; Go software is developed and built within a particular file system hierarchy ;; Go software is developed and built within a particular file system hierarchy
;; structure called a 'workspace' [1]. This workspace is found by Go ;; structure called a 'workspace' [1]. This workspace can be found by Go via
;; via the GOPATH environment variable. Typically, all Go source code ;; the GOPATH environment variable. Typically, all Go source code and compiled
;; and compiled objects are kept in a single workspace, but it is ;; objects are kept in a single workspace, but GOPATH may be a list of
;; possible for GOPATH to contain a list of directories, and that is ;; directories [2]. In this go-build-system we create a filesystem union of
;; what we do in this go-build-system. [2] ;; the Go-language dependencies. Previously, we made GOPATH a list of store
;; directories, but stopped because Go programs started keeping references to
;; these directories in Go 1.11:
;; <https://bugs.gnu.org/33620>.
;; ;;
;; Go software, whether a package or a command, is uniquely named using ;; Go software, whether a package or a command, is uniquely named using an
;; an 'import path'. The import path is based on the URL of the ;; 'import path'. The import path is based on the URL of the software's source.
;; software's source. Since most source code is provided over the ;; Because most source code is provided over the internet, the import path is
;; internet, the import path is typically a combination of the remote ;; typically a combination of the remote URL and the source repository's file
;; URL and the source repository's file system structure. For example, ;; system structure. For example, the Go port of the common `du` command is
;; the Go port of the common `du` command is hosted on github.com, at ;; hosted on github.com, at <https://github.com/calmh/du>. Thus, the import
;; <https://github.com/calmh/du>. Thus, the import path is ;; path is <github.com/calmh/du>. [3]
;; <github.com/calmh/du>. [3]
;; ;;
;; It may be possible to programatically guess a package's import path ;; It may be possible to automatically guess a package's import path based on
;; based on the source URL, but we don't try that in this revision of ;; the source URL, but we don't try that in this revision of the
;; the go-build-system. ;; go-build-system.
;; ;;
;; Modules of modular Go libraries are named uniquely with their ;; Modules of modular Go libraries are named uniquely with their
;; file system paths. For example, the supplemental but "standardized" ;; file system paths. For example, the supplemental but "standardized"
@ -75,6 +78,17 @@
;; file system union of the required modules of such libraries. I think ;; file system union of the required modules of such libraries. I think
;; this could be improved in future revisions of the go-build-system. ;; this could be improved in future revisions of the go-build-system.
;; ;;
;; TODO:
;; * Avoid copying dependencies into the build environment and / or avoid using
;; a tmpdir when creating the inputs union.
;; * Use Go modules [4]
;; * Re-use compiled packages [5]
;; * Avoid the go-inputs hack
;; * Stop needing remove-go-references (-trimpath ? )
;; * Remove module packages, only offering the full Git repos? This is
;; more idiomatic, I think, because Go downloads Git repos, not modules.
;; What are the trade-offs?
;;
;; [0] `go build`: ;; [0] `go build`:
;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies ;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies
;; `go install`: ;; `go install`:
@ -107,18 +121,44 @@
;; ;;
;; [2] https://golang.org/doc/code.html#GOPATH ;; [2] https://golang.org/doc/code.html#GOPATH
;; [3] https://golang.org/doc/code.html#ImportPaths ;; [3] https://golang.org/doc/code.html#ImportPaths
;; [4] https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more
;; [5] https://bugs.gnu.org/32919
;; ;;
;; Code: ;; Code:
(define* (setup-go-environment #:key inputs outputs #:allow-other-keys)
"Prepare a Go build environment for INPUTS and OUTPUTS. Build a filesystem
union of INPUTS. Export GOPATH, which helps the compiler find the source code
of the package being built and its dependencies, and GOBIN, which determines
where executables (\"commands\") are installed to. This phase is sometimes used
by packages that use (guix build-system gnu) but have a handful of Go
dependencies, so it should be self-contained."
;; Using the current working directory as GOPATH makes it easier for packagers
;; who need to manipulate the unpacked source code.
(setenv "GOPATH" (getcwd))
(setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin"))
(let ((tmpdir (tmpnam)))
(match (go-inputs inputs)
(((names . directories) ...)
(union-build tmpdir (filter directory-exists? directories)
#:create-all-directories? #t
#:log-port (%make-void-port "w"))))
;; XXX A little dance because (guix build union) doesn't use mkdir-p.
(copy-recursively tmpdir
(string-append (getenv "GOPATH"))
#:keep-mtime? #t)
(delete-file-recursively tmpdir))
#t)
(define* (unpack #:key source import-path unpack-path #:allow-other-keys) (define* (unpack #:key source import-path unpack-path #:allow-other-keys)
"Unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is the UNPACK-PATH is "Relative to $GOPATH, unpack SOURCE in the UNPACK-PATH, or the IMPORT-PATH is
unset. When SOURCE is a directory, copy it instead of unpacking." the UNPACK-PATH is unset. When SOURCE is a directory, copy it instead of
unpacking."
(if (string-null? import-path) (if (string-null? import-path)
((display "WARNING: The Go import path is unset.\n"))) ((display "WARNING: The Go import path is unset.\n")))
(if (string-null? unpack-path) (if (string-null? unpack-path)
(set! unpack-path import-path)) (set! unpack-path import-path))
(mkdir "src") (let ((dest (string-append (getenv "GOPATH") "/src/" unpack-path)))
(let ((dest (string-append "src/" unpack-path)))
(mkdir-p dest) (mkdir-p dest)
(if (file-is-directory? source) (if (file-is-directory? source)
(begin (begin
@ -128,15 +168,6 @@ unset. When SOURCE is a directory, copy it instead of unpacking."
(invoke "unzip" "-d" dest source) (invoke "unzip" "-d" dest source)
(invoke "tar" "-C" dest "-xvf" source))))) (invoke "tar" "-C" dest "-xvf" source)))))
(define* (install-source #:key install-source? outputs #:allow-other-keys)
"Install the source code to the output directory."
(let* ((out (assoc-ref outputs "out"))
(source "src")
(dest (string-append out "/" source)))
(when install-source?
(copy-recursively source dest #:keep-mtime? #t))
#t))
(define (go-package? name) (define (go-package? name)
(string-prefix? "go-" name)) (string-prefix? "go-" name))
@ -155,27 +186,6 @@ unset. When SOURCE is a directory, copy it instead of unpacking."
(_ #f)) (_ #f))
inputs)))) inputs))))
(define* (setup-environment #:key inputs outputs #:allow-other-keys)
"Export the variables GOPATH and GOBIN, which are based on INPUTS and OUTPUTS,
respectively."
(let ((out (assoc-ref outputs "out")))
;; GOPATH is where Go looks for the source code of the build's dependencies.
(set-path-environment-variable "GOPATH"
;; XXX Matching "." hints that we could do
;; something simpler here...
(list ".")
(match (go-inputs inputs)
(((_ . dir) ...)
dir)))
;; Add the source code of the package being built to GOPATH.
(if (getenv "GOPATH")
(setenv "GOPATH" (string-append (getcwd) ":" (getenv "GOPATH")))
(setenv "GOPATH" (getcwd)))
;; Where to install compiled executable files ('commands' in Go parlance').
(setenv "GOBIN" (string-append out "/bin"))
#t))
(define* (build #:key import-path #:allow-other-keys) (define* (build #:key import-path #:allow-other-keys)
"Build the package named by IMPORT-PATH." "Build the package named by IMPORT-PATH."
(with-throw-handler (with-throw-handler
@ -193,22 +203,26 @@ respectively."
"Here are the results of `go env`:\n")) "Here are the results of `go env`:\n"))
(invoke "go" "env")))) (invoke "go" "env"))))
;; Can this also install commands???
(define* (check #:key tests? import-path #:allow-other-keys) (define* (check #:key tests? import-path #:allow-other-keys)
"Run the tests for the package named by IMPORT-PATH." "Run the tests for the package named by IMPORT-PATH."
(when tests? (when tests?
(invoke "go" "test" import-path)) (invoke "go" "test" import-path))
#t) #t)
(define* (install #:key outputs #:allow-other-keys) (define* (install #:key install-source? outputs import-path unpack-path #:allow-other-keys)
"Install the compiled libraries. `go install` installs these files to "Install the source code of IMPORT-PATH to the primary output directory.
$GOPATH/pkg, so we have to copy them into the output directory manually. Compiled executable files (Go \"commands\") should have already been installed
Compiled executable files should have already been installed to the store based to the store based on $GOBIN in the build phase.
on $GOBIN in the build phase." XXX We can't make us of compiled libraries (Go \"packages\")."
;; TODO: From go-1.10 onward, the pkg folder should not be needed (see (when install-source?
;; https://lists.gnu.org/archive/html/guix-devel/2018-11/msg00208.html). (if (string-null? import-path)
;; Remove it? ((display "WARNING: The Go import path is unset.\n")))
(when (file-exists? "pkg") (let* ((out (assoc-ref outputs "out"))
(copy-recursively "pkg" (string-append (assoc-ref outputs "out") "/pkg"))) (source (string-append (getenv "GOPATH") "/src/" import-path))
(dest (string-append out "/src/" import-path)))
(mkdir-p dest)
(copy-recursively source dest #:keep-mtime? #t)))
#t) #t)
(define* (remove-store-reference file file-name (define* (remove-store-reference file file-name
@ -269,9 +283,8 @@ files in OUTPUTS."
(delete 'bootstrap) (delete 'bootstrap)
(delete 'configure) (delete 'configure)
(delete 'patch-generated-file-shebangs) (delete 'patch-generated-file-shebangs)
(add-before 'unpack 'setup-go-environment setup-go-environment)
(replace 'unpack unpack) (replace 'unpack unpack)
(add-after 'unpack 'install-source install-source)
(add-before 'build 'setup-environment setup-environment)
(replace 'build build) (replace 'build build)
(replace 'check check) (replace 'check check)
(replace 'install install) (replace 'install install)