rekahsoft
/
guix
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

import: pypi: Parse wheel METADATA instead of metadata.json. 

With newer Wheel releases, there is no more metadata.json file; the METADATA
file should be used instead (see: https://github.com/pypa/wheel/issues/195).

This change updates our PyPI importer so that it uses the latter.

* guix/import/pypi.scm (define-module): Remove unnecessary modules and export
the PARSE-WHEEL-METADATA procedure.
(parse-wheel-metadata): Add procedure.
(guess-requirements): Use it.
* tests/pypi.scm (test-metadata): Test it.
This commit is contained in:
Maxim Cournoyer 2019-03-28 00:26:03 -04:00
parent c799ad7276
commit f0190a5dcd
No known key found for this signature in database
GPG Key ID: 1260E46482E63562
2 changed files with 101 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
guix/import/pypi.scm
View File

@ -21,9 +21,7 @@
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (guix import pypi)
#:use-module (ice-9 binary-ports)
#:use-module (ice-9 match)
#:use-module (ice-9 pretty-print)
#:use-module (ice-9 regex)
#:use-module (ice-9 receive)
#:use-module ((ice-9 rdelim) #:select (read-line))
@ -31,9 +29,6 @@
#:use-module (srfi srfi-26)
#:use-module (srfi srfi-34)
#:use-module (srfi srfi-35)
#:use-module (rnrs bytevectors)
#:use-module (json)
#:use-module (web uri)
#:use-module (guix ui)
#:use-module (guix utils)
#:use-module ((guix build utils)
@ -49,6 +44,7 @@
#:use-module ((guix licenses) #:prefix license:)
#:use-module (guix build-system python)
#:export (parse-requires.txt
parse-wheel-metadata
specification->requirement-name
guix-package->pypi-name
pypi-recursive-import
@ -177,18 +173,49 @@ requirement names."
;; Stop when a section is encountered, as sections contain optional
;; (extra) requirements. Non-optional requirements must appear
;; before any section is defined.
(if (or (eof-object? line) (section-header? line))
;; Duplicates can occur, since the same requirement can be
;; listed multiple times with different conditional markers, e.g.
;; pytest >= 3 ; python_version >= "3.3"
;; pytest < 3 ; python_version < "3.3"
(reverse (delete-duplicates result))
(cond
((or (string-null? line) (comment? line))
(loop result))
(else
(loop (cons (specification->requirement-name line)
result))))))))))
(cond
((or (eof-object? line) (section-header? line))
;; Duplicates can occur, since the same requirement can be
;; listed multiple times with different conditional markers, e.g.
;; pytest >= 3 ; python_version >= "3.3"
;; pytest < 3 ; python_version < "3.3"
(reverse (delete-duplicates result)))
((or (string-null? line) (comment? line))
(loop result))
(else
(loop (cons (specification->requirement-name line)
result)))))))))
(define (parse-wheel-metadata metadata)
"Given METADATA, a Wheel metadata file, return a list of requirement names."
;; METADATA is a RFC-2822-like, header based file.
(define (requires-dist-header? line)
;; Return #t if the given LINE is a Requires-Dist header.
(string-match "^Requires-Dist: " line))
(define (requires-dist-value line)
(string-drop line (string-length "Requires-Dist: ")))
(define (extra? line)
;; Return #t if the given LINE is an "extra" requirement.
(string-match "extra == '(.*)'" line))
(call-with-input-file metadata
(lambda (port)
(let loop ((requirements '()))
(let ((line (read-line port)))
;; Stop at the first 'Provides-Extra' section: the non-optional
;; requirements appear before the optional ones.
(cond
((eof-object? line)
(reverse (delete-duplicates requirements)))
((and (requires-dist-header? line) (not (extra? line)))
(loop (cons (specification->requirement-name
(requires-dist-value line))
requirements)))
(else
(loop requirements))))))))
(define (guess-requirements source-url wheel-url archive)
"Given SOURCE-URL, WHEEL-URL and a ARCHIVE of the package, return a list
@ -197,25 +224,18 @@ be extracted in a temporary directory."
(define (read-wheel-metadata wheel-archive)
;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's
;; requirements.
;; requirements, or #f if the metadata file contained therein couldn't be
;; extracted.
(let* ((dirname (wheel-url->extracted-directory wheel-url))
(json-file (string-append dirname "/metadata.json")))
(and (zero? (system* "unzip" "-q" wheel-archive json-file))
(dynamic-wind
(const #t)
(lambda ()
(call-with-input-file json-file
(lambda (port)
(let* ((metadata (json->scm port))
(run_requires (hash-ref metadata "run_requires"))
(requirements (if run_requires
(hash-ref (list-ref run_requires 0)
"requires")
'())))
(map specification->requirement-name requirements)))))
(lambda ()
(delete-file json-file)
(rmdir dirname))))))
(metadata (string-append dirname "/METADATA")))
(call-with-temporary-directory
(lambda (dir)
(if (zero? (system* "unzip" "-q" wheel-archive "-d" dir metadata))
(parse-wheel-metadata (string-append dir "/" metadata))
(begin
(warning
(G_ "Failed to extract file: ~a from wheel.~%") metadata)
#f))))))
(define (guess-requirements-from-wheel)
;; Return the package's requirements using the wheel, or #f if an error

60
tests/pypi.scm
View File

@ -22,6 +22,7 @@
#:use-module (guix base32)
#:use-module (guix memoization)
#:use-module (gcrypt hash)
#:use-module (guix memoization)
#:use-module (guix tests)
#:use-module (guix build-system python)
#:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
@ -79,17 +80,33 @@ bar != 2
pytest (>=2.5.0)
")
(define test-metadata
"{
\"run_requires\": [
{
\"requires\": [
\"bar\",
\"baz (>13.37)\"
]
}
]
}")
(define test-metadata "\
Classifier: Programming Language :: Python :: 3.7
Requires-Dist: baz ~= 3
Requires-Dist: bar != 2
Provides-Extra: test
pytest (>=2.5.0)
")
(define test-metadata-with-extras "
Classifier: Programming Language :: Python :: 3.7
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Requires-Dist: wrapt (<2,>=1)
Requires-Dist: bar
Provides-Extra: dev
Requires-Dist: tox ; extra == 'dev'
Requires-Dist: bumpversion (<1) ; extra == 'dev'
")
;;; Provides-Extra can appear before Requires-Dist.
(define test-metadata-with-extras-jedi "\
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Provides-Extra: testing
Requires-Dist: parso (>=0.3.0)
Provides-Extra: testing
Requires-Dist: pytest (>=3.1.0); extra == 'testing'
")
(test-begin "pypi")
@ -128,6 +145,18 @@ pytest (>=2.5.0)
call-with-input-string)
(parse-requires.txt test-requires-with-sections)))
(test-equal "parse-wheel-metadata, with extras"
'("wrapt" "bar")
(mock ((ice-9 ports) call-with-input-file
call-with-input-string)
(parse-wheel-metadata test-metadata-with-extras)))
(test-equal "parse-wheel-metadata, with extras - Jedi"
'("parso")
(mock ((ice-9 ports) call-with-input-file
call-with-input-string)
(parse-wheel-metadata test-metadata-with-extras-jedi)))
(test-assert "pypi->guix-package"
;; Replace network resources with sample data.
(mock ((guix import utils) url-fetch
@ -191,7 +220,7 @@ pytest (>=2.5.0)
(mkdir-p "foo-1.0.0/foo.egg-info/")
(with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
(lambda ()
(display test-requires.txt)))
(display "wrong data to make sure we're testing wheels ")))
(parameterize ((current-output-port (%make-void-port "rw+")))
(system* "tar" "czvf" file-name "foo-1.0.0/"))
(delete-file-recursively "foo-1.0.0")
@ -200,13 +229,13 @@ pytest (>=2.5.0)
("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
(begin
(mkdir "foo-1.0.0.dist-info")
(with-output-to-file "foo-1.0.0.dist-info/metadata.json"
(with-output-to-file "foo-1.0.0.dist-info/METADATA"
(lambda ()
(display test-metadata)))
(let ((zip-file (string-append file-name ".zip")))
;; zip always adds a "zip" extension to the file it creates,
;; so we need to rename it.
(system* "zip" zip-file "foo-1.0.0.dist-info/metadata.json")
(system* "zip" zip-file "foo-1.0.0.dist-info/METADATA")
(rename-file zip-file file-name))
(delete-file-recursively "foo-1.0.0.dist-info")))
(_ (error "Unexpected URL: " url)))))
@ -218,6 +247,9 @@ pytest (>=2.5.0)
(string-length test-json)))
("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
(_ (error "Unexpected URL: " url)))))
;; Not clearing the memoization cache here would mean returning the value
;; computed in the previous test.
(invalidate-memoization! pypi->guix-package)
(match (pypi->guix-package "foo")
(('package
('name "python-foo")