req-compile 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {req_compile-1.2.0 → req_compile-1.4.0}/CHANGELOG.rst +28 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/MANIFEST.in +6 -1
- {req_compile-1.2.0/req_compile.egg-info → req_compile-1.4.0}/PKG-INFO +42 -7
- {req_compile-1.2.0 → req_compile-1.4.0}/README.rst +13 -6
- {req_compile-1.2.0 → req_compile-1.4.0}/pyproject.toml +6 -4
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/compile.py +209 -11
- req_compile-1.4.0/req_compile/errors.py +134 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/solution.py +66 -20
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/target.py +13 -9
- {req_compile-1.2.0 → req_compile-1.4.0/req_compile.egg-info}/PKG-INFO +42 -7
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/SOURCES.txt +1 -18
- {req_compile-1.2.0 → req_compile-1.4.0}/version.bzl +1 -1
- req_compile-1.2.0/req_compile/errors.py +0 -55
- req_compile-1.2.0/tests/test_cmdline.py +0 -136
- req_compile-1.2.0/tests/test_compile.py +0 -309
- req_compile-1.2.0/tests/test_containers.py +0 -18
- req_compile-1.2.0/tests/test_cross_platform.py +0 -47
- req_compile-1.2.0/tests/test_dists.py +0 -449
- req_compile-1.2.0/tests/test_egg_info.py +0 -173
- req_compile-1.2.0/tests/test_extractor.py +0 -127
- req_compile-1.2.0/tests/test_filename.py +0 -35
- req_compile-1.2.0/tests/test_integration.py +0 -74
- req_compile-1.2.0/tests/test_merge_requirements.py +0 -92
- req_compile-1.2.0/tests/test_metadata.py +0 -235
- req_compile-1.2.0/tests/test_no_setup_py.py +0 -20
- req_compile-1.2.0/tests/test_repositories.py +0 -195
- req_compile-1.2.0/tests/test_solution_target.py +0 -182
- req_compile-1.2.0/tests/test_target.py +0 -406
- req_compile-1.2.0/tests/test_utils.py +0 -61
- req_compile-1.2.0/tests/test_versions.py +0 -134
- {req_compile-1.2.0 → req_compile-1.4.0}/LICENSE.txt +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/__init__.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/__main__.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/candidates.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/cmdline.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/config.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/containers.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/dists.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/filename.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/__init__.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/dist_info.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/extractor.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/metadata.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/patch.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/pyproject.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/source.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/py.typed +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/__init__.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/findlinks.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/multi.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/pypi.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/repository.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/source.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/utils.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/versions.py +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/dependency_links.txt +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/entry_points.txt +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/requires.txt +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/top_level.txt +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/requirements.in +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/setup.cfg +0 -0
- {req_compile-1.2.0 → req_compile-1.4.0}/setup.py +0 -0
|
@@ -1,5 +1,33 @@
|
|
|
1
|
+
New in version 1.3.1:
|
|
2
|
+
|
|
3
|
+
* ``py_reqs_compiler`` compiles the solutions of several platforms together: ``requirements_txts`` maps a
|
|
4
|
+
solution file to each platform. The solutions use the same version of every package, a package keeps the
|
|
5
|
+
version the first platform resolves it to and a version another platform cannot use is dropped for all of
|
|
6
|
+
them. The compilation fails, naming the platforms, when no version of a package works everywhere. A
|
|
7
|
+
``py_reqs_solution_test`` of such a target checks every solution, including that they agree.
|
|
8
|
+
* The Bazel compiler accepts ``--solution``, ``--platform`` and ``--output`` once per solution.
|
|
9
|
+
* ``perform_compile`` accepts soft pins, which yield with ``PinConflictException`` when a package cannot
|
|
10
|
+
satisfy them, and ``perform_compile_group`` compiles the same requirements for several targets with one
|
|
11
|
+
version per package.
|
|
12
|
+
|
|
1
13
|
New in version 1.2.0:
|
|
2
14
|
|
|
15
|
+
* ``requirements.parse`` now maps lock files to ids in ``requirements_locks`` and config settings to those
|
|
16
|
+
ids in the new ``config_settings`` attribute. Both are label keyed, so each is resolved by the module
|
|
17
|
+
defining the hub and any config setting visible to it can be used. ``//conditions:default`` names a
|
|
18
|
+
fallback lock. Without ``config_settings``, locks compiled with the ``platform`` attribute of
|
|
19
|
+
``py_reqs_compiler`` are selected by the platform read from the header of the file. Label strings as
|
|
20
|
+
``requirements_locks`` values still work but are deprecated.
|
|
21
|
+
* Every distinct wheel is downloaded once into a repository named ``{package}_{version}_{sha256[:8]}``
|
|
22
|
+
and shared by all hubs of all modules pinning it. Each hub keeps its own dependency edges, so extras
|
|
23
|
+
and version differences between hubs are preserved. Hub names must be unique across modules.
|
|
24
|
+
* ``package_annotation`` applies to the hubs of the module declaring it unless its new ``hubs``
|
|
25
|
+
attribute names particular hubs or ``"*"`` for every hub. Hubs annotated differently get their own
|
|
26
|
+
repository for the package.
|
|
27
|
+
* Hubs whose locks do not all contain the same packages are supported. A package missing from a lock
|
|
28
|
+
is incompatible with that lock's platform, so targets depending on it are skipped there.
|
|
29
|
+
* Removed the ``defs_id_map`` attribute of the ``parse`` tag and the WORKSPACE-only ``repositories()``
|
|
30
|
+
flow. ``py_requirements_repository`` is now only instantiated by the module extension.
|
|
3
31
|
* Cross-platform compilation. ``--platform``, ``--python-version`` and ``--implementation`` select
|
|
4
32
|
wheels and evaluate environment markers for an environment other than the one running ``req-compile``.
|
|
5
33
|
``--platform`` takes a wheel platform tag such as ``manylinux_2_28_x86_64`` or ``macosx_11_0_arm64``,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: req-compile
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Python requirements compiler
|
|
5
5
|
Home-page: https://github.com/periareon/req-compile
|
|
6
6
|
Author: Spencer Putt
|
|
@@ -38,8 +38,36 @@ Dynamic: requires-dist
|
|
|
38
38
|
Dynamic: requires-python
|
|
39
39
|
Dynamic: summary
|
|
40
40
|
|
|
41
|
+
New in version 1.3.1:
|
|
42
|
+
|
|
43
|
+
* ``py_reqs_compiler`` compiles the solutions of several platforms together: ``requirements_txts`` maps a
|
|
44
|
+
solution file to each platform. The solutions use the same version of every package, a package keeps the
|
|
45
|
+
version the first platform resolves it to and a version another platform cannot use is dropped for all of
|
|
46
|
+
them. The compilation fails, naming the platforms, when no version of a package works everywhere. A
|
|
47
|
+
``py_reqs_solution_test`` of such a target checks every solution, including that they agree.
|
|
48
|
+
* The Bazel compiler accepts ``--solution``, ``--platform`` and ``--output`` once per solution.
|
|
49
|
+
* ``perform_compile`` accepts soft pins, which yield with ``PinConflictException`` when a package cannot
|
|
50
|
+
satisfy them, and ``perform_compile_group`` compiles the same requirements for several targets with one
|
|
51
|
+
version per package.
|
|
52
|
+
|
|
41
53
|
New in version 1.2.0:
|
|
42
54
|
|
|
55
|
+
* ``requirements.parse`` now maps lock files to ids in ``requirements_locks`` and config settings to those
|
|
56
|
+
ids in the new ``config_settings`` attribute. Both are label keyed, so each is resolved by the module
|
|
57
|
+
defining the hub and any config setting visible to it can be used. ``//conditions:default`` names a
|
|
58
|
+
fallback lock. Without ``config_settings``, locks compiled with the ``platform`` attribute of
|
|
59
|
+
``py_reqs_compiler`` are selected by the platform read from the header of the file. Label strings as
|
|
60
|
+
``requirements_locks`` values still work but are deprecated.
|
|
61
|
+
* Every distinct wheel is downloaded once into a repository named ``{package}_{version}_{sha256[:8]}``
|
|
62
|
+
and shared by all hubs of all modules pinning it. Each hub keeps its own dependency edges, so extras
|
|
63
|
+
and version differences between hubs are preserved. Hub names must be unique across modules.
|
|
64
|
+
* ``package_annotation`` applies to the hubs of the module declaring it unless its new ``hubs``
|
|
65
|
+
attribute names particular hubs or ``"*"`` for every hub. Hubs annotated differently get their own
|
|
66
|
+
repository for the package.
|
|
67
|
+
* Hubs whose locks do not all contain the same packages are supported. A package missing from a lock
|
|
68
|
+
is incompatible with that lock's platform, so targets depending on it are skipped there.
|
|
69
|
+
* Removed the ``defs_id_map`` attribute of the ``parse`` tag and the WORKSPACE-only ``repositories()``
|
|
70
|
+
flow. ``py_requirements_repository`` is now only instantiated by the module extension.
|
|
43
71
|
* Cross-platform compilation. ``--platform``, ``--python-version`` and ``--implementation`` select
|
|
44
72
|
wheels and evaluate environment markers for an environment other than the one running ``req-compile``.
|
|
45
73
|
``--platform`` takes a wheel platform tag such as ``manylinux_2_28_x86_64`` or ``macosx_11_0_arm64``,
|
|
@@ -114,19 +142,26 @@ To get started with Bazel, add the following to your ``MODULE.bazel`` file:
|
|
|
114
142
|
# Find {version} on the req-compile GitHub releases page.
|
|
115
143
|
bazel_dep(name = "rules_req_compile", version = "{version}")
|
|
116
144
|
|
|
117
|
-
requirements = use_extension("//extensions:python.bzl", "requirements")
|
|
145
|
+
requirements = use_extension("@rules_req_compile//extensions:python.bzl", "requirements")
|
|
118
146
|
requirements.parse(
|
|
119
147
|
name = "my_pip_deps",
|
|
120
148
|
requirements_locks = {
|
|
121
|
-
"//3rdparty:requirements.
|
|
122
|
-
"//3rdparty:requirements.
|
|
123
|
-
"//3rdparty:requirements.
|
|
149
|
+
"//3rdparty:requirements.linux_x86_64.txt": "linux_x86_64",
|
|
150
|
+
"//3rdparty:requirements.macos_aarch64.txt": "macos_aarch64",
|
|
151
|
+
"//3rdparty:requirements.windows_x86_64.txt": "windows_x86_64",
|
|
124
152
|
},
|
|
125
153
|
)
|
|
126
154
|
use_repo(requirements, "my_pip_deps")
|
|
127
155
|
|
|
128
|
-
|
|
129
|
-
``py_reqs_compiler
|
|
156
|
+
Each lock is the solution for one platform and is given an id. Locks compiled with the ``platform``
|
|
157
|
+
attribute of ``py_reqs_compiler`` record their platform, which is used to select them. To decide
|
|
158
|
+
yourself, map your own ``config_setting`` labels to the lock ids with ``config_settings``. The solutions
|
|
159
|
+
for every platform can be compiled from a single machine.
|
|
160
|
+
|
|
161
|
+
Every wheel is downloaded once and shared by all the hubs of all the modules pinning it, while each hub
|
|
162
|
+
keeps the dependencies it resolved. Packages missing from the lock of a platform make the targets using
|
|
163
|
+
them incompatible with that platform instead of failing the build.
|
|
164
|
+
See the `Bazel rules documentation <docs/docs.md>`_ for details.
|
|
130
165
|
|
|
131
166
|
|
|
132
167
|
Why use it?
|
|
@@ -36,19 +36,26 @@ To get started with Bazel, add the following to your ``MODULE.bazel`` file:
|
|
|
36
36
|
# Find {version} on the req-compile GitHub releases page.
|
|
37
37
|
bazel_dep(name = "rules_req_compile", version = "{version}")
|
|
38
38
|
|
|
39
|
-
requirements = use_extension("//extensions:python.bzl", "requirements")
|
|
39
|
+
requirements = use_extension("@rules_req_compile//extensions:python.bzl", "requirements")
|
|
40
40
|
requirements.parse(
|
|
41
41
|
name = "my_pip_deps",
|
|
42
42
|
requirements_locks = {
|
|
43
|
-
"//3rdparty:requirements.
|
|
44
|
-
"//3rdparty:requirements.
|
|
45
|
-
"//3rdparty:requirements.
|
|
43
|
+
"//3rdparty:requirements.linux_x86_64.txt": "linux_x86_64",
|
|
44
|
+
"//3rdparty:requirements.macos_aarch64.txt": "macos_aarch64",
|
|
45
|
+
"//3rdparty:requirements.windows_x86_64.txt": "windows_x86_64",
|
|
46
46
|
},
|
|
47
47
|
)
|
|
48
48
|
use_repo(requirements, "my_pip_deps")
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
``py_reqs_compiler
|
|
50
|
+
Each lock is the solution for one platform and is given an id. Locks compiled with the ``platform``
|
|
51
|
+
attribute of ``py_reqs_compiler`` record their platform, which is used to select them. To decide
|
|
52
|
+
yourself, map your own ``config_setting`` labels to the lock ids with ``config_settings``. The solutions
|
|
53
|
+
for every platform can be compiled from a single machine.
|
|
54
|
+
|
|
55
|
+
Every wheel is downloaded once and shared by all the hubs of all the modules pinning it, while each hub
|
|
56
|
+
keeps the dependencies it resolved. Packages missing from the lock of a platform make the targets using
|
|
57
|
+
them incompatible with that platform instead of failing the build.
|
|
58
|
+
See the `Bazel rules documentation <docs/docs.md>`_ for details.
|
|
52
59
|
|
|
53
60
|
|
|
54
61
|
Why use it?
|
|
@@ -2,8 +2,13 @@
|
|
|
2
2
|
requires = ["setuptools", "wheel"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
|
+
[tool.isort]
|
|
6
|
+
profile = "black"
|
|
7
|
+
|
|
5
8
|
[tool.mypy]
|
|
6
|
-
|
|
9
|
+
# `req_compile.private` deliberately has no `__init__.py` so that `setup.py`'s
|
|
10
|
+
# `find_packages` leaves the Bazel rule helpers out of the distribution.
|
|
11
|
+
namespace_packages = true
|
|
7
12
|
|
|
8
13
|
[tool.pylint.main]
|
|
9
14
|
max-line-length = 120
|
|
@@ -21,6 +26,3 @@ disable = [
|
|
|
21
26
|
"consider-using-f-string",
|
|
22
27
|
"useless-import-alias", # Conflict with ruff check.
|
|
23
28
|
]
|
|
24
|
-
|
|
25
|
-
[tool.isort]
|
|
26
|
-
profile = "black"
|
|
@@ -6,15 +6,22 @@ import logging
|
|
|
6
6
|
import operator
|
|
7
7
|
import os
|
|
8
8
|
from collections import defaultdict
|
|
9
|
-
from typing import Dict, Iterable, Mapping, Optional, Set, Tuple
|
|
9
|
+
from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple
|
|
10
10
|
|
|
11
11
|
import packaging.requirements
|
|
12
|
+
import packaging.version
|
|
12
13
|
|
|
13
14
|
import req_compile.containers
|
|
14
15
|
import req_compile.dists
|
|
15
16
|
from req_compile.containers import RequirementContainer
|
|
16
17
|
from req_compile.dists import DependencyNode, DistributionCollection
|
|
17
|
-
from req_compile.errors import
|
|
18
|
+
from req_compile.errors import (
|
|
19
|
+
MetadataError,
|
|
20
|
+
NoCandidateException,
|
|
21
|
+
NoCommonVersionException,
|
|
22
|
+
PinConflictException,
|
|
23
|
+
Rejection,
|
|
24
|
+
)
|
|
18
25
|
from req_compile.repos.repository import Repository
|
|
19
26
|
from req_compile.repos.source import SourceRepository
|
|
20
27
|
from req_compile.target import TargetEnvironment
|
|
@@ -30,6 +37,9 @@ from req_compile.versions import is_possible
|
|
|
30
37
|
|
|
31
38
|
MAX_COMPILE_DEPTH = int(os.environ.get("REQ_COMPILE_MAX_DEPTH", "80"))
|
|
32
39
|
MAX_DOWNGRADE = int(os.environ.get("REQ_COMPILE_MAX_DOWNGRADE", "20"))
|
|
40
|
+
# The number of times solutions compiled together are resolved again after one
|
|
41
|
+
# of them rejected a version another had chosen.
|
|
42
|
+
MAX_GROUP_PASSES = int(os.environ.get("REQ_COMPILE_MAX_GROUP_PASSES", "20"))
|
|
33
43
|
|
|
34
44
|
LOG = logging.getLogger("req_compile.compile")
|
|
35
45
|
|
|
@@ -50,14 +60,24 @@ class CompileOptions:
|
|
|
50
60
|
extras: Optional[Iterable[str]] = None
|
|
51
61
|
allow_circular_dependencies: bool = True
|
|
52
62
|
pinned_requirements: Mapping[NormName, packaging.requirements.Requirement] = {}
|
|
63
|
+
# Requirements applied to a project only if it is required, which yield when
|
|
64
|
+
# the project cannot satisfy them: `PinConflictException` is raised instead of
|
|
65
|
+
# resolving the reverse dependencies differently. Solutions compiled together
|
|
66
|
+
# use these to keep their versions aligned, see `perform_compile_group`.
|
|
67
|
+
soft_pins: Mapping[NormName, packaging.requirements.Requirement] = {}
|
|
53
68
|
only_binary: Set[NormName] = set()
|
|
54
69
|
|
|
55
70
|
|
|
56
|
-
def _get_strictest_reverse_dep(
|
|
71
|
+
def _get_strictest_reverse_dep(
|
|
72
|
+
node: DependencyNode,
|
|
73
|
+
soft_pin: Optional[packaging.requirements.Requirement] = None,
|
|
74
|
+
) -> Optional[DependencyNode]:
|
|
57
75
|
"""Get the strictest constraint from all reverse dependencies.
|
|
58
76
|
|
|
59
77
|
Args:
|
|
60
78
|
node: The node to check.
|
|
79
|
+
soft_pin: A soft pin on the node, scored like a reverse dependency that
|
|
80
|
+
cannot itself be walked back.
|
|
61
81
|
|
|
62
82
|
Returns:
|
|
63
83
|
The single reverse dependency that was the most conflicting. If no reverse
|
|
@@ -67,6 +87,10 @@ def _get_strictest_reverse_dep(node: DependencyNode) -> Optional[DependencyNode]
|
|
|
67
87
|
|
|
68
88
|
violate_score: Dict[DependencyNode, int] = defaultdict(int)
|
|
69
89
|
for idx, revnode in enumerate(nodes):
|
|
90
|
+
if soft_pin is not None and not is_possible(
|
|
91
|
+
merge_requirements(revnode.dependencies[node], soft_pin)
|
|
92
|
+
):
|
|
93
|
+
violate_score[revnode] += 1
|
|
70
94
|
for next_node in nodes[idx + 1 :]:
|
|
71
95
|
if not is_possible(
|
|
72
96
|
merge_requirements(
|
|
@@ -158,6 +182,20 @@ def compile_roots(
|
|
|
158
182
|
pin = options.pinned_requirements.get(spec_name, spec_req)
|
|
159
183
|
spec_req = merge_requirements(spec_req, pin)
|
|
160
184
|
|
|
185
|
+
# The requirement before any soft pin, which decides whether a failure is
|
|
186
|
+
# the pin's fault.
|
|
187
|
+
unpinned_req = spec_req
|
|
188
|
+
soft_pin = options.soft_pins.get(spec_name)
|
|
189
|
+
if soft_pin is not None:
|
|
190
|
+
spec_req = merge_requirements(spec_req, soft_pin)
|
|
191
|
+
|
|
192
|
+
def _no_candidate(no_candidate: bool) -> NoCandidateException:
|
|
193
|
+
if soft_pin is not None and is_possible(unpinned_req):
|
|
194
|
+
return PinConflictException(
|
|
195
|
+
spec_name, soft_pin, unpinned_req, node, no_candidate
|
|
196
|
+
)
|
|
197
|
+
return NoCandidateException(spec_req)
|
|
198
|
+
|
|
161
199
|
if not is_possible(spec_req):
|
|
162
200
|
logger.info("Requirement conflict for %s (%s)", node, spec_req)
|
|
163
201
|
# Try walking back. Some reverse dependency contributed a requirement
|
|
@@ -167,13 +205,13 @@ def compile_roots(
|
|
|
167
205
|
# version from being selected.
|
|
168
206
|
# Search the full space is expensive and possibly harmful because of how heavily
|
|
169
207
|
# it must hit remote indices.
|
|
170
|
-
reverse_dep = _get_strictest_reverse_dep(node)
|
|
208
|
+
reverse_dep = _get_strictest_reverse_dep(node, soft_pin)
|
|
171
209
|
if (
|
|
172
210
|
reverse_dep is None
|
|
173
211
|
or reverse_dep.metadata is None
|
|
174
212
|
or reverse_dep.metadata.version is None
|
|
175
213
|
):
|
|
176
|
-
raise
|
|
214
|
+
raise _no_candidate(no_candidate=False)
|
|
177
215
|
|
|
178
216
|
if _walkback_budget is not None and _walkback_budget <= 0:
|
|
179
217
|
logger.info(
|
|
@@ -181,7 +219,7 @@ def compile_roots(
|
|
|
181
219
|
node,
|
|
182
220
|
spec_req,
|
|
183
221
|
)
|
|
184
|
-
ex =
|
|
222
|
+
ex = _no_candidate(no_candidate=False)
|
|
185
223
|
ex.conflicting_node = node
|
|
186
224
|
ex.walkback_project = reverse_dep.metadata.name
|
|
187
225
|
raise ex
|
|
@@ -241,11 +279,22 @@ def compile_roots(
|
|
|
241
279
|
dists.remove_dists(bad_constraint, remove_upstream=False)
|
|
242
280
|
return
|
|
243
281
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
282
|
+
try:
|
|
283
|
+
metadata, cached = repo.get_dist(
|
|
284
|
+
spec_req,
|
|
285
|
+
allow_source_dist=spec_name not in options.only_binary,
|
|
286
|
+
max_downgrade=max_downgrade,
|
|
287
|
+
)
|
|
288
|
+
except NoCandidateException:
|
|
289
|
+
if soft_pin is None:
|
|
290
|
+
raise
|
|
291
|
+
# The pin is only at fault if the requirement is solvable without it.
|
|
292
|
+
repo.get_dist(
|
|
293
|
+
unpinned_req,
|
|
294
|
+
allow_source_dist=spec_name not in options.only_binary,
|
|
295
|
+
max_downgrade=max_downgrade,
|
|
296
|
+
)
|
|
297
|
+
raise _no_candidate(no_candidate=True) from None
|
|
249
298
|
logger.debug(
|
|
250
299
|
"Acquired candidate %s %s [%s] (%s)",
|
|
251
300
|
metadata,
|
|
@@ -312,6 +361,7 @@ def perform_compile(
|
|
|
312
361
|
only_binary: Optional[Set[NormName]] = None,
|
|
313
362
|
max_downgrade: Optional[int] = None,
|
|
314
363
|
target: Optional[TargetEnvironment] = None,
|
|
364
|
+
pins: Optional[Mapping[NormName, packaging.requirements.Requirement]] = None,
|
|
315
365
|
) -> Tuple[DistributionCollection, Set[DependencyNode]]:
|
|
316
366
|
"""Perform a compilation using the given inputs and constraints.
|
|
317
367
|
|
|
@@ -331,6 +381,11 @@ def perform_compile(
|
|
|
331
381
|
max_downgrade: The maximum number of version downgrades that will be allowed for conflicts.
|
|
332
382
|
target: The environment to compile for. Defaults to the target of `repo`, which
|
|
333
383
|
itself defaults to the host.
|
|
384
|
+
pins: Soft pins keyed by normalized name. Unlike constraints, a soft pin
|
|
385
|
+
never requires a package, is not recorded in the solution and yields
|
|
386
|
+
when the package cannot satisfy it: `PinConflictException` is raised
|
|
387
|
+
instead of resolving the dependents differently. See
|
|
388
|
+
`CompileOptions.soft_pins`.
|
|
334
389
|
|
|
335
390
|
Returns:
|
|
336
391
|
the solution and root nodes used to generate it
|
|
@@ -370,6 +425,7 @@ def perform_compile(
|
|
|
370
425
|
if all_pinned and constraint_reqs:
|
|
371
426
|
LOG.info("All constraints were pins - no need to solve the constraints")
|
|
372
427
|
options.pinned_requirements = pinned_requirements
|
|
428
|
+
options.soft_pins = dict(pins or {})
|
|
373
429
|
|
|
374
430
|
if max_downgrade is None:
|
|
375
431
|
max_downgrade = MAX_DOWNGRADE
|
|
@@ -402,6 +458,148 @@ def perform_compile(
|
|
|
402
458
|
return results, roots
|
|
403
459
|
|
|
404
460
|
|
|
461
|
+
def _soft_pins(
|
|
462
|
+
agreed: Mapping[NormName, packaging.version.Version],
|
|
463
|
+
rejected: Mapping[NormName, Sequence[Rejection]],
|
|
464
|
+
) -> Dict[NormName, packaging.requirements.Requirement]:
|
|
465
|
+
"""The soft pins keeping solutions compiled together aligned.
|
|
466
|
+
|
|
467
|
+
A project with an agreed version is pinned to it. A project without one, whose
|
|
468
|
+
versions some solution rejected, is kept away from those versions.
|
|
469
|
+
"""
|
|
470
|
+
pins = {}
|
|
471
|
+
for name, version in agreed.items():
|
|
472
|
+
pins[name] = parse_requirement(f"{name}=={version}")
|
|
473
|
+
for name, rejections in rejected.items():
|
|
474
|
+
if name in agreed or not rejections:
|
|
475
|
+
continue
|
|
476
|
+
pins[name] = parse_requirement(
|
|
477
|
+
name + ",".join(f"!={rejection.version}" for rejection in rejections)
|
|
478
|
+
)
|
|
479
|
+
return pins
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def perform_compile_group(
|
|
483
|
+
input_reqs: Iterable[RequirementContainer],
|
|
484
|
+
repos: Sequence[Repository],
|
|
485
|
+
constraint_reqs: Optional[Iterable[RequirementContainer]] = None,
|
|
486
|
+
remove_constraints: bool = False,
|
|
487
|
+
extras: Optional[Iterable[str]] = None,
|
|
488
|
+
allow_circular_dependencies: bool = True,
|
|
489
|
+
only_binary: Optional[Set[NormName]] = None,
|
|
490
|
+
max_downgrade: Optional[int] = None,
|
|
491
|
+
pins: Optional[Mapping[NormName, packaging.version.Version]] = None,
|
|
492
|
+
max_passes: int = MAX_GROUP_PASSES,
|
|
493
|
+
) -> List[Tuple[DistributionCollection, Set[DependencyNode]]]:
|
|
494
|
+
"""Compile the same requirements for several targets, with one version per project.
|
|
495
|
+
|
|
496
|
+
Each repository is compiled for its own target, in order. A project keeps the
|
|
497
|
+
version the first target that needs it resolves it to, so the solutions never
|
|
498
|
+
disagree. When a later target cannot use that version at all, e.g. there is no
|
|
499
|
+
artifact for its platform or its dependencies require another version, the
|
|
500
|
+
version is rejected for every target and the group is compiled again, until a
|
|
501
|
+
pass completes without any rejection. A target may also resolve the dependents
|
|
502
|
+
of a pinned project to other versions than an earlier target did rather than
|
|
503
|
+
reject the version, in which case the next pass aligns the earlier target.
|
|
504
|
+
|
|
505
|
+
Args:
|
|
506
|
+
input_reqs: The requirements, shared by every target.
|
|
507
|
+
repos: One repository per target, compiled in this order. Each compiles for
|
|
508
|
+
its own target, see `Repository.target`.
|
|
509
|
+
constraint_reqs: Constraints to use when compiling.
|
|
510
|
+
remove_constraints: Whether to remove the constraints from the solutions.
|
|
511
|
+
extras: Extras to apply automatically to source projects.
|
|
512
|
+
allow_circular_dependencies: Whether to allow circular dependencies.
|
|
513
|
+
only_binary: Set of projects that should only consider binary distributions.
|
|
514
|
+
max_downgrade: The maximum number of version downgrades that will be allowed
|
|
515
|
+
for conflicts, and the maximum number of versions of a project the
|
|
516
|
+
targets may reject before giving up.
|
|
517
|
+
pins: Versions to start from, keyed by normalized name, e.g. those of the
|
|
518
|
+
solutions being recompiled. They are soft pins: a project is only pinned
|
|
519
|
+
if it is required, and a version that a target cannot use is rejected
|
|
520
|
+
like any other.
|
|
521
|
+
max_passes: The maximum number of times the group is compiled.
|
|
522
|
+
|
|
523
|
+
Returns:
|
|
524
|
+
The solution and root nodes of each repository, in the order given.
|
|
525
|
+
|
|
526
|
+
Raises:
|
|
527
|
+
NoCommonVersionException: If no version of a project can be used by every
|
|
528
|
+
target that needs it.
|
|
529
|
+
NoCandidateException: If the requirements cannot be resolved for a target.
|
|
530
|
+
"""
|
|
531
|
+
input_reqs = list(input_reqs)
|
|
532
|
+
if max_downgrade is None:
|
|
533
|
+
max_downgrade = MAX_DOWNGRADE
|
|
534
|
+
agreed: Dict[NormName, packaging.version.Version] = dict(pins or {})
|
|
535
|
+
rejected: Dict[NormName, List[Rejection]] = defaultdict(list)
|
|
536
|
+
last: Optional[Rejection] = None
|
|
537
|
+
|
|
538
|
+
for _ in range(max_passes):
|
|
539
|
+
rejections = 0
|
|
540
|
+
results: List[Tuple[DistributionCollection, Set[DependencyNode]]] = []
|
|
541
|
+
for repo in repos:
|
|
542
|
+
target = repo.target
|
|
543
|
+
while True:
|
|
544
|
+
try:
|
|
545
|
+
results.append(
|
|
546
|
+
perform_compile(
|
|
547
|
+
input_reqs,
|
|
548
|
+
repo,
|
|
549
|
+
constraint_reqs=constraint_reqs,
|
|
550
|
+
remove_constraints=remove_constraints,
|
|
551
|
+
extras=extras,
|
|
552
|
+
allow_circular_dependencies=allow_circular_dependencies,
|
|
553
|
+
only_binary=only_binary,
|
|
554
|
+
max_downgrade=max_downgrade,
|
|
555
|
+
target=target,
|
|
556
|
+
pins=_soft_pins(agreed, rejected),
|
|
557
|
+
)
|
|
558
|
+
)
|
|
559
|
+
break
|
|
560
|
+
except PinConflictException as ex:
|
|
561
|
+
rejections += 1
|
|
562
|
+
version = agreed.pop(ex.name, None)
|
|
563
|
+
rejected[ex.name].append(Rejection(target, version, ex))
|
|
564
|
+
if version is None or len(rejected[ex.name]) > max_downgrade:
|
|
565
|
+
# Every version this target can use was rejected by
|
|
566
|
+
# another, or the search is not converging.
|
|
567
|
+
raise NoCommonVersionException(
|
|
568
|
+
ex.name, rejected[ex.name]
|
|
569
|
+
) from ex
|
|
570
|
+
LOG.info(
|
|
571
|
+
"%s cannot use %s (%s), resolving it again for every target",
|
|
572
|
+
target.name,
|
|
573
|
+
ex.pin,
|
|
574
|
+
ex,
|
|
575
|
+
)
|
|
576
|
+
last = rejected[ex.name][-1]
|
|
577
|
+
|
|
578
|
+
for node in results[-1][0]:
|
|
579
|
+
if (
|
|
580
|
+
node.metadata is None
|
|
581
|
+
or node.metadata.meta
|
|
582
|
+
or node.metadata.version is None
|
|
583
|
+
):
|
|
584
|
+
continue
|
|
585
|
+
version = agreed.setdefault(node.key, node.metadata.version)
|
|
586
|
+
assert (
|
|
587
|
+
version == node.metadata.version
|
|
588
|
+
), f"{target.name} resolved {node.key} to {node.metadata.version}, not {version}"
|
|
589
|
+
|
|
590
|
+
if not rejections:
|
|
591
|
+
return results
|
|
592
|
+
|
|
593
|
+
# Every pass rejected a version, so there is a last rejection to report.
|
|
594
|
+
assert last is not None
|
|
595
|
+
LOG.error(
|
|
596
|
+
"The solutions still disagree after %d passes. REQ_COMPILE_MAX_GROUP_PASSES "
|
|
597
|
+
"raises the limit.",
|
|
598
|
+
max_passes,
|
|
599
|
+
)
|
|
600
|
+
raise NoCommonVersionException(last.cause.name, rejected[last.cause.name])
|
|
601
|
+
|
|
602
|
+
|
|
405
603
|
def _add_constraints(
|
|
406
604
|
all_pinned: bool,
|
|
407
605
|
constraint_reqs: Optional[Iterable[RequirementContainer]],
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Errors describing problems that can occur when extracting metadata"""
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, List, NamedTuple, Optional
|
|
4
|
+
|
|
5
|
+
import packaging.requirements
|
|
6
|
+
import packaging.version
|
|
7
|
+
|
|
8
|
+
from req_compile.utils import NormName
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from req_compile.dists import DependencyNode
|
|
12
|
+
from req_compile.target import TargetEnvironment
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ExceptionWithDetails(Exception):
|
|
16
|
+
def __init__(self) -> None:
|
|
17
|
+
super(ExceptionWithDetails, self).__init__()
|
|
18
|
+
self.results: Optional[Any] = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class MetadataError(ExceptionWithDetails):
|
|
22
|
+
def __init__(
|
|
23
|
+
self, name: str, version: Optional[packaging.version.Version], ex: Exception
|
|
24
|
+
) -> None:
|
|
25
|
+
super(MetadataError, self).__init__()
|
|
26
|
+
self.name = name
|
|
27
|
+
self.version = version
|
|
28
|
+
self.ex = ex
|
|
29
|
+
|
|
30
|
+
def __str__(self) -> str:
|
|
31
|
+
return "Failed to parse metadata for package {} ({}) - {}: {}".format(
|
|
32
|
+
self.name, self.version, self.ex.__class__.__name__, str(self.ex)
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class NoCandidateException(ExceptionWithDetails):
|
|
37
|
+
def __init__(
|
|
38
|
+
self, req: packaging.requirements.Requirement, results: Any = None
|
|
39
|
+
) -> None:
|
|
40
|
+
super(NoCandidateException, self).__init__()
|
|
41
|
+
self.req = req
|
|
42
|
+
self.results = results
|
|
43
|
+
self.check_level = 0
|
|
44
|
+
# List of versions marked do not use because they produce conflicts with other deps.
|
|
45
|
+
self.do_not_use: list[packaging.version.Version] = []
|
|
46
|
+
# Node that conflicted with this req, if applicable.
|
|
47
|
+
self.conflicting_node: Optional["DependencyNode"] = None
|
|
48
|
+
# Project name that was walked back, if applicable.
|
|
49
|
+
self.walkback_project: Optional[str] = None
|
|
50
|
+
|
|
51
|
+
def __str__(self) -> str:
|
|
52
|
+
if self.req.specifier:
|
|
53
|
+
return 'NoCandidateException - no candidate for "{}" satisfies {}'.format(
|
|
54
|
+
self.req.name, self.req.specifier
|
|
55
|
+
)
|
|
56
|
+
return 'NoCandidateException - no candidates found for "{}"'.format(
|
|
57
|
+
self.req.name
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class PinConflictException(NoCandidateException):
|
|
62
|
+
"""A soft pin could not be honored for a requirement that is otherwise solvable.
|
|
63
|
+
|
|
64
|
+
Raised in place of `NoCandidateException` when the requirement without the
|
|
65
|
+
pin can be satisfied, so the caller can lift the pin and resolve again. See
|
|
66
|
+
`CompileOptions.soft_pins`.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
name: NormName,
|
|
72
|
+
pin: packaging.requirements.Requirement,
|
|
73
|
+
req: packaging.requirements.Requirement,
|
|
74
|
+
node: "DependencyNode",
|
|
75
|
+
no_candidate: bool,
|
|
76
|
+
) -> None:
|
|
77
|
+
"""
|
|
78
|
+
Args:
|
|
79
|
+
name: The normalized name of the pinned project.
|
|
80
|
+
pin: The soft pin that could not be honored.
|
|
81
|
+
req: The requirement without the pin, as the reverse dependencies and
|
|
82
|
+
hard pins constrain it.
|
|
83
|
+
node: The node the pin applies to.
|
|
84
|
+
no_candidate: True when the pin conflicts with nothing but no candidate
|
|
85
|
+
satisfies it, e.g. the pinned version has no artifact for the
|
|
86
|
+
target. False when the pin conflicts with `req`.
|
|
87
|
+
"""
|
|
88
|
+
super().__init__(req)
|
|
89
|
+
self.name = name
|
|
90
|
+
self.pin = pin
|
|
91
|
+
self.node = node
|
|
92
|
+
self.no_candidate = no_candidate
|
|
93
|
+
|
|
94
|
+
def __str__(self) -> str:
|
|
95
|
+
if self.no_candidate:
|
|
96
|
+
return f"no candidate for {self.pin}"
|
|
97
|
+
return f"{self.pin} conflicts with {self.req}"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class Rejection(NamedTuple):
|
|
101
|
+
"""A version of a project that one of the solutions compiled together cannot use."""
|
|
102
|
+
|
|
103
|
+
target: "TargetEnvironment"
|
|
104
|
+
# The version the other solutions had agreed on, or None if none had and the
|
|
105
|
+
# target cannot use any version left after the earlier rejections.
|
|
106
|
+
version: Optional[packaging.version.Version]
|
|
107
|
+
cause: PinConflictException
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class NoCommonVersionException(ExceptionWithDetails):
|
|
111
|
+
"""Solutions compiled together cannot agree on the version of a project."""
|
|
112
|
+
|
|
113
|
+
def __init__(self, name: NormName, rejections: List[Rejection]) -> None:
|
|
114
|
+
"""
|
|
115
|
+
Args:
|
|
116
|
+
name: The normalized name of the project.
|
|
117
|
+
rejections: The versions some target could not use, in the order they
|
|
118
|
+
were rejected, ending with the failure that stopped the search.
|
|
119
|
+
"""
|
|
120
|
+
super().__init__()
|
|
121
|
+
self.name = name
|
|
122
|
+
self.rejections = list(rejections)
|
|
123
|
+
|
|
124
|
+
def __str__(self) -> str:
|
|
125
|
+
lines = [f"No version of {self.name} can be used by every solution:"]
|
|
126
|
+
for rejection in self.rejections:
|
|
127
|
+
if rejection.version is None:
|
|
128
|
+
lines.append(f" {rejection.target.name}: {rejection.cause}")
|
|
129
|
+
else:
|
|
130
|
+
lines.append(
|
|
131
|
+
f" {rejection.target.name} cannot use "
|
|
132
|
+
f"{self.name}=={rejection.version}: {rejection.cause}"
|
|
133
|
+
)
|
|
134
|
+
return "\n".join(lines)
|