req-compile 1.2.0__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {req_compile-1.2.0 → req_compile-1.4.0}/CHANGELOG.rst +28 -0
  2. {req_compile-1.2.0 → req_compile-1.4.0}/MANIFEST.in +6 -1
  3. {req_compile-1.2.0/req_compile.egg-info → req_compile-1.4.0}/PKG-INFO +42 -7
  4. {req_compile-1.2.0 → req_compile-1.4.0}/README.rst +13 -6
  5. {req_compile-1.2.0 → req_compile-1.4.0}/pyproject.toml +6 -4
  6. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/compile.py +209 -11
  7. req_compile-1.4.0/req_compile/errors.py +134 -0
  8. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/solution.py +66 -20
  9. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/target.py +13 -9
  10. {req_compile-1.2.0 → req_compile-1.4.0/req_compile.egg-info}/PKG-INFO +42 -7
  11. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/SOURCES.txt +1 -18
  12. {req_compile-1.2.0 → req_compile-1.4.0}/version.bzl +1 -1
  13. req_compile-1.2.0/req_compile/errors.py +0 -55
  14. req_compile-1.2.0/tests/test_cmdline.py +0 -136
  15. req_compile-1.2.0/tests/test_compile.py +0 -309
  16. req_compile-1.2.0/tests/test_containers.py +0 -18
  17. req_compile-1.2.0/tests/test_cross_platform.py +0 -47
  18. req_compile-1.2.0/tests/test_dists.py +0 -449
  19. req_compile-1.2.0/tests/test_egg_info.py +0 -173
  20. req_compile-1.2.0/tests/test_extractor.py +0 -127
  21. req_compile-1.2.0/tests/test_filename.py +0 -35
  22. req_compile-1.2.0/tests/test_integration.py +0 -74
  23. req_compile-1.2.0/tests/test_merge_requirements.py +0 -92
  24. req_compile-1.2.0/tests/test_metadata.py +0 -235
  25. req_compile-1.2.0/tests/test_no_setup_py.py +0 -20
  26. req_compile-1.2.0/tests/test_repositories.py +0 -195
  27. req_compile-1.2.0/tests/test_solution_target.py +0 -182
  28. req_compile-1.2.0/tests/test_target.py +0 -406
  29. req_compile-1.2.0/tests/test_utils.py +0 -61
  30. req_compile-1.2.0/tests/test_versions.py +0 -134
  31. {req_compile-1.2.0 → req_compile-1.4.0}/LICENSE.txt +0 -0
  32. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/__init__.py +0 -0
  33. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/__main__.py +0 -0
  34. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/candidates.py +0 -0
  35. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/cmdline.py +0 -0
  36. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/config.py +0 -0
  37. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/containers.py +0 -0
  38. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/dists.py +0 -0
  39. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/filename.py +0 -0
  40. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/__init__.py +0 -0
  41. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/dist_info.py +0 -0
  42. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/extractor.py +0 -0
  43. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/metadata.py +0 -0
  44. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/patch.py +0 -0
  45. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/pyproject.py +0 -0
  46. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/metadata/source.py +0 -0
  47. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/py.typed +0 -0
  48. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/__init__.py +0 -0
  49. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/findlinks.py +0 -0
  50. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/multi.py +0 -0
  51. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/pypi.py +0 -0
  52. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/repository.py +0 -0
  53. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/repos/source.py +0 -0
  54. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/utils.py +0 -0
  55. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile/versions.py +0 -0
  56. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/dependency_links.txt +0 -0
  57. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/entry_points.txt +0 -0
  58. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/requires.txt +0 -0
  59. {req_compile-1.2.0 → req_compile-1.4.0}/req_compile.egg-info/top_level.txt +0 -0
  60. {req_compile-1.2.0 → req_compile-1.4.0}/requirements.in +0 -0
  61. {req_compile-1.2.0 → req_compile-1.4.0}/setup.cfg +0 -0
  62. {req_compile-1.2.0 → req_compile-1.4.0}/setup.py +0 -0
@@ -1,5 +1,33 @@
1
+ New in version 1.3.1:
2
+
3
+ * ``py_reqs_compiler`` compiles the solutions of several platforms together: ``requirements_txts`` maps a
4
+ solution file to each platform. The solutions use the same version of every package, a package keeps the
5
+ version the first platform resolves it to and a version another platform cannot use is dropped for all of
6
+ them. The compilation fails, naming the platforms, when no version of a package works everywhere. A
7
+ ``py_reqs_solution_test`` of such a target checks every solution, including that they agree.
8
+ * The Bazel compiler accepts ``--solution``, ``--platform`` and ``--output`` once per solution.
9
+ * ``perform_compile`` accepts soft pins, which yield with ``PinConflictException`` when a package cannot
10
+ satisfy them, and ``perform_compile_group`` compiles the same requirements for several targets with one
11
+ version per package.
12
+
1
13
  New in version 1.2.0:
2
14
 
15
+ * ``requirements.parse`` now maps lock files to ids in ``requirements_locks`` and config settings to those
16
+ ids in the new ``config_settings`` attribute. Both are label keyed, so each is resolved by the module
17
+ defining the hub and any config setting visible to it can be used. ``//conditions:default`` names a
18
+ fallback lock. Without ``config_settings``, locks compiled with the ``platform`` attribute of
19
+ ``py_reqs_compiler`` are selected by the platform read from the header of the file. Label strings as
20
+ ``requirements_locks`` values still work but are deprecated.
21
+ * Every distinct wheel is downloaded once into a repository named ``{package}_{version}_{sha256[:8]}``
22
+ and shared by all hubs of all modules pinning it. Each hub keeps its own dependency edges, so extras
23
+ and version differences between hubs are preserved. Hub names must be unique across modules.
24
+ * ``package_annotation`` applies to the hubs of the module declaring it unless its new ``hubs``
25
+ attribute names particular hubs or ``"*"`` for every hub. Hubs annotated differently get their own
26
+ repository for the package.
27
+ * Hubs whose locks do not all contain the same packages are supported. A package missing from a lock
28
+ is incompatible with that lock's platform, so targets depending on it are skipped there.
29
+ * Removed the ``defs_id_map`` attribute of the ``parse`` tag and the WORKSPACE-only ``repositories()``
30
+ flow. ``py_requirements_repository`` is now only instantiated by the module extension.
3
31
  * Cross-platform compilation. ``--platform``, ``--python-version`` and ``--implementation`` select
4
32
  wheels and evaluate environment markers for an environment other than the one running ``req-compile``.
5
33
  ``--platform`` takes a wheel platform tag such as ``manylinux_2_28_x86_64`` or ``macosx_11_0_arm64``,
@@ -1,5 +1,10 @@
1
1
  include *.py
2
- prune tests private .idea requirements.txt
2
+ prune tests
3
+ prune examples
4
+ prune rule_tests
5
+ prune req_compile/private
6
+ prune .idea
7
+ exclude requirements.txt
3
8
  include CHANGELOG.rst
4
9
  include LICENSE.txt
5
10
  include pyproject.toml
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: req-compile
3
- Version: 1.2.0
3
+ Version: 1.4.0
4
4
  Summary: Python requirements compiler
5
5
  Home-page: https://github.com/periareon/req-compile
6
6
  Author: Spencer Putt
@@ -38,8 +38,36 @@ Dynamic: requires-dist
38
38
  Dynamic: requires-python
39
39
  Dynamic: summary
40
40
 
41
+ New in version 1.3.1:
42
+
43
+ * ``py_reqs_compiler`` compiles the solutions of several platforms together: ``requirements_txts`` maps a
44
+ solution file to each platform. The solutions use the same version of every package, a package keeps the
45
+ version the first platform resolves it to and a version another platform cannot use is dropped for all of
46
+ them. The compilation fails, naming the platforms, when no version of a package works everywhere. A
47
+ ``py_reqs_solution_test`` of such a target checks every solution, including that they agree.
48
+ * The Bazel compiler accepts ``--solution``, ``--platform`` and ``--output`` once per solution.
49
+ * ``perform_compile`` accepts soft pins, which yield with ``PinConflictException`` when a package cannot
50
+ satisfy them, and ``perform_compile_group`` compiles the same requirements for several targets with one
51
+ version per package.
52
+
41
53
  New in version 1.2.0:
42
54
 
55
+ * ``requirements.parse`` now maps lock files to ids in ``requirements_locks`` and config settings to those
56
+ ids in the new ``config_settings`` attribute. Both are label keyed, so each is resolved by the module
57
+ defining the hub and any config setting visible to it can be used. ``//conditions:default`` names a
58
+ fallback lock. Without ``config_settings``, locks compiled with the ``platform`` attribute of
59
+ ``py_reqs_compiler`` are selected by the platform read from the header of the file. Label strings as
60
+ ``requirements_locks`` values still work but are deprecated.
61
+ * Every distinct wheel is downloaded once into a repository named ``{package}_{version}_{sha256[:8]}``
62
+ and shared by all hubs of all modules pinning it. Each hub keeps its own dependency edges, so extras
63
+ and version differences between hubs are preserved. Hub names must be unique across modules.
64
+ * ``package_annotation`` applies to the hubs of the module declaring it unless its new ``hubs``
65
+ attribute names particular hubs or ``"*"`` for every hub. Hubs annotated differently get their own
66
+ repository for the package.
67
+ * Hubs whose locks do not all contain the same packages are supported. A package missing from a lock
68
+ is incompatible with that lock's platform, so targets depending on it are skipped there.
69
+ * Removed the ``defs_id_map`` attribute of the ``parse`` tag and the WORKSPACE-only ``repositories()``
70
+ flow. ``py_requirements_repository`` is now only instantiated by the module extension.
43
71
  * Cross-platform compilation. ``--platform``, ``--python-version`` and ``--implementation`` select
44
72
  wheels and evaluate environment markers for an environment other than the one running ``req-compile``.
45
73
  ``--platform`` takes a wheel platform tag such as ``manylinux_2_28_x86_64`` or ``macosx_11_0_arm64``,
@@ -114,19 +142,26 @@ To get started with Bazel, add the following to your ``MODULE.bazel`` file:
114
142
  # Find {version} on the req-compile GitHub releases page.
115
143
  bazel_dep(name = "rules_req_compile", version = "{version}")
116
144
 
117
- requirements = use_extension("//extensions:python.bzl", "requirements")
145
+ requirements = use_extension("@rules_req_compile//extensions:python.bzl", "requirements")
118
146
  requirements.parse(
119
147
  name = "my_pip_deps",
120
148
  requirements_locks = {
121
- "//3rdparty:requirements.linux.txt": "@platforms//os:linux",
122
- "//3rdparty:requirements.macos.txt": "@platforms//os:macos",
123
- "//3rdparty:requirements.windows.txt": "@platforms//os:windows",
149
+ "//3rdparty:requirements.linux_x86_64.txt": "linux_x86_64",
150
+ "//3rdparty:requirements.macos_aarch64.txt": "macos_aarch64",
151
+ "//3rdparty:requirements.windows_x86_64.txt": "windows_x86_64",
124
152
  },
125
153
  )
126
154
  use_repo(requirements, "my_pip_deps")
127
155
 
128
- The solutions for every platform can be compiled from a single machine with the ``platform`` attribute of
129
- ``py_reqs_compiler``. See the `Bazel rules documentation <docs/docs.md>`_ for details.
156
+ Each lock is the solution for one platform and is given an id. Locks compiled with the ``platform``
157
+ attribute of ``py_reqs_compiler`` record their platform, which is used to select them. To decide
158
+ yourself, map your own ``config_setting`` labels to the lock ids with ``config_settings``. The solutions
159
+ for every platform can be compiled from a single machine.
160
+
161
+ Every wheel is downloaded once and shared by all the hubs of all the modules pinning it, while each hub
162
+ keeps the dependencies it resolved. Packages missing from the lock of a platform make the targets using
163
+ them incompatible with that platform instead of failing the build.
164
+ See the `Bazel rules documentation <docs/docs.md>`_ for details.
130
165
 
131
166
 
132
167
  Why use it?
@@ -36,19 +36,26 @@ To get started with Bazel, add the following to your ``MODULE.bazel`` file:
36
36
  # Find {version} on the req-compile GitHub releases page.
37
37
  bazel_dep(name = "rules_req_compile", version = "{version}")
38
38
 
39
- requirements = use_extension("//extensions:python.bzl", "requirements")
39
+ requirements = use_extension("@rules_req_compile//extensions:python.bzl", "requirements")
40
40
  requirements.parse(
41
41
  name = "my_pip_deps",
42
42
  requirements_locks = {
43
- "//3rdparty:requirements.linux.txt": "@platforms//os:linux",
44
- "//3rdparty:requirements.macos.txt": "@platforms//os:macos",
45
- "//3rdparty:requirements.windows.txt": "@platforms//os:windows",
43
+ "//3rdparty:requirements.linux_x86_64.txt": "linux_x86_64",
44
+ "//3rdparty:requirements.macos_aarch64.txt": "macos_aarch64",
45
+ "//3rdparty:requirements.windows_x86_64.txt": "windows_x86_64",
46
46
  },
47
47
  )
48
48
  use_repo(requirements, "my_pip_deps")
49
49
 
50
- The solutions for every platform can be compiled from a single machine with the ``platform`` attribute of
51
- ``py_reqs_compiler``. See the `Bazel rules documentation <docs/docs.md>`_ for details.
50
+ Each lock is the solution for one platform and is given an id. Locks compiled with the ``platform``
51
+ attribute of ``py_reqs_compiler`` record their platform, which is used to select them. To decide
52
+ yourself, map your own ``config_setting`` labels to the lock ids with ``config_settings``. The solutions
53
+ for every platform can be compiled from a single machine.
54
+
55
+ Every wheel is downloaded once and shared by all the hubs of all the modules pinning it, while each hub
56
+ keeps the dependencies it resolved. Packages missing from the lock of a platform make the targets using
57
+ them incompatible with that platform instead of failing the build.
58
+ See the `Bazel rules documentation <docs/docs.md>`_ for details.
52
59
 
53
60
 
54
61
  Why use it?
@@ -2,8 +2,13 @@
2
2
  requires = ["setuptools", "wheel"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
+ [tool.isort]
6
+ profile = "black"
7
+
5
8
  [tool.mypy]
6
- namespace_packages = false
9
+ # `req_compile.private` deliberately has no `__init__.py` so that `setup.py`'s
10
+ # `find_packages` leaves the Bazel rule helpers out of the distribution.
11
+ namespace_packages = true
7
12
 
8
13
  [tool.pylint.main]
9
14
  max-line-length = 120
@@ -21,6 +26,3 @@ disable = [
21
26
  "consider-using-f-string",
22
27
  "useless-import-alias", # Conflict with ruff check.
23
28
  ]
24
-
25
- [tool.isort]
26
- profile = "black"
@@ -6,15 +6,22 @@ import logging
6
6
  import operator
7
7
  import os
8
8
  from collections import defaultdict
9
- from typing import Dict, Iterable, Mapping, Optional, Set, Tuple
9
+ from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple
10
10
 
11
11
  import packaging.requirements
12
+ import packaging.version
12
13
 
13
14
  import req_compile.containers
14
15
  import req_compile.dists
15
16
  from req_compile.containers import RequirementContainer
16
17
  from req_compile.dists import DependencyNode, DistributionCollection
17
- from req_compile.errors import MetadataError, NoCandidateException
18
+ from req_compile.errors import (
19
+ MetadataError,
20
+ NoCandidateException,
21
+ NoCommonVersionException,
22
+ PinConflictException,
23
+ Rejection,
24
+ )
18
25
  from req_compile.repos.repository import Repository
19
26
  from req_compile.repos.source import SourceRepository
20
27
  from req_compile.target import TargetEnvironment
@@ -30,6 +37,9 @@ from req_compile.versions import is_possible
30
37
 
31
38
  MAX_COMPILE_DEPTH = int(os.environ.get("REQ_COMPILE_MAX_DEPTH", "80"))
32
39
  MAX_DOWNGRADE = int(os.environ.get("REQ_COMPILE_MAX_DOWNGRADE", "20"))
40
+ # The number of times solutions compiled together are resolved again after one
41
+ # of them rejected a version another had chosen.
42
+ MAX_GROUP_PASSES = int(os.environ.get("REQ_COMPILE_MAX_GROUP_PASSES", "20"))
33
43
 
34
44
  LOG = logging.getLogger("req_compile.compile")
35
45
 
@@ -50,14 +60,24 @@ class CompileOptions:
50
60
  extras: Optional[Iterable[str]] = None
51
61
  allow_circular_dependencies: bool = True
52
62
  pinned_requirements: Mapping[NormName, packaging.requirements.Requirement] = {}
63
+ # Requirements applied to a project only if it is required, which yield when
64
+ # the project cannot satisfy them: `PinConflictException` is raised instead of
65
+ # resolving the reverse dependencies differently. Solutions compiled together
66
+ # use these to keep their versions aligned, see `perform_compile_group`.
67
+ soft_pins: Mapping[NormName, packaging.requirements.Requirement] = {}
53
68
  only_binary: Set[NormName] = set()
54
69
 
55
70
 
56
- def _get_strictest_reverse_dep(node: DependencyNode) -> Optional[DependencyNode]:
71
+ def _get_strictest_reverse_dep(
72
+ node: DependencyNode,
73
+ soft_pin: Optional[packaging.requirements.Requirement] = None,
74
+ ) -> Optional[DependencyNode]:
57
75
  """Get the strictest constraint from all reverse dependencies.
58
76
 
59
77
  Args:
60
78
  node: The node to check.
79
+ soft_pin: A soft pin on the node, scored like a reverse dependency that
80
+ cannot itself be walked back.
61
81
 
62
82
  Returns:
63
83
  The single reverse dependency that was the most conflicting. If no reverse
@@ -67,6 +87,10 @@ def _get_strictest_reverse_dep(node: DependencyNode) -> Optional[DependencyNode]
67
87
 
68
88
  violate_score: Dict[DependencyNode, int] = defaultdict(int)
69
89
  for idx, revnode in enumerate(nodes):
90
+ if soft_pin is not None and not is_possible(
91
+ merge_requirements(revnode.dependencies[node], soft_pin)
92
+ ):
93
+ violate_score[revnode] += 1
70
94
  for next_node in nodes[idx + 1 :]:
71
95
  if not is_possible(
72
96
  merge_requirements(
@@ -158,6 +182,20 @@ def compile_roots(
158
182
  pin = options.pinned_requirements.get(spec_name, spec_req)
159
183
  spec_req = merge_requirements(spec_req, pin)
160
184
 
185
+ # The requirement before any soft pin, which decides whether a failure is
186
+ # the pin's fault.
187
+ unpinned_req = spec_req
188
+ soft_pin = options.soft_pins.get(spec_name)
189
+ if soft_pin is not None:
190
+ spec_req = merge_requirements(spec_req, soft_pin)
191
+
192
+ def _no_candidate(no_candidate: bool) -> NoCandidateException:
193
+ if soft_pin is not None and is_possible(unpinned_req):
194
+ return PinConflictException(
195
+ spec_name, soft_pin, unpinned_req, node, no_candidate
196
+ )
197
+ return NoCandidateException(spec_req)
198
+
161
199
  if not is_possible(spec_req):
162
200
  logger.info("Requirement conflict for %s (%s)", node, spec_req)
163
201
  # Try walking back. Some reverse dependency contributed a requirement
@@ -167,13 +205,13 @@ def compile_roots(
167
205
  # version from being selected.
168
206
  # Search the full space is expensive and possibly harmful because of how heavily
169
207
  # it must hit remote indices.
170
- reverse_dep = _get_strictest_reverse_dep(node)
208
+ reverse_dep = _get_strictest_reverse_dep(node, soft_pin)
171
209
  if (
172
210
  reverse_dep is None
173
211
  or reverse_dep.metadata is None
174
212
  or reverse_dep.metadata.version is None
175
213
  ):
176
- raise NoCandidateException(spec_req)
214
+ raise _no_candidate(no_candidate=False)
177
215
 
178
216
  if _walkback_budget is not None and _walkback_budget <= 0:
179
217
  logger.info(
@@ -181,7 +219,7 @@ def compile_roots(
181
219
  node,
182
220
  spec_req,
183
221
  )
184
- ex = NoCandidateException(spec_req)
222
+ ex = _no_candidate(no_candidate=False)
185
223
  ex.conflicting_node = node
186
224
  ex.walkback_project = reverse_dep.metadata.name
187
225
  raise ex
@@ -241,11 +279,22 @@ def compile_roots(
241
279
  dists.remove_dists(bad_constraint, remove_upstream=False)
242
280
  return
243
281
 
244
- metadata, cached = repo.get_dist(
245
- spec_req,
246
- allow_source_dist=spec_name not in options.only_binary,
247
- max_downgrade=max_downgrade,
248
- )
282
+ try:
283
+ metadata, cached = repo.get_dist(
284
+ spec_req,
285
+ allow_source_dist=spec_name not in options.only_binary,
286
+ max_downgrade=max_downgrade,
287
+ )
288
+ except NoCandidateException:
289
+ if soft_pin is None:
290
+ raise
291
+ # The pin is only at fault if the requirement is solvable without it.
292
+ repo.get_dist(
293
+ unpinned_req,
294
+ allow_source_dist=spec_name not in options.only_binary,
295
+ max_downgrade=max_downgrade,
296
+ )
297
+ raise _no_candidate(no_candidate=True) from None
249
298
  logger.debug(
250
299
  "Acquired candidate %s %s [%s] (%s)",
251
300
  metadata,
@@ -312,6 +361,7 @@ def perform_compile(
312
361
  only_binary: Optional[Set[NormName]] = None,
313
362
  max_downgrade: Optional[int] = None,
314
363
  target: Optional[TargetEnvironment] = None,
364
+ pins: Optional[Mapping[NormName, packaging.requirements.Requirement]] = None,
315
365
  ) -> Tuple[DistributionCollection, Set[DependencyNode]]:
316
366
  """Perform a compilation using the given inputs and constraints.
317
367
 
@@ -331,6 +381,11 @@ def perform_compile(
331
381
  max_downgrade: The maximum number of version downgrades that will be allowed for conflicts.
332
382
  target: The environment to compile for. Defaults to the target of `repo`, which
333
383
  itself defaults to the host.
384
+ pins: Soft pins keyed by normalized name. Unlike constraints, a soft pin
385
+ never requires a package, is not recorded in the solution and yields
386
+ when the package cannot satisfy it: `PinConflictException` is raised
387
+ instead of resolving the dependents differently. See
388
+ `CompileOptions.soft_pins`.
334
389
 
335
390
  Returns:
336
391
  the solution and root nodes used to generate it
@@ -370,6 +425,7 @@ def perform_compile(
370
425
  if all_pinned and constraint_reqs:
371
426
  LOG.info("All constraints were pins - no need to solve the constraints")
372
427
  options.pinned_requirements = pinned_requirements
428
+ options.soft_pins = dict(pins or {})
373
429
 
374
430
  if max_downgrade is None:
375
431
  max_downgrade = MAX_DOWNGRADE
@@ -402,6 +458,148 @@ def perform_compile(
402
458
  return results, roots
403
459
 
404
460
 
461
+ def _soft_pins(
462
+ agreed: Mapping[NormName, packaging.version.Version],
463
+ rejected: Mapping[NormName, Sequence[Rejection]],
464
+ ) -> Dict[NormName, packaging.requirements.Requirement]:
465
+ """The soft pins keeping solutions compiled together aligned.
466
+
467
+ A project with an agreed version is pinned to it. A project without one, whose
468
+ versions some solution rejected, is kept away from those versions.
469
+ """
470
+ pins = {}
471
+ for name, version in agreed.items():
472
+ pins[name] = parse_requirement(f"{name}=={version}")
473
+ for name, rejections in rejected.items():
474
+ if name in agreed or not rejections:
475
+ continue
476
+ pins[name] = parse_requirement(
477
+ name + ",".join(f"!={rejection.version}" for rejection in rejections)
478
+ )
479
+ return pins
480
+
481
+
482
+ def perform_compile_group(
483
+ input_reqs: Iterable[RequirementContainer],
484
+ repos: Sequence[Repository],
485
+ constraint_reqs: Optional[Iterable[RequirementContainer]] = None,
486
+ remove_constraints: bool = False,
487
+ extras: Optional[Iterable[str]] = None,
488
+ allow_circular_dependencies: bool = True,
489
+ only_binary: Optional[Set[NormName]] = None,
490
+ max_downgrade: Optional[int] = None,
491
+ pins: Optional[Mapping[NormName, packaging.version.Version]] = None,
492
+ max_passes: int = MAX_GROUP_PASSES,
493
+ ) -> List[Tuple[DistributionCollection, Set[DependencyNode]]]:
494
+ """Compile the same requirements for several targets, with one version per project.
495
+
496
+ Each repository is compiled for its own target, in order. A project keeps the
497
+ version the first target that needs it resolves it to, so the solutions never
498
+ disagree. When a later target cannot use that version at all, e.g. there is no
499
+ artifact for its platform or its dependencies require another version, the
500
+ version is rejected for every target and the group is compiled again, until a
501
+ pass completes without any rejection. A target may also resolve the dependents
502
+ of a pinned project to other versions than an earlier target did rather than
503
+ reject the version, in which case the next pass aligns the earlier target.
504
+
505
+ Args:
506
+ input_reqs: The requirements, shared by every target.
507
+ repos: One repository per target, compiled in this order. Each compiles for
508
+ its own target, see `Repository.target`.
509
+ constraint_reqs: Constraints to use when compiling.
510
+ remove_constraints: Whether to remove the constraints from the solutions.
511
+ extras: Extras to apply automatically to source projects.
512
+ allow_circular_dependencies: Whether to allow circular dependencies.
513
+ only_binary: Set of projects that should only consider binary distributions.
514
+ max_downgrade: The maximum number of version downgrades that will be allowed
515
+ for conflicts, and the maximum number of versions of a project the
516
+ targets may reject before giving up.
517
+ pins: Versions to start from, keyed by normalized name, e.g. those of the
518
+ solutions being recompiled. They are soft pins: a project is only pinned
519
+ if it is required, and a version that a target cannot use is rejected
520
+ like any other.
521
+ max_passes: The maximum number of times the group is compiled.
522
+
523
+ Returns:
524
+ The solution and root nodes of each repository, in the order given.
525
+
526
+ Raises:
527
+ NoCommonVersionException: If no version of a project can be used by every
528
+ target that needs it.
529
+ NoCandidateException: If the requirements cannot be resolved for a target.
530
+ """
531
+ input_reqs = list(input_reqs)
532
+ if max_downgrade is None:
533
+ max_downgrade = MAX_DOWNGRADE
534
+ agreed: Dict[NormName, packaging.version.Version] = dict(pins or {})
535
+ rejected: Dict[NormName, List[Rejection]] = defaultdict(list)
536
+ last: Optional[Rejection] = None
537
+
538
+ for _ in range(max_passes):
539
+ rejections = 0
540
+ results: List[Tuple[DistributionCollection, Set[DependencyNode]]] = []
541
+ for repo in repos:
542
+ target = repo.target
543
+ while True:
544
+ try:
545
+ results.append(
546
+ perform_compile(
547
+ input_reqs,
548
+ repo,
549
+ constraint_reqs=constraint_reqs,
550
+ remove_constraints=remove_constraints,
551
+ extras=extras,
552
+ allow_circular_dependencies=allow_circular_dependencies,
553
+ only_binary=only_binary,
554
+ max_downgrade=max_downgrade,
555
+ target=target,
556
+ pins=_soft_pins(agreed, rejected),
557
+ )
558
+ )
559
+ break
560
+ except PinConflictException as ex:
561
+ rejections += 1
562
+ version = agreed.pop(ex.name, None)
563
+ rejected[ex.name].append(Rejection(target, version, ex))
564
+ if version is None or len(rejected[ex.name]) > max_downgrade:
565
+ # Every version this target can use was rejected by
566
+ # another, or the search is not converging.
567
+ raise NoCommonVersionException(
568
+ ex.name, rejected[ex.name]
569
+ ) from ex
570
+ LOG.info(
571
+ "%s cannot use %s (%s), resolving it again for every target",
572
+ target.name,
573
+ ex.pin,
574
+ ex,
575
+ )
576
+ last = rejected[ex.name][-1]
577
+
578
+ for node in results[-1][0]:
579
+ if (
580
+ node.metadata is None
581
+ or node.metadata.meta
582
+ or node.metadata.version is None
583
+ ):
584
+ continue
585
+ version = agreed.setdefault(node.key, node.metadata.version)
586
+ assert (
587
+ version == node.metadata.version
588
+ ), f"{target.name} resolved {node.key} to {node.metadata.version}, not {version}"
589
+
590
+ if not rejections:
591
+ return results
592
+
593
+ # Every pass rejected a version, so there is a last rejection to report.
594
+ assert last is not None
595
+ LOG.error(
596
+ "The solutions still disagree after %d passes. REQ_COMPILE_MAX_GROUP_PASSES "
597
+ "raises the limit.",
598
+ max_passes,
599
+ )
600
+ raise NoCommonVersionException(last.cause.name, rejected[last.cause.name])
601
+
602
+
405
603
  def _add_constraints(
406
604
  all_pinned: bool,
407
605
  constraint_reqs: Optional[Iterable[RequirementContainer]],
@@ -0,0 +1,134 @@
1
+ """Errors describing problems that can occur when extracting metadata"""
2
+
3
+ from typing import TYPE_CHECKING, Any, List, NamedTuple, Optional
4
+
5
+ import packaging.requirements
6
+ import packaging.version
7
+
8
+ from req_compile.utils import NormName
9
+
10
+ if TYPE_CHECKING:
11
+ from req_compile.dists import DependencyNode
12
+ from req_compile.target import TargetEnvironment
13
+
14
+
15
+ class ExceptionWithDetails(Exception):
16
+ def __init__(self) -> None:
17
+ super(ExceptionWithDetails, self).__init__()
18
+ self.results: Optional[Any] = None
19
+
20
+
21
+ class MetadataError(ExceptionWithDetails):
22
+ def __init__(
23
+ self, name: str, version: Optional[packaging.version.Version], ex: Exception
24
+ ) -> None:
25
+ super(MetadataError, self).__init__()
26
+ self.name = name
27
+ self.version = version
28
+ self.ex = ex
29
+
30
+ def __str__(self) -> str:
31
+ return "Failed to parse metadata for package {} ({}) - {}: {}".format(
32
+ self.name, self.version, self.ex.__class__.__name__, str(self.ex)
33
+ )
34
+
35
+
36
+ class NoCandidateException(ExceptionWithDetails):
37
+ def __init__(
38
+ self, req: packaging.requirements.Requirement, results: Any = None
39
+ ) -> None:
40
+ super(NoCandidateException, self).__init__()
41
+ self.req = req
42
+ self.results = results
43
+ self.check_level = 0
44
+ # List of versions marked do not use because they produce conflicts with other deps.
45
+ self.do_not_use: list[packaging.version.Version] = []
46
+ # Node that conflicted with this req, if applicable.
47
+ self.conflicting_node: Optional["DependencyNode"] = None
48
+ # Project name that was walked back, if applicable.
49
+ self.walkback_project: Optional[str] = None
50
+
51
+ def __str__(self) -> str:
52
+ if self.req.specifier:
53
+ return 'NoCandidateException - no candidate for "{}" satisfies {}'.format(
54
+ self.req.name, self.req.specifier
55
+ )
56
+ return 'NoCandidateException - no candidates found for "{}"'.format(
57
+ self.req.name
58
+ )
59
+
60
+
61
+ class PinConflictException(NoCandidateException):
62
+ """A soft pin could not be honored for a requirement that is otherwise solvable.
63
+
64
+ Raised in place of `NoCandidateException` when the requirement without the
65
+ pin can be satisfied, so the caller can lift the pin and resolve again. See
66
+ `CompileOptions.soft_pins`.
67
+ """
68
+
69
+ def __init__(
70
+ self,
71
+ name: NormName,
72
+ pin: packaging.requirements.Requirement,
73
+ req: packaging.requirements.Requirement,
74
+ node: "DependencyNode",
75
+ no_candidate: bool,
76
+ ) -> None:
77
+ """
78
+ Args:
79
+ name: The normalized name of the pinned project.
80
+ pin: The soft pin that could not be honored.
81
+ req: The requirement without the pin, as the reverse dependencies and
82
+ hard pins constrain it.
83
+ node: The node the pin applies to.
84
+ no_candidate: True when the pin conflicts with nothing but no candidate
85
+ satisfies it, e.g. the pinned version has no artifact for the
86
+ target. False when the pin conflicts with `req`.
87
+ """
88
+ super().__init__(req)
89
+ self.name = name
90
+ self.pin = pin
91
+ self.node = node
92
+ self.no_candidate = no_candidate
93
+
94
+ def __str__(self) -> str:
95
+ if self.no_candidate:
96
+ return f"no candidate for {self.pin}"
97
+ return f"{self.pin} conflicts with {self.req}"
98
+
99
+
100
+ class Rejection(NamedTuple):
101
+ """A version of a project that one of the solutions compiled together cannot use."""
102
+
103
+ target: "TargetEnvironment"
104
+ # The version the other solutions had agreed on, or None if none had and the
105
+ # target cannot use any version left after the earlier rejections.
106
+ version: Optional[packaging.version.Version]
107
+ cause: PinConflictException
108
+
109
+
110
+ class NoCommonVersionException(ExceptionWithDetails):
111
+ """Solutions compiled together cannot agree on the version of a project."""
112
+
113
+ def __init__(self, name: NormName, rejections: List[Rejection]) -> None:
114
+ """
115
+ Args:
116
+ name: The normalized name of the project.
117
+ rejections: The versions some target could not use, in the order they
118
+ were rejected, ending with the failure that stopped the search.
119
+ """
120
+ super().__init__()
121
+ self.name = name
122
+ self.rejections = list(rejections)
123
+
124
+ def __str__(self) -> str:
125
+ lines = [f"No version of {self.name} can be used by every solution:"]
126
+ for rejection in self.rejections:
127
+ if rejection.version is None:
128
+ lines.append(f" {rejection.target.name}: {rejection.cause}")
129
+ else:
130
+ lines.append(
131
+ f" {rejection.target.name} cannot use "
132
+ f"{self.name}=={rejection.version}: {rejection.cause}"
133
+ )
134
+ return "\n".join(lines)