pex 2.59.5__py2.py3-none-any.whl → 2.60.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pex might be problematic. Click here for more details.

Files changed (112) hide show
  1. pex/cache/dirs.py +14 -4
  2. pex/cli/commands/lock.py +8 -5
  3. pex/common.py +57 -7
  4. pex/dist_metadata.py +48 -6
  5. pex/docs/html/_pagefind/fragment/en_52292af.pf_fragment +0 -0
  6. pex/docs/html/_pagefind/fragment/en_6190e2d.pf_fragment +0 -0
  7. pex/docs/html/_pagefind/fragment/en_8d14bba.pf_fragment +0 -0
  8. pex/docs/html/_pagefind/fragment/en_9ba8f7b.pf_fragment +0 -0
  9. pex/docs/html/_pagefind/fragment/en_c350870.pf_fragment +0 -0
  10. pex/docs/html/_pagefind/fragment/en_cb99877.pf_fragment +0 -0
  11. pex/docs/html/_pagefind/fragment/en_cf3d25b.pf_fragment +0 -0
  12. pex/docs/html/_pagefind/fragment/en_df34874.pf_fragment +0 -0
  13. pex/docs/html/_pagefind/index/{en_974dc5a.pf_index → en_853e43e.pf_index} +0 -0
  14. pex/docs/html/_pagefind/pagefind-entry.json +1 -1
  15. pex/docs/html/_pagefind/pagefind.en_71c76562fd.pf_meta +0 -0
  16. pex/docs/html/_static/documentation_options.js +1 -1
  17. pex/docs/html/api/vars.html +5 -5
  18. pex/docs/html/buildingpex.html +5 -5
  19. pex/docs/html/genindex.html +5 -5
  20. pex/docs/html/index.html +5 -5
  21. pex/docs/html/recipes.html +5 -5
  22. pex/docs/html/scie.html +5 -5
  23. pex/docs/html/search.html +5 -5
  24. pex/docs/html/whatispex.html +5 -5
  25. pex/entry_points_txt.py +98 -0
  26. pex/environment.py +13 -10
  27. pex/finders.py +1 -1
  28. pex/installed_wheel.py +127 -0
  29. pex/interpreter.py +17 -5
  30. pex/interpreter_constraints.py +4 -4
  31. pex/pep_376.py +37 -380
  32. pex/pep_427.py +757 -246
  33. pex/pex_builder.py +4 -4
  34. pex/pex_info.py +8 -3
  35. pex/resolve/venv_resolver.py +46 -34
  36. pex/resolver.py +10 -3
  37. pex/sysconfig.py +5 -3
  38. pex/third_party/__init__.py +1 -1
  39. pex/tools/commands/repository.py +47 -24
  40. pex/vendor/__init__.py +1 -8
  41. pex/vendor/__main__.py +62 -41
  42. pex/vendor/_vendored/ansicolors/.layout.json +1 -1
  43. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.dist-info/RECORD +11 -0
  44. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.pex-info/original-whl-info.json +1 -0
  45. pex/vendor/_vendored/appdirs/.layout.json +1 -1
  46. pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/RECORD +7 -0
  47. pex/vendor/_vendored/appdirs/appdirs-1.4.4.pex-info/original-whl-info.json +1 -0
  48. pex/vendor/_vendored/attrs/.layout.json +1 -1
  49. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/RECORD +37 -0
  50. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.pex-info/original-whl-info.json +1 -0
  51. pex/vendor/_vendored/packaging_20_9/.layout.json +1 -1
  52. pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/RECORD +20 -0
  53. pex/vendor/_vendored/packaging_20_9/packaging-20.9.pex-info/original-whl-info.json +1 -0
  54. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD +7 -0
  55. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.pex-info/original-whl-info.json +1 -0
  56. pex/vendor/_vendored/packaging_21_3/.layout.json +1 -1
  57. pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/RECORD +20 -0
  58. pex/vendor/_vendored/packaging_21_3/packaging-21.3.pex-info/original-whl-info.json +1 -0
  59. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.dist-info/RECORD +18 -0
  60. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.pex-info/original-whl-info.json +1 -0
  61. pex/vendor/_vendored/packaging_24_0/.layout.json +1 -1
  62. pex/vendor/_vendored/packaging_24_0/packaging-24.0.dist-info/RECORD +22 -0
  63. pex/vendor/_vendored/packaging_24_0/packaging-24.0.pex-info/original-whl-info.json +1 -0
  64. pex/vendor/_vendored/packaging_25_0/.layout.json +1 -1
  65. pex/vendor/_vendored/packaging_25_0/packaging-25.0.dist-info/RECORD +24 -0
  66. pex/vendor/_vendored/packaging_25_0/packaging-25.0.pex-info/original-whl-info.json +1 -0
  67. pex/vendor/_vendored/pip/.layout.json +1 -1
  68. pex/vendor/_vendored/pip/pip-20.3.4.dist-info/RECORD +388 -0
  69. pex/vendor/_vendored/pip/pip-20.3.4.pex-info/original-whl-info.json +1 -0
  70. pex/vendor/_vendored/setuptools/.layout.json +1 -1
  71. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/RECORD +107 -0
  72. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.pex-info/original-whl-info.json +1 -0
  73. pex/vendor/_vendored/toml/.layout.json +1 -1
  74. pex/vendor/_vendored/toml/toml-0.10.2.dist-info/RECORD +11 -0
  75. pex/vendor/_vendored/toml/toml-0.10.2.pex-info/original-whl-info.json +1 -0
  76. pex/vendor/_vendored/tomli/.layout.json +1 -1
  77. pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/RECORD +10 -0
  78. pex/vendor/_vendored/tomli/tomli-2.0.1.pex-info/original-whl-info.json +1 -0
  79. pex/venv/installer.py +9 -5
  80. pex/version.py +1 -1
  81. pex/wheel.py +79 -15
  82. pex/whl.py +67 -0
  83. pex/windows/__init__.py +14 -11
  84. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/METADATA +4 -4
  85. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/RECORD +90 -74
  86. pex/docs/html/_pagefind/fragment/en_34b3bf8.pf_fragment +0 -0
  87. pex/docs/html/_pagefind/fragment/en_3cefc8e.pf_fragment +0 -0
  88. pex/docs/html/_pagefind/fragment/en_44ba8a7.pf_fragment +0 -0
  89. pex/docs/html/_pagefind/fragment/en_8eb9a56.pf_fragment +0 -0
  90. pex/docs/html/_pagefind/fragment/en_db171fd.pf_fragment +0 -0
  91. pex/docs/html/_pagefind/fragment/en_ecf679c.pf_fragment +0 -0
  92. pex/docs/html/_pagefind/fragment/en_fb971c7.pf_fragment +0 -0
  93. pex/docs/html/_pagefind/fragment/en_fd8f242.pf_fragment +0 -0
  94. pex/docs/html/_pagefind/pagefind.en_3549188bce.pf_meta +0 -0
  95. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.dist-info/INSTALLER +0 -1
  96. pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/INSTALLER +0 -1
  97. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER +0 -1
  98. pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER +0 -1
  99. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER +0 -1
  100. pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER +0 -1
  101. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.dist-info/INSTALLER +0 -1
  102. pex/vendor/_vendored/packaging_24_0/packaging-24.0.dist-info/INSTALLER +0 -1
  103. pex/vendor/_vendored/packaging_25_0/packaging-25.0.dist-info/INSTALLER +0 -1
  104. pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER +0 -1
  105. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER +0 -1
  106. pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER +0 -1
  107. pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER +0 -1
  108. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/WHEEL +0 -0
  109. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/entry_points.txt +0 -0
  110. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/licenses/LICENSE +0 -0
  111. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/pylock/pylock.toml +0 -0
  112. {pex-2.59.5.dist-info → pex-2.60.1.dist-info}/top_level.txt +0 -0
pex/pex_builder.py CHANGED
@@ -33,11 +33,11 @@ from pex.enum import Enum
33
33
  from pex.executables import chmod_plus_x, create_sh_python_redirector_shebang
34
34
  from pex.finders import get_entry_point_from_console_script, get_script_from_distributions
35
35
  from pex.fs import safe_rename, safe_symlink
36
+ from pex.installed_wheel import InstalledWheel
36
37
  from pex.interpreter import PythonInterpreter
37
38
  from pex.layout import Layout
38
39
  from pex.orderedset import OrderedSet
39
40
  from pex.os import WINDOWS
40
- from pex.pep_376 import InstalledWheel
41
41
  from pex.pex import PEX
42
42
  from pex.pex_info import PexInfo
43
43
  from pex.sh_boot import create_sh_boot_script
@@ -790,9 +790,9 @@ class PEXBuilder(object):
790
790
  self._chroot.zip(
791
791
  os.path.join(atomic_zip_dir.work_dir, location),
792
792
  deterministic=deterministic,
793
- exclude_file=is_pyc_temporary_file
794
- if bytecode_compile
795
- else is_pyc_file,
793
+ exclude_file=(
794
+ is_pyc_temporary_file if bytecode_compile else is_pyc_file
795
+ ),
796
796
  strip_prefix=os.path.join(pex_info.internal_cache, location),
797
797
  labels=(location,),
798
798
  compress=compress,
pex/pex_info.py CHANGED
@@ -604,9 +604,14 @@ class PexInfo(object):
604
604
  data["distributions"] = self._distributions.copy()
605
605
  return data
606
606
 
607
- def dump(self, **extra_json_dumps_kwargs):
608
- # type: (**Any) -> str
609
- return json.dumps(self.as_json_dict(), sort_keys=True, **extra_json_dumps_kwargs)
607
+ def dump(self, indent=None):
608
+ # type: (Optional[int]) -> str
609
+ return json.dumps(
610
+ self.as_json_dict(),
611
+ sort_keys=True,
612
+ indent=indent,
613
+ separators=None if indent else (",", ":"),
614
+ )
610
615
 
611
616
  def copy(self):
612
617
  # type: () -> PexInfo
@@ -5,7 +5,6 @@ from __future__ import absolute_import
5
5
 
6
6
  import functools
7
7
  import hashlib
8
- import io
9
8
  import itertools
10
9
  import os
11
10
  from collections import defaultdict, deque
@@ -14,7 +13,7 @@ from pex import pex_warnings
14
13
  from pex.atomic_directory import atomic_directory
15
14
  from pex.cache.dirs import CacheDir, InstalledWheelDir
16
15
  from pex.common import safe_relative_symlink
17
- from pex.compatibility import PY2, commonpath
16
+ from pex.compatibility import commonpath
18
17
  from pex.dependency_configuration import DependencyConfiguration
19
18
  from pex.dist_metadata import (
20
19
  Constraint,
@@ -26,9 +25,11 @@ from pex.dist_metadata import (
26
25
  )
27
26
  from pex.exceptions import production_assert, reportable_unexpected_error_msg
28
27
  from pex.fingerprinted_distribution import FingerprintedDistribution
28
+ from pex.installed_wheel import InstalledWheel
29
+ from pex.interpreter import PythonInterpreter
29
30
  from pex.jobs import DEFAULT_MAX_JOBS, iter_map_parallel
30
31
  from pex.orderedset import OrderedSet
31
- from pex.pep_376 import InstalledWheel, Record
32
+ from pex.pep_376 import Record
32
33
  from pex.pep_427 import InstallableType, InstallableWheel, InstallPaths, install_wheel_chroot
33
34
  from pex.pep_503 import ProjectName
34
35
  from pex.pip.version import PipVersion
@@ -43,6 +44,7 @@ from pex.targets import LocalInterpreter, Target, Targets
43
44
  from pex.typing import TYPE_CHECKING
44
45
  from pex.venv.virtualenv import Virtualenv
45
46
  from pex.wheel import Wheel
47
+ from pex.whl import repacked_whl
46
48
 
47
49
  if TYPE_CHECKING:
48
50
  from typing import DefaultDict, Deque, FrozenSet, Iterable, Iterator, List, Mapping, Set, Union
@@ -69,9 +71,14 @@ def _normalize_record(
69
71
  return record_data
70
72
 
71
73
  scripts_dir = os.path.realpath(install_paths.scripts)
74
+ record_lines = record_data.decode("utf-8").splitlines(True) # N.B. no kw in 2.7: keepends=True
75
+ eol = os.sep
76
+ if record_lines:
77
+ eol = "\r\n" if record_lines[0].endswith("\r\n") else "\n"
78
+
72
79
  installed_files = [
73
80
  installed_file
74
- for installed_file in Record.read(lines=iter(record_data.decode("utf-8").splitlines()))
81
+ for installed_file in Record.read(lines=iter(record_lines))
75
82
  if (
76
83
  (os.path.basename(installed_file.path) not in entry_point_scripts)
77
84
  or (
@@ -85,28 +92,25 @@ def _normalize_record(
85
92
  )
86
93
  )
87
94
  ]
88
-
89
- if PY2:
90
- record_fp = io.BytesIO()
91
- Record.write_fp(fp=record_fp, installed_files=installed_files)
92
- return record_fp.getvalue()
93
- else:
94
- record_fp = io.StringIO()
95
- Record.write_fp(fp=record_fp, installed_files=installed_files)
96
- return record_fp.getvalue().encode("utf-8")
95
+ return Record.write_bytes(installed_files=installed_files, eol=eol)
97
96
 
98
97
 
99
98
  def _install_distribution(
100
- venv_install_paths, # type: InstallPaths
99
+ interpreter, # type: PythonInterpreter
100
+ result_type, # type: InstallableType.Value
101
+ use_system_time, # type: bool
101
102
  distribution, # type: Distribution
102
103
  ):
103
- # type: (...) -> InstalledWheel
104
+ # type: (...) -> FingerprintedDistribution
104
105
 
105
106
  production_assert(distribution.type is DistributionType.INSTALLED)
106
107
  production_assert(distribution.metadata.files.metadata.type is MetadataType.DIST_INFO)
107
108
 
108
- wheel = InstallableWheel(
109
- wheel=Wheel.from_distribution(distribution), install_paths=venv_install_paths
109
+ venv_install_paths = InstallPaths.interpreter(
110
+ interpreter, project_name=distribution.metadata.project_name
111
+ )
112
+ wheel = InstallableWheel.from_whl(
113
+ whl=Wheel.from_distribution(distribution), install_paths=venv_install_paths
110
114
  )
111
115
  record_data = wheel.metadata_files.read("RECORD")
112
116
  if not record_data:
@@ -148,28 +152,37 @@ def _install_distribution(
148
152
  installed_wheel_dir,
149
153
  os.path.join(symlink_atomic_dir.work_dir, wheel.wheel_file_name),
150
154
  )
151
- return InstalledWheel.load(installed_wheel_dir)
155
+
156
+ installed_wheel = InstalledWheel.load(installed_wheel_dir)
157
+ if not installed_wheel.fingerprint:
158
+ raise AssertionError(reportable_unexpected_error_msg())
159
+
160
+ if result_type is InstallableType.INSTALLED_WHEEL_CHROOT:
161
+ return FingerprintedDistribution(
162
+ distribution=Distribution.load(installed_wheel.prefix_dir),
163
+ fingerprint=installed_wheel.fingerprint,
164
+ )
165
+ return repacked_whl(
166
+ installed_wheel, fingerprint=installed_wheel.fingerprint, use_system_time=use_system_time
167
+ )
152
168
 
153
169
 
154
170
  def _install_venv_distributions(
155
171
  venv, # type: Virtualenv
156
172
  distributions, # type: Iterable[Distribution]
157
173
  max_install_jobs=DEFAULT_MAX_JOBS, # type: int
174
+ result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value
175
+ use_system_time=False, # type: bool
158
176
  ):
159
177
  # type: (...) -> Iterator[FingerprintedDistribution]
160
178
 
161
- venv_install_paths = InstallPaths.interpreter(venv.interpreter)
162
- for installed_wheel in iter_map_parallel(
179
+ return iter_map_parallel(
163
180
  inputs=distributions,
164
- function=functools.partial(_install_distribution, venv_install_paths),
181
+ function=functools.partial(
182
+ _install_distribution, venv.interpreter, result_type, use_system_time
183
+ ),
165
184
  max_jobs=max_install_jobs,
166
- ):
167
- if not installed_wheel.fingerprint:
168
- raise AssertionError(reportable_unexpected_error_msg())
169
- yield FingerprintedDistribution(
170
- distribution=Distribution.load(installed_wheel.prefix_dir),
171
- fingerprint=installed_wheel.fingerprint,
172
- )
185
+ )
173
186
 
174
187
 
175
188
  @attr.s(frozen=True)
@@ -240,6 +253,7 @@ def _resolve_distributions(
240
253
  allow_prereleases=False, # type: bool
241
254
  compile=False, # type: bool
242
255
  ignore_errors=False, # type: bool
256
+ result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value
243
257
  ):
244
258
  # type: (...) -> Iterator[Union[Distribution, FingerprintedDistribution, Error]]
245
259
 
@@ -315,6 +329,7 @@ def _resolve_distributions(
315
329
  transitive=False,
316
330
  compile=compile,
317
331
  ignore_errors=ignore_errors,
332
+ result_type=result_type,
318
333
  )
319
334
  for dist in result.distributions:
320
335
  to_resolve.extend(
@@ -354,12 +369,6 @@ def resolve_from_venv(
354
369
  ):
355
370
  # type: (...) -> Union[ResolveResult, Error]
356
371
 
357
- if result_type is InstallableType.WHEEL_FILE:
358
- return Error(
359
- "Cannot resolve .whl files from virtual environment at {venv_dir}; its distributions "
360
- "are all installed.".format(venv_dir=venv.venv_dir)
361
- )
362
-
363
372
  target = LocalInterpreter.create(venv.interpreter)
364
373
  if not targets.is_empty:
365
374
  return Error(
@@ -446,6 +455,7 @@ def resolve_from_venv(
446
455
  allow_prereleases=pip_configuration.allow_prereleases,
447
456
  compile=compile,
448
457
  ignore_errors=ignore_errors,
458
+ result_type=result_type,
449
459
  ):
450
460
  if isinstance(distribution_or_error, Error):
451
461
  return distribution_or_error
@@ -491,6 +501,8 @@ def resolve_from_venv(
491
501
  venv=venv,
492
502
  distributions=venv_distributions,
493
503
  max_install_jobs=pip_configuration.max_jobs,
504
+ result_type=result_type,
505
+ use_system_time=True,
494
506
  )
495
507
  ),
496
508
  fingerprinted_distributions,
pex/resolver.py CHANGED
@@ -37,10 +37,10 @@ from pex.dist_metadata import (
37
37
  )
38
38
  from pex.exceptions import production_assert
39
39
  from pex.fingerprinted_distribution import FingerprintedDistribution
40
+ from pex.installed_wheel import InstalledWheel
40
41
  from pex.jobs import Raise, SpawnedJob, execute_parallel, iter_map_parallel
41
42
  from pex.network_configuration import NetworkConfiguration
42
43
  from pex.orderedset import OrderedSet
43
- from pex.pep_376 import InstalledWheel
44
44
  from pex.pep_425 import CompatibilityTags
45
45
  from pex.pep_427 import InstallableType, WheelError, install_wheel_chroot
46
46
  from pex.pep_503 import ProjectName
@@ -622,7 +622,7 @@ class BuildResult(object):
622
622
  target=self.request.target.render_description(),
623
623
  )
624
624
  )
625
- return InstallRequest.create(self.request.target, wheel_path)
625
+ return InstallRequest.create(self.request.target, wheel_path, was_built_locally=True)
626
626
 
627
627
 
628
628
  @attr.s(frozen=True)
@@ -632,6 +632,7 @@ class InstallRequest(object):
632
632
  cls,
633
633
  target, # type: Union[DownloadTarget, Target]
634
634
  wheel_path, # type: str
635
+ was_built_locally=False, # type: bool
635
636
  ):
636
637
  # type: (...) -> InstallRequest
637
638
  fingerprint = fingerprint_path(wheel_path)
@@ -639,11 +640,13 @@ class InstallRequest(object):
639
640
  download_target=_as_download_target(target),
640
641
  wheel_path=wheel_path,
641
642
  fingerprint=fingerprint,
643
+ was_built_locally=was_built_locally,
642
644
  )
643
645
 
644
646
  download_target = attr.ib(converter=_as_download_target) # type: DownloadTarget
645
647
  wheel_path = attr.ib() # type: str
646
648
  fingerprint = attr.ib() # type: str
649
+ was_built_locally = attr.ib(default=False) # type: bool
647
650
 
648
651
  @property
649
652
  def target(self):
@@ -987,7 +990,11 @@ def _perform_install(
987
990
  ):
988
991
  # type: (...) -> InstallResult
989
992
  install_result = install_request.result(installed_wheels_dir)
990
- install_wheel_chroot(wheel=install_request.wheel_path, destination=install_result.build_chroot)
993
+ install_wheel_chroot(
994
+ wheel=install_request.wheel_path,
995
+ destination=install_result.build_chroot,
996
+ normalize_file_stat=install_request.was_built_locally,
997
+ )
991
998
  return install_result
992
999
 
993
1000
 
pex/sysconfig.py CHANGED
@@ -82,7 +82,7 @@ class _PlatformValue(Enum.Value):
82
82
  self.arch = arch
83
83
 
84
84
  @property
85
- def extension(self):
85
+ def exe_extension(self):
86
86
  # type: () -> str
87
87
  return ".exe" if self.os is Os.WINDOWS else ""
88
88
 
@@ -93,12 +93,14 @@ class _PlatformValue(Enum.Value):
93
93
 
94
94
  def binary_name(self, binary_name):
95
95
  # type: (_Text) -> _Text
96
- return "{binary_name}{extension}".format(binary_name=binary_name, extension=self.extension)
96
+ return "{binary_name}{extension}".format(
97
+ binary_name=binary_name, extension=self.exe_extension
98
+ )
97
99
 
98
100
  def qualified_binary_name(self, binary_name):
99
101
  # type: (_Text) -> _Text
100
102
  return "{binary_name}-{platform}{extension}".format(
101
- binary_name=binary_name, platform=self, extension=self.extension
103
+ binary_name=binary_name, platform=self, extension=self.exe_extension
102
104
  )
103
105
 
104
106
  def qualified_file_name(self, file_name):
@@ -639,7 +639,7 @@ def expose_installed_wheels(
639
639
 
640
640
  from pex.atomic_directory import atomic_directory
641
641
  from pex.cache.dirs import InstalledWheelDir
642
- from pex.pep_376 import InstalledWheel
642
+ from pex.installed_wheel import InstalledWheel
643
643
 
644
644
  for path in expose(dists, interpreter=interpreter):
645
645
  # TODO(John Sirois): Maybe consolidate with pex.resolver.BuildAndInstallRequest.
@@ -18,12 +18,14 @@ from pex import dist_metadata
18
18
  from pex.atomic_directory import atomic_directory
19
19
  from pex.cache.dirs import CacheDir
20
20
  from pex.commands.command import JsonMixin, OutputMixin
21
- from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdir, safe_mkdtemp, safe_open
21
+ from pex.common import pluralize, safe_mkdir, safe_mkdtemp, safe_open
22
22
  from pex.compatibility import Queue
23
23
  from pex.dist_metadata import Distribution
24
24
  from pex.environment import PEXEnvironment
25
+ from pex.installed_wheel import InstalledWheel
25
26
  from pex.interpreter import PythonInterpreter
26
- from pex.jobs import Job, Retain, SpawnedJob, execute_parallel
27
+ from pex.jobs import Job, iter_map_parallel
28
+ from pex.pep_427 import WheelInstallError, repack
27
29
  from pex.pex import PEX
28
30
  from pex.result import Error, Ok, Result
29
31
  from pex.tools.command import PEXCommand
@@ -31,7 +33,7 @@ from pex.typing import TYPE_CHECKING, cast
31
33
  from pex.venv.virtualenv import InstallationChoice, Virtualenv
32
34
 
33
35
  if TYPE_CHECKING:
34
- from typing import IO, Any, Callable, Iterable, Iterator, List, Text, Tuple
36
+ from typing import IO, Any, Callable, Iterable, Iterator, List, Tuple
35
37
 
36
38
  import attr # vendor:skip
37
39
 
@@ -121,6 +123,29 @@ class FindLinksRepo(object):
121
123
  self._server_process.kill()
122
124
 
123
125
 
126
+ def _extract_wheel(
127
+ distribution, # type: Distribution
128
+ dest_dir, # type: str
129
+ use_system_time=False, # type: bool
130
+ ):
131
+ # type: (...) -> Tuple[Distribution, Result]
132
+
133
+ try:
134
+ whl = repack(
135
+ installed_wheel=InstalledWheel.load(distribution.location),
136
+ dest_dir=dest_dir,
137
+ use_system_time=use_system_time,
138
+ )
139
+ except (InstalledWheel.LoadError, WheelInstallError) as e:
140
+ result = Error(str(e)) # type: Result
141
+ else:
142
+ result = Ok(
143
+ "{distribution}: Repacked wheel as {whl}".format(distribution=distribution, whl=whl)
144
+ )
145
+
146
+ return distribution, result
147
+
148
+
124
149
  class Repository(JsonMixin, OutputMixin, PEXCommand):
125
150
  """Interact with the Python distribution repository contained in a PEX file."""
126
151
 
@@ -270,36 +295,23 @@ class Repository(JsonMixin, OutputMixin, PEXCommand):
270
295
  if self.options.sources:
271
296
  self._extract_sdist(pex, dest_dir)
272
297
 
273
- def spawn_extract(distribution):
274
- # type: (Distribution) -> SpawnedJob[Text]
275
- env = os.environ.copy()
276
- if not self.options.use_system_time:
277
- env.update(REPRODUCIBLE_BUILDS_ENV)
278
- job = spawn_python_job_with_setuptools_and_wheel(
279
- args=["-m", "wheel", "pack", "--dest-dir", dest_dir, distribution.location],
280
- interpreter=pex.interpreter,
281
- stdout=subprocess.PIPE,
282
- env=env,
283
- )
284
- return SpawnedJob.stdout(
285
- job, result_func=lambda out: "{}: {}".format(distribution, out.decode())
286
- )
287
-
288
298
  with self._distributions_output(pex) as (distributions, output):
289
299
  errors = [] # type: List[Distribution]
290
- for result in execute_parallel(
291
- distributions, spawn_extract, error_handler=Retain[Distribution]()
300
+ for distribution, result in iter_map_parallel(
301
+ distributions,
302
+ functools.partial(
303
+ _extract_wheel, dest_dir=dest_dir, use_system_time=self.options.use_system_time
304
+ ),
292
305
  ):
293
- if isinstance(result, tuple):
294
- distribution, error = result
306
+ if isinstance(result, Error):
295
307
  errors.append(distribution)
296
308
  output.write(
297
309
  "Failed to build a wheel for {distribution}: {error}\n".format(
298
- distribution=distribution, error=error
310
+ distribution=distribution, error=result
299
311
  )
300
312
  )
301
313
  else:
302
- output.write(result)
314
+ output.write(str(result))
303
315
  if errors:
304
316
  return Error(
305
317
  "Failed to build wheels for {count} {distributions}.".format(
@@ -435,6 +447,17 @@ class Repository(JsonMixin, OutputMixin, PEXCommand):
435
447
  with open(os.path.join(chroot, "setup.py"), "w") as fp:
436
448
  fp.write("import setuptools; setuptools.setup()")
437
449
 
450
+ with open(os.path.join(chroot, "pyproject.toml"), "w") as fp:
451
+ fp.write(
452
+ dedent(
453
+ """\
454
+ [build-system]
455
+ requires = ["setuptools"]
456
+ backend = "setuptools.build_meta"
457
+ """
458
+ )
459
+ )
460
+
438
461
  spawn_python_job_with_setuptools_and_wheel(
439
462
  args=["setup.py", "sdist", "--dist-dir", dest_dir],
440
463
  interpreter=pex.interpreter,
pex/vendor/__init__.py CHANGED
@@ -318,7 +318,6 @@ def vendor_runtime(
318
318
  dest_basedir, # type: str
319
319
  label, # type: str
320
320
  root_module_names, # type: Iterable[str]
321
- include_dist_info=(), # type: Iterable[str]
322
321
  ):
323
322
  # type: (...) -> Set[str]
324
323
  """Includes portions of vendored distributions in a chroot.
@@ -331,7 +330,6 @@ def vendor_runtime(
331
330
  :param dest_basedir: The prefix to store the vendored code under in the ``chroot``.
332
331
  :param label: The chroot label for the vendored code fileset.
333
332
  :param root_module_names: The names of the root vendored modules to include in the chroot.
334
- :param include_dist_info: Include the .dist-info dirs associated with these root module names.
335
333
  :returns: The set of absolute paths of the source files that were vendored.
336
334
  :raise: :class:`ValueError` if any of the given ``root_module_names`` could not be found amongst
337
335
  the vendored code and added to the chroot.
@@ -370,12 +368,7 @@ def vendor_runtime(
370
368
  vendor_module_names[name] = True
371
369
  TRACER.log("Vendoring {} from {} @ {}".format(name, spec, spec.target_dir), V=3)
372
370
 
373
- dirs[:] = packages + [
374
- d
375
- for project in include_dist_info
376
- for d in dirs
377
- if d.startswith(project) and d.endswith(".dist-info")
378
- ]
371
+ dirs[:] = packages
379
372
  files[:] = modules
380
373
 
381
374
  # We copy over sources and data only; no pyc files.
pex/vendor/__main__.py CHANGED
@@ -6,11 +6,12 @@ from __future__ import absolute_import, print_function
6
6
  import glob
7
7
  import os
8
8
  import pkgutil
9
- import re
10
9
  import subprocess
11
10
  import sys
11
+ import zipfile
12
12
  from argparse import ArgumentParser
13
13
  from collections import OrderedDict, defaultdict
14
+ from contextlib import closing
14
15
  from typing import FrozenSet, Iterator, List
15
16
 
16
17
  import libcst
@@ -36,7 +37,15 @@ from libcst import (
36
37
  SimpleString,
37
38
  )
38
39
 
39
- from pex.common import safe_delete, safe_mkdir, safe_mkdtemp, safe_open, safe_rmtree
40
+ from pex.common import (
41
+ REPRODUCIBLE_BUILDS_ENV,
42
+ safe_delete,
43
+ safe_mkdir,
44
+ safe_mkdtemp,
45
+ safe_open,
46
+ safe_rmtree,
47
+ )
48
+ from pex.pep_427 import ZipMetadata
40
49
  from pex.typing import TYPE_CHECKING
41
50
  from pex.vendor import VendorSpec, iter_vendor_specs
42
51
 
@@ -362,10 +371,10 @@ def vendorize(root_dir, vendor_specs, prefix, update):
362
371
  # importable code must lie at the top of its vendored chroot. Although
363
372
  # `pex.pep_472.install_wheel_chroot` encodes the logic to achieve this layout, we can't run
364
373
  # that without 1st approximating that layout!. We take the tack of performing an importable
365
- # installation using `pip wheel ...` + `wheel unpack ...`. Although simply un-packing a wheel
374
+ # installation using `pip wheel ...` + `unzip ...`. Although simply unzipping a wheel
366
375
  # does not make it importable in general, it works for our pure-python vendored code.
367
376
 
368
- unpacked_wheel_chroots_by_vendor_spec = defaultdict(list)
377
+ unzipped_wheel_chroots_by_vendor_spec = defaultdict(list)
369
378
  for vendor_spec in vendor_specs:
370
379
  # NB: We set --no-build-isolation to prevent pip from installing the requirements listed in
371
380
  # its [build-system] config in its pyproject.toml.
@@ -387,6 +396,7 @@ def vendorize(root_dir, vendor_specs, prefix, update):
387
396
  cmd = [
388
397
  "pip",
389
398
  "wheel",
399
+ "--prefer-binary",
390
400
  "--no-build-isolation",
391
401
  "--no-cache-dir",
392
402
  "--wheel-dir",
@@ -408,7 +418,9 @@ def vendorize(root_dir, vendor_specs, prefix, update):
408
418
  if os.path.isfile(constraints_file):
409
419
  cmd.extend(["--constraint", constraints_file])
410
420
 
411
- result = subprocess.call(cmd)
421
+ env = os.environ.copy()
422
+ env.update(REPRODUCIBLE_BUILDS_ENV)
423
+ result = subprocess.call(cmd, env=env)
412
424
  if result != 0:
413
425
  raise VendorizeError("Failed to vendor {!r}".format(vendor_spec))
414
426
 
@@ -416,22 +428,17 @@ def vendorize(root_dir, vendor_specs, prefix, update):
416
428
  # later.
417
429
  safe_mkdir(vendor_spec.target_dir)
418
430
  for wheel_file in glob.glob(os.path.join(wheels_dir, "*.whl")):
419
- extract_dir = os.path.join(wheels_dir, ".extracted")
420
- output = subprocess.check_output(
421
- ["wheel", "unpack", "--dest", extract_dir, wheel_file]
422
- ).decode("utf-8")
423
- match = re.match(r"^Unpacking to: (?P<unpack_dir>.+)\.\.\.OK$", output)
424
- assert match is not None, (
425
- "Failed to determine {wheel_file} unpack dir from wheel unpack output:\n"
426
- "{output}".format(wheel_file=wheel_file, output=output)
431
+ unzipped_wheel_dir = os.path.join(
432
+ wheels_dir, ".extracted", os.path.basename(wheel_file)
433
+ )
434
+ with closing(zipfile.ZipFile(wheel_file)) as zf:
435
+ zf.extractall(unzipped_wheel_dir)
436
+ unzipped_wheel_chroots_by_vendor_spec[vendor_spec].append(
437
+ (unzipped_wheel_dir, wheel_file)
427
438
  )
428
- unpacked_to_dir = os.path.join(extract_dir, match["unpack_dir"])
429
- unpacked_wheel_dir = os.path.join(extract_dir, os.path.basename(wheel_file))
430
- os.rename(unpacked_to_dir, unpacked_wheel_dir)
431
- unpacked_wheel_chroots_by_vendor_spec[vendor_spec].append(unpacked_wheel_dir)
432
- for path in os.listdir(unpacked_wheel_dir):
439
+ for path in os.listdir(unzipped_wheel_dir):
433
440
  os.symlink(
434
- os.path.join(unpacked_wheel_dir, path),
441
+ os.path.join(unzipped_wheel_dir, path),
435
442
  os.path.join(vendor_spec.target_dir, path),
436
443
  )
437
444
 
@@ -484,7 +491,8 @@ def vendorize(root_dir, vendor_specs, prefix, update):
484
491
  # Import all code needed below now before we move any vendored bits it depends on temporarily
485
492
  # back to the prefix site-packages dir.
486
493
  from pex.dist_metadata import ProjectNameAndVersion, Requirement
487
- from pex.pep_427 import install_wheel_chroot
494
+ from pex.pep_427 import InstallableWheel, InstallPaths, install_wheel_chroot
495
+ from pex.wheel import Wheel
488
496
 
489
497
  for vendor_spec in vendor_specs:
490
498
  print(
@@ -512,29 +520,40 @@ def vendorize(root_dir, vendor_specs, prefix, update):
512
520
  # We want the primary artifact to own any special Pex wheel chroot metadata; so we arrange
513
521
  # a list of installs that place it last.
514
522
  primary_project = Requirement.parse(vendor_spec.requirement).project_name
515
- wheel_chroots_by_project_name = {
516
- ProjectNameAndVersion.from_filename(
517
- wheel_chroot
518
- ).canonicalized_project_name: wheel_chroot
519
- for wheel_chroot in unpacked_wheel_chroots_by_vendor_spec[vendor_spec]
520
- }
521
- primary_wheel_chroot = wheel_chroots_by_project_name.pop(primary_project)
523
+ wheel_chroots_by_project_name = OrderedDict()
524
+ for wheel_chroot, wheel_file in unzipped_wheel_chroots_by_vendor_spec[vendor_spec]:
525
+ pnav = ProjectNameAndVersion.from_filename(wheel_chroot)
526
+ wheel_chroots_by_project_name[pnav.canonicalized_project_name] = (
527
+ pnav.canonicalized_project_name,
528
+ pnav.canonicalized_version,
529
+ wheel_chroot,
530
+ wheel_file,
531
+ )
532
+
533
+ (
534
+ project_name,
535
+ version,
536
+ primary_wheel_chroot,
537
+ primary_wheel_file,
538
+ ) = wheel_chroots_by_project_name.pop(primary_project)
522
539
  wheels_chroots_to_install = list(wheel_chroots_by_project_name.values())
523
- wheels_chroots_to_install.append(primary_wheel_chroot)
524
-
525
- for wheel_chroot in wheels_chroots_to_install:
526
- dest_dir = safe_mkdtemp()
527
- subprocess.check_call(["wheel", "pack", "--dest-dir", dest_dir, wheel_chroot])
528
- wheel_files = glob.glob(os.path.join(dest_dir, "*.whl"))
529
- assert len(wheel_files) == 1, (
530
- "Expected re-packing {wheel_chroot} to produce one `.whl` file but found {count}:\n"
531
- "{wheel_files}"
532
- ).format(
533
- wheel_chroot=wheel_chroot,
534
- count=len(wheel_files),
535
- wheel_files="\n".join(os.path.basename(wheel_file) for wheel_file in wheel_files),
540
+ wheels_chroots_to_install.append(
541
+ (project_name, version, primary_wheel_chroot, primary_wheel_file)
542
+ )
543
+
544
+ for project_name, version, wheel_chroot, wheel_file in wheels_chroots_to_install:
545
+ with closing(zipfile.ZipFile(wheel_file)) as zf:
546
+ zip_metadata = ZipMetadata.from_zip(filename=wheel_file, info_list=zf.infolist())
547
+ install_wheel_chroot(
548
+ wheel=InstallableWheel(
549
+ wheel=Wheel.load(wheel_chroot),
550
+ install_paths=InstallPaths.wheel(
551
+ destination=wheel_chroot, project_name=project_name, version=version
552
+ ),
553
+ zip_metadata=zip_metadata,
554
+ ),
555
+ destination=vendor_spec.target_dir,
536
556
  )
537
- install_wheel_chroot(wheel=wheel_files[0], destination=vendor_spec.target_dir)
538
557
 
539
558
 
540
559
  if __name__ == "__main__":
@@ -551,6 +570,8 @@ if __name__ == "__main__":
551
570
  root_directory = VendorSpec.ROOT
552
571
  try:
553
572
  safe_rmtree(VendorSpec.vendor_root())
573
+ # Stabilize our umask to get the same file perms on any run.
574
+ os.umask(0o022)
554
575
  vendorize(
555
576
  root_dir=root_directory,
556
577
  vendor_specs=list(iter_vendor_specs()),
@@ -1 +1 @@
1
- {"fingerprint": "a175b8cdfaabde6458bf70a07cf4ea1a51d4a8edf2502dff5777a049b610f354", "record_relpath": "ansicolors-1.1.8.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}
1
+ {"fingerprint":"93bbf4f9990c23d7f074400fbc7b72699025043cff2ba03c34196ad876cc265c","record_relpath":"ansicolors-1.1.8.dist-info/RECORD","root_is_purelib":true,"stash_dir":".prefix"}
@@ -0,0 +1,11 @@
1
+ ansicolors-1.1.8.dist-info/DESCRIPTION.rst,sha256=hgL3qWOqDdnS5m7OGbUJvvZ_gOnggqyKN0LzedMq3o4,7900
2
+ ansicolors-1.1.8.dist-info/METADATA,sha256=91m-dxXKkt0Fg_Zv_HA7Zlxx2f1bGDFRn0cMD-6YkvI,9006
3
+ ansicolors-1.1.8.dist-info/RECORD,,
4
+ ansicolors-1.1.8.dist-info/WHEEL,sha256=o2k-Qa-RMNIJmUdIc7KU6VWR_ErNRbWNlxDIpl7lm34,110
5
+ ansicolors-1.1.8.dist-info/metadata.json,sha256=jEvYWhL1nvf1IOuhZN76kIr3cS4QQN9_aql483E8Sd8,1181
6
+ ansicolors-1.1.8.dist-info/top_level.txt,sha256=igtcW1b_UlN1_x_2Ooi1-5-0ZZzr7UbkgT0OyOkOz1A,7
7
+ colors/__init__.py,sha256=7UV3RlAaU9ihmC33l1KnlJ19mV79q9knPrayzG0LA4g,119
8
+ colors/colors.py,sha256=VNM8dhjwb51Q3rbNFLhAfl-Xi3vd88NLnLT8BavVpXg,5603
9
+ colors/csscolors.py,sha256=pYW3Yisj3bGz-ocg23jxmqQ7YjNFoaZuLW38Lrqd8pg,6584
10
+ colors/version.py,sha256=1AmxhanHVlK7-ylUJmxvVmAhH674NXaZPyMgOYTXThQ,22
11
+ ansicolors-1.1.8.pex-info/original-whl-info.json,sha256=HbV2rpplaYIBPX3BXgmvHbJ3jE4Kai_UN1AI0iFOiAg,717
@@ -0,0 +1 @@
1
+ {"entries":[["colors/__init__.py",[2017,5,17,21,12,12],2175008768],["colors/colors.py",[2017,6,2,20,55,34],2175008768],["colors/csscolors.py",[2017,6,2,19,55,26],2175008768],["colors/version.py",[2017,6,2,21,17,26],2175008768],["ansicolors-1.1.8.dist-info/DESCRIPTION.rst",[2017,6,2,21,22,12],2175008768],["ansicolors-1.1.8.dist-info/metadata.json",[2017,6,2,21,22,12],2175008768],["ansicolors-1.1.8.dist-info/top_level.txt",[2017,6,2,21,22,12],2175008768],["ansicolors-1.1.8.dist-info/WHEEL",[2017,6,2,21,22,12],2175008768],["ansicolors-1.1.8.dist-info/METADATA",[2017,6,2,21,22,12],2175008768],["ansicolors-1.1.8.dist-info/RECORD",[2017,6,2,21,22,12],2175008768]],"filename":"ansicolors-1.1.8-py2.py3-none-any.whl"}