pex 2.54.2__py2.py3-none-any.whl → 2.69.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pex might be problematic. Click here for more details.

Files changed (180) hide show
  1. pex/auth.py +1 -1
  2. pex/bin/pex.py +15 -2
  3. pex/build_backend/configuration.py +5 -5
  4. pex/build_backend/wrap.py +27 -23
  5. pex/build_system/pep_517.py +4 -1
  6. pex/cache/dirs.py +17 -12
  7. pex/cli/commands/lock.py +302 -165
  8. pex/cli/commands/pip/core.py +4 -12
  9. pex/cli/commands/pip/wheel.py +1 -1
  10. pex/cli/commands/run.py +13 -20
  11. pex/cli/commands/venv.py +85 -16
  12. pex/cli/pex.py +11 -4
  13. pex/common.py +57 -7
  14. pex/compatibility.py +1 -1
  15. pex/dependency_configuration.py +87 -15
  16. pex/dist_metadata.py +143 -25
  17. pex/docs/html/_pagefind/fragment/en_4250138.pf_fragment +0 -0
  18. pex/docs/html/_pagefind/fragment/en_7125dad.pf_fragment +0 -0
  19. pex/docs/html/_pagefind/fragment/en_785d562.pf_fragment +0 -0
  20. pex/docs/html/_pagefind/fragment/en_8e94bb8.pf_fragment +0 -0
  21. pex/docs/html/_pagefind/fragment/en_a0396bb.pf_fragment +0 -0
  22. pex/docs/html/_pagefind/fragment/en_a8a3588.pf_fragment +0 -0
  23. pex/docs/html/_pagefind/fragment/en_c07d988.pf_fragment +0 -0
  24. pex/docs/html/_pagefind/fragment/en_d718411.pf_fragment +0 -0
  25. pex/docs/html/_pagefind/index/en_a2e3c5e.pf_index +0 -0
  26. pex/docs/html/_pagefind/pagefind-entry.json +1 -1
  27. pex/docs/html/_pagefind/pagefind.en_4ce1afa9e3.pf_meta +0 -0
  28. pex/docs/html/_static/documentation_options.js +1 -1
  29. pex/docs/html/_static/pygments.css +164 -146
  30. pex/docs/html/_static/styles/furo.css +1 -1
  31. pex/docs/html/_static/styles/furo.css.map +1 -1
  32. pex/docs/html/api/vars.html +25 -34
  33. pex/docs/html/buildingpex.html +25 -34
  34. pex/docs/html/genindex.html +24 -33
  35. pex/docs/html/index.html +25 -34
  36. pex/docs/html/recipes.html +25 -34
  37. pex/docs/html/scie.html +25 -34
  38. pex/docs/html/search.html +24 -33
  39. pex/docs/html/whatispex.html +25 -34
  40. pex/entry_points_txt.py +98 -0
  41. pex/environment.py +54 -33
  42. pex/finders.py +1 -1
  43. pex/hashing.py +71 -9
  44. pex/installed_wheel.py +141 -0
  45. pex/interpreter.py +41 -38
  46. pex/interpreter_constraints.py +25 -25
  47. pex/interpreter_implementation.py +40 -0
  48. pex/jobs.py +13 -6
  49. pex/pep_376.py +68 -384
  50. pex/pep_425.py +11 -2
  51. pex/pep_427.py +937 -205
  52. pex/pep_508.py +4 -5
  53. pex/pex_builder.py +5 -8
  54. pex/pex_info.py +14 -9
  55. pex/pip/dependencies/__init__.py +85 -13
  56. pex/pip/dependencies/requires.py +38 -3
  57. pex/pip/foreign_platform/__init__.py +4 -3
  58. pex/pip/installation.py +2 -2
  59. pex/pip/local_project.py +6 -14
  60. pex/pip/package_repositories/__init__.py +78 -0
  61. pex/pip/package_repositories/link_collector.py +96 -0
  62. pex/pip/tool.py +139 -33
  63. pex/pip/vcs.py +109 -43
  64. pex/pip/version.py +8 -1
  65. pex/requirements.py +121 -16
  66. pex/resolve/config.py +5 -1
  67. pex/resolve/configured_resolve.py +32 -10
  68. pex/resolve/configured_resolver.py +10 -39
  69. pex/resolve/downloads.py +4 -3
  70. pex/resolve/lock_downloader.py +16 -23
  71. pex/resolve/lock_resolver.py +41 -51
  72. pex/resolve/locked_resolve.py +89 -32
  73. pex/resolve/locker.py +145 -101
  74. pex/resolve/locker_patches.py +123 -197
  75. pex/resolve/lockfile/create.py +232 -87
  76. pex/resolve/lockfile/download_manager.py +5 -1
  77. pex/resolve/lockfile/json_codec.py +103 -28
  78. pex/resolve/lockfile/model.py +13 -35
  79. pex/resolve/lockfile/pep_751.py +117 -98
  80. pex/resolve/lockfile/requires_dist.py +17 -262
  81. pex/resolve/lockfile/subset.py +11 -0
  82. pex/resolve/lockfile/targets.py +445 -0
  83. pex/resolve/lockfile/updater.py +22 -10
  84. pex/resolve/package_repository.py +406 -0
  85. pex/resolve/pex_repository_resolver.py +1 -1
  86. pex/resolve/pre_resolved_resolver.py +19 -16
  87. pex/resolve/project.py +233 -47
  88. pex/resolve/requirement_configuration.py +28 -10
  89. pex/resolve/resolver_configuration.py +18 -32
  90. pex/resolve/resolver_options.py +234 -28
  91. pex/resolve/resolvers.py +3 -12
  92. pex/resolve/target_options.py +18 -2
  93. pex/resolve/target_system.py +908 -0
  94. pex/resolve/venv_resolver.py +670 -0
  95. pex/resolver.py +673 -209
  96. pex/scie/__init__.py +40 -1
  97. pex/scie/model.py +2 -0
  98. pex/scie/science.py +25 -3
  99. pex/sdist.py +219 -0
  100. pex/sh_boot.py +24 -21
  101. pex/sysconfig.py +5 -3
  102. pex/targets.py +31 -10
  103. pex/third_party/__init__.py +1 -1
  104. pex/tools/commands/repository.py +48 -25
  105. pex/vendor/__init__.py +4 -9
  106. pex/vendor/__main__.py +65 -41
  107. pex/vendor/_vendored/ansicolors/.layout.json +1 -1
  108. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.dist-info/RECORD +11 -0
  109. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.pex-info/original-whl-info.json +1 -0
  110. pex/vendor/_vendored/appdirs/.layout.json +1 -1
  111. pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/RECORD +7 -0
  112. pex/vendor/_vendored/appdirs/appdirs-1.4.4.pex-info/original-whl-info.json +1 -0
  113. pex/vendor/_vendored/attrs/.layout.json +1 -1
  114. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/RECORD +37 -0
  115. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.pex-info/original-whl-info.json +1 -0
  116. pex/vendor/_vendored/packaging_20_9/.layout.json +1 -1
  117. pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/RECORD +20 -0
  118. pex/vendor/_vendored/packaging_20_9/packaging-20.9.pex-info/original-whl-info.json +1 -0
  119. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD +7 -0
  120. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.pex-info/original-whl-info.json +1 -0
  121. pex/vendor/_vendored/packaging_21_3/.layout.json +1 -1
  122. pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/RECORD +20 -0
  123. pex/vendor/_vendored/packaging_21_3/packaging-21.3.pex-info/original-whl-info.json +1 -0
  124. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.dist-info/RECORD +18 -0
  125. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.pex-info/original-whl-info.json +1 -0
  126. pex/vendor/_vendored/packaging_24_0/.layout.json +1 -1
  127. pex/vendor/_vendored/packaging_24_0/packaging-24.0.dist-info/RECORD +22 -0
  128. pex/vendor/_vendored/packaging_24_0/packaging-24.0.pex-info/original-whl-info.json +1 -0
  129. pex/vendor/_vendored/packaging_25_0/.layout.json +1 -1
  130. pex/vendor/_vendored/packaging_25_0/packaging-25.0.dist-info/RECORD +24 -0
  131. pex/vendor/_vendored/packaging_25_0/packaging-25.0.pex-info/original-whl-info.json +1 -0
  132. pex/vendor/_vendored/pip/.layout.json +1 -1
  133. pex/vendor/_vendored/pip/pip/_vendor/certifi/cacert.pem +63 -1
  134. pex/vendor/_vendored/pip/pip-20.3.4.dist-info/RECORD +388 -0
  135. pex/vendor/_vendored/pip/pip-20.3.4.pex-info/original-whl-info.json +1 -0
  136. pex/vendor/_vendored/setuptools/.layout.json +1 -1
  137. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/RECORD +107 -0
  138. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.pex-info/original-whl-info.json +1 -0
  139. pex/vendor/_vendored/toml/.layout.json +1 -1
  140. pex/vendor/_vendored/toml/toml-0.10.2.dist-info/RECORD +11 -0
  141. pex/vendor/_vendored/toml/toml-0.10.2.pex-info/original-whl-info.json +1 -0
  142. pex/vendor/_vendored/tomli/.layout.json +1 -1
  143. pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/RECORD +10 -0
  144. pex/vendor/_vendored/tomli/tomli-2.0.1.pex-info/original-whl-info.json +1 -0
  145. pex/venv/installer.py +46 -19
  146. pex/venv/venv_pex.py +6 -3
  147. pex/version.py +1 -1
  148. pex/wheel.py +188 -40
  149. pex/whl.py +67 -0
  150. pex/windows/__init__.py +14 -11
  151. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/METADATA +6 -5
  152. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/RECORD +157 -133
  153. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/entry_points.txt +1 -0
  154. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/pylock/pylock.toml +1 -1
  155. pex/docs/html/_pagefind/fragment/en_42c9d8c.pf_fragment +0 -0
  156. pex/docs/html/_pagefind/fragment/en_45dd5a2.pf_fragment +0 -0
  157. pex/docs/html/_pagefind/fragment/en_4ca74d2.pf_fragment +0 -0
  158. pex/docs/html/_pagefind/fragment/en_77273d5.pf_fragment +0 -0
  159. pex/docs/html/_pagefind/fragment/en_87a59c5.pf_fragment +0 -0
  160. pex/docs/html/_pagefind/fragment/en_8dc89b5.pf_fragment +0 -0
  161. pex/docs/html/_pagefind/fragment/en_9d1319b.pf_fragment +0 -0
  162. pex/docs/html/_pagefind/fragment/en_e55df9d.pf_fragment +0 -0
  163. pex/docs/html/_pagefind/index/en_1e98c6f.pf_index +0 -0
  164. pex/docs/html/_pagefind/pagefind.en_d1c488ecae.pf_meta +0 -0
  165. pex/vendor/_vendored/ansicolors/ansicolors-1.1.8.dist-info/INSTALLER +0 -1
  166. pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/INSTALLER +0 -1
  167. pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER +0 -1
  168. pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER +0 -1
  169. pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER +0 -1
  170. pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER +0 -1
  171. pex/vendor/_vendored/packaging_21_3/pyparsing-3.0.7.dist-info/INSTALLER +0 -1
  172. pex/vendor/_vendored/packaging_24_0/packaging-24.0.dist-info/INSTALLER +0 -1
  173. pex/vendor/_vendored/packaging_25_0/packaging-25.0.dist-info/INSTALLER +0 -1
  174. pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER +0 -1
  175. pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER +0 -1
  176. pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER +0 -1
  177. pex/vendor/_vendored/tomli/tomli-2.0.1.dist-info/INSTALLER +0 -1
  178. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/WHEEL +0 -0
  179. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/licenses/LICENSE +0 -0
  180. {pex-2.54.2.dist-info → pex-2.69.0.dist-info}/top_level.txt +0 -0
pex/pep_427.py CHANGED
@@ -3,49 +3,67 @@
3
3
 
4
4
  from __future__ import absolute_import, print_function
5
5
 
6
+ import errno
6
7
  import itertools
8
+ import json
7
9
  import os.path
8
10
  import re
9
11
  import shutil
10
12
  import subprocess
11
13
  import sys
12
- from contextlib import closing
13
- from fileinput import FileInput
14
- from textwrap import dedent
15
-
16
- from pex import pex_warnings, windows
17
- from pex.common import is_pyc_file, iter_copytree, open_zip, safe_open, touch
18
- from pex.compatibility import commonpath, get_stdout_bytes_buffer, safe_commonpath
19
- from pex.dist_metadata import (
20
- CallableEntryPoint,
21
- Distribution,
22
- NamedEntryPoint,
23
- ProjectNameAndVersion,
14
+ import time
15
+ import zipfile
16
+
17
+ from pex import pex_warnings
18
+ from pex.common import (
19
+ CopyMode,
20
+ ZipFileType,
21
+ deterministic_walk,
22
+ open_zip,
23
+ safe_copy,
24
+ safe_mkdir,
25
+ safe_mkdtemp,
26
+ safe_open,
27
+ safe_relative_symlink,
28
+ safe_rmtree,
29
+ touch,
24
30
  )
31
+ from pex.compatibility import commonpath, string
32
+ from pex.dist_metadata import DistMetadata, Distribution, MetadataFiles
33
+ from pex.entry_points_txt import install_scripts
25
34
  from pex.enum import Enum
26
- from pex.executables import chmod_plus_x
35
+ from pex.exceptions import production_assert, reportable_unexpected_error_msg
36
+ from pex.executables import chmod_plus_x, is_python_script
37
+ from pex.installed_wheel import InstalledWheel
27
38
  from pex.interpreter import PythonInterpreter
28
- from pex.os import WINDOWS
29
- from pex.pep_376 import InstalledFile, InstalledWheel, Record
39
+ from pex.pep_376 import InstalledDirectory, InstalledFile, Record, create_installed_file
40
+ from pex.pep_440 import Version
30
41
  from pex.pep_503 import ProjectName
31
- from pex.sysconfig import SCRIPT_DIR
42
+ from pex.sysconfig import SCRIPT_DIR, SysPlatform
32
43
  from pex.typing import TYPE_CHECKING, cast
44
+ from pex.venv.virtualenv import Virtualenv
33
45
  from pex.wheel import Wheel
34
46
 
35
47
  if TYPE_CHECKING:
36
48
  from typing import ( # noqa
49
+ Any,
37
50
  Callable,
38
51
  DefaultDict,
52
+ Dict,
39
53
  Iterable,
40
54
  Iterator,
41
55
  List,
42
56
  Mapping,
43
57
  Optional,
58
+ Set,
44
59
  Text,
45
60
  Tuple,
61
+ Union,
46
62
  )
47
63
 
48
64
  import attr # vendor:skip
65
+
66
+ from pex.installed_wheel import InstalledWheel # noqa
49
67
  else:
50
68
  from pex.third_party import attr
51
69
 
@@ -65,6 +83,46 @@ class InstallableType(Enum["InstallableType.Value"]):
65
83
  InstallableType.seal()
66
84
 
67
85
 
86
+ def _headers_install_path_for_wheel(
87
+ base, # type: str
88
+ wheel, # type: Wheel
89
+ ):
90
+ # type: (...) -> str
91
+
92
+ major = "X" # type: Any
93
+ minor = "Y" # type: Any
94
+ compatible_python_versions = tuple(frozenset(wheel.iter_compatible_python_versions()))
95
+ if len(compatible_python_versions) == 1 and len(compatible_python_versions[0]) >= 2:
96
+ major, minor = compatible_python_versions[0][:2]
97
+
98
+ return _headers_install_path(base, version=(major, minor), project_name=wheel.project_name)
99
+
100
+
101
+ def _headers_install_path(
102
+ base, # type: str
103
+ version, # type: Tuple[Any, Any]
104
+ project_name, # type: ProjectName
105
+ ):
106
+ # type: (...) -> str
107
+
108
+ # N.B.: You'd think sysconfig_paths["include"] would be the right answer here but both
109
+ # `pip`, and by emulation, `uv pip`, use `<venv>/include/site/pythonX.Y/<project name>`.
110
+ #
111
+ # The "mess" is admitted and described at length here:
112
+ # + https://discuss.python.org/t/clarification-on-a-wheels-header-data/9305
113
+ # + https://discuss.python.org/t/deprecating-the-headers-wheel-data-key/23712
114
+ #
115
+ # Both discussions died out with no path resolved to clean up the mess.
116
+
117
+ return os.path.join(
118
+ base,
119
+ "include",
120
+ "site",
121
+ "python{major}.{minor}".format(major=version[0], minor=version[1]),
122
+ project_name.raw,
123
+ )
124
+
125
+
68
126
  @attr.s(frozen=True)
69
127
  class InstallPaths(object):
70
128
 
@@ -74,28 +132,103 @@ class InstallPaths(object):
74
132
  def chroot(
75
133
  cls,
76
134
  destination, # type: str
77
- project_name, # type: ProjectName
135
+ wheel, # type: Wheel
78
136
  ):
79
137
  # type: (...) -> InstallPaths
138
+
80
139
  base = os.path.join(destination, cls.CHROOT_STASH)
140
+
141
+ if wheel.root_is_purelib:
142
+ purelib = destination
143
+ platlib = os.path.join(base, "platlib")
144
+ path_names = ("headers", "scripts", "platlib", "data", "purelib")
145
+ else:
146
+ purelib = os.path.join(base, "purelib")
147
+ platlib = destination
148
+ path_names = ("headers", "scripts", "purelib", "data", "platlib")
149
+
81
150
  return cls(
82
- purelib=destination,
83
- platlib=destination,
84
- headers=os.path.join(base, "include", "site", "pythonX.Y", project_name.raw),
151
+ purelib=purelib,
152
+ platlib=platlib,
153
+ headers=_headers_install_path_for_wheel(base, wheel),
85
154
  scripts=os.path.join(base, SCRIPT_DIR),
86
- data=base,
155
+ data=os.path.join(base, "data"),
156
+ path_names=path_names,
87
157
  )
88
158
 
89
159
  @classmethod
90
- def interpreter(cls, interpreter):
91
- # type: (PythonInterpreter) -> InstallPaths
160
+ def interpreter(
161
+ cls,
162
+ interpreter, # type: PythonInterpreter
163
+ project_name, # type: ProjectName
164
+ root_is_purelib, # type: bool
165
+ ):
166
+ # type: (...) -> InstallPaths
167
+
92
168
  sysconfig_paths = interpreter.identity.paths
169
+
170
+ if root_is_purelib:
171
+ path_names = ("purelib", "platlib", "headers", "scripts", "data")
172
+ else:
173
+ path_names = ("platlib", "purelib", "headers", "scripts", "data")
174
+
93
175
  return cls(
94
176
  purelib=sysconfig_paths["purelib"],
95
177
  platlib=sysconfig_paths["platlib"],
96
- headers=sysconfig_paths["include"],
178
+ headers=_headers_install_path(
179
+ interpreter.prefix,
180
+ version=(interpreter.version[0], interpreter.version[1]),
181
+ project_name=project_name,
182
+ ),
97
183
  scripts=sysconfig_paths["scripts"],
98
184
  data=sysconfig_paths["data"],
185
+ path_names=path_names,
186
+ )
187
+
188
+ @classmethod
189
+ def flat(
190
+ cls,
191
+ destination, # type: str
192
+ wheel, # type: Wheel
193
+ ):
194
+ # type: (...) -> InstallPaths
195
+ return cls(
196
+ purelib=destination,
197
+ platlib=destination,
198
+ headers=_headers_install_path_for_wheel(destination, wheel),
199
+ scripts=os.path.join(destination, SCRIPT_DIR),
200
+ data=destination,
201
+ path_names=("headers", "scripts", "data", "purelib", "platlib"),
202
+ )
203
+
204
+ @classmethod
205
+ def wheel(
206
+ cls,
207
+ destination, # type: str
208
+ wheel, # type: Union[Wheel, InstallableWheel]
209
+ ):
210
+ # type: (...) -> InstallPaths
211
+
212
+ data = os.path.join(
213
+ destination, "{wheel_prefix}.data".format(wheel_prefix=wheel.wheel_prefix)
214
+ )
215
+
216
+ if wheel.root_is_purelib:
217
+ purelib = destination
218
+ platlib = os.path.join(data, "platlib")
219
+ path_names = ("headers", "scripts", "platlib", "data", "purelib")
220
+ else:
221
+ purelib = os.path.join(data, "purelib")
222
+ platlib = destination
223
+ path_names = ("headers", "scripts", "purelib", "data", "platlib")
224
+
225
+ return cls(
226
+ purelib=purelib,
227
+ platlib=platlib,
228
+ headers=os.path.join(data, "headers"),
229
+ scripts=os.path.join(data, "scripts"),
230
+ data=os.path.join(data, "data"),
231
+ path_names=path_names,
99
232
  )
100
233
 
101
234
  purelib = attr.ib() # type: str
@@ -103,6 +236,7 @@ class InstallPaths(object):
103
236
  headers = attr.ib() # type: str
104
237
  scripts = attr.ib() # type: str
105
238
  data = attr.ib() # type: str
239
+ _path_names = attr.ib() # type: Tuple[str, ...]
106
240
 
107
241
  def __getitem__(self, item):
108
242
  # type: (Text) -> str
@@ -118,153 +252,816 @@ class InstallPaths(object):
118
252
  return self.data
119
253
  raise KeyError("Not a known install path: {item}".format(item=item))
120
254
 
255
+ def __iter__(self):
256
+ # type: () -> Iterator[Tuple[str, str]]
257
+ for path_name in self._path_names:
258
+ yield path_name, self[path_name]
259
+
260
+ def __str__(self):
261
+ # type: () -> str
262
+ return "\n".join(
263
+ "{path}={value}".format(path=path_name, value=value) for path_name, value in self
264
+ )
265
+
266
+
267
+ @attr.s(frozen=True)
268
+ class ZipEntryInfo(object):
269
+ @classmethod
270
+ def from_zip_info(
271
+ cls,
272
+ zip_info, # type: zipfile.ZipInfo
273
+ normalize_file_stat=False, # type: bool
274
+ ):
275
+ # type: (...) -> ZipEntryInfo
276
+ return cls(
277
+ filename=zip_info.filename,
278
+ date_time=zip_info.date_time,
279
+ external_attr=(
280
+ ZipFileType.from_zip_info(zip_info).deterministic_external_attr
281
+ if normalize_file_stat
282
+ else zip_info.external_attr
283
+ ),
284
+ )
285
+
286
+ @classmethod
287
+ def from_json(cls, data):
288
+ # type: (Any) -> ZipEntryInfo
289
+
290
+ if not isinstance(data, list) or not len(data) == 3:
291
+ raise ValueError(
292
+ "Invalid ZipEntryInfo JSON data. Expected a 3-item list, given {value} of type "
293
+ "{type}.".format(value=data, type=type(data))
294
+ )
295
+
296
+ filename, date_time, external_attr = data
297
+ if not isinstance(filename, string):
298
+ raise ValueError(
299
+ "Invalid ZipEntryInfo JSON data. Expected a `filename` string property; found "
300
+ "{value} of type {type}.".format(value=filename, type=type(filename))
301
+ )
302
+
303
+ if (
304
+ not isinstance(date_time, list)
305
+ or not len(date_time) == 6
306
+ or not all(isinstance(component, int) for component in date_time)
307
+ ):
308
+ raise ValueError(
309
+ "Invalid ZipEntryInfo JSON data. Expected a `date_time` list of six integers "
310
+ "property; found {value} of type {type}.".format(
311
+ value=date_time, type=type(date_time)
312
+ )
313
+ )
314
+
315
+ if not isinstance(external_attr, int):
316
+ raise ValueError(
317
+ "Invalid ZipEntryInfo JSON data. Expected an `external_attr` integer property; "
318
+ "found {value} of type {type}.".format(
319
+ value=external_attr, type=type(external_attr)
320
+ )
321
+ )
322
+
323
+ return cls(
324
+ filename=filename,
325
+ date_time=cast("Tuple[int, int, int, int, int, int]", tuple(date_time)),
326
+ external_attr=external_attr,
327
+ )
328
+
329
+ filename = attr.ib() # type: Text
330
+ date_time = attr.ib() # type: Tuple[int, int, int, int, int, int]
331
+ external_attr = attr.ib() # type: int
332
+
333
+ @property
334
+ def is_dir(self):
335
+ # type: () -> bool
336
+ return self.filename.endswith("/")
337
+
338
+ def date_time_as_struct_time(self):
339
+ # type: () -> time.struct_time
340
+ return time.struct_time(self.date_time + (0, 0, -1))
341
+
342
+ def external_attr_as_stat_mode(self):
343
+ # type: () -> int
344
+ return self.external_attr >> 16
345
+
346
+ def to_json(self):
347
+ # type: () -> Any
348
+ return self.filename, self.date_time, self.external_attr
349
+
350
+
351
+ @attr.s(frozen=True)
352
+ class ZipMetadata(object):
353
+ FILENAME = "original-whl-info.json"
354
+
355
+ @classmethod
356
+ def from_zip(
357
+ cls,
358
+ filename, # type: str
359
+ info_list, # type: Iterable[zipfile.ZipInfo]
360
+ normalize_file_stat=False, # type: bool
361
+ ):
362
+ # type: (...) -> ZipMetadata
363
+ return cls(
364
+ filename=os.path.basename(filename),
365
+ entry_info=tuple(
366
+ ZipEntryInfo.from_zip_info(zip_info, normalize_file_stat=normalize_file_stat)
367
+ for zip_info in info_list
368
+ ),
369
+ )
370
+
371
+ @classmethod
372
+ def read(cls, wheel):
373
+ # type: (Wheel) -> Optional[ZipMetadata]
374
+
375
+ data = wheel.read_pex_metadata(cls.FILENAME)
376
+ if not data:
377
+ return None
378
+ zip_metadata = json.loads(data)
379
+ if not isinstance(zip_metadata, dict):
380
+ raise ValueError(
381
+ "Invalid ZipMetadata JSON data. Expected an object; found "
382
+ "{value} of type {type}.".format(value=zip_metadata, type=type(zip_metadata))
383
+ )
384
+
385
+ filename = zip_metadata.pop("filename", None)
386
+ if not isinstance(filename, string):
387
+ raise ValueError(
388
+ "Invalid ZipMetadata JSON data. Expected an object with a string-valued 'filename' "
389
+ "property; instead found {value} of type {type}.".format(
390
+ value=zip_metadata, type=type(zip_metadata)
391
+ )
392
+ )
393
+
394
+ entries = zip_metadata.pop("entries", None)
395
+ if not isinstance(entries, list):
396
+ raise ValueError(
397
+ "Invalid ZipMetadata JSON data. Expected an object with a list-valued 'entries' "
398
+ "property; instead found {value} of type {type}.".format(
399
+ value=zip_metadata, type=type(zip_metadata)
400
+ )
401
+ )
402
+
403
+ if zip_metadata:
404
+ raise ValueError(
405
+ "Invalid ZipMetadata JSON data. Unrecognized object keys: {keys}".format(
406
+ keys=", ".join(zip_metadata)
407
+ )
408
+ )
409
+
410
+ return cls(
411
+ filename=filename,
412
+ entry_info=tuple(ZipEntryInfo.from_json(zip_entry_info) for zip_entry_info in entries),
413
+ )
414
+
415
+ filename = attr.ib() # type: str
416
+ entry_info = attr.ib() # type: Tuple[ZipEntryInfo, ...]
417
+
418
+ def __iter__(self):
419
+ # type: () -> Iterator[ZipEntryInfo]
420
+ return iter(self.entry_info)
421
+
422
+ def write(
423
+ self,
424
+ dest, # type: str
425
+ wheel, # type: Wheel
426
+ ):
427
+ # type: (...) -> str
428
+ path = os.path.join(dest, wheel.pex_metadata_path(self.FILENAME))
429
+ with safe_open(path, "w") as fp:
430
+ json.dump(
431
+ {
432
+ "filename": self.filename,
433
+ "entries": [entry_info.to_json() for entry_info in self.entry_info],
434
+ },
435
+ fp,
436
+ sort_keys=True,
437
+ separators=(",", ":"),
438
+ )
439
+ return path
440
+
441
+
442
+ @attr.s(frozen=True)
443
+ class InstallableWheel(object):
444
+ @classmethod
445
+ def from_whl(
446
+ cls,
447
+ whl, # type: Union[str, Wheel]
448
+ install_paths=None, # type: Optional[InstallPaths]
449
+ ):
450
+ # type: (...) -> InstallableWheel
451
+ wheel = whl if isinstance(whl, Wheel) else Wheel.load(whl)
452
+ zip_metadata = ZipMetadata.read(wheel)
453
+ return cls(wheel=wheel, install_paths=install_paths, zip_metadata=zip_metadata)
454
+
455
+ @classmethod
456
+ def from_installed_wheel(cls, installed_wheel):
457
+ # type: (InstalledWheel) -> InstallableWheel
458
+ wheel = Wheel.load(installed_wheel.prefix_dir)
459
+ return cls.from_whl(
460
+ whl=wheel, install_paths=InstallPaths.chroot(installed_wheel.prefix_dir, wheel=wheel)
461
+ )
462
+
463
+ wheel = attr.ib() # type: Wheel
464
+ is_whl = attr.ib(init=False) # type: bool
465
+ install_paths = attr.ib(default=None) # type: Optional[InstallPaths]
466
+ zip_metadata = attr.ib(default=None) # type: Optional[ZipMetadata]
467
+
468
+ def record_zip_metadata(self, dest):
469
+ # type: (str) -> Optional[str]
470
+ if self.zip_metadata:
471
+ return self.zip_metadata.write(dest, self.wheel)
472
+ return None
473
+
474
+ @property
475
+ def project_name(self):
476
+ # type: () -> ProjectName
477
+ return self.wheel.project_name
478
+
479
+ @property
480
+ def version(self):
481
+ # type: () -> Version
482
+ return self.wheel.version
483
+
484
+ def __attrs_post_init__(self):
485
+ # type: () -> None
486
+ is_whl = zipfile.is_zipfile(self.wheel.location)
487
+
488
+ if is_whl and self.install_paths:
489
+ raise ValueError(
490
+ "A wheel file should have no installed paths but given the following paths for "
491
+ "{wheel}:\n"
492
+ "{install_paths}".format(
493
+ wheel=self.wheel.location, install_paths=self.install_paths
494
+ )
495
+ )
496
+
497
+ if not is_whl and not self.install_paths:
498
+ raise ValueError(
499
+ "The wheel for {source} is installed but not given its install paths".format(
500
+ source=self.source
501
+ )
502
+ )
503
+
504
+ object.__setattr__(self, "is_whl", is_whl)
505
+
506
+ def iter_install_paths_by_name(self):
507
+ # type: () -> Iterator[Tuple[str, str]]
508
+ if self.install_paths:
509
+ for path_name, path in self.install_paths:
510
+ yield path_name, path
511
+
512
+ @property
513
+ def location(self):
514
+ # type: () -> str
515
+ return self.wheel.location
516
+
517
+ @property
518
+ def source(self):
519
+ # type: () -> str
520
+ return self.wheel.source
521
+
522
+ @property
523
+ def metadata_files(self):
524
+ # type: () -> MetadataFiles
525
+ return self.wheel.metadata_files
526
+
527
+ @property
528
+ def root_is_purelib(self):
529
+ # type: () -> bool
530
+ return self.wheel.root_is_purelib
531
+
532
+ @property
533
+ def data_dir(self):
534
+ # type: () -> str
535
+ return self.wheel.data_dir
536
+
537
+ @property
538
+ def wheel_prefix(self):
539
+ # type: () -> str
540
+ return self.wheel.wheel_prefix
541
+
542
+ @property
543
+ def wheel_file_name(self):
544
+ # type: () -> str
545
+ return self.zip_metadata.filename if self.zip_metadata else self.wheel.wheel_file_name
546
+
547
+ def dist_metadata(self):
548
+ # type: () -> DistMetadata
549
+ return self.wheel.dist_metadata()
550
+
551
+ def metadata_path(self, *components):
552
+ # type: (*str) -> str
553
+ return self.wheel.metadata_path(*components)
554
+
555
+ def distribution(self):
556
+ # type: () -> Distribution
557
+ return Distribution(location=self.location, metadata=self.dist_metadata())
558
+
559
+ def pex_metadata_path(self, *components):
560
+ # type: (*str) -> str
561
+ return self.wheel.pex_metadata_path(*components)
562
+
121
563
 
122
564
  class WheelInstallError(WheelError):
123
565
  """Indicates an error installing a `.whl` file."""
124
566
 
125
567
 
568
+ def reinstall_flat(
569
+ installed_wheel, # type: InstalledWheel
570
+ target_dir, # type: str
571
+ copy_mode=CopyMode.LINK, # type: CopyMode.Value
572
+ ):
573
+ # type: (...) -> Iterator[Tuple[Text, Text]]
574
+ """Re-installs the installed wheel in a flat target directory.
575
+
576
+ N.B.: A record of reinstalled files is returned in the form of an iterator that must be
577
+ consumed to drive the installation to completion.
578
+
579
+ If there is an error re-installing a file due to it already existing in the target
580
+ directory, the error is suppressed, and it's expected that the caller detects this by
581
+ comparing the record of installed files against those installed previously.
582
+
583
+ :return: An iterator over src -> dst pairs.
584
+ """
585
+ for src, dst in install_wheel_flat(
586
+ wheel=InstallableWheel.from_installed_wheel(installed_wheel),
587
+ destination=target_dir,
588
+ copy_mode=copy_mode,
589
+ ):
590
+ yield src, dst
591
+
592
+
593
+ def reinstall_venv(
594
+ installed_wheel, # type: InstalledWheel
595
+ venv, # type: Virtualenv
596
+ copy_mode=CopyMode.LINK, # type: CopyMode.Value
597
+ rel_extra_path=None, # type: Optional[str]
598
+ ):
599
+ # type: (...) -> Iterator[Tuple[Text, Text]]
600
+ """Re-installs the installed wheel in a venv.
601
+
602
+ N.B.: A record of reinstalled files is returned in the form of an iterator that must be
603
+ consumed to drive the installation to completion.
604
+
605
+ If there is an error re-installing a file due to it already existing in the destination
606
+ venv, the error is suppressed, and it's expected that the caller detects this by comparing
607
+ the record of installed files against those installed previously.
608
+
609
+ :return: An iterator over src -> dst pairs.
610
+ """
611
+
612
+ for src, dst in install_wheel_interpreter(
613
+ wheel=InstallableWheel.from_installed_wheel(installed_wheel),
614
+ interpreter=venv.interpreter,
615
+ copy_mode=copy_mode,
616
+ rel_extra_path=rel_extra_path,
617
+ compile=False,
618
+ ):
619
+ yield src, dst
620
+
621
+
622
+ def repack(
623
+ installed_wheel, # type: InstalledWheel
624
+ dest_dir, # type: str
625
+ use_system_time=False, # type: bool
626
+ override_wheel_file_name=None, # type: Optional[str]
627
+ ):
628
+ # type: (...) -> str
629
+ return create_whl(
630
+ wheel=InstallableWheel.from_installed_wheel(installed_wheel),
631
+ destination=dest_dir,
632
+ use_system_time=use_system_time,
633
+ override_wheel_file_name=override_wheel_file_name,
634
+ )
635
+
636
+
126
637
  def install_wheel_chroot(
127
- wheel_path, # type: str
638
+ wheel, # type: Union[str, InstallableWheel]
128
639
  destination, # type: str
129
- compile=False, # type: bool
130
- requested=True, # type: bool
640
+ normalize_file_stat=False, # type: bool
641
+ re_hash=False, # type: bool
131
642
  ):
132
643
  # type: (...) -> InstalledWheel
133
644
 
134
- wheel = install_wheel(
135
- wheel_path,
136
- InstallPaths.chroot(
137
- destination,
138
- project_name=ProjectNameAndVersion.from_filename(wheel_path).canonicalized_project_name,
139
- ),
140
- compile=compile,
141
- requested=requested,
645
+ wheel_to_install = (
646
+ wheel if isinstance(wheel, InstallableWheel) else InstallableWheel.from_whl(wheel)
647
+ )
648
+ chroot_install_paths = InstallPaths.chroot(destination, wheel=wheel_to_install.wheel)
649
+ install_wheel(
650
+ wheel_to_install,
651
+ chroot_install_paths,
652
+ record_entry_info=True,
653
+ normalize_file_stat=normalize_file_stat,
654
+ re_hash=re_hash,
142
655
  )
143
656
 
144
- record_relpath = wheel.metadata_files.metadata_file_rel_path("RECORD")
657
+ record_relpath = wheel_to_install.metadata_files.metadata_file_rel_path("RECORD")
145
658
  assert (
146
659
  record_relpath is not None
147
660
  ), "The {module}.install_wheel function should always create a RECORD.".format(module=__name__)
661
+
662
+ root_is_purelib = wheel_to_install.root_is_purelib
663
+
664
+ entry_names = ("purelib", "platlib") if root_is_purelib else ("platlib", "purelib")
665
+ sys_path_entries = [] # type: List[str]
666
+ for entry_name in entry_names:
667
+ entry = chroot_install_paths[entry_name]
668
+ if os.path.isdir(entry):
669
+ sys_path_entries.append(os.path.relpath(entry, destination))
670
+
148
671
  return InstalledWheel.save(
149
672
  prefix_dir=destination,
150
673
  stash_dir=InstallPaths.CHROOT_STASH,
151
674
  record_relpath=record_relpath,
152
- root_is_purelib=wheel.root_is_purelib,
675
+ root_is_purelib=root_is_purelib,
676
+ sys_path_entries=tuple(sys_path_entries),
153
677
  )
154
678
 
155
679
 
156
680
  def install_wheel_interpreter(
157
- wheel_path, # type: str
681
+ wheel, # type: Union[str, InstallableWheel]
158
682
  interpreter, # type: PythonInterpreter
683
+ copy_mode=CopyMode.LINK, # type: CopyMode.Value
684
+ rel_extra_path=None, # type: Optional[str]
159
685
  compile=True, # type: bool
160
686
  requested=True, # type: bool
161
687
  ):
162
- # type: (...) -> Wheel
688
+ # type: (...) -> Tuple[Tuple[Text, Text], ...]
163
689
 
690
+ wheel_to_install = (
691
+ wheel if isinstance(wheel, InstallableWheel) else InstallableWheel.from_whl(wheel)
692
+ )
164
693
  return install_wheel(
165
- wheel_path,
166
- InstallPaths.interpreter(interpreter),
694
+ wheel_to_install,
695
+ InstallPaths.interpreter(
696
+ interpreter,
697
+ project_name=wheel_to_install.project_name,
698
+ root_is_purelib=wheel_to_install.root_is_purelib,
699
+ ),
700
+ copy_mode=copy_mode,
167
701
  interpreter=interpreter,
702
+ rel_extra_path=rel_extra_path,
168
703
  compile=compile,
169
704
  requested=requested,
705
+ record_entry_info=True,
706
+ )
707
+
708
+
709
+ def install_wheel_flat(
710
+ wheel, # type: Union[str, InstallableWheel]
711
+ destination, # type: str
712
+ copy_mode=CopyMode.LINK, # type: CopyMode.Value
713
+ compile=False, # type: bool
714
+ ):
715
+ # type: (...) -> Tuple[Tuple[Text, Text], ...]
716
+
717
+ wheel_to_install = (
718
+ wheel if isinstance(wheel, InstallableWheel) else InstallableWheel.from_whl(wheel)
719
+ )
720
+ return install_wheel(
721
+ wheel_to_install,
722
+ InstallPaths.flat(destination, wheel=wheel_to_install.wheel),
723
+ copy_mode=copy_mode,
724
+ compile=compile,
725
+ )
726
+
727
+
728
+ def create_whl(
729
+ wheel, # type: Union[str, InstallableWheel]
730
+ destination, # type: str
731
+ compile=False, # type: bool
732
+ use_system_time=False, # type: bool
733
+ override_wheel_file_name=None, # type: Optional[str]
734
+ ):
735
+ # type: (...) -> str
736
+
737
+ if not isinstance(wheel, InstallableWheel) and zipfile.is_zipfile(wheel):
738
+ wheel_dst = os.path.join(destination, os.path.basename(wheel))
739
+ safe_copy(wheel, wheel_dst)
740
+ return wheel_dst
741
+
742
+ wheel_to_create = (
743
+ wheel if isinstance(wheel, InstallableWheel) else InstallableWheel.from_whl(wheel)
170
744
  )
745
+ whl_file_name = override_wheel_file_name or wheel_to_create.wheel_file_name
746
+ whl_chroot = os.path.join(safe_mkdtemp(prefix="pex_create_whl."), whl_file_name)
747
+ install_wheel(
748
+ wheel_to_create,
749
+ InstallPaths.wheel(destination=whl_chroot, wheel=wheel_to_create),
750
+ compile=compile,
751
+ install_entry_point_scripts=False,
752
+ )
753
+ record_data = Wheel.load(whl_chroot).metadata_files.read("RECORD")
754
+ if record_data is None:
755
+ raise AssertionError(reportable_unexpected_error_msg())
756
+
757
+ wheel_path = os.path.join(destination, whl_file_name)
758
+ with open_zip(wheel_path, "w") as zip_fp:
759
+ if use_system_time and wheel_to_create.zip_metadata:
760
+ for zip_entry_info in wheel_to_create.zip_metadata:
761
+ src = os.path.join(whl_chroot, zip_entry_info.filename)
762
+ if not os.path.exists(src):
763
+ production_assert(
764
+ zip_entry_info.is_dir,
765
+ "The wheel entry {filename} is unexpectedly missing from {source}.",
766
+ filename=zip_entry_info.filename,
767
+ source=wheel_to_create.source,
768
+ )
769
+ safe_mkdir(src)
770
+ zip_fp.write_ex(
771
+ src,
772
+ zip_entry_info.filename,
773
+ date_time=zip_entry_info.date_time_as_struct_time(),
774
+ file_mode=zip_entry_info.external_attr_as_stat_mode(),
775
+ )
776
+ else:
777
+ for installed_file in Record.read(lines=iter(record_data.decode("utf-8").splitlines())):
778
+ path = (
779
+ installed_file.dir_info.path
780
+ if isinstance(installed_file, InstalledDirectory)
781
+ else installed_file.path
782
+ )
783
+ src = os.path.join(whl_chroot, path)
784
+ if not os.path.exists(src):
785
+ production_assert(
786
+ isinstance(installed_file, InstalledDirectory),
787
+ "The wheel entry {filename} is unexpectedly missing from {source}.",
788
+ filename=path,
789
+ source=wheel_to_create.source,
790
+ )
791
+ safe_mkdir(src)
792
+ if use_system_time:
793
+ zip_fp.write(src, path)
794
+ else:
795
+ zip_fp.write_deterministic(src, path)
796
+ return wheel_path
797
+
798
+
799
+ def _detect_record_eol(path):
800
+ # type: (Text) -> str
801
+
802
+ with open(path, "rb") as fp:
803
+ line = fp.readline()
804
+ return "\r\n" if line.endswith(b"\r\n") else "\n"
805
+
806
+
807
+ def _iter_installed_files(
808
+ chroot, # type: str
809
+ exclude_rel_paths=(), # type: Iterable[str]
810
+ ):
811
+ # type: (...) -> Iterator[InstalledFile]
812
+ exclude = frozenset(exclude_rel_paths)
813
+ for root, _, files in deterministic_walk(chroot):
814
+ for path in files:
815
+ rel_path = os.path.relpath(os.path.join(root, path), chroot)
816
+ if rel_path in exclude:
817
+ continue
818
+ yield InstalledFile(rel_path)
171
819
 
172
820
 
173
821
  def install_wheel(
174
- wheel_path, # type: str
822
+ wheel, # type: InstallableWheel
175
823
  install_paths, # type: InstallPaths
824
+ copy_mode=CopyMode.LINK, # type: CopyMode.Value
176
825
  interpreter=None, # type: Optional[PythonInterpreter]
826
+ rel_extra_path=None, # type: Optional[str]
177
827
  compile=False, # type: bool
178
828
  requested=True, # type: bool
829
+ install_entry_point_scripts=True, # type: bool
830
+ record_entry_info=False, # type: bool
831
+ normalize_file_stat=False, # type: bool
832
+ re_hash=False, # type: bool
179
833
  ):
180
- # type: (...) -> Wheel
834
+ # type: (...) -> Tuple[Tuple[Text, Text], ...]
181
835
 
182
836
  # See: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl
183
- wheel = Wheel.load(wheel_path)
837
+
184
838
  dest = install_paths.purelib if wheel.root_is_purelib else install_paths.platlib
839
+ if rel_extra_path:
840
+ dest = os.path.join(dest, rel_extra_path)
841
+ if wheel.root_is_purelib:
842
+ install_paths = attr.evolve(install_paths, purelib=dest)
843
+ else:
844
+ install_paths = attr.evolve(install_paths, platlib=dest)
185
845
 
186
- record_relpath = wheel.metadata_path("RECORD")
187
- record_abspath = os.path.join(dest, record_relpath)
846
+ data_dir = None # type: Optional[str]
847
+ if wheel.is_whl:
848
+ whl = wheel.location
849
+ zip_metadata = None # type: Optional[ZipMetadata]
850
+ with open_zip(whl) as zf:
851
+ # 1. Unpack
852
+ zf.extractall(dest)
853
+ data_dir = os.path.join(dest, wheel.data_dir)
854
+ if record_entry_info:
855
+ zip_metadata = ZipMetadata.from_zip(
856
+ filename=whl, info_list=zf.infolist(), normalize_file_stat=normalize_file_stat
857
+ )
858
+
859
+ # TODO(John Sirois): Consider verifying signatures.
860
+ # N.B.: Pip does not and its also not clear what good this does. A zip can be easily
861
+ # poked on a per-entry basis allowing forging a RECORD entry and its associated file.
862
+ # Only an outer fingerprint of the whole wheel really solves this sort of tampering.
188
863
 
189
- data_rel_path = wheel.data_dir
190
- data_path = os.path.join(dest, data_rel_path)
864
+ unpacked_wheel = Wheel.load(dest, project_name=wheel.project_name)
865
+ wheel = InstallableWheel(
866
+ wheel=unpacked_wheel,
867
+ install_paths=InstallPaths.wheel(dest, wheel=unpacked_wheel),
868
+ zip_metadata=zip_metadata,
869
+ )
191
870
 
192
- installed_files = [] # type: List[InstalledFile]
871
+ # Deal with bad whl `RECORD`s. We happen to hit one from selenium-4.1.2-py3-none-any.whl
872
+ # in our tests. The selenium >=4,<4.1.3 wheels are all published with absolute paths for
873
+ # all the .py file RECORD entries. The .dist-info and .data entries are fine though.
874
+ record_data = wheel.metadata_files.read("RECORD")
193
875
 
194
- def record_files(
195
- root_dir, # type: Text
196
- names, # type: Iterable[Text]
197
- ):
198
- # type: (...) -> None
199
- for name in sorted(names):
200
- if is_pyc_file(name):
201
- # These files are both optional to RECORD and should never be present in wheels
202
- # anyway per the spec.
203
- continue
204
- file_abspath = os.path.join(root_dir, name)
205
- if record_relpath == name:
206
- # We'll generate a new RECORD below.
207
- os.unlink(file_abspath)
208
- continue
209
- installed_files.append(
210
- InstalledWheel.create_installed_file(path=file_abspath, dest_dir=dest)
876
+ record_lines = [] # type: List[Text]
877
+ eol = os.sep
878
+ if record_data:
879
+ record_lines = record_data.decode("utf-8").splitlines(
880
+ True # N.B. no kw in 2.7: keepends=True
881
+ )
882
+ eol = "\r\n" if record_lines[0].endswith("\r\n") else "\n"
883
+
884
+ if not record_data or any(
885
+ os.path.isabs(
886
+ installed_file.dir_info.path
887
+ if isinstance(installed_file, InstalledDirectory)
888
+ else installed_file.path
889
+ )
890
+ for installed_file in Record.read(lines=iter(record_lines))
891
+ ):
892
+ prefix = "The RECORD in {whl}".format(whl=os.path.basename(whl))
893
+ suffix = "so wheel re-packing will not be round-trippable."
894
+ if not record_data:
895
+ pex_warnings.warn(
896
+ "{the_record} is missing; {and_so}.".format(the_record=prefix, and_so=suffix)
897
+ )
898
+ else:
899
+ pex_warnings.warn(
900
+ "{the_record} has at least some invalid entries with absolute paths; "
901
+ "{and_so}.".format(the_record=prefix, and_so=suffix)
902
+ )
903
+ # Write a minimal repaired record to drive the spread operation below.
904
+ Record.write(
905
+ dst=os.path.join(dest, wheel.metadata_path("RECORD")),
906
+ installed_files=list(_iter_installed_files(dest)),
907
+ eol=eol,
908
+ )
909
+
910
+ if not wheel.install_paths:
911
+ raise AssertionError(reportable_unexpected_error_msg())
912
+
913
+ record_data = wheel.metadata_files.read("RECORD")
914
+ if not record_data:
915
+ try:
916
+ installed_wheel = InstalledWheel.load(wheel.location)
917
+ except InstalledWheel.LoadError:
918
+ raise WheelInstallError(
919
+ "Cannot re-install wheel for {source} because it has no installation RECORD "
920
+ "metadata.".format(source=wheel.source)
921
+ )
922
+ else:
923
+ # This is a legacy installed wheel layout with no RECORD; so we concoct one
924
+ layout_file_rel_path = os.path.relpath(
925
+ installed_wheel.layout_file(wheel.location), wheel.location
926
+ )
927
+ record_data = Record.write_bytes(
928
+ installed_files=_iter_installed_files(
929
+ chroot=wheel.location, exclude_rel_paths=[layout_file_rel_path]
930
+ )
211
931
  )
212
932
 
213
- with open_zip(wheel_path) as zf:
214
- zf.extractall(dest)
215
- # TODO(John Sirois): Consider verifying signatures.
216
- # N.B.: Pip does not and its also not clear what good this does. A zip can be easily poked
217
- # on a per-entry basis allowing forging a RECORD entry and its associated file. Only an
218
- # outer fingerprint of the whole wheel really solves this sort of tampering.
219
- record_files(
220
- root_dir=dest,
221
- names=[
222
- name
223
- for name in zf.namelist()
224
- if not name.endswith("/")
225
- and data_rel_path != safe_commonpath((data_rel_path, name))
226
- ],
933
+ # 2. Spread
934
+ entry_points = wheel.distribution().get_entry_map()
935
+ script_names = frozenset(
936
+ SysPlatform.CURRENT.binary_name(script)
937
+ for script in itertools.chain.from_iterable(
938
+ entry_points.get(key, {}) for key in ("console_scripts", "gui_scripts")
227
939
  )
228
- if os.path.isdir(data_path):
229
- for entry in sorted(os.listdir(data_path)):
230
- try:
231
- dest_dir = install_paths[entry]
232
- except KeyError as e:
233
- raise WheelInstallError(
234
- "The wheel at {wheel_path} is invalid and cannot be installed: "
235
- "{err}".format(wheel_path=wheel_path, err=e)
940
+ )
941
+
942
+ def is_entry_point_script(script_path):
943
+ # type: (Text) -> bool
944
+ return os.path.basename(script_path) in script_names
945
+
946
+ record_relpath = wheel.metadata_path("RECORD")
947
+ record_eol = os.linesep
948
+
949
+ dist_info_dir_relpath = wheel.metadata_path()
950
+ pex_info_dir_relpath = wheel.pex_metadata_path()
951
+ installer_relpath = wheel.metadata_path("INSTALLER")
952
+ requested_relpath = wheel.metadata_path("REQUESTED")
953
+ zip_metadata_relpath = wheel.pex_metadata_path(ZipMetadata.FILENAME)
954
+
955
+ installed_files = [] # type: List[Union[InstalledFile, InstalledDirectory]]
956
+ provenance = [] # type: List[Tuple[Text, Text]]
957
+ symlinked = set() # type: Set[Text]
958
+ for installed_file_or_dir in Record.read(lines=iter(record_data.decode("utf-8").splitlines())):
959
+ if isinstance(installed_file_or_dir, InstalledDirectory):
960
+ installed_files.append(installed_file_or_dir)
961
+ continue
962
+
963
+ installed_file = installed_file_or_dir
964
+ if installed_file.path == record_relpath:
965
+ record_eol = _detect_record_eol(os.path.join(wheel.location, installed_file.path))
966
+ installed_files.append(InstalledFile(path=record_relpath, hash=None, size=None))
967
+ # We'll generate these metadata files below as needed.
968
+ continue
969
+ if installed_file.path in (installer_relpath, requested_relpath, zip_metadata_relpath):
970
+ # We'll generate these metadata files below as needed.
971
+ continue
972
+
973
+ if not compile and installed_file.path.endswith(".pyc"):
974
+ continue
975
+
976
+ src_file = os.path.realpath(os.path.join(wheel.location, installed_file.path))
977
+ dst_components = None # type: Optional[Tuple[Text, Text, bool]]
978
+ for path_name, installed_path in wheel.iter_install_paths_by_name():
979
+ installed_path = os.path.realpath(installed_path)
980
+ if installed_path == commonpath((installed_path, src_file)):
981
+ rewrite_script = False
982
+ if "scripts" == path_name:
983
+ if is_entry_point_script(src_file):
984
+ # This entry point script will be installed afresh below as needed.
985
+ break
986
+ rewrite_script = interpreter is not None and is_python_script(
987
+ src_file, check_executable=False
988
+ )
989
+
990
+ dst_rel_path = os.path.relpath(src_file, installed_path)
991
+ dst_components = path_name, dst_rel_path, rewrite_script
992
+ break
993
+ else:
994
+ raise WheelInstallError(
995
+ "Encountered a file from {source} with no identifiable target install path: "
996
+ "{file}".format(source=wheel.source, file=installed_file.path)
997
+ )
998
+ if dst_components:
999
+ dst_path_name, dst_rel_path, rewrite_script = dst_components
1000
+ dst_file = os.path.join(install_paths[dst_path_name], dst_rel_path)
1001
+
1002
+ def create_dst_installed_file(regenerate_hash):
1003
+ # type: (bool) -> InstalledFile
1004
+ return (
1005
+ create_installed_file(path=dst_file, dest_dir=dest)
1006
+ if regenerate_hash
1007
+ else InstalledFile(
1008
+ path=os.path.relpath(dst_file, dest),
1009
+ hash=installed_file.hash,
1010
+ size=installed_file.size,
236
1011
  )
237
- entry_path = os.path.join(data_path, entry)
238
- copied = [dst for _, dst in iter_copytree(entry_path, dest_dir)]
239
- if copied and "scripts" == entry:
240
- for script in copied:
241
- chmod_plus_x(script)
242
- if interpreter:
243
- with closing(FileInput(files=copied, inplace=True, mode="rb")) as script_fi:
244
- for line in cast("Iterator[bytes]", script_fi):
245
- buffer = get_stdout_bytes_buffer()
246
- if script_fi.isfirstline() and re.match(br"^#!pythonw?", line):
247
- _, _, shebang_args = line.partition(b" ")
248
- buffer.write(
249
- "{shebang}\n".format(
250
- shebang=interpreter.shebang(
251
- args=shebang_args.decode("utf-8")
252
- )
253
- ).encode("utf-8")
254
- )
255
- else:
256
- # N.B.: These lines include the newline already.
257
- buffer.write(cast(bytes, line))
258
-
259
- record_files(
260
- root_dir=dest_dir,
261
- names=[
262
- os.path.relpath(os.path.join(root, f), entry_path)
263
- for root, _, files in os.walk(entry_path)
264
- for f in files
265
- ],
266
1012
  )
267
- shutil.rmtree(data_path)
1013
+
1014
+ if rewrite_script and interpreter is not None:
1015
+ with open(src_file, mode="rb") as in_fp, safe_open(dst_file, "wb") as out_fp:
1016
+ first_line = in_fp.readline()
1017
+ if first_line and re.match(br"^#!pythonw?", first_line):
1018
+ _, _, shebang_args = first_line.partition(b" ")
1019
+ encoding_line = ""
1020
+ next_line = in_fp.readline()
1021
+ # See: https://peps.python.org/pep-0263/
1022
+ if next_line and re.match(
1023
+ br"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)", next_line
1024
+ ):
1025
+ encoding_line = str(next_line.decode("ascii"))
1026
+ out_fp.write(
1027
+ "{shebang}\n".format(
1028
+ shebang=interpreter.shebang(
1029
+ args=shebang_args.decode("utf-8"), encoding_line=encoding_line
1030
+ )
1031
+ ).encode("utf-8")
1032
+ )
1033
+ if not encoding_line and next_line:
1034
+ out_fp.write(next_line)
1035
+ shutil.copyfileobj(in_fp, out_fp)
1036
+ chmod_plus_x(out_fp.name)
1037
+
1038
+ # We modified the script shebang; so we need to re-hash / re-size.
1039
+ dst_installed_file = create_dst_installed_file(regenerate_hash=True)
1040
+ elif copy_mode is CopyMode.SYMLINK:
1041
+ top_level = dst_rel_path.split(os.sep)[0]
1042
+ if top_level in (dist_info_dir_relpath, pex_info_dir_relpath):
1043
+ safe_relative_symlink(src_file, dst_file)
1044
+ elif top_level not in symlinked:
1045
+ top_level_src = os.path.join(wheel.install_paths[dst_path_name], top_level)
1046
+ top_level_dst = os.path.join(install_paths[dst_path_name], top_level)
1047
+ try:
1048
+ safe_relative_symlink(top_level_src, top_level_dst)
1049
+ symlinked.add(top_level)
1050
+ except OSError as e:
1051
+ if e.errno != errno.EEXIST:
1052
+ raise
1053
+ dst_installed_file = create_dst_installed_file(regenerate_hash=re_hash)
1054
+ else:
1055
+ safe_mkdir(os.path.dirname(dst_file))
1056
+ if copy_mode is CopyMode.LINK:
1057
+ safe_copy(src_file, dst_file, overwrite=False)
1058
+ elif not os.path.exists(dst_file):
1059
+ shutil.copy(src_file, dst_file)
1060
+ dst_installed_file = create_dst_installed_file(regenerate_hash=re_hash)
1061
+ installed_files.append(dst_installed_file)
1062
+ provenance.append((src_file, dst_file))
1063
+ if data_dir:
1064
+ safe_rmtree(data_dir)
268
1065
 
269
1066
  if compile:
270
1067
  args = [
@@ -276,7 +1073,7 @@ def install_wheel(
276
1073
  py_files = [
277
1074
  os.path.join(dest, installed_file.path)
278
1075
  for installed_file in installed_files
279
- if installed_file.path.endswith(".py")
1076
+ if isinstance(installed_file, InstalledFile) and installed_file.path.endswith(".py")
280
1077
  ]
281
1078
  process = subprocess.Popen(
282
1079
  args=args + py_files, stdout=subprocess.PIPE, stderr=subprocess.PIPE
@@ -285,7 +1082,7 @@ def install_wheel(
285
1082
  if process.returncode != 0:
286
1083
  pex_warnings.warn(
287
1084
  "Failed to compile some .py files for install of {wheel} to {dest}:\n"
288
- "{stderr}".format(wheel=wheel_path, dest=dest, stderr=stderr.decode("utf-8"))
1085
+ "{stderr}".format(wheel=wheel.source, dest=dest, stderr=stderr.decode("utf-8"))
289
1086
  )
290
1087
  for root, _, files in os.walk(commonpath(py_files)):
291
1088
  for f in files:
@@ -293,95 +1090,30 @@ def install_wheel(
293
1090
  file = InstalledFile(path=os.path.relpath(os.path.join(root, f), dest))
294
1091
  installed_files.append(file)
295
1092
 
296
- dist = Distribution(location=dest, metadata=wheel.dist_metadata())
297
- entry_points = dist.get_entry_map()
298
- installed_files.extend(
299
- InstalledWheel.create_installed_file(path=script_abspath, dest_dir=dest)
300
- for script_abspath in install_scripts(install_paths, entry_points, interpreter)
301
- )
302
-
303
- with safe_open(os.path.join(dest, wheel.metadata_path("INSTALLER")), "w") as fp:
304
- print("pex", file=fp)
305
- installed_files.append(InstalledWheel.create_installed_file(path=fp.name, dest_dir=dest))
1093
+ if install_entry_point_scripts:
1094
+ for script_src, script_abspath in install_scripts(
1095
+ install_paths.scripts, entry_points, interpreter, overwrite=False
1096
+ ):
1097
+ installed_files.append(create_installed_file(path=script_abspath, dest_dir=dest))
1098
+ provenance.append((script_src, script_abspath))
306
1099
 
307
1100
  if interpreter:
308
- # Finalize a proper venv install with REQUESTED and a RECORD to support uninstalling.
1101
+ # Finalize a proper venv install with INSTALLER and REQUESTED (if it was).
1102
+ with safe_open(os.path.join(dest, installer_relpath), "w") as fp:
1103
+ print("pex", file=fp)
1104
+ installed_files.append(create_installed_file(path=fp.name, dest_dir=dest))
309
1105
  if requested:
310
- requested_path = os.path.join(dest, wheel.metadata_path("REQUESTED"))
1106
+ requested_path = os.path.join(dest, requested_relpath)
311
1107
  touch(requested_path)
312
- installed_files.append(
313
- InstalledWheel.create_installed_file(path=requested_path, dest_dir=dest)
314
- )
315
-
316
- installed_files.append(InstalledFile(path=record_relpath, hash=None, size=None))
317
- Record.write(dst=record_abspath, installed_files=installed_files)
1108
+ installed_files.append(create_installed_file(path=requested_path, dest_dir=dest))
318
1109
 
319
- return wheel
1110
+ if record_entry_info:
1111
+ zip_metadata_path = wheel.record_zip_metadata(dest)
1112
+ if zip_metadata_path:
1113
+ installed_files.append(create_installed_file(path=zip_metadata_path, dest_dir=dest))
320
1114
 
1115
+ Record.write(
1116
+ dst=os.path.join(dest, record_relpath), installed_files=installed_files, eol=record_eol
1117
+ )
321
1118
 
322
- def install_scripts(
323
- install_paths, # type: InstallPaths
324
- entry_points, # type: Mapping[str, Mapping[str, NamedEntryPoint]]
325
- interpreter=None, # type: Optional[PythonInterpreter]
326
- ):
327
- # type: (...) -> Iterator[str]
328
-
329
- shebang = interpreter.shebang() if interpreter else "#!python"
330
- for named_entry_point, gui in itertools.chain.from_iterable(
331
- ((value, gui) for value in entry_points.get(key, {}).values())
332
- for key, gui in (("console_scripts", False), ("gui_scripts", True))
333
- ):
334
- entry_point = named_entry_point.entry_point
335
- if isinstance(entry_point, CallableEntryPoint):
336
- script = dedent(
337
- """\
338
- {shebang}
339
- # -*- coding: utf-8 -*-
340
- import importlib
341
- import sys
342
-
343
- entry_point = importlib.import_module({modname!r})
344
- for attr in {attrs!r}:
345
- entry_point = getattr(entry_point, attr)
346
-
347
- if __name__ == "__main__":
348
- import os
349
- pex_root_fallback = os.environ.get("_PEX_ROOT_FALLBACK")
350
- if pex_root_fallback:
351
- import atexit
352
- import shutil
353
-
354
- atexit.register(shutil.rmtree, pex_root_fallback, True)
355
-
356
- sys.exit(entry_point())
357
- """
358
- ).format(shebang=shebang, modname=entry_point.module, attrs=entry_point.attrs)
359
- else:
360
- script = dedent(
361
- """\
362
- {shebang}
363
- # -*- coding: utf-8 -*-
364
- import runpy
365
- import sys
366
-
367
- if __name__ == "__main__":
368
- import os
369
- pex_root_fallback = os.environ.get("_PEX_ROOT_FALLBACK")
370
- if pex_root_fallback:
371
- import atexit
372
- import shutil
373
-
374
- atexit.register(shutil.rmtree, pex_root_fallback, True)
375
-
376
- runpy.run_module({modname!r}, run_name="__main__", alter_sys=True)
377
- sys.exit(0)
378
- """
379
- ).format(shebang=shebang, modname=entry_point.module)
380
- script_abspath = os.path.join(install_paths.scripts, named_entry_point.name)
381
- if WINDOWS:
382
- script_abspath = windows.create_script(script_abspath, script, gui=gui)
383
- else:
384
- with safe_open(script_abspath, "w") as fp:
385
- fp.write(script)
386
- chmod_plus_x(fp.name)
387
- yield script_abspath
1119
+ return tuple(provenance)