csspin-python 2.1.0__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csspin_python/behave.py +11 -11
- csspin_python/devpi.py +6 -3
- csspin_python/playwright.py +34 -10
- csspin_python/playwright_schema.yaml +3 -1
- csspin_python/pytest.py +48 -8
- csspin_python/pytest_schema.yaml +19 -0
- csspin_python/python.py +213 -181
- csspin_python/python_schema.yaml +8 -15
- csspin_python/radon.py +8 -6
- csspin_python/uv_provisioner.py +187 -0
- csspin_python/uv_provisioner_schema.yaml +24 -0
- {csspin_python-2.1.0.dist-info → csspin_python-3.0.0.dist-info}/METADATA +14 -4
- csspin_python-3.0.0.dist-info/RECORD +21 -0
- csspin_python-2.1.0.dist-info/RECORD +0 -19
- {csspin_python-2.1.0.dist-info → csspin_python-3.0.0.dist-info}/WHEEL +0 -0
- {csspin_python-2.1.0.dist-info → csspin_python-3.0.0.dist-info}/licenses/LICENSE +0 -0
- {csspin_python-2.1.0.dist-info → csspin_python-3.0.0.dist-info}/top_level.txt +0 -0
csspin_python/python.py
CHANGED
|
@@ -68,6 +68,7 @@ point to the base installation.
|
|
|
68
68
|
|
|
69
69
|
import abc
|
|
70
70
|
import configparser
|
|
71
|
+
import hashlib
|
|
71
72
|
import logging
|
|
72
73
|
import os
|
|
73
74
|
import re
|
|
@@ -75,6 +76,14 @@ import shutil
|
|
|
75
76
|
import sys
|
|
76
77
|
from subprocess import check_output
|
|
77
78
|
from textwrap import dedent, indent
|
|
79
|
+
from typing import Iterable, Type, Union
|
|
80
|
+
|
|
81
|
+
try:
|
|
82
|
+
from typing import Self # type: ignore[attr-defined]
|
|
83
|
+
except ImportError:
|
|
84
|
+
from typing import TypeVar
|
|
85
|
+
|
|
86
|
+
Self = TypeVar("Self") # type: ignore[misc]
|
|
78
87
|
|
|
79
88
|
from click.exceptions import Abort
|
|
80
89
|
from csspin import (
|
|
@@ -144,10 +153,6 @@ defaults = config(
|
|
|
144
153
|
python="{python.scriptdir}/python{platform.exe}",
|
|
145
154
|
provisioner=None,
|
|
146
155
|
provisioner_memo="{spin.spin_dir}/python_provisioner.memo",
|
|
147
|
-
current_package=config(
|
|
148
|
-
install=True,
|
|
149
|
-
extras=[],
|
|
150
|
-
),
|
|
151
156
|
aws_auth=config(
|
|
152
157
|
enabled=False,
|
|
153
158
|
memo="{spin.spin_dir}/aws_auth.memo",
|
|
@@ -185,16 +190,16 @@ defaults = config(
|
|
|
185
190
|
|
|
186
191
|
|
|
187
192
|
@task()
|
|
188
|
-
def python(args):
|
|
193
|
+
def python(args: Iterable[object]) -> None:
|
|
189
194
|
"""Run the Python interpreter used for this projects."""
|
|
190
195
|
sh("python", *args)
|
|
191
196
|
|
|
192
197
|
|
|
193
198
|
@task("python:wheel", when="package")
|
|
194
199
|
def wheel(
|
|
195
|
-
cfg,
|
|
196
|
-
paths: argument(type=str, nargs=-1, required=False), #
|
|
197
|
-
):
|
|
200
|
+
cfg: ConfigTree,
|
|
201
|
+
paths: argument(type=str, nargs=-1, required=False), # type: ignore[valid-type]
|
|
202
|
+
) -> None:
|
|
198
203
|
"""Build a wheel of the current project and any additional wheels."""
|
|
199
204
|
setenv(PIP_INDEX_URL=cfg.python.index_url)
|
|
200
205
|
search_paths = paths or cfg.python.build_wheels
|
|
@@ -226,8 +231,13 @@ def wheel(
|
|
|
226
231
|
|
|
227
232
|
|
|
228
233
|
@task()
|
|
229
|
-
def env():
|
|
230
|
-
"""
|
|
234
|
+
def env() -> None:
|
|
235
|
+
"""
|
|
236
|
+
Generate command to activate the virtual environment
|
|
237
|
+
|
|
238
|
+
NOTE: spin itself should not be run from within the activated virtual
|
|
239
|
+
environment!
|
|
240
|
+
"""
|
|
231
241
|
if sys.platform == "win32":
|
|
232
242
|
# Don't care about cmd
|
|
233
243
|
print(normpath("{python.scriptdir}", "activate.ps1"))
|
|
@@ -235,12 +245,12 @@ def env():
|
|
|
235
245
|
print(f". {normpath('{python.scriptdir}', 'activate')}")
|
|
236
246
|
|
|
237
247
|
|
|
238
|
-
def pyenv_install(cfg):
|
|
248
|
+
def pyenv_install(cfg: ConfigTree) -> None:
|
|
239
249
|
"""Install and setup the virtual environment using pyenv"""
|
|
240
250
|
with namespaces(cfg.python):
|
|
241
251
|
if cfg.python.user_pyenv:
|
|
242
252
|
info("Using your existing pyenv installation ...")
|
|
243
|
-
sh(
|
|
253
|
+
sh("pyenv", "install", "--skip-existing", {cfg.python.version})
|
|
244
254
|
cfg.python.interpreter = backtick("pyenv which python --nosystem").strip()
|
|
245
255
|
else:
|
|
246
256
|
info("Installing Python {version} to {inst_dir}")
|
|
@@ -248,16 +258,18 @@ def pyenv_install(cfg):
|
|
|
248
258
|
# pyenv is by far the most robust way to install a
|
|
249
259
|
# version of Python.
|
|
250
260
|
if not exists("{pyenv.path}"):
|
|
251
|
-
sh(
|
|
261
|
+
sh("git", "clone", cfg.python.pyenv.url, cfg.python.pyenv.path)
|
|
252
262
|
else:
|
|
253
263
|
with cd(cfg.python.pyenv.path):
|
|
254
|
-
sh("git pull")
|
|
264
|
+
sh("git", "pull")
|
|
255
265
|
# we should set
|
|
256
266
|
setenv(PYTHON_BUILD_CACHE_PATH=mkdir(cfg.python.pyenv.cache))
|
|
257
267
|
setenv(PYTHON_CFLAGS="-DOPENSSL_NO_COMP")
|
|
258
268
|
try:
|
|
259
269
|
sh(
|
|
260
|
-
|
|
270
|
+
cfg.python.pyenv.python_build,
|
|
271
|
+
cfg.python.version,
|
|
272
|
+
cfg.python.inst_dir,
|
|
261
273
|
)
|
|
262
274
|
except Abort:
|
|
263
275
|
error("Failed to build the Python interpreter - removing it")
|
|
@@ -265,7 +277,7 @@ def pyenv_install(cfg):
|
|
|
265
277
|
raise
|
|
266
278
|
|
|
267
279
|
|
|
268
|
-
def nuget_install(cfg):
|
|
280
|
+
def nuget_install(cfg: ConfigTree) -> None:
|
|
269
281
|
"""Install the virtual environment using nuget"""
|
|
270
282
|
if not exists(cfg.python.nuget.exe):
|
|
271
283
|
download(cfg.python.nuget.url, cfg.python.nuget.exe)
|
|
@@ -283,7 +295,7 @@ def nuget_install(cfg):
|
|
|
283
295
|
"-source",
|
|
284
296
|
cfg.python.nuget.source,
|
|
285
297
|
)
|
|
286
|
-
sh(
|
|
298
|
+
sh(cfg.python.interpreter, "-m", "ensurepip", "--upgrade")
|
|
287
299
|
sh(
|
|
288
300
|
cfg.python.interpreter,
|
|
289
301
|
"-mpip",
|
|
@@ -300,13 +312,11 @@ def provision(cfg: ConfigTree) -> None:
|
|
|
300
312
|
"""Provision the python plugin"""
|
|
301
313
|
with memoizer(cfg.python.provisioner_memo) as memo:
|
|
302
314
|
if cfg.python.provisioner is None:
|
|
303
|
-
cfg.python.provisioner = SimpleProvisioner()
|
|
315
|
+
cfg.python.provisioner = SimpleProvisioner(cfg)
|
|
304
316
|
if not memo.check(cfg.python.provisioner):
|
|
305
317
|
memo.add(cfg.python.provisioner)
|
|
306
318
|
|
|
307
|
-
info("Checking {python.interpreter}")
|
|
308
319
|
if not shutil.which(cfg.python.interpreter):
|
|
309
|
-
info("Provisioning '{python.interpreter}'")
|
|
310
320
|
cfg.python.provisioner.provision_python(cfg)
|
|
311
321
|
|
|
312
322
|
venv_provision(cfg)
|
|
@@ -314,7 +324,7 @@ def provision(cfg: ConfigTree) -> None:
|
|
|
314
324
|
cfg.python.site_packages = get_site_packages(interpreter=cfg.python.python)
|
|
315
325
|
|
|
316
326
|
|
|
317
|
-
def configure(cfg):
|
|
327
|
+
def configure(cfg: ConfigTree) -> None:
|
|
318
328
|
"""Configure the python plugin"""
|
|
319
329
|
if not cfg.python.version and not cfg.python.use:
|
|
320
330
|
die(
|
|
@@ -348,7 +358,7 @@ def configure(cfg):
|
|
|
348
358
|
_check_aws_token_validity(cfg)
|
|
349
359
|
|
|
350
360
|
|
|
351
|
-
def init(cfg):
|
|
361
|
+
def init(cfg: ConfigTree) -> None:
|
|
352
362
|
"""Initialize the python plugin"""
|
|
353
363
|
if not cfg.python.use:
|
|
354
364
|
logging.debug("Checking for %s", cfg.python.interpreter)
|
|
@@ -365,7 +375,7 @@ def init(cfg):
|
|
|
365
375
|
ACTIVATED = False
|
|
366
376
|
|
|
367
377
|
|
|
368
|
-
def venv_init(cfg):
|
|
378
|
+
def venv_init(cfg: ConfigTree) -> None:
|
|
369
379
|
"""Activate the virtual environment"""
|
|
370
380
|
global ACTIVATED # pylint: disable=global-statement
|
|
371
381
|
if os.environ.get("VIRTUAL_ENV", "") != cfg.python.venv and not ACTIVATED:
|
|
@@ -386,7 +396,38 @@ def venv_init(cfg):
|
|
|
386
396
|
ACTIVATED = True
|
|
387
397
|
|
|
388
398
|
|
|
389
|
-
|
|
399
|
+
class ActivateScriptPatcher(abc.ABC):
|
|
400
|
+
activatescript: Union[str, Path]
|
|
401
|
+
setpattern: str
|
|
402
|
+
resetpattern: str
|
|
403
|
+
old_env_pattern: str
|
|
404
|
+
patchmarker: str
|
|
405
|
+
replacements: list[tuple[str, str]]
|
|
406
|
+
script: str
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
@abc.abstractmethod
|
|
410
|
+
def interpolate_environ_value(value: str) -> str:
|
|
411
|
+
"""
|
|
412
|
+
Translate value so the script can handle uninterpolated "{ENVVAR}" literals in value
|
|
413
|
+
|
|
414
|
+
Example:
|
|
415
|
+
# Assume the following subset of os.environ
|
|
416
|
+
os.environ = {
|
|
417
|
+
"PATH": "/bin:/usr/bin",
|
|
418
|
+
"COMPILER_PATHS": "/compiler/A/bin:/compiler/B/bin",
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
# Now, setenv has been called with
|
|
422
|
+
# setenv(PATH="{python.scriptdir}:{COMPILER_PATHS}:{PATH}") thus the
|
|
423
|
+
# value of ``PATH`` in ``EXPORTS`` equals "/venv/bin:{COMPILER_PATHS}:{PATH}" as
|
|
424
|
+
# ``COMPILER_PATHS`` and ``PATH`` haven't been interpolated yet.
|
|
425
|
+
interpolate_environ_value(value) => /venv/bin:/compiler/A/bin:/compiler/B/bin:/bin:/usr/bin
|
|
426
|
+
"""
|
|
427
|
+
return value
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def patch_activate(schema: Type[ActivateScriptPatcher]) -> None:
|
|
390
431
|
"""Patch the activate script"""
|
|
391
432
|
if exists(schema.activatescript):
|
|
392
433
|
setters = []
|
|
@@ -397,9 +438,9 @@ def patch_activate(schema):
|
|
|
397
438
|
setters.append(schema.setpattern.format(name=name, value=value))
|
|
398
439
|
resetters.add(schema.resetpattern.format(name=name))
|
|
399
440
|
old_value_setters.add(schema.old_env_pattern.format(name=name))
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
441
|
+
resetters_string = "\n".join(resetters)
|
|
442
|
+
setters_string = "\n".join(setters)
|
|
443
|
+
old_value_setters_string = "\n".join(old_value_setters)
|
|
403
444
|
original = readtext(schema.activatescript)
|
|
404
445
|
if schema.patchmarker not in original:
|
|
405
446
|
shutil.copyfile(
|
|
@@ -418,36 +459,13 @@ def patch_activate(schema):
|
|
|
418
459
|
newscript = schema.script.format(
|
|
419
460
|
patchmarker=schema.patchmarker,
|
|
420
461
|
original=original,
|
|
421
|
-
resetters=
|
|
422
|
-
old_value_setters=
|
|
423
|
-
setters=
|
|
462
|
+
resetters=resetters_string,
|
|
463
|
+
old_value_setters=old_value_setters_string,
|
|
464
|
+
setters=setters_string,
|
|
424
465
|
)
|
|
425
466
|
writetext(f"{schema.activatescript}", newscript)
|
|
426
467
|
|
|
427
468
|
|
|
428
|
-
class ActivateScriptPatcher(abc.ABC):
|
|
429
|
-
@staticmethod
|
|
430
|
-
@abc.abstractmethod
|
|
431
|
-
def interpolate_environ_value(value):
|
|
432
|
-
"""
|
|
433
|
-
Translate value so the script can handle uninterpolated "{ENVVAR}" literals in value
|
|
434
|
-
|
|
435
|
-
Example:
|
|
436
|
-
# Assume the following subset of os.environ
|
|
437
|
-
os.environ = {
|
|
438
|
-
"PATH": "/bin:/usr/bin",
|
|
439
|
-
"COMPILER_PATHS": "/compiler/A/bin:/compiler/B/bin",
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
# Now, setenv has been called with
|
|
443
|
-
# setenv(PATH="{python.scriptdir}:{COMPILER_PATHS}:{PATH}") thus the
|
|
444
|
-
# value of ``PATH`` in ``EXPORTS`` equals "/venv/bin:{COMPILER_PATHS}:{PATH}" as
|
|
445
|
-
# ``COMPILER_PATHS`` and ``PATH`` haven't been interpolated yet.
|
|
446
|
-
interpolate_environ_value(value) => /venv/bin:/compiler/A/bin:/compiler/B/bin:/bin:/usr/bin
|
|
447
|
-
"""
|
|
448
|
-
return value
|
|
449
|
-
|
|
450
|
-
|
|
451
469
|
class BashActivate(ActivateScriptPatcher):
|
|
452
470
|
patchmarker = "\n## Patched by csspin_python.python\n"
|
|
453
471
|
activatescript = Path("{python.scriptdir}") / "activate"
|
|
@@ -510,7 +528,7 @@ class BashActivate(ActivateScriptPatcher):
|
|
|
510
528
|
)
|
|
511
529
|
|
|
512
530
|
@staticmethod
|
|
513
|
-
def interpolate_environ_value(value):
|
|
531
|
+
def interpolate_environ_value(value: str) -> str:
|
|
514
532
|
if not value:
|
|
515
533
|
return ""
|
|
516
534
|
keys = re.findall(r"{(?P<key>\w+?)}", value)
|
|
@@ -564,7 +582,7 @@ class PowershellActivate(ActivateScriptPatcher):
|
|
|
564
582
|
)
|
|
565
583
|
|
|
566
584
|
@staticmethod
|
|
567
|
-
def interpolate_environ_value(value):
|
|
585
|
+
def interpolate_environ_value(value: str) -> str:
|
|
568
586
|
if not value:
|
|
569
587
|
return ""
|
|
570
588
|
keys = re.findall(r"{(?P<key>\w+?)}", value)
|
|
@@ -577,7 +595,7 @@ class PowershellActivate(ActivateScriptPatcher):
|
|
|
577
595
|
class BatchActivate(ActivateScriptPatcher):
|
|
578
596
|
patchmarker = "\nREM Patched by csspin_python.python\n"
|
|
579
597
|
activatescript = Path("{python.scriptdir}") / "activate.bat"
|
|
580
|
-
replacements =
|
|
598
|
+
replacements = []
|
|
581
599
|
old_env_pattern = dedent(
|
|
582
600
|
"""
|
|
583
601
|
if defined _OLD_SPIN_VALUE_{name} goto ENDIFSPIN{name}1
|
|
@@ -614,7 +632,7 @@ class BatchActivate(ActivateScriptPatcher):
|
|
|
614
632
|
)
|
|
615
633
|
|
|
616
634
|
@staticmethod
|
|
617
|
-
def interpolate_environ_value(value):
|
|
635
|
+
def interpolate_environ_value(value: str) -> str:
|
|
618
636
|
if not value:
|
|
619
637
|
return ""
|
|
620
638
|
keys = re.findall(r"{(?P<key>\w+?)}", value)
|
|
@@ -627,7 +645,7 @@ class BatchActivate(ActivateScriptPatcher):
|
|
|
627
645
|
class BatchDeactivate(ActivateScriptPatcher):
|
|
628
646
|
patchmarker = "\nREM Patched by csspin_python.python\n"
|
|
629
647
|
activatescript = Path("{python.scriptdir}") / "deactivate.bat"
|
|
630
|
-
replacements =
|
|
648
|
+
replacements = []
|
|
631
649
|
old_env_pattern = ""
|
|
632
650
|
setpattern = ""
|
|
633
651
|
resetpattern = dedent(
|
|
@@ -658,7 +676,7 @@ class BatchDeactivate(ActivateScriptPatcher):
|
|
|
658
676
|
class PythonActivate(ActivateScriptPatcher):
|
|
659
677
|
patchmarker = "# Patched by csspin_python.python\n"
|
|
660
678
|
activatescript = Path("{python.scriptdir}") / "activate_this.py"
|
|
661
|
-
replacements =
|
|
679
|
+
replacements = []
|
|
662
680
|
old_env_pattern = ""
|
|
663
681
|
setpattern = 'os.environ["{name}"] = fr"{value}"'
|
|
664
682
|
resetpattern = ""
|
|
@@ -671,7 +689,7 @@ class PythonActivate(ActivateScriptPatcher):
|
|
|
671
689
|
)
|
|
672
690
|
|
|
673
691
|
@staticmethod
|
|
674
|
-
def interpolate_environ_value(value):
|
|
692
|
+
def interpolate_environ_value(value: str) -> str:
|
|
675
693
|
if not value:
|
|
676
694
|
return ""
|
|
677
695
|
keys = re.findall(r"{(?P<key>\w+?)}", value)
|
|
@@ -681,7 +699,7 @@ class PythonActivate(ActivateScriptPatcher):
|
|
|
681
699
|
return value
|
|
682
700
|
|
|
683
701
|
|
|
684
|
-
def get_site_packages(interpreter):
|
|
702
|
+
def get_site_packages(interpreter: Path) -> Path:
|
|
685
703
|
"""Return the path to the virtual environments site-packages."""
|
|
686
704
|
return Path(
|
|
687
705
|
check_output(
|
|
@@ -696,7 +714,7 @@ def get_site_packages(interpreter):
|
|
|
696
714
|
)
|
|
697
715
|
|
|
698
716
|
|
|
699
|
-
def finalize_provision(cfg):
|
|
717
|
+
def finalize_provision(cfg: ConfigTree) -> None:
|
|
700
718
|
"""Patching the activate scripts and preparing the site-packages"""
|
|
701
719
|
cfg.python.provisioner.install(cfg)
|
|
702
720
|
|
|
@@ -723,7 +741,7 @@ def finalize_provision(cfg):
|
|
|
723
741
|
|
|
724
742
|
class ProvisionerProtocol:
|
|
725
743
|
"""An implementation of this protocol is used to provision
|
|
726
|
-
|
|
744
|
+
requirements to a virtual environment.
|
|
727
745
|
|
|
728
746
|
Separate plugins, can implement this interface and overwrite
|
|
729
747
|
cfg.python.provisioner.
|
|
@@ -732,8 +750,9 @@ class ProvisionerProtocol:
|
|
|
732
750
|
The provisioner will be memoized, so make sure it works with ``pickle.dumps``.
|
|
733
751
|
"""
|
|
734
752
|
|
|
735
|
-
|
|
736
|
-
|
|
753
|
+
_requirements: set[str] = set()
|
|
754
|
+
|
|
755
|
+
def provision_python(self: Self, cfg: ConfigTree) -> None:
|
|
737
756
|
"""Provision the project's python interpreter"""
|
|
738
757
|
if sys.platform == "win32":
|
|
739
758
|
nuget_install(cfg)
|
|
@@ -742,10 +761,8 @@ class ProvisionerProtocol:
|
|
|
742
761
|
pyenv_install(cfg)
|
|
743
762
|
|
|
744
763
|
# noinspection PyMethodMayBeStatic
|
|
745
|
-
def provision_venv(self, cfg: ConfigTree) -> None:
|
|
764
|
+
def provision_venv(self: Self, cfg: ConfigTree) -> None:
|
|
746
765
|
"""Provision the virtual environment of the project"""
|
|
747
|
-
# virtualenv is guaranteed to be available like this
|
|
748
|
-
# as we declared it as one of spin's dependencies
|
|
749
766
|
cmd = [
|
|
750
767
|
sys.executable,
|
|
751
768
|
"-mvirtualenv",
|
|
@@ -761,43 +778,44 @@ class ProvisionerProtocol:
|
|
|
761
778
|
env={"PYTHONPATH": cfg.spin.spin_dir / "plugins"},
|
|
762
779
|
)
|
|
763
780
|
|
|
764
|
-
def prerequisites(self, cfg: ConfigTree) -> None:
|
|
781
|
+
def prerequisites(self: Self, cfg: ConfigTree) -> None:
|
|
765
782
|
"""Provide requirements for the provisioning strategy."""
|
|
766
783
|
|
|
767
|
-
def
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
def sync(self, cfg: ConfigTree) -> None:
|
|
777
|
-
"""Synchronize the environment with the locked dependencies."""
|
|
784
|
+
def add(
|
|
785
|
+
self: Self, cfg: ConfigTree, req: str # pylint: disable=unused-argument
|
|
786
|
+
) -> None:
|
|
787
|
+
"""
|
|
788
|
+
Add a single requirement `req`, that will be installed into the
|
|
789
|
+
environment.
|
|
790
|
+
"""
|
|
791
|
+
self._requirements.add(req)
|
|
778
792
|
|
|
779
|
-
def install(self, cfg: ConfigTree) -> None:
|
|
780
|
-
"""Install the
|
|
793
|
+
def install(self: Self, cfg: ConfigTree) -> None:
|
|
794
|
+
"""Install the requirements"""
|
|
781
795
|
|
|
782
|
-
|
|
783
|
-
def cleanup(self, cfg: ConfigTree) -> None:
|
|
796
|
+
def cleanup(self: Self, cfg: ConfigTree) -> None:
|
|
784
797
|
"""Cleanup the provisioned environment"""
|
|
785
798
|
rmtree(cfg.python.venv)
|
|
786
799
|
|
|
787
800
|
|
|
788
801
|
class SimpleProvisioner(ProvisionerProtocol):
|
|
789
|
-
"""
|
|
802
|
+
"""
|
|
803
|
+
The simplest Python provisioner, using pip.
|
|
790
804
|
|
|
791
|
-
This provisioner will never uninstall
|
|
792
|
-
|
|
805
|
+
This provisioner will never uninstall requirements that are no longer
|
|
806
|
+
required.
|
|
793
807
|
"""
|
|
794
808
|
|
|
795
|
-
def __init__(self):
|
|
796
|
-
self.
|
|
797
|
-
self.
|
|
798
|
-
|
|
809
|
+
def __init__(self: Self, cfg: ConfigTree) -> None:
|
|
810
|
+
self._m = Memoizer(interpolate1("{python.memo}"))
|
|
811
|
+
self._install_command = Command(
|
|
812
|
+
"pip",
|
|
813
|
+
None if cfg.verbosity > Verbosity.NORMAL else "-q",
|
|
814
|
+
"--disable-pip-version-check",
|
|
815
|
+
"install",
|
|
816
|
+
)
|
|
799
817
|
|
|
800
|
-
def prerequisites(self, cfg):
|
|
818
|
+
def prerequisites(self: Self, cfg: ConfigTree) -> None:
|
|
801
819
|
# We'll need pip
|
|
802
820
|
sh(
|
|
803
821
|
"python",
|
|
@@ -811,82 +829,97 @@ class SimpleProvisioner(ProvisionerProtocol):
|
|
|
811
829
|
"pip",
|
|
812
830
|
)
|
|
813
831
|
|
|
814
|
-
def
|
|
815
|
-
|
|
832
|
+
def install(self: Self, cfg: ConfigTree) -> None:
|
|
833
|
+
if requirements := self._filter(
|
|
834
|
+
self._requirements, self._m, cfg.spin.project_root
|
|
835
|
+
):
|
|
836
|
+
self._install_command(*self._split(requirements))
|
|
837
|
+
self._m.clear()
|
|
838
|
+
for req in requirements:
|
|
839
|
+
self._m.add(_req_for_memo(req, cfg.spin.project_root))
|
|
816
840
|
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
841
|
+
@staticmethod
|
|
842
|
+
def _split(requirements: Iterable[str]) -> list[str]:
|
|
843
|
+
"""Used to pass whitespace-less args to :func:`csspin.sh()`."""
|
|
844
|
+
requirement_list = []
|
|
845
|
+
for requirement in requirements:
|
|
846
|
+
requirement_list.extend(requirement.split())
|
|
847
|
+
return requirement_list
|
|
822
848
|
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
849
|
+
@staticmethod
|
|
850
|
+
def _filter(
|
|
851
|
+
requirements: set[str], memo: Memoizer, project_root: Union[Path, str]
|
|
852
|
+
) -> set[str]:
|
|
853
|
+
"""
|
|
854
|
+
We want to filter all requirements prior to installing them, because we
|
|
855
|
+
only want to run the install, when there are changes, as it takes pip
|
|
856
|
+
quite some time to check, whether it has to do something.
|
|
857
|
+
"""
|
|
858
|
+
if all(memo.check(_req_for_memo(req, project_root)) for req in requirements):
|
|
859
|
+
return set()
|
|
860
|
+
else:
|
|
861
|
+
return requirements
|
|
829
862
|
|
|
830
|
-
def install(self, cfg):
|
|
831
|
-
quietflag = None if cfg.verbosity > Verbosity.NORMAL else "-q"
|
|
832
|
-
self.__execute_installation(self.devpackages, quietflag, cfg.python.index_url)
|
|
833
863
|
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
"pip",
|
|
841
|
-
quietflag,
|
|
842
|
-
"--disable-pip-version-check",
|
|
843
|
-
"install",
|
|
844
|
-
"--index-url",
|
|
845
|
-
cfg.python.index_url,
|
|
846
|
-
"-e",
|
|
847
|
-
]
|
|
848
|
-
if cfg.python.current_package.extras:
|
|
849
|
-
cmd.append(f".[{','.join(cfg.python.current_package.extras)}]")
|
|
850
|
-
else:
|
|
851
|
-
cmd.append(".")
|
|
852
|
-
sh(*cmd)
|
|
864
|
+
def _file_hash(filename: Union[Path, str]) -> str:
|
|
865
|
+
"""
|
|
866
|
+
Calculate a sha256 hash of a file's content and return its hexdigest.
|
|
867
|
+
"""
|
|
868
|
+
with open(filename, mode="br") as fd:
|
|
869
|
+
return hashlib.sha256(fd.read()).hexdigest() # nosec: hashlib
|
|
853
870
|
|
|
854
|
-
# Verify dependency compatibility of installed packages
|
|
855
|
-
pip_check = sh(
|
|
856
|
-
"pip",
|
|
857
|
-
"--disable-pip-version-check",
|
|
858
|
-
"check",
|
|
859
|
-
check=False,
|
|
860
|
-
capture_output=True,
|
|
861
|
-
)
|
|
862
|
-
if pip_check.returncode:
|
|
863
|
-
die(pip_check.stdout)
|
|
864
|
-
|
|
865
|
-
def _split(self, reqset):
|
|
866
|
-
"""to pass whitespace-less args to sh()"""
|
|
867
|
-
reqlist = []
|
|
868
|
-
for req in reqset:
|
|
869
|
-
reqlist.extend(req.split())
|
|
870
|
-
return reqlist
|
|
871
|
-
|
|
872
|
-
def __execute_installation(self, packages, quietflag, index_url):
|
|
873
|
-
"""Install packages that are not yet memoized"""
|
|
874
|
-
if to_install := {package for package in packages if not self.m.check(package)}:
|
|
875
|
-
sh(
|
|
876
|
-
"pip",
|
|
877
|
-
quietflag,
|
|
878
|
-
"--disable-pip-version-check",
|
|
879
|
-
"install",
|
|
880
|
-
"--index-url",
|
|
881
|
-
index_url,
|
|
882
|
-
*self._split(to_install),
|
|
883
|
-
)
|
|
884
|
-
for package in to_install:
|
|
885
|
-
self.m.add(package)
|
|
886
|
-
self.m.save()
|
|
887
871
|
|
|
872
|
+
def _split_requirement_option(req: str, project_root: Path) -> Union[Path, None]:
|
|
873
|
+
"""
|
|
874
|
+
Takes an element of ``python.requirements`` and checks if it
|
|
875
|
+
is an argument for pip that contains a filename. If so,
|
|
876
|
+
the filename will be returned, ``None`` otherwise.
|
|
877
|
+
|
|
878
|
+
The following options are respected:
|
|
879
|
+
- ``-r``/``--requirement``
|
|
880
|
+
- ``-c``/``--constraint``
|
|
888
881
|
|
|
889
|
-
|
|
882
|
+
If a file for an option cannot be found, the plugin will
|
|
883
|
+
:func:`csspin.die()`.
|
|
884
|
+
"""
|
|
885
|
+
if (
|
|
886
|
+
req.startswith(option := "-r")
|
|
887
|
+
or req.startswith(option := "--requirement")
|
|
888
|
+
or req.startswith(option := "-c")
|
|
889
|
+
or req.startswith(option := "--constraint")
|
|
890
|
+
):
|
|
891
|
+
# The pattern has to enforce the " "/"=" for the long-options
|
|
892
|
+
match_many = "+" if option.startswith("--") else "*"
|
|
893
|
+
pattern = rf"{option}[ =]{match_many}(?P<filename>.*)"
|
|
894
|
+
match = re.match(pattern, req)
|
|
895
|
+
if not match:
|
|
896
|
+
die(f"{req} could not be validated.")
|
|
897
|
+
else:
|
|
898
|
+
file = project_root / match.group("filename")
|
|
899
|
+
if not file.exists():
|
|
900
|
+
die(f"{file} does not exist.")
|
|
901
|
+
return file
|
|
902
|
+
return None
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
def _req_for_memo(
|
|
906
|
+
req: str, project_root: Union[Path, str]
|
|
907
|
+
) -> str: # pylint: disable=inconsistent-return-statements
|
|
908
|
+
"""
|
|
909
|
+
Return a memoizable representation of a python requirement. In case a
|
|
910
|
+
requirement is on of the following options, the function returns requirement
|
|
911
|
+
with a hash of the files' content appended. Otherwise the requirement itself
|
|
912
|
+
will be returned.
|
|
913
|
+
"""
|
|
914
|
+
if file := _split_requirement_option(req, project_root):
|
|
915
|
+
return f"{req}{_file_hash(file)}"
|
|
916
|
+
else:
|
|
917
|
+
return req
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def venv_provision( # pylint: disable=too-many-branches,missing-function-docstring
|
|
921
|
+
cfg: ConfigTree,
|
|
922
|
+
) -> None:
|
|
890
923
|
fresh_env = False
|
|
891
924
|
|
|
892
925
|
info("Checking venv '{python.venv}'")
|
|
@@ -916,16 +949,10 @@ def venv_provision(cfg): # pylint: disable=too-many-branches,missing-function-d
|
|
|
916
949
|
logging.debug(f"{plugin_module.__name__}.venv_hook()")
|
|
917
950
|
hook(cfg)
|
|
918
951
|
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
# Install packages required by the project ('requirements')
|
|
952
|
+
# Add packages required by the project ('requirements')
|
|
922
953
|
for req in cfg.python.get("requirements", []):
|
|
923
954
|
cfg.python.provisioner.add(cfg, interpolate1(req))
|
|
924
955
|
|
|
925
|
-
# Install development packages required by the project ('devpackages')
|
|
926
|
-
for pkgspec in cfg.python.get("devpackages", []):
|
|
927
|
-
cfg.python.provisioner.add(cfg, interpolate1(pkgspec), True)
|
|
928
|
-
|
|
929
956
|
# Install packages required by plugins used
|
|
930
957
|
# ('<plugin>.requires.python')
|
|
931
958
|
for plugin in cfg.spin.topo_plugins:
|
|
@@ -933,9 +960,6 @@ def venv_provision(cfg): # pylint: disable=too-many-branches,missing-function-d
|
|
|
933
960
|
for req in get_requires(plugin_module.defaults, "python"):
|
|
934
961
|
cfg.python.provisioner.add(cfg, interpolate1(req))
|
|
935
962
|
|
|
936
|
-
cfg.python.provisioner.lock_extras(cfg)
|
|
937
|
-
cfg.python.provisioner.sync(cfg)
|
|
938
|
-
|
|
939
963
|
|
|
940
964
|
def cleanup(cfg: ConfigTree) -> None:
|
|
941
965
|
"""Remove directories and files generated by the python plugin."""
|
|
@@ -961,7 +985,7 @@ def cleanup(cfg: ConfigTree) -> None:
|
|
|
961
985
|
rmtree(current_path / filename)
|
|
962
986
|
|
|
963
987
|
|
|
964
|
-
def _get_pipconf(cfg):
|
|
988
|
+
def _get_pipconf(cfg: ConfigTree) -> Path:
|
|
965
989
|
"""Retrieve the pipconf configuration file path."""
|
|
966
990
|
if sys.platform == "win32":
|
|
967
991
|
pipconf = interpolate1(Path(cfg.python.venv)) / "pip.ini"
|
|
@@ -971,7 +995,7 @@ def _get_pipconf(cfg):
|
|
|
971
995
|
return pipconf
|
|
972
996
|
|
|
973
997
|
|
|
974
|
-
def _configure_pipconf(cfg, update=False):
|
|
998
|
+
def _configure_pipconf(cfg: ConfigTree, update: bool = False) -> None:
|
|
975
999
|
"""Configure the pip configuration file"""
|
|
976
1000
|
config_parser = configparser.ConfigParser()
|
|
977
1001
|
config_parser.read_string(cfg.python.pipconf)
|
|
@@ -985,7 +1009,7 @@ def _configure_pipconf(cfg, update=False):
|
|
|
985
1009
|
config_parser.write(fd)
|
|
986
1010
|
|
|
987
1011
|
|
|
988
|
-
def _obfuscate_index_url(index_url):
|
|
1012
|
+
def _obfuscate_index_url(index_url: str) -> None:
|
|
989
1013
|
"""Add the CodeArtifact token to the secrets."""
|
|
990
1014
|
|
|
991
1015
|
from csspin import secrets
|
|
@@ -993,7 +1017,7 @@ def _obfuscate_index_url(index_url):
|
|
|
993
1017
|
secrets.add(index_url.split(":")[2].split("@")[0]) # Codeartifact token
|
|
994
1018
|
|
|
995
1019
|
|
|
996
|
-
def _check_aws_token_validity(cfg):
|
|
1020
|
+
def _check_aws_token_validity(cfg: ConfigTree) -> None:
|
|
997
1021
|
"""
|
|
998
1022
|
If csspin-python[aws_auth] is installed, we can use csaccess to get the
|
|
999
1023
|
CodeArtifact authentication token.
|
|
@@ -1016,7 +1040,9 @@ def _check_aws_token_validity(cfg):
|
|
|
1016
1040
|
for item in memo.items():
|
|
1017
1041
|
if isinstance(item, str) and item.startswith(f"{timestamp_key}:"):
|
|
1018
1042
|
last_time = int(item.split(":", 1)[1])
|
|
1019
|
-
if current_time - last_time <
|
|
1043
|
+
if current_time - last_time < int(
|
|
1044
|
+
interpolate1(cfg.python.aws_auth.key_duration)
|
|
1045
|
+
):
|
|
1020
1046
|
pipconf = _get_pipconf(cfg)
|
|
1021
1047
|
config_parser = configparser.ConfigParser()
|
|
1022
1048
|
config_parser.read(pipconf)
|
|
@@ -1034,12 +1060,18 @@ def _check_aws_token_validity(cfg):
|
|
|
1034
1060
|
info("Updating Codeartifact token.")
|
|
1035
1061
|
from urllib.parse import urljoin
|
|
1036
1062
|
|
|
1063
|
+
opts = {
|
|
1064
|
+
"static_oidc": interpolate1(cfg.python.aws_auth.static_oidc).lower()
|
|
1065
|
+
== "true"
|
|
1066
|
+
}
|
|
1067
|
+
if cfg.python.aws_auth.client_id:
|
|
1068
|
+
opts["client_id"] = interpolate1(cfg.python.aws_auth.client_id)
|
|
1069
|
+
if cfg.python.aws_auth.role_arn:
|
|
1070
|
+
opts["aws_role_arn"] = interpolate1(cfg.python.aws_auth.role_arn)
|
|
1071
|
+
|
|
1072
|
+
index_base_url = get_ca_pypi_url_programmatic(**opts)
|
|
1037
1073
|
index_url = urljoin(
|
|
1038
|
-
|
|
1039
|
-
static_oidc=cfg.python.aws_auth.static_oidc
|
|
1040
|
-
)
|
|
1041
|
-
+ "/",
|
|
1042
|
-
cfg.python.aws_auth.index,
|
|
1074
|
+
index_base_url + "/", interpolate1(cfg.python.aws_auth.index)
|
|
1043
1075
|
)
|
|
1044
1076
|
cfg.python.index_url = index_url
|
|
1045
1077
|
_obfuscate_index_url(index_url)
|