calkit-python 0.49.0__py3-none-any.whl → 0.49.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/__init__.py +2 -2
- calkit/cli/check.py +12 -18
- calkit/cli/core.py +7 -17
- calkit/cli/list.py +67 -0
- calkit/cli/main/core.py +7 -4
- calkit/core.py +89 -60
- calkit/dvc/core.py +2 -2
- calkit/environments.py +3 -0
- calkit/models/core.py +15 -3
- calkit/models/requirements.py +40 -6
- calkit/{dependencies.py → requirements.py} +190 -65
- calkit/tests/cli/main/test_core.py +3 -5
- calkit/tests/cli/test_check.py +5 -2
- calkit/tests/cli/test_list.py +81 -0
- calkit/tests/models/test_core.py +1 -1
- calkit/tests/test_environments.py +5 -1
- calkit/tests/{test_dependencies.py → test_requirements.py} +171 -46
- calkit/tests/test_workspace.py +25 -0
- calkit/workspace.py +22 -2
- {calkit_python-0.49.0.dist-info → calkit_python-0.49.1.dist-info}/METADATA +1 -1
- {calkit_python-0.49.0.dist-info → calkit_python-0.49.1.dist-info}/RECORD +37 -37
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.49.0.data → calkit_python-0.49.1.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.49.0.dist-info → calkit_python-0.49.1.dist-info}/WHEEL +0 -0
- {calkit_python-0.49.0.dist-info → calkit_python-0.49.1.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.49.0.dist-info → calkit_python-0.49.1.dist-info}/licenses/LICENSE +0 -0
calkit/__init__.py
CHANGED
|
@@ -13,7 +13,6 @@ if TYPE_CHECKING:
|
|
|
13
13
|
conda,
|
|
14
14
|
config,
|
|
15
15
|
datasets,
|
|
16
|
-
dependencies,
|
|
17
16
|
detect,
|
|
18
17
|
docker,
|
|
19
18
|
docx,
|
|
@@ -44,6 +43,7 @@ if TYPE_CHECKING:
|
|
|
44
43
|
questions,
|
|
45
44
|
releases,
|
|
46
45
|
reproducibility,
|
|
46
|
+
requirements,
|
|
47
47
|
resources,
|
|
48
48
|
schema,
|
|
49
49
|
server,
|
|
@@ -60,7 +60,6 @@ _SUBMODULES = {
|
|
|
60
60
|
"conda",
|
|
61
61
|
"config",
|
|
62
62
|
"datasets",
|
|
63
|
-
"dependencies",
|
|
64
63
|
"detect",
|
|
65
64
|
"docker",
|
|
66
65
|
"docx",
|
|
@@ -91,6 +90,7 @@ _SUBMODULES = {
|
|
|
91
90
|
"questions",
|
|
92
91
|
"releases",
|
|
93
92
|
"reproducibility",
|
|
93
|
+
"requirements",
|
|
94
94
|
"resources",
|
|
95
95
|
"schema",
|
|
96
96
|
"server",
|
calkit/cli/check.py
CHANGED
|
@@ -2199,7 +2199,7 @@ def check_project_requirements(
|
|
|
2199
2199
|
typer.echo("Checking project requirements")
|
|
2200
2200
|
dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
|
|
2201
2201
|
if no_cache:
|
|
2202
|
-
calkit.
|
|
2202
|
+
calkit.requirements.cache_clear()
|
|
2203
2203
|
try:
|
|
2204
2204
|
calkit.check_requirements(use_cache=not no_cache)
|
|
2205
2205
|
except Exception as e:
|
|
@@ -2240,7 +2240,9 @@ def check_stage_runs() -> None:
|
|
|
2240
2240
|
recorded = stage_runs.get(name)
|
|
2241
2241
|
if not isinstance(recorded, dict):
|
|
2242
2242
|
problems.append(f"Stage '{name}' has no recorded run")
|
|
2243
|
-
elif recorded.get("
|
|
2243
|
+
elif recorded.get("dvc_lock_hash") != calkit.dvc.stage_lock_hash(
|
|
2244
|
+
locked
|
|
2245
|
+
):
|
|
2244
2246
|
problems.append(
|
|
2245
2247
|
f"Stage '{name}' has changed in dvc.lock since its last "
|
|
2246
2248
|
"recorded run"
|
|
@@ -2263,28 +2265,20 @@ def check_env_vars(
|
|
|
2263
2265
|
] = False,
|
|
2264
2266
|
) -> None:
|
|
2265
2267
|
"""Check that the project's required environmental variables exist."""
|
|
2268
|
+
from calkit.core import _normalize_requirement
|
|
2269
|
+
|
|
2266
2270
|
typer.echo("Checking project environmental variables")
|
|
2267
2271
|
dotenv.load_dotenv(dotenv_path=".env")
|
|
2268
2272
|
ck_info = calkit.load_calkit_info()
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2273
|
+
for raw in calkit.get_requirements(ck_info):
|
|
2274
|
+
req = _normalize_requirement(raw)
|
|
2275
|
+
if req["kind"] != "env-var":
|
|
2276
|
+
continue
|
|
2277
|
+
name = req["name"]
|
|
2272
2278
|
if verbose:
|
|
2273
2279
|
typer.echo(f"Checking for environmental variable '{name}'")
|
|
2274
|
-
# Pull the dep's attrs to honor any per-var default.
|
|
2275
|
-
attrs: dict = {}
|
|
2276
|
-
for dep in deps:
|
|
2277
|
-
if isinstance(dep, dict) and dep.get("name") == name:
|
|
2278
|
-
attrs = dep
|
|
2279
|
-
break
|
|
2280
|
-
if isinstance(dep, dict) and list(dep.keys()) == [name]:
|
|
2281
|
-
attrs = dep[name] or {}
|
|
2282
|
-
break
|
|
2283
2280
|
if name not in os.environ:
|
|
2284
|
-
|
|
2285
|
-
value = calkit.dependencies.prompt_and_store_env_var(
|
|
2286
|
-
name, default=attrs.get("default")
|
|
2287
|
-
)
|
|
2281
|
+
value = calkit.requirements.prompt_for_env_var_req(req)
|
|
2288
2282
|
if value is None:
|
|
2289
2283
|
raise_error(f"No value provided for '{name}'")
|
|
2290
2284
|
message = "✅ All set!"
|
calkit/cli/core.py
CHANGED
|
@@ -6,7 +6,7 @@ import os
|
|
|
6
6
|
import re
|
|
7
7
|
import subprocess
|
|
8
8
|
from enum import Enum
|
|
9
|
-
from typing import
|
|
9
|
+
from typing import Any, NoReturn
|
|
10
10
|
|
|
11
11
|
import typer
|
|
12
12
|
from typer.core import TyperCommand, TyperGroup
|
|
@@ -28,10 +28,6 @@ class EnvDefaultsModeChoice(str, Enum):
|
|
|
28
28
|
merge = "merge"
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
if TYPE_CHECKING:
|
|
32
|
-
import click
|
|
33
|
-
|
|
34
|
-
|
|
35
31
|
class AliasGroup(TyperGroup):
|
|
36
32
|
"""TyperGroup that resolves command aliases defined with '|' in the name."""
|
|
37
33
|
|
|
@@ -87,21 +83,19 @@ class OptionalValueCommand(TyperCommand):
|
|
|
87
83
|
|
|
88
84
|
|
|
89
85
|
def complete_stage_names(
|
|
90
|
-
ctx:
|
|
91
|
-
|
|
92
|
-
incomplete: str,
|
|
93
|
-
) -> list:
|
|
86
|
+
ctx: object, param: object, incomplete: str
|
|
87
|
+
) -> list[str]:
|
|
94
88
|
"""Return pipeline stage names for shell tab-completion.
|
|
95
89
|
|
|
96
90
|
``ctx`` and ``param`` are part of the Click completion-callback signature
|
|
97
|
-
but are unused here, so they accept
|
|
98
|
-
tests
|
|
91
|
+
but are unused here, so they accept anything, e.g., ``None`` when called
|
|
92
|
+
directly in tests. Typer bundles its own Click, so its types would only
|
|
93
|
+
tie this to that copy.
|
|
99
94
|
"""
|
|
100
95
|
if not os.path.isfile("calkit.yaml"):
|
|
101
96
|
return []
|
|
102
97
|
try:
|
|
103
98
|
import ruamel.yaml
|
|
104
|
-
from click.shell_completion import CompletionItem
|
|
105
99
|
|
|
106
100
|
ryaml = ruamel.yaml.YAML()
|
|
107
101
|
with open("calkit.yaml") as f:
|
|
@@ -122,11 +116,7 @@ def complete_stage_names(
|
|
|
122
116
|
sp_stages = sp_info.get("pipeline", {}).get("stages", {})
|
|
123
117
|
for stage in sp_stages:
|
|
124
118
|
candidates.append(f"{sp_name}:{stage}")
|
|
125
|
-
return [
|
|
126
|
-
CompletionItem(name)
|
|
127
|
-
for name in candidates
|
|
128
|
-
if name.startswith(incomplete)
|
|
129
|
-
]
|
|
119
|
+
return [name for name in candidates if name.startswith(incomplete)]
|
|
130
120
|
except Exception:
|
|
131
121
|
return []
|
|
132
122
|
|
calkit/cli/list.py
CHANGED
|
@@ -394,6 +394,73 @@ def list_environments(
|
|
|
394
394
|
typer.echo(f" {k}: {v}")
|
|
395
395
|
|
|
396
396
|
|
|
397
|
+
@list_app.command(name="requirements|reqs")
|
|
398
|
+
def list_requirements(
|
|
399
|
+
json_output: Annotated[
|
|
400
|
+
bool, typer.Option("--json", help="Output result as JSON.")
|
|
401
|
+
] = False,
|
|
402
|
+
) -> None:
|
|
403
|
+
"""List what a machine needs to run the project.
|
|
404
|
+
|
|
405
|
+
That's the project's requirements, then those of each environment and
|
|
406
|
+
stage, each with where it was declared. Nothing is checked; use
|
|
407
|
+
'calkit check reqs' for that.
|
|
408
|
+
"""
|
|
409
|
+
from calkit.core import _normalize_requirement
|
|
410
|
+
|
|
411
|
+
ck_info = calkit.load_calkit_info()
|
|
412
|
+
sources: list[tuple[str, list]] = [
|
|
413
|
+
("project", calkit.get_requirements(ck_info))
|
|
414
|
+
]
|
|
415
|
+
for name, env in (ck_info.get("environments") or {}).items():
|
|
416
|
+
if not isinstance(env, dict):
|
|
417
|
+
continue
|
|
418
|
+
if env.get("requirements"):
|
|
419
|
+
sources.append((f"environment '{name}'", env["requirements"]))
|
|
420
|
+
# An option a switch defines in place is named after its position
|
|
421
|
+
for n, option in enumerate(env.get("switch") or [], start=1):
|
|
422
|
+
use = option.get("use") if isinstance(option, dict) else None
|
|
423
|
+
if isinstance(use, dict) and use.get("requirements"):
|
|
424
|
+
sources.append(
|
|
425
|
+
(f"environment '{name}.{n}'", use["requirements"])
|
|
426
|
+
)
|
|
427
|
+
stages = (ck_info.get("pipeline") or {}).get("stages") or {}
|
|
428
|
+
for name, stage in stages.items():
|
|
429
|
+
if isinstance(stage, dict) and stage.get("requirements"):
|
|
430
|
+
sources.append((f"stage '{name}'", stage["requirements"]))
|
|
431
|
+
result = []
|
|
432
|
+
for source, reqs in sources:
|
|
433
|
+
for raw in reqs:
|
|
434
|
+
try:
|
|
435
|
+
req = _normalize_requirement(raw)
|
|
436
|
+
except ValueError as e:
|
|
437
|
+
raise_error(f"In {source}: {e}")
|
|
438
|
+
result.append({"source": source, **req})
|
|
439
|
+
if json_output:
|
|
440
|
+
echo_json(result)
|
|
441
|
+
return
|
|
442
|
+
property_kinds = calkit.system_property_requirement_kinds()
|
|
443
|
+
last_source = None
|
|
444
|
+
for req in result:
|
|
445
|
+
req = dict(req)
|
|
446
|
+
source = req.pop("source")
|
|
447
|
+
if source != last_source:
|
|
448
|
+
if last_source is not None:
|
|
449
|
+
typer.echo()
|
|
450
|
+
typer.echo(source[0].upper() + source[1:] + ":")
|
|
451
|
+
last_source = source
|
|
452
|
+
name = req.pop("name")
|
|
453
|
+
kind = req.pop("kind")
|
|
454
|
+
# A property requirement's name is its kind
|
|
455
|
+
typer.echo(
|
|
456
|
+
f" - {name}" if kind in property_kinds else f" - {name} ({kind})"
|
|
457
|
+
)
|
|
458
|
+
for k, v in req.items():
|
|
459
|
+
if isinstance(v, str):
|
|
460
|
+
v = " ".join(v.split())
|
|
461
|
+
typer.echo(f" {k}: {v}")
|
|
462
|
+
|
|
463
|
+
|
|
397
464
|
@list_app.command(name="templates")
|
|
398
465
|
def list_templates(
|
|
399
466
|
kind: Annotated[
|
calkit/cli/main/core.py
CHANGED
|
@@ -211,7 +211,7 @@ def _exec_with_version(version_spec: str) -> None:
|
|
|
211
211
|
Strips ``--use-version`` from ``sys.argv`` before re-invoking so the
|
|
212
212
|
child process doesn't loop. Requires ``uv`` to be available on PATH.
|
|
213
213
|
"""
|
|
214
|
-
from calkit.
|
|
214
|
+
from calkit.requirements import _format_uvx_from
|
|
215
215
|
|
|
216
216
|
if shutil.which("uvx") is None:
|
|
217
217
|
raise_error(
|
|
@@ -548,7 +548,7 @@ def get_status(
|
|
|
548
548
|
# than enforces, so anything still unset is a warning: it must not stop
|
|
549
549
|
# the rest of the status from being shown.
|
|
550
550
|
if ck_info:
|
|
551
|
-
missing_env_vars = calkit.
|
|
551
|
+
missing_env_vars = calkit.requirements.resolve_env_var_deps(ck_info)
|
|
552
552
|
if missing_env_vars:
|
|
553
553
|
warn(
|
|
554
554
|
"Missing environmental variable(s): "
|
|
@@ -2082,10 +2082,13 @@ def _tell_hub_we_pushed(targets: list[str], git_args: list[str]) -> None:
|
|
|
2082
2082
|
# The remote git actually pushed to: whatever was named on the
|
|
2083
2083
|
# command line, else the branch's own upstream, else origin.
|
|
2084
2084
|
named = [a for a in git_args if not a.startswith("-")]
|
|
2085
|
+
upstream = repo.active_branch.tracking_branch()
|
|
2085
2086
|
if named:
|
|
2086
2087
|
body["remote"] = named[0]
|
|
2088
|
+
elif upstream is not None:
|
|
2089
|
+
body["remote"] = upstream.remote_name
|
|
2087
2090
|
else:
|
|
2088
|
-
body["remote"] =
|
|
2091
|
+
body["remote"] = "origin"
|
|
2089
2092
|
except Exception:
|
|
2090
2093
|
body.setdefault("remote", "origin")
|
|
2091
2094
|
try:
|
|
@@ -3673,7 +3676,7 @@ def run(
|
|
|
3673
3676
|
if lock_dir not in lock_stages:
|
|
3674
3677
|
lock_stages[lock_dir] = calkit.dvc.read_lock_stages(lock_dir)
|
|
3675
3678
|
if isinstance(lock_stages[lock_dir].get(lock_name), dict):
|
|
3676
|
-
entry["
|
|
3679
|
+
entry["dvc_lock_hash"] = calkit.dvc.stage_lock_hash(
|
|
3677
3680
|
lock_stages[lock_dir][lock_name]
|
|
3678
3681
|
)
|
|
3679
3682
|
entry["calkit_version"] = system_info.get("calkit_version")
|
calkit/core.py
CHANGED
|
@@ -790,7 +790,9 @@ def _normalize_requirement(req) -> dict:
|
|
|
790
790
|
required there. A ``setup`` requirement may omit it, since a single
|
|
791
791
|
anonymous setup step is common and forcing users to invent a name adds
|
|
792
792
|
friction; a stable one is synthesized from a short hash of
|
|
793
|
-
``check_command``.
|
|
793
|
+
``check_command``. An ``aws-credentials`` requirement may omit it too,
|
|
794
|
+
and is named after its profile and account. A machine-property
|
|
795
|
+
requirement has no name at all --
|
|
794
796
|
``kind: cpu-count`` already says everything there is to say about which
|
|
795
797
|
property it constrains -- so the kind is used as the name in messages.
|
|
796
798
|
"""
|
|
@@ -802,12 +804,11 @@ def _normalize_requirement(req) -> dict:
|
|
|
802
804
|
"'kind', e.g., '- kind: env-var' with 'name: TOKEN'"
|
|
803
805
|
)
|
|
804
806
|
|
|
805
|
-
|
|
806
|
-
if isinstance(dep, str):
|
|
807
|
+
if isinstance(req, str):
|
|
807
808
|
# Split on the first version operator so a string like
|
|
808
809
|
# ``calkit>=0.38`` produces both a clean name and a version spec
|
|
809
810
|
# the caller can validate.
|
|
810
|
-
m = re.match(r"^([A-Za-z0-9_.\-]+)(.*)$",
|
|
811
|
+
m = re.match(r"^([A-Za-z0-9_.\-]+)(.*)$", req.strip())
|
|
811
812
|
if m is None:
|
|
812
813
|
raise malformed()
|
|
813
814
|
out: dict = {"name": m.group(1), "kind": "app"}
|
|
@@ -815,14 +816,14 @@ def _normalize_requirement(req) -> dict:
|
|
|
815
816
|
if spec:
|
|
816
817
|
out["version_spec"] = spec
|
|
817
818
|
return out
|
|
818
|
-
if not isinstance(
|
|
819
|
+
if not isinstance(req, dict):
|
|
819
820
|
raise malformed()
|
|
820
|
-
keys = list(
|
|
821
|
+
keys = list(req.keys())
|
|
821
822
|
# Flat form with explicit kind: only requires ``name`` for kinds where
|
|
822
823
|
# name is the identity (app, env-var). Setup requirements may omit it,
|
|
823
824
|
# and property requirements have none to give.
|
|
824
825
|
if "kind" in keys:
|
|
825
|
-
out = dict(
|
|
826
|
+
out = dict(req)
|
|
826
827
|
if "name" not in out:
|
|
827
828
|
kind = out["kind"]
|
|
828
829
|
if kind in system_property_requirement_kinds():
|
|
@@ -833,18 +834,27 @@ def _normalize_requirement(req) -> dict:
|
|
|
833
834
|
check_command.encode("utf-8")
|
|
834
835
|
).hexdigest()[:8]
|
|
835
836
|
out["name"] = f"setup-{short}"
|
|
837
|
+
elif kind == "aws-credentials":
|
|
838
|
+
out["name"] = "-".join(
|
|
839
|
+
["aws-credentials"]
|
|
840
|
+
+ [
|
|
841
|
+
str(out[key])
|
|
842
|
+
for key in ("profile", "account_id")
|
|
843
|
+
if out.get(key)
|
|
844
|
+
]
|
|
845
|
+
)
|
|
836
846
|
else:
|
|
837
|
-
raise ValueError(f"Requirement missing required 'name': {
|
|
847
|
+
raise ValueError(f"Requirement missing required 'name': {req}")
|
|
838
848
|
return out
|
|
839
849
|
if "name" in keys:
|
|
840
|
-
out = dict(
|
|
850
|
+
out = dict(req)
|
|
841
851
|
out.setdefault("kind", "app")
|
|
842
852
|
return out
|
|
843
853
|
if len(keys) != 1:
|
|
844
854
|
raise malformed()
|
|
845
855
|
# Single-key form: {name: {kind: ..., ...}}
|
|
846
856
|
name = keys[0]
|
|
847
|
-
attrs =
|
|
857
|
+
attrs = req[name] or {}
|
|
848
858
|
if not isinstance(attrs, dict):
|
|
849
859
|
raise malformed()
|
|
850
860
|
out = dict(attrs)
|
|
@@ -1118,24 +1128,26 @@ def check_requirements(
|
|
|
1118
1128
|
machine it names.
|
|
1119
1129
|
|
|
1120
1130
|
``setup`` requirements are verified via
|
|
1121
|
-
:func:`calkit.
|
|
1131
|
+
:func:`calkit.requirements.check_setup_dep`, which runs the
|
|
1122
1132
|
``check_command`` and -- on an interactive TTY -- optionally runs the
|
|
1123
1133
|
declared ``setup_command`` after asking the user. Non-interactive
|
|
1124
1134
|
failures abort with the fix-it command printed for the user to run.
|
|
1135
|
+
``aws-credentials`` requirements are checked the same way, by
|
|
1136
|
+
:func:`calkit.requirements.check_aws_credentials_req`.
|
|
1125
1137
|
"""
|
|
1126
1138
|
if requirements is not None:
|
|
1127
1139
|
# An environment's own list is checked exactly as declared: it
|
|
1128
1140
|
# describes a machine that runs stages, which is a smaller claim
|
|
1129
1141
|
# than being where the project lives.
|
|
1130
|
-
|
|
1142
|
+
reqs = list(requirements)
|
|
1131
1143
|
else:
|
|
1132
1144
|
if ck_info is None:
|
|
1133
1145
|
ck_info = load_calkit_info(wdir=wdir)
|
|
1134
|
-
|
|
1146
|
+
reqs = get_requirements(ck_info)
|
|
1135
1147
|
# Git is how a project exists at all, so the host needs it whether
|
|
1136
1148
|
# or not anyone wrote it down.
|
|
1137
|
-
if "git" not in
|
|
1138
|
-
|
|
1149
|
+
if "git" not in reqs:
|
|
1150
|
+
reqs.append("git")
|
|
1139
1151
|
# Resolve TTY interactivity once so a single per-call answer drives
|
|
1140
1152
|
# every prompt (env-var, app installer, setup step).
|
|
1141
1153
|
if interactive is None:
|
|
@@ -1145,7 +1157,8 @@ def check_requirements(
|
|
|
1145
1157
|
# before we start installing things on it, then env-vars (some
|
|
1146
1158
|
# installers and setup commands read from them), then apps (env
|
|
1147
1159
|
# managers like pixi / uv need to exist before setup steps that run
|
|
1148
|
-
# inside an env), then setup steps
|
|
1160
|
+
# inside an env), then setup steps, then AWS credentials, which need
|
|
1161
|
+
# the AWS CLI and may be set up by a step. The setup-step
|
|
1149
1162
|
# ``check_command`` typically wraps ``calkit xenv``, which validates
|
|
1150
1163
|
# its own environment, so we don't need a separate env-check phase.
|
|
1151
1164
|
property_kinds = system_property_requirement_kinds()
|
|
@@ -1154,19 +1167,20 @@ def check_requirements(
|
|
|
1154
1167
|
"env-var": [],
|
|
1155
1168
|
"app": [],
|
|
1156
1169
|
"setup": [],
|
|
1170
|
+
"aws-credentials": [],
|
|
1157
1171
|
}
|
|
1158
|
-
for
|
|
1159
|
-
|
|
1160
|
-
kind =
|
|
1172
|
+
for raw in reqs:
|
|
1173
|
+
req = _normalize_requirement(raw)
|
|
1174
|
+
kind = req["kind"]
|
|
1161
1175
|
if kind in property_kinds:
|
|
1162
|
-
buckets["_property"].append(
|
|
1176
|
+
buckets["_property"].append(req)
|
|
1163
1177
|
elif kind not in buckets:
|
|
1164
1178
|
# Unknown / legacy kinds (e.g., ``calkit-config``) fall through
|
|
1165
1179
|
# to ``check_dep_exists`` in original order so we don't change
|
|
1166
1180
|
# behavior for them silently.
|
|
1167
|
-
buckets.setdefault("_other", []).append(
|
|
1181
|
+
buckets.setdefault("_other", []).append(req)
|
|
1168
1182
|
else:
|
|
1169
|
-
buckets[kind].append(
|
|
1183
|
+
buckets[kind].append(req)
|
|
1170
1184
|
if buckets["_property"]:
|
|
1171
1185
|
# Read the machine only when something asks about it; describing a
|
|
1172
1186
|
# system shells out for a version from every package manager it can
|
|
@@ -1177,43 +1191,45 @@ def check_requirements(
|
|
|
1177
1191
|
if system_info is not None
|
|
1178
1192
|
else get_machine_properties()
|
|
1179
1193
|
)
|
|
1180
|
-
for
|
|
1181
|
-
check_property_requirement(
|
|
1182
|
-
for
|
|
1183
|
-
|
|
1184
|
-
if
|
|
1194
|
+
for req in buckets["_property"]:
|
|
1195
|
+
check_property_requirement(req, info, described_as=described_as)
|
|
1196
|
+
for req in buckets["env-var"]:
|
|
1197
|
+
req_name = req["name"]
|
|
1198
|
+
if req_name in os.environ:
|
|
1185
1199
|
continue
|
|
1186
1200
|
# On a TTY, prompt the user and persist to .env so the very next
|
|
1187
1201
|
# ``calkit run`` works without a separate setup step. Non-TTY
|
|
1188
1202
|
# (CI) falls through to the legacy "not found" abort.
|
|
1189
1203
|
if interactive:
|
|
1190
|
-
from calkit.
|
|
1204
|
+
from calkit.requirements import prompt_for_env_var_req
|
|
1191
1205
|
|
|
1192
|
-
|
|
1193
|
-
value = prompt_and_store_env_var(
|
|
1194
|
-
dep_name, default=dep.get("default")
|
|
1195
|
-
)
|
|
1196
|
-
if value is not None:
|
|
1206
|
+
if prompt_for_env_var_req(req) is not None:
|
|
1197
1207
|
continue
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1208
|
+
msg = f"env-var '{req_name}' not found"
|
|
1209
|
+
if req.get("setup_command"):
|
|
1210
|
+
msg += (
|
|
1211
|
+
f"; get a value with '{req['setup_command']}' and set it "
|
|
1212
|
+
f"with 'calkit set-env-var {req_name} <value>'"
|
|
1213
|
+
)
|
|
1214
|
+
raise ValueError(msg)
|
|
1215
|
+
for req in buckets["app"]:
|
|
1216
|
+
req_name = req["name"]
|
|
1201
1217
|
# The ``calkit`` app is always satisfied by the running process,
|
|
1202
1218
|
# but a declared ``version_spec`` (e.g. ``calkit>=0.38``) is
|
|
1203
1219
|
# checked against the installed version so projects can pin a
|
|
1204
1220
|
# minimum CLI without writing a custom setup step.
|
|
1205
|
-
if
|
|
1206
|
-
spec =
|
|
1221
|
+
if req_name == "calkit":
|
|
1222
|
+
spec = req.get("version_spec")
|
|
1207
1223
|
if spec:
|
|
1208
|
-
from calkit.
|
|
1224
|
+
from calkit.requirements import check_calkit_version
|
|
1209
1225
|
|
|
1210
1226
|
check_calkit_version(spec)
|
|
1211
1227
|
continue
|
|
1212
|
-
spec =
|
|
1213
|
-
if check_dep_exists(
|
|
1228
|
+
spec = req.get("version_spec")
|
|
1229
|
+
if check_dep_exists(req_name, "app", system_info=system_info):
|
|
1214
1230
|
if spec:
|
|
1215
1231
|
check_app_version(
|
|
1216
|
-
|
|
1232
|
+
req_name,
|
|
1217
1233
|
spec,
|
|
1218
1234
|
system_info=system_info,
|
|
1219
1235
|
described_as=described_as,
|
|
@@ -1222,14 +1238,14 @@ def check_requirements(
|
|
|
1222
1238
|
# Offer the registered native installer when we have one.
|
|
1223
1239
|
from calkit import install as _install
|
|
1224
1240
|
|
|
1225
|
-
if _install.get_installer(
|
|
1226
|
-
print(f"App '{
|
|
1241
|
+
if _install.get_installer(req_name) is not None:
|
|
1242
|
+
print(f"App '{req_name}' is not installed.")
|
|
1227
1243
|
if _install.prompt_and_install(
|
|
1228
|
-
|
|
1229
|
-
) and check_dep_exists(
|
|
1244
|
+
req_name, interactive=interactive
|
|
1245
|
+
) and check_dep_exists(req_name, "app", system_info=system_info):
|
|
1230
1246
|
if spec:
|
|
1231
1247
|
check_app_version(
|
|
1232
|
-
|
|
1248
|
+
req_name,
|
|
1233
1249
|
spec,
|
|
1234
1250
|
system_info=system_info,
|
|
1235
1251
|
described_as=described_as,
|
|
@@ -1237,28 +1253,41 @@ def check_requirements(
|
|
|
1237
1253
|
continue
|
|
1238
1254
|
# Some apps have no installer here but a known way in, e.g., a
|
|
1239
1255
|
# system package manager on Linux
|
|
1240
|
-
hint = _install.get_unsupported_message(
|
|
1256
|
+
hint = _install.get_unsupported_message(req_name)
|
|
1241
1257
|
raise ValueError(
|
|
1242
|
-
f"app '{
|
|
1258
|
+
f"app '{req_name}' not found on {described_as}"
|
|
1243
1259
|
+ (f"; {hint}" if hint else "")
|
|
1244
1260
|
)
|
|
1245
|
-
for
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
if not check_dep_exists(
|
|
1249
|
-
raise ValueError(f"{
|
|
1250
|
-
for
|
|
1251
|
-
from calkit.
|
|
1261
|
+
for req in buckets.get("_other", []):
|
|
1262
|
+
req_name = req["name"]
|
|
1263
|
+
req_kind = req["kind"]
|
|
1264
|
+
if not check_dep_exists(req_name, req_kind, system_info=system_info):
|
|
1265
|
+
raise ValueError(f"{req_kind} '{req_name}' not found")
|
|
1266
|
+
for req in buckets["setup"]:
|
|
1267
|
+
from calkit.requirements import check_setup_dep
|
|
1252
1268
|
|
|
1253
1269
|
ok = check_setup_dep(
|
|
1254
|
-
|
|
1270
|
+
req,
|
|
1271
|
+
interactive=interactive,
|
|
1272
|
+
use_cache=use_cache,
|
|
1273
|
+
wdir=wdir,
|
|
1274
|
+
)
|
|
1275
|
+
if not ok:
|
|
1276
|
+
raise ValueError(
|
|
1277
|
+
f"setup requirement '{req['name']}' is not satisfied"
|
|
1278
|
+
)
|
|
1279
|
+
for req in buckets["aws-credentials"]:
|
|
1280
|
+
from calkit.requirements import check_aws_credentials_req
|
|
1281
|
+
|
|
1282
|
+
ok = check_aws_credentials_req(
|
|
1283
|
+
req,
|
|
1255
1284
|
interactive=interactive,
|
|
1256
1285
|
use_cache=use_cache,
|
|
1257
1286
|
wdir=wdir,
|
|
1258
1287
|
)
|
|
1259
1288
|
if not ok:
|
|
1260
1289
|
raise ValueError(
|
|
1261
|
-
f"
|
|
1290
|
+
f"aws-credentials requirement '{req['name']}' is not satisfied"
|
|
1262
1291
|
)
|
|
1263
1292
|
|
|
1264
1293
|
|
|
@@ -1272,12 +1301,12 @@ def get_env_var_dep_names(ck_info: dict | None = None) -> list[str]:
|
|
|
1272
1301
|
if ck_info is None:
|
|
1273
1302
|
ck_info = load_calkit_info()
|
|
1274
1303
|
env_vars = []
|
|
1275
|
-
for
|
|
1304
|
+
for req in get_requirements(ck_info):
|
|
1276
1305
|
# Delegate shape-parsing to ``_normalize_requirement`` so this stays
|
|
1277
1306
|
# in lockstep with ``check_requirements`` -- string entries,
|
|
1278
1307
|
# single-key dicts, flat dicts, and nameless setup steps all flow
|
|
1279
1308
|
# through one path.
|
|
1280
|
-
normalized = _normalize_requirement(
|
|
1309
|
+
normalized = _normalize_requirement(req)
|
|
1281
1310
|
if normalized["kind"] == "env-var":
|
|
1282
1311
|
env_vars.append(normalized["name"])
|
|
1283
1312
|
return env_vars
|
calkit/dvc/core.py
CHANGED
|
@@ -949,7 +949,7 @@ def split_lock_stage_name(stage_name: str) -> tuple[str | None, str]:
|
|
|
949
949
|
return None, stage_name
|
|
950
950
|
|
|
951
951
|
|
|
952
|
-
def
|
|
952
|
+
def stage_lock_hash(entry: dict) -> str:
|
|
953
953
|
"""A checksum of what a stage's ``dvc.lock`` entry says it ran and made.
|
|
954
954
|
|
|
955
955
|
Built from its command, params, and each dep's and out's path and hash,
|
|
@@ -974,7 +974,7 @@ def stage_lock_md5(entry: dict) -> str:
|
|
|
974
974
|
"outs": files("outs"),
|
|
975
975
|
}
|
|
976
976
|
content = json.dumps(data, sort_keys=True, default=str).encode()
|
|
977
|
-
return "
|
|
977
|
+
return "sha256:" + hashlib.sha256(content).hexdigest()
|
|
978
978
|
|
|
979
979
|
|
|
980
980
|
def configure_autostage(wdir: str | None = None) -> int:
|
calkit/environments.py
CHANGED
|
@@ -955,6 +955,7 @@ def resolve_switch(env_name: str, envs: dict) -> str | None:
|
|
|
955
955
|
AppExistsCondition,
|
|
956
956
|
EnvVarEqualsCondition,
|
|
957
957
|
EnvVarExistsCondition,
|
|
958
|
+
EnvVarSetCondition,
|
|
958
959
|
HostnameMatchesCondition,
|
|
959
960
|
MachineIdEqualsCondition,
|
|
960
961
|
OsIsCondition,
|
|
@@ -969,6 +970,8 @@ def resolve_switch(env_name: str, envs: dict) -> str | None:
|
|
|
969
970
|
shutil.which(a) is not None
|
|
970
971
|
for a in any_of(condition.app_exists)
|
|
971
972
|
)
|
|
973
|
+
if isinstance(condition, EnvVarSetCondition):
|
|
974
|
+
return any(n in os.environ for n in any_of(condition.env_var_set))
|
|
972
975
|
if isinstance(condition, EnvVarExistsCondition):
|
|
973
976
|
return any(
|
|
974
977
|
n in os.environ for n in any_of(condition.env_var_exists)
|
calkit/models/core.py
CHANGED
|
@@ -28,6 +28,7 @@ from calkit.models.io import EnvVarInput
|
|
|
28
28
|
from calkit.models.iteration import ParametersType
|
|
29
29
|
from calkit.models.pipeline import Pipeline, RelativeChildPathString
|
|
30
30
|
from calkit.models.requirements import ( # noqa: F401
|
|
31
|
+
AwsCredentialsRequirement,
|
|
31
32
|
Dependency,
|
|
32
33
|
DependencyAttrs,
|
|
33
34
|
Requirement,
|
|
@@ -1289,10 +1290,20 @@ class AppExistsCondition(Model):
|
|
|
1289
1290
|
)
|
|
1290
1291
|
|
|
1291
1292
|
|
|
1293
|
+
class EnvVarSetCondition(Model):
|
|
1294
|
+
model_config = ConfigDict(extra="forbid")
|
|
1295
|
+
env_var_set: str | list[str] = Field(
|
|
1296
|
+
description="Environmental variable that must be set."
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
|
|
1292
1300
|
class EnvVarExistsCondition(Model):
|
|
1301
|
+
"""The old name for ``env_var_set``, still honored."""
|
|
1302
|
+
|
|
1293
1303
|
model_config = ConfigDict(extra="forbid")
|
|
1294
1304
|
env_var_exists: str | list[str] = Field(
|
|
1295
|
-
description="
|
|
1305
|
+
description="Deprecated alias for 'env_var_set'.",
|
|
1306
|
+
json_schema_extra={"deprecated": True},
|
|
1296
1307
|
)
|
|
1297
1308
|
|
|
1298
1309
|
|
|
@@ -1329,7 +1340,7 @@ class MachineIdEqualsCondition(Model):
|
|
|
1329
1340
|
|
|
1330
1341
|
_SWITCH_CONDITIONS: list[type[Model]] = [
|
|
1331
1342
|
AppExistsCondition,
|
|
1332
|
-
|
|
1343
|
+
EnvVarSetCondition,
|
|
1333
1344
|
EnvVarEqualsCondition,
|
|
1334
1345
|
HostnameMatchesCondition,
|
|
1335
1346
|
OsIsCondition,
|
|
@@ -1348,6 +1359,7 @@ def _switch_condition_name(value: object) -> str | None:
|
|
|
1348
1359
|
|
|
1349
1360
|
SwitchCondition = Annotated[
|
|
1350
1361
|
Annotated[AppExistsCondition, Tag("app_exists")]
|
|
1362
|
+
| Annotated[EnvVarSetCondition, Tag("env_var_set")]
|
|
1351
1363
|
| Annotated[EnvVarExistsCondition, Tag("env_var_exists")]
|
|
1352
1364
|
| Annotated[EnvVarEqualsCondition, Tag("env_var_equals")]
|
|
1353
1365
|
| Annotated[HostnameMatchesCondition, Tag("hostname_matches")]
|
|
@@ -1408,7 +1420,7 @@ class SwitchEnvironment(Environment):
|
|
|
1408
1420
|
pair:
|
|
1409
1421
|
|
|
1410
1422
|
- ``app_exists``: an app on ``PATH``.
|
|
1411
|
-
- ``
|
|
1423
|
+
- ``env_var_set``: an environmental variable that is set.
|
|
1412
1424
|
- ``env_var_equals``: ``NAME=value`` for an environmental variable.
|
|
1413
1425
|
- ``hostname_matches``: a glob the hostname matches, ignoring case.
|
|
1414
1426
|
- ``os_is``: ``linux``, ``macos`` or ``windows``.
|