calkit-python 0.48.3__py3-none-any.whl → 0.49.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/__init__.py +65 -1
- calkit/cli/check.py +333 -30
- calkit/cli/delete.py +25 -0
- calkit/cli/describe.py +9 -1
- calkit/cli/latex.py +90 -144
- calkit/cli/list.py +52 -0
- calkit/cli/main/core.py +589 -126
- calkit/cli/main/xr.py +55 -29
- calkit/cli/new.py +67 -0
- calkit/cli/overleaf.py +139 -109
- calkit/cli/scheduler.py +217 -2
- calkit/cli/update.py +161 -6
- calkit/conda.py +455 -41
- calkit/conditions.py +251 -0
- calkit/config.py +2 -25
- calkit/core.py +226 -54
- calkit/dependencies.py +6 -16
- calkit/docker.py +112 -0
- calkit/dvc/core.py +224 -4
- calkit/environments.py +1223 -72
- calkit/latex.py +191 -24
- calkit/models/core.py +346 -220
- calkit/models/io.py +26 -2
- calkit/models/pipeline.py +202 -23
- calkit/models/requirements.py +216 -0
- calkit/overleaf.py +89 -8
- calkit/pipeline.py +499 -15
- calkit/questions.py +30 -160
- calkit/tests/cli/main/test_core.py +548 -7
- calkit/tests/cli/main/test_xr.py +8 -0
- calkit/tests/cli/test_check.py +326 -0
- calkit/tests/cli/test_delete.py +53 -0
- calkit/tests/cli/test_describe.py +17 -0
- calkit/tests/cli/test_latex.py +80 -0
- calkit/tests/cli/test_list.py +61 -0
- calkit/tests/cli/test_new.py +60 -0
- calkit/tests/cli/test_overleaf.py +60 -1
- calkit/tests/cli/test_scheduler.py +146 -0
- calkit/tests/cli/test_update.py +162 -0
- calkit/tests/models/test_core.py +136 -1
- calkit/tests/test_conda.py +325 -0
- calkit/tests/test_conditions.py +114 -0
- calkit/tests/test_core.py +125 -3
- calkit/tests/test_docker.py +81 -0
- calkit/tests/test_environments.py +830 -1
- calkit/tests/test_overleaf.py +76 -0
- calkit/tests/test_pipeline.py +456 -3
- calkit/tests/test_questions.py +79 -127
- calkit/tests/test_workspace.py +40 -2
- calkit/upgrade.py +1 -1
- calkit/workspace.py +88 -26
- {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/METADATA +4 -3
- {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/RECORD +69 -66
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/WHEEL +0 -0
- {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/licenses/LICENSE +0 -0
calkit/cli/__init__.py
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
1
5
|
from .core import * # noqa: F403
|
|
2
6
|
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
import typer
|
|
9
|
+
|
|
3
10
|
|
|
4
11
|
def run() -> None:
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
5
15
|
# Intercept ``--use-version`` before Typer/Click parses argv: Click
|
|
6
16
|
# processes eager options like ``--help`` and ``--version`` (and the
|
|
7
17
|
# group's ``no_args_is_help``) ahead of the callback body, so an
|
|
@@ -11,7 +21,61 @@ def run() -> None:
|
|
|
11
21
|
_maybe_exec_with_version()
|
|
12
22
|
from .main import app
|
|
13
23
|
|
|
14
|
-
|
|
24
|
+
# Completion lists commands, so needs them all
|
|
25
|
+
if any(k.startswith("_") and k.endswith("_COMPLETE") for k in os.environ):
|
|
26
|
+
app()
|
|
27
|
+
else:
|
|
28
|
+
_only_invoked(app, sys.argv[1:])()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _only_invoked(
|
|
32
|
+
app: typer.Typer, args: list[str], top: bool = True
|
|
33
|
+
) -> typer.Typer:
|
|
34
|
+
"""Narrow ``app`` to the command or group ``args`` invoke.
|
|
35
|
+
|
|
36
|
+
Typer builds every command's options before running one, which for all
|
|
37
|
+
of Calkit's takes a few tenths of a second, so this leaves out the ones
|
|
38
|
+
that won't run. Anything else, e.g., ``--help`` or a misspelled command,
|
|
39
|
+
keeps them all, so help and suggestions list everything.
|
|
40
|
+
decisions/calkit.yaml has a question on how much that saves.
|
|
41
|
+
"""
|
|
42
|
+
import copy
|
|
43
|
+
import re
|
|
44
|
+
|
|
45
|
+
from typer.main import get_command_name, solve_typer_info_defaults
|
|
46
|
+
|
|
47
|
+
from .main.core import APP_OPTIONS_WITH_VALUES
|
|
48
|
+
|
|
49
|
+
# Find the command name, skipping the top level's options
|
|
50
|
+
takes_value = {
|
|
51
|
+
flag for flags in APP_OPTIONS_WITH_VALUES.values() for flag in flags
|
|
52
|
+
}
|
|
53
|
+
i = 0
|
|
54
|
+
while i < len(args) and args[i].startswith("-") and top:
|
|
55
|
+
i += 2 if args[i] in takes_value else 1
|
|
56
|
+
if i >= len(args) or args[i].startswith("-"):
|
|
57
|
+
return app
|
|
58
|
+
name, rest = args[i], args[i + 1 :]
|
|
59
|
+
|
|
60
|
+
def names(spec: str | None) -> list[str]:
|
|
61
|
+
return re.split(r" ?[,|] ?", spec) if spec else []
|
|
62
|
+
|
|
63
|
+
narrowed = copy.copy(app)
|
|
64
|
+
for cmd in app.registered_commands:
|
|
65
|
+
default = cmd.callback and get_command_name(cmd.callback.__name__)
|
|
66
|
+
if name in names(cmd.name or default):
|
|
67
|
+
narrowed.registered_commands = [cmd]
|
|
68
|
+
narrowed.registered_groups = []
|
|
69
|
+
return narrowed
|
|
70
|
+
for group in app.registered_groups:
|
|
71
|
+
sub_app = group.typer_instance
|
|
72
|
+
if sub_app and name in names(solve_typer_info_defaults(group).name):
|
|
73
|
+
group = copy.copy(group)
|
|
74
|
+
group.typer_instance = _only_invoked(sub_app, rest, top=False)
|
|
75
|
+
narrowed.registered_commands = []
|
|
76
|
+
narrowed.registered_groups = [group]
|
|
77
|
+
return narrowed
|
|
78
|
+
return app
|
|
15
79
|
|
|
16
80
|
|
|
17
81
|
def _maybe_exec_with_version() -> None:
|
calkit/cli/check.py
CHANGED
|
@@ -6,6 +6,7 @@ import functools
|
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
8
|
import platform as _platform
|
|
9
|
+
import re
|
|
9
10
|
import shutil
|
|
10
11
|
import subprocess
|
|
11
12
|
import textwrap
|
|
@@ -18,7 +19,15 @@ import calkit
|
|
|
18
19
|
from calkit.cli import AliasGroup, raise_error, warn
|
|
19
20
|
from calkit.core import get_md5
|
|
20
21
|
|
|
21
|
-
|
|
22
|
+
|
|
23
|
+
def stage_calkit_files(*args: object, **kwargs: object) -> None:
|
|
24
|
+
"""Stage what a check wrote, which stages depend on."""
|
|
25
|
+
calkit.environments.stage_calkit_files()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
check_app = typer.Typer(
|
|
29
|
+
cls=AliasGroup, no_args_is_help=True, result_callback=stage_calkit_files
|
|
30
|
+
)
|
|
22
31
|
|
|
23
32
|
|
|
24
33
|
def _juliaup_version_installed(julia_version: str) -> bool:
|
|
@@ -62,6 +71,7 @@ def _check_julia_env(
|
|
|
62
71
|
julia_version: str | None = None,
|
|
63
72
|
verbose: bool = False,
|
|
64
73
|
cache_key: str | None = None,
|
|
74
|
+
force: bool = False,
|
|
65
75
|
) -> str:
|
|
66
76
|
"""Check a Julia environment and instantiate only when needed."""
|
|
67
77
|
abs_env_path = os.path.abspath(env_path)
|
|
@@ -80,7 +90,7 @@ def _check_julia_env(
|
|
|
80
90
|
cache_env_name = cache_key or (
|
|
81
91
|
f"julia::{abs_env_path}::{julia_version or ''}"
|
|
82
92
|
)
|
|
83
|
-
if calkit.environments.check_cache(
|
|
93
|
+
if not force and calkit.environments.check_cache(
|
|
84
94
|
env_name=cache_env_name,
|
|
85
95
|
env=env,
|
|
86
96
|
wdir=env_dir,
|
|
@@ -471,8 +481,20 @@ def check_environment(
|
|
|
471
481
|
verbose: Annotated[
|
|
472
482
|
bool, typer.Option("--verbose", help="Print verbose output.")
|
|
473
483
|
] = False,
|
|
484
|
+
force: Annotated[
|
|
485
|
+
bool,
|
|
486
|
+
typer.Option(
|
|
487
|
+
"--force",
|
|
488
|
+
"-f",
|
|
489
|
+
help="Rebuild the environment from scratch, e.g., after "
|
|
490
|
+
"something it was built from changed outside the project.",
|
|
491
|
+
),
|
|
492
|
+
] = False,
|
|
474
493
|
) -> str | None:
|
|
475
|
-
"""Check that an environment is up-to-date.
|
|
494
|
+
"""Check that an environment is up-to-date.
|
|
495
|
+
|
|
496
|
+
An environment is also rebuilt when one of its 'inputs' changes.
|
|
497
|
+
"""
|
|
476
498
|
from calkit.environments import (
|
|
477
499
|
get_all_conda_lock_fpaths,
|
|
478
500
|
get_all_docker_lock_fpaths,
|
|
@@ -480,7 +502,6 @@ def check_environment(
|
|
|
480
502
|
get_default_venv_prefix,
|
|
481
503
|
get_env_lock_fpath,
|
|
482
504
|
write_scheduler_env_lock,
|
|
483
|
-
write_system_env_lock,
|
|
484
505
|
)
|
|
485
506
|
|
|
486
507
|
dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
|
|
@@ -491,9 +512,28 @@ def check_environment(
|
|
|
491
512
|
if isinstance(envs, list):
|
|
492
513
|
raise_error("Error: Environments should be a dict, not a list")
|
|
493
514
|
assert isinstance(envs, dict)
|
|
494
|
-
|
|
515
|
+
found = calkit.environments.get_env(env_name, envs)
|
|
516
|
+
if found is None:
|
|
495
517
|
raise_error(f"Environment '{env_name}' does not exist")
|
|
496
|
-
env =
|
|
518
|
+
env = found
|
|
519
|
+
# Docker records its inputs in its lock, which rebuilds it already
|
|
520
|
+
try:
|
|
521
|
+
rebuild = force or (
|
|
522
|
+
env["kind"] != "docker"
|
|
523
|
+
and calkit.environments.inputs_changed_since_build(env_name, env)
|
|
524
|
+
)
|
|
525
|
+
except ValueError as e:
|
|
526
|
+
raise_error(f"Environment '{env_name}': {e}")
|
|
527
|
+
if rebuild and env["kind"] not in (
|
|
528
|
+
"docker",
|
|
529
|
+
calkit.environments.SWITCH_KIND,
|
|
530
|
+
):
|
|
531
|
+
try:
|
|
532
|
+
removed = calkit.environments.remove_built_env(env_name, envs)
|
|
533
|
+
except (ValueError, OSError, subprocess.CalledProcessError) as e:
|
|
534
|
+
raise_error(f"Failed to remove environment '{env_name}': {e}")
|
|
535
|
+
if removed and verbose:
|
|
536
|
+
typer.echo(f"Removed {removed} to rebuild it")
|
|
497
537
|
if env["kind"] == "docker":
|
|
498
538
|
lock_fpath = get_env_lock_fpath(
|
|
499
539
|
env=env, env_name=env_name, as_posix=False
|
|
@@ -535,6 +575,7 @@ def check_environment(
|
|
|
535
575
|
registry=env.get("registry"),
|
|
536
576
|
lock_archs=calkit.docker.get_lock_archs(env),
|
|
537
577
|
quiet=not verbose,
|
|
578
|
+
rebuild=rebuild,
|
|
538
579
|
)
|
|
539
580
|
elif env["kind"] == "conda":
|
|
540
581
|
lock_fpath = get_env_lock_fpath(
|
|
@@ -615,6 +656,7 @@ def check_environment(
|
|
|
615
656
|
output_fpath=get_env_lock_fpath(
|
|
616
657
|
env=env, env_name=env_name, as_posix=False
|
|
617
658
|
), # type: ignore
|
|
659
|
+
rebuild=rebuild,
|
|
618
660
|
)
|
|
619
661
|
elif env["kind"] == "julia":
|
|
620
662
|
env_path = env.get("path")
|
|
@@ -630,6 +672,7 @@ def check_environment(
|
|
|
630
672
|
julia_version=julia_version,
|
|
631
673
|
verbose=verbose,
|
|
632
674
|
cache_key=env_name,
|
|
675
|
+
force=rebuild,
|
|
633
676
|
)
|
|
634
677
|
elif env["kind"] in ("slurm", "pbs"):
|
|
635
678
|
# Job-scheduler envs have no external manifest to validate; the
|
|
@@ -637,6 +680,81 @@ def check_environment(
|
|
|
637
680
|
# env config so DVC stages that depend on the env get invalidated
|
|
638
681
|
# when the config changes.
|
|
639
682
|
write_scheduler_env_lock(env_name=env_name, env=env)
|
|
683
|
+
elif env["kind"] == calkit.environments.SWITCH_KIND:
|
|
684
|
+
# Checking a switch means checking what it picks here. Until a stage
|
|
685
|
+
# has run with it, i.e., none depends on its lock, options on other
|
|
686
|
+
# machines are locked too, where they can be reached, and every
|
|
687
|
+
# option's lock is part of the switch's.
|
|
688
|
+
switch_lock = os.path.join(
|
|
689
|
+
calkit.environments.get_env_lock_dir(), env_name, "switch.json"
|
|
690
|
+
)
|
|
691
|
+
try:
|
|
692
|
+
with open("dvc.lock") as f:
|
|
693
|
+
ran = switch_lock.replace(os.sep, "/") in f.read()
|
|
694
|
+
except OSError:
|
|
695
|
+
ran = False
|
|
696
|
+
try:
|
|
697
|
+
options = calkit.environments.get_switch_options(env_name, envs)
|
|
698
|
+
except ValueError as e:
|
|
699
|
+
raise_error(str(e))
|
|
700
|
+
for option in [] if ran else options:
|
|
701
|
+
option_env = calkit.environments.get_env(option, envs) or {}
|
|
702
|
+
option_lock = get_env_lock_fpath(
|
|
703
|
+
env=option_env, env_name=option, as_posix=False
|
|
704
|
+
)
|
|
705
|
+
if (
|
|
706
|
+
option_env.get("kind") != "system"
|
|
707
|
+
or not option_env.get("lock")
|
|
708
|
+
or not option_env.get("host")
|
|
709
|
+
or calkit.environments.env_is_local(option_env)
|
|
710
|
+
or (option_lock is not None and os.path.isfile(option_lock))
|
|
711
|
+
):
|
|
712
|
+
continue
|
|
713
|
+
try:
|
|
714
|
+
check_environment(env_name=option, verbose=verbose)
|
|
715
|
+
except typer.Exit:
|
|
716
|
+
warn(
|
|
717
|
+
f"Option '{option}' of '{env_name}' couldn't be locked "
|
|
718
|
+
"from here, so it will be on its first check there"
|
|
719
|
+
)
|
|
720
|
+
try:
|
|
721
|
+
calkit.environments.write_switch_env_lock(
|
|
722
|
+
env_name, envs, relocked=None if ran else options
|
|
723
|
+
)
|
|
724
|
+
picked = calkit.environments.resolve_switch(env_name, envs)
|
|
725
|
+
except ValueError as e:
|
|
726
|
+
raise_error(str(e))
|
|
727
|
+
if picked is None:
|
|
728
|
+
warn(calkit.environments.describe_switch_no_match(env_name, env))
|
|
729
|
+
return None
|
|
730
|
+
if verbose:
|
|
731
|
+
typer.echo(f"Environment '{env_name}' picks '{picked}' here")
|
|
732
|
+
if env.get("lock"):
|
|
733
|
+
machine_env = calkit.environments.switch_machine_lock_env(env)
|
|
734
|
+
try:
|
|
735
|
+
mismatch = calkit.environments.gate_system_env_lock(
|
|
736
|
+
env_name,
|
|
737
|
+
machine_env,
|
|
738
|
+
system_info=calkit.environments.picked_machine_info(
|
|
739
|
+
picked,
|
|
740
|
+
(calkit.environments.get_env(picked, envs) or {}),
|
|
741
|
+
ck_info,
|
|
742
|
+
lock=env.get("lock"),
|
|
743
|
+
),
|
|
744
|
+
)
|
|
745
|
+
except ValueError as e:
|
|
746
|
+
raise_error(f"Environment '{env_name}': {e}")
|
|
747
|
+
if mismatch:
|
|
748
|
+
warn(
|
|
749
|
+
calkit.environments.describe_system_env_lock_mismatch(
|
|
750
|
+
env_name, mismatch
|
|
751
|
+
)
|
|
752
|
+
)
|
|
753
|
+
lock_fpath = check_environment(
|
|
754
|
+
env_name=picked, verbose=verbose, force=rebuild
|
|
755
|
+
)
|
|
756
|
+
calkit.environments.record_build_inputs(env_name, env)
|
|
757
|
+
return lock_fpath
|
|
640
758
|
elif env["kind"] == "system":
|
|
641
759
|
# Nothing is installed or built for a system env; checking it means
|
|
642
760
|
# making sure the machine is as the project requires, then reading
|
|
@@ -647,7 +765,17 @@ def check_environment(
|
|
|
647
765
|
calkit.check_requirements(
|
|
648
766
|
requirements=env.get("requirements", [])
|
|
649
767
|
)
|
|
650
|
-
|
|
768
|
+
# A mismatch warns, and stages that need to run are
|
|
769
|
+
# skipped, or fail at the gate in 'xenv'
|
|
770
|
+
mismatch = calkit.environments.gate_system_env_lock(
|
|
771
|
+
env_name, env
|
|
772
|
+
)
|
|
773
|
+
if mismatch:
|
|
774
|
+
warn(
|
|
775
|
+
calkit.environments.describe_system_env_lock_mismatch(
|
|
776
|
+
env_name, mismatch
|
|
777
|
+
)
|
|
778
|
+
)
|
|
651
779
|
except ValueError as e:
|
|
652
780
|
# A requirement that isn't met, or a property that can't be
|
|
653
781
|
# locked -- a misspelled one, or a tool that isn't
|
|
@@ -662,9 +790,9 @@ def check_environment(
|
|
|
662
790
|
# from that machine, since those are the ones the results
|
|
663
791
|
# depend on.
|
|
664
792
|
import calkit.workspace as workspace
|
|
665
|
-
from calkit.
|
|
793
|
+
from calkit.core import is_tty
|
|
666
794
|
|
|
667
|
-
interactive =
|
|
795
|
+
interactive = is_tty()
|
|
668
796
|
ck_info = calkit.load_calkit_info()
|
|
669
797
|
try:
|
|
670
798
|
# Anything written as ${CK_SSH_HOST} is the first thing to
|
|
@@ -713,7 +841,12 @@ def check_environment(
|
|
|
713
841
|
)
|
|
714
842
|
system_info = None
|
|
715
843
|
if needs_system_info:
|
|
716
|
-
system_info = workspace.remote_system_info(
|
|
844
|
+
system_info = workspace.remote_system_info(
|
|
845
|
+
ws,
|
|
846
|
+
apps=calkit.environments.lock_app_names(
|
|
847
|
+
env.get("lock")
|
|
848
|
+
),
|
|
849
|
+
)
|
|
717
850
|
# Before anything else is decided from it, so a
|
|
718
851
|
# mismatched machine is reported rather than measured
|
|
719
852
|
# and recorded as what results depend on
|
|
@@ -727,19 +860,22 @@ def check_environment(
|
|
|
727
860
|
system_info=system_info,
|
|
728
861
|
verbose=verbose,
|
|
729
862
|
)
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
write_system_env_lock(
|
|
733
|
-
env_name=env_name,
|
|
734
|
-
env=env,
|
|
735
|
-
system_info=system_info,
|
|
863
|
+
mismatch = calkit.environments.gate_system_env_lock(
|
|
864
|
+
env_name, env, system_info=system_info
|
|
736
865
|
)
|
|
866
|
+
if mismatch:
|
|
867
|
+
warn(
|
|
868
|
+
calkit.environments.describe_system_env_lock_mismatch(
|
|
869
|
+
env_name, mismatch
|
|
870
|
+
)
|
|
871
|
+
)
|
|
737
872
|
except ValueError as e:
|
|
738
873
|
raise_error(f"Environment '{env_name}': {e}")
|
|
739
874
|
elif env["kind"] == "nix":
|
|
740
875
|
check_nix_env(env=env, verbose=verbose)
|
|
741
876
|
else:
|
|
742
877
|
raise_error(f"Environment kind '{env['kind']}' not supported")
|
|
878
|
+
calkit.environments.record_build_inputs(env_name, env)
|
|
743
879
|
return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)
|
|
744
880
|
|
|
745
881
|
|
|
@@ -1116,15 +1252,28 @@ def check_docker_env(
|
|
|
1116
1252
|
quiet: Annotated[
|
|
1117
1253
|
bool, typer.Option("--quiet", "-q", help="Be quiet.")
|
|
1118
1254
|
] = False,
|
|
1255
|
+
rebuild: Annotated[
|
|
1256
|
+
bool,
|
|
1257
|
+
typer.Option(
|
|
1258
|
+
"--rebuild",
|
|
1259
|
+
help="Build the image from scratch, pulling its base image, "
|
|
1260
|
+
"rather than reusing a cached or locked one.",
|
|
1261
|
+
),
|
|
1262
|
+
] = False,
|
|
1119
1263
|
) -> None:
|
|
1120
1264
|
"""Check that Docker environment is up-to-date."""
|
|
1121
1265
|
from calkit import docker as ck_docker
|
|
1266
|
+
from calkit.core import is_tty
|
|
1122
1267
|
from calkit.environments import get_docker_arch
|
|
1123
1268
|
|
|
1124
1269
|
if fpath is None and lock_fpath is None:
|
|
1125
1270
|
raise_error(
|
|
1126
1271
|
"Lock file output path must be provided if input Dockerfile is not"
|
|
1127
1272
|
)
|
|
1273
|
+
try:
|
|
1274
|
+
ck_docker.ensure_ready(interactive=is_tty())
|
|
1275
|
+
except ck_docker.DockerNotReadyError as e:
|
|
1276
|
+
raise_error(str(e))
|
|
1128
1277
|
outfile = open(os.devnull, "w") if quiet else None
|
|
1129
1278
|
if fpath is not None:
|
|
1130
1279
|
typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
|
|
@@ -1201,6 +1350,10 @@ def check_docker_env(
|
|
|
1201
1350
|
)
|
|
1202
1351
|
lock = None
|
|
1203
1352
|
lock_is_current_arch = False
|
|
1353
|
+
# Without a Dockerfile, rebuilding means pulling the image again
|
|
1354
|
+
if rebuild:
|
|
1355
|
+
lock = None
|
|
1356
|
+
lock_is_current_arch = False
|
|
1204
1357
|
# Work out where this image lives in a registry, so it can be pulled
|
|
1205
1358
|
# instead of rebuilt, and pushed after being built. Only an image built
|
|
1206
1359
|
# from a Dockerfile goes to the project's registry; one named directly
|
|
@@ -1350,10 +1503,14 @@ def check_docker_env(
|
|
|
1350
1503
|
"--push",
|
|
1351
1504
|
"-f",
|
|
1352
1505
|
dockerfile_name,
|
|
1353
|
-
".",
|
|
1354
1506
|
]
|
|
1507
|
+
if rebuild:
|
|
1508
|
+
cmd += ["--no-cache", "--pull"]
|
|
1509
|
+
cmd.append(".")
|
|
1355
1510
|
else:
|
|
1356
1511
|
cmd = ["docker", "build", "-t", tag, "-f", dockerfile_name]
|
|
1512
|
+
if rebuild:
|
|
1513
|
+
cmd += ["--no-cache", "--pull"]
|
|
1357
1514
|
build_platform = (
|
|
1358
1515
|
build_platforms[0] if build_platforms else platform
|
|
1359
1516
|
)
|
|
@@ -1792,21 +1949,26 @@ def check_venv(
|
|
|
1792
1949
|
if lock_fpath is None:
|
|
1793
1950
|
fname, ext = os.path.splitext(path)
|
|
1794
1951
|
lock_fpath = fname + "-lock" + ext
|
|
1795
|
-
|
|
1952
|
+
|
|
1953
|
+
def full(fpath: str) -> str:
|
|
1954
|
+
return os.path.join(wdir or "", fpath)
|
|
1955
|
+
|
|
1956
|
+
lock_fpath_full = full(lock_fpath)
|
|
1957
|
+
lock_dir = os.path.dirname(lock_fpath_full)
|
|
1796
1958
|
if lock_dir:
|
|
1797
1959
|
os.makedirs(lock_dir, exist_ok=True)
|
|
1798
1960
|
# Use main lock file if exists, else try alternatives (including legacy)
|
|
1799
1961
|
reqs_to_use = lock_fpath
|
|
1800
1962
|
used_legacy_lock = None
|
|
1801
|
-
if not os.path.isfile(
|
|
1963
|
+
if not os.path.isfile(lock_fpath_full):
|
|
1802
1964
|
for alt_fpath in alt_lock_fpaths:
|
|
1803
|
-
if os.path.isfile(alt_fpath):
|
|
1965
|
+
if os.path.isfile(full(alt_fpath)):
|
|
1804
1966
|
reqs_to_use = alt_fpath
|
|
1805
1967
|
if verbose:
|
|
1806
1968
|
typer.echo(f"Using alternative lock file: {alt_fpath}")
|
|
1807
1969
|
break
|
|
1808
1970
|
for legacy_fpath in alt_lock_fpaths_delete:
|
|
1809
|
-
if os.path.isfile(legacy_fpath):
|
|
1971
|
+
if os.path.isfile(full(legacy_fpath)):
|
|
1810
1972
|
reqs_to_use = legacy_fpath
|
|
1811
1973
|
used_legacy_lock = legacy_fpath
|
|
1812
1974
|
if verbose:
|
|
@@ -1814,20 +1976,89 @@ def check_venv(
|
|
|
1814
1976
|
break
|
|
1815
1977
|
activate_cmd = calkit.environments.get_venv_activate_cmd(prefix)
|
|
1816
1978
|
|
|
1979
|
+
def venv_python_version() -> str | None:
|
|
1980
|
+
"""The Python version a plain venv was created with."""
|
|
1981
|
+
if _platform.system() == "Windows":
|
|
1982
|
+
exe = os.path.join(prefix_full_path, "Scripts", "python.exe")
|
|
1983
|
+
else:
|
|
1984
|
+
exe = os.path.join(prefix_full_path, "bin", "python")
|
|
1985
|
+
try:
|
|
1986
|
+
return subprocess.check_output(
|
|
1987
|
+
[
|
|
1988
|
+
exe,
|
|
1989
|
+
"-c",
|
|
1990
|
+
"import platform; print(platform.python_version())",
|
|
1991
|
+
],
|
|
1992
|
+
text=True,
|
|
1993
|
+
).strip()
|
|
1994
|
+
except (OSError, subprocess.CalledProcessError):
|
|
1995
|
+
return None
|
|
1996
|
+
|
|
1997
|
+
def is_local(line: str) -> bool:
|
|
1998
|
+
"""Whether a frozen requirement names a path on this machine."""
|
|
1999
|
+
line = line.strip()
|
|
2000
|
+
if line.startswith(("-e ", "--editable")):
|
|
2001
|
+
return re.match(r"^(-e|--editable)[\s=]+\w+\+", line) is None
|
|
2002
|
+
return re.search(r"@\s*file:", line) is not None
|
|
2003
|
+
|
|
2004
|
+
recorded_hash = calkit.environments.read_env_spec_hash(lock_fpath, wdir)
|
|
2005
|
+
spec_hash = calkit.environments.env_spec_hash(full(path), python=python)
|
|
2006
|
+
# A lock resolved from the current spec is an input, not rewritten here
|
|
2007
|
+
lock_is_current = (
|
|
2008
|
+
os.path.isfile(lock_fpath_full) and recorded_hash == spec_hash
|
|
2009
|
+
)
|
|
2010
|
+
|
|
1817
2011
|
def pip_install_and_freeze(reqs_arg: str) -> None:
|
|
1818
2012
|
check_cmd = (
|
|
1819
|
-
f"{activate_cmd} "
|
|
1820
|
-
f"
|
|
1821
|
-
f"&& {pip_freeze_cmd} > {lock_fpath} "
|
|
1822
|
-
"&& deactivate"
|
|
2013
|
+
f"{activate_cmd} && {pip_cmd} install {pip_install_args} "
|
|
2014
|
+
f"{reqs_arg}"
|
|
1823
2015
|
)
|
|
2016
|
+
if not lock_is_current:
|
|
2017
|
+
check_cmd += f" && {pip_freeze_cmd} > {lock_fpath}"
|
|
2018
|
+
check_cmd += " && deactivate"
|
|
1824
2019
|
if verbose:
|
|
1825
2020
|
typer.echo(f"Running command: {check_cmd}")
|
|
2021
|
+
try:
|
|
2022
|
+
with open(lock_fpath_full, encoding="utf-8") as f:
|
|
2023
|
+
lock_before = f.read()
|
|
2024
|
+
except OSError:
|
|
2025
|
+
lock_before = None
|
|
1826
2026
|
subprocess.run(check_cmd, shell=True, cwd=wdir, check=True)
|
|
2027
|
+
if lock_is_current:
|
|
2028
|
+
return
|
|
2029
|
+
# Local packages are installed from the spec, since their paths
|
|
2030
|
+
# differ between machines
|
|
2031
|
+
with open(lock_fpath_full, encoding="utf-8") as f:
|
|
2032
|
+
frozen = f.read()
|
|
2033
|
+
lock = "".join(
|
|
2034
|
+
ln for ln in frozen.splitlines(keepends=True) if not is_local(ln)
|
|
2035
|
+
)
|
|
2036
|
+
if lock != frozen:
|
|
2037
|
+
with open(lock_fpath_full, "w", encoding="utf-8") as f:
|
|
2038
|
+
f.write(lock)
|
|
2039
|
+
lock_changed = lock != lock_before
|
|
2040
|
+
# All platforms are relocked together, so they agree; an existing
|
|
2041
|
+
# lock that matches is adopted as is, and one with no record is
|
|
2042
|
+
# only added to
|
|
2043
|
+
spec_changed = recorded_hash != spec_hash
|
|
2044
|
+
adopted = recorded_hash is None and not lock_changed
|
|
2045
|
+
if (lock_changed or spec_changed) and not adopted:
|
|
2046
|
+
calkit.environments.write_cross_platform_venv_locks(
|
|
2047
|
+
spec_fpath=path,
|
|
2048
|
+
lock_fpath=lock_fpath,
|
|
2049
|
+
python_version=python or venv_python_version(),
|
|
2050
|
+
wdir=wdir,
|
|
2051
|
+
verbose=verbose,
|
|
2052
|
+
relock=spec_changed and recorded_hash is not None,
|
|
2053
|
+
)
|
|
2054
|
+
# Last, so an interrupted relock isn't taken as current
|
|
2055
|
+
calkit.environments.write_env_spec_hash(
|
|
2056
|
+
lock_fpath, path, python=python, wdir=wdir
|
|
2057
|
+
)
|
|
1827
2058
|
# Delete legacy lock file after use
|
|
1828
2059
|
if used_legacy_lock:
|
|
1829
2060
|
try:
|
|
1830
|
-
os.remove(used_legacy_lock)
|
|
2061
|
+
os.remove(full(used_legacy_lock))
|
|
1831
2062
|
if verbose:
|
|
1832
2063
|
typer.echo(
|
|
1833
2064
|
"Deleted legacy lock file after use: "
|
|
@@ -1840,9 +2071,19 @@ def check_venv(
|
|
|
1840
2071
|
f"{used_legacy_lock}: {e}"
|
|
1841
2072
|
)
|
|
1842
2073
|
|
|
1843
|
-
#
|
|
2074
|
+
# A lock resolved from an older spec would bring back what the spec no
|
|
2075
|
+
# longer has, as would the env built from it, so both start over
|
|
1844
2076
|
dep_file_txt = f"-r {path}"
|
|
1845
|
-
if
|
|
2077
|
+
if recorded_hash is not None and recorded_hash != spec_hash:
|
|
2078
|
+
if verbose:
|
|
2079
|
+
typer.echo(f"Rebuilding {kind} at {prefix} since its spec changed")
|
|
2080
|
+
try:
|
|
2081
|
+
shutil.rmtree(prefix_full_path)
|
|
2082
|
+
create_venv()
|
|
2083
|
+
except OSError as e:
|
|
2084
|
+
warn(f"Failed to remove {kind} at {prefix}: {e}")
|
|
2085
|
+
elif os.path.isfile(full(reqs_to_use)):
|
|
2086
|
+
# If the lock file exists, try to install with that
|
|
1846
2087
|
dep_file_txt += f" -r {reqs_to_use}"
|
|
1847
2088
|
try:
|
|
1848
2089
|
pip_install_and_freeze(dep_file_txt)
|
|
@@ -1861,6 +2102,14 @@ def check_venv(
|
|
|
1861
2102
|
create_venv()
|
|
1862
2103
|
pip_install_and_freeze(dep_file_txt)
|
|
1863
2104
|
except (subprocess.CalledProcessError, OSError):
|
|
2105
|
+
if lock_is_current:
|
|
2106
|
+
raise_error(
|
|
2107
|
+
f"Failed to create {kind} at {prefix} from its lock file "
|
|
2108
|
+
f"({lock_fpath}), which matches the spec ({path}). "
|
|
2109
|
+
"Rebuilding from the spec would leave the environment "
|
|
2110
|
+
"out of sync with its lock; delete the lock file to "
|
|
2111
|
+
"re-resolve it, which reruns stages that depend on it"
|
|
2112
|
+
)
|
|
1864
2113
|
warn(
|
|
1865
2114
|
f"Failed to create environment from lock file ({reqs_to_use}); "
|
|
1866
2115
|
f"attempting rebuild from input file {path}"
|
|
@@ -1879,15 +2128,20 @@ def check_matlab_env(
|
|
|
1879
2128
|
typer.Option("--name", "-n", help="Environment name in calkit.yaml."),
|
|
1880
2129
|
],
|
|
1881
2130
|
output_fpath: Annotated[str, typer.Option("--output", "-o")],
|
|
2131
|
+
rebuild: Annotated[
|
|
2132
|
+
bool,
|
|
2133
|
+
typer.Option("--rebuild", help="Rebuild the image from scratch."),
|
|
2134
|
+
] = False,
|
|
1882
2135
|
) -> None:
|
|
1883
2136
|
"""Check a MATLAB environment matches its spec and export a JSON lock
|
|
1884
2137
|
file.
|
|
1885
2138
|
"""
|
|
1886
2139
|
ck_info = calkit.load_calkit_info()
|
|
1887
2140
|
environments = ck_info.get("environments", {})
|
|
1888
|
-
|
|
2141
|
+
found = calkit.environments.get_env(env_name, environments)
|
|
2142
|
+
if found is None:
|
|
1889
2143
|
raise_error(f"Environment '{env_name}' not found in calkit.yaml")
|
|
1890
|
-
env =
|
|
2144
|
+
env = found
|
|
1891
2145
|
if env.get("kind") != "matlab":
|
|
1892
2146
|
raise_error(f"Environment '{env_name}' is not a MATLAB environment")
|
|
1893
2147
|
if "version" not in env:
|
|
@@ -1913,6 +2167,7 @@ def check_matlab_env(
|
|
|
1913
2167
|
fpath=dockerfile_fpath,
|
|
1914
2168
|
lock_fpath=output_fpath,
|
|
1915
2169
|
platform="linux/amd64", # Only one available for now
|
|
2170
|
+
rebuild=rebuild,
|
|
1916
2171
|
)
|
|
1917
2172
|
|
|
1918
2173
|
|
|
@@ -1953,6 +2208,54 @@ def check_project_requirements(
|
|
|
1953
2208
|
calkit.echo(message)
|
|
1954
2209
|
|
|
1955
2210
|
|
|
2211
|
+
@check_app.command(name="stage-runs")
|
|
2212
|
+
def check_stage_runs() -> None:
|
|
2213
|
+
"""Check each stage in dvc.lock matches the recorded run that made it.
|
|
2214
|
+
|
|
2215
|
+
A stage's entry in .calkit/stage-runs.json records a checksum of its
|
|
2216
|
+
dvc.lock entry after it ran, so one that disagrees got its dvc.lock
|
|
2217
|
+
entry some other way, e.g., 'dvc commit' after an output was edited, an
|
|
2218
|
+
edit or merge of dvc.lock, or running it outside Calkit. This catches
|
|
2219
|
+
mistakes and shortcuts, not forgery, since a matching entry can be
|
|
2220
|
+
written by anyone.
|
|
2221
|
+
"""
|
|
2222
|
+
try:
|
|
2223
|
+
with open(calkit.STAGE_RUNS_FPATH) as f:
|
|
2224
|
+
stage_runs = json.load(f)
|
|
2225
|
+
except OSError:
|
|
2226
|
+
stage_runs = {}
|
|
2227
|
+
except json.JSONDecodeError as e:
|
|
2228
|
+
raise_error(f"Can't read {calkit.STAGE_RUNS_FPATH}: {e}")
|
|
2229
|
+
# By directory, for a subproject's stages
|
|
2230
|
+
lock_stages = {None: calkit.dvc.read_lock_stages()}
|
|
2231
|
+
problems = []
|
|
2232
|
+
for name in sorted(set(lock_stages[None]) | set(stage_runs)):
|
|
2233
|
+
lock_dir, lock_name = calkit.dvc.split_lock_stage_name(name)
|
|
2234
|
+
if lock_dir not in lock_stages:
|
|
2235
|
+
lock_stages[lock_dir] = calkit.dvc.read_lock_stages(lock_dir)
|
|
2236
|
+
locked = lock_stages[lock_dir].get(lock_name)
|
|
2237
|
+
# One no longer in dvc.lock has nothing to vouch for
|
|
2238
|
+
if not isinstance(locked, dict):
|
|
2239
|
+
continue
|
|
2240
|
+
recorded = stage_runs.get(name)
|
|
2241
|
+
if not isinstance(recorded, dict):
|
|
2242
|
+
problems.append(f"Stage '{name}' has no recorded run")
|
|
2243
|
+
elif recorded.get("dvc_lock") != calkit.dvc.stage_lock_md5(locked):
|
|
2244
|
+
problems.append(
|
|
2245
|
+
f"Stage '{name}' has changed in dvc.lock since its last "
|
|
2246
|
+
"recorded run"
|
|
2247
|
+
)
|
|
2248
|
+
for problem in problems:
|
|
2249
|
+
typer.echo(problem)
|
|
2250
|
+
if problems:
|
|
2251
|
+
raise_error(
|
|
2252
|
+
f"{len(problems)} stage(s) in dvc.lock don't match a recorded "
|
|
2253
|
+
f"run in {calkit.STAGE_RUNS_FPATH}; rerunning them with "
|
|
2254
|
+
"'calkit run --force <stage>' records them"
|
|
2255
|
+
)
|
|
2256
|
+
typer.echo("Every stage in dvc.lock matches its recorded run")
|
|
2257
|
+
|
|
2258
|
+
|
|
1956
2259
|
@check_app.command(name="env-vars")
|
|
1957
2260
|
def check_env_vars(
|
|
1958
2261
|
verbose: Annotated[
|
calkit/cli/delete.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import typer
|
|
6
|
+
from typing_extensions import Annotated
|
|
6
7
|
|
|
7
8
|
import calkit
|
|
8
9
|
from calkit.cli import AliasGroup, raise_error
|
|
@@ -31,3 +32,27 @@ def delete_question(
|
|
|
31
32
|
with open("calkit.yaml", "w") as f:
|
|
32
33
|
ryaml.dump(ck_info, f)
|
|
33
34
|
typer.echo(f"Removed question: {removed}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@delete_app.command(name="comment")
|
|
38
|
+
def delete_comment(
|
|
39
|
+
path: Annotated[
|
|
40
|
+
str | None,
|
|
41
|
+
typer.Argument(help="File the thread is in, if not found by --id."),
|
|
42
|
+
] = None,
|
|
43
|
+
id: Annotated[
|
|
44
|
+
str | None,
|
|
45
|
+
typer.Option(
|
|
46
|
+
"--id",
|
|
47
|
+
help="ID of the thread, which finds it in any of the project's documents.",
|
|
48
|
+
),
|
|
49
|
+
] = None,
|
|
50
|
+
line: Annotated[
|
|
51
|
+
int | None,
|
|
52
|
+
typer.Option("--line", "-l", help="Number of any line in the thread."),
|
|
53
|
+
] = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Delete a review comment thread and its replies."""
|
|
56
|
+
from calkit.cli.latex import edit_comment
|
|
57
|
+
|
|
58
|
+
edit_comment(path, id, line, None)
|