calkit-python 0.48.3__py3-none-any.whl → 0.49.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. calkit/cli/__init__.py +65 -1
  2. calkit/cli/check.py +333 -30
  3. calkit/cli/delete.py +25 -0
  4. calkit/cli/describe.py +9 -1
  5. calkit/cli/latex.py +90 -144
  6. calkit/cli/list.py +52 -0
  7. calkit/cli/main/core.py +589 -126
  8. calkit/cli/main/xr.py +55 -29
  9. calkit/cli/new.py +67 -0
  10. calkit/cli/overleaf.py +139 -109
  11. calkit/cli/scheduler.py +217 -2
  12. calkit/cli/update.py +161 -6
  13. calkit/conda.py +455 -41
  14. calkit/conditions.py +251 -0
  15. calkit/config.py +2 -25
  16. calkit/core.py +226 -54
  17. calkit/dependencies.py +6 -16
  18. calkit/docker.py +112 -0
  19. calkit/dvc/core.py +224 -4
  20. calkit/environments.py +1223 -72
  21. calkit/latex.py +191 -24
  22. calkit/models/core.py +346 -220
  23. calkit/models/io.py +26 -2
  24. calkit/models/pipeline.py +202 -23
  25. calkit/models/requirements.py +216 -0
  26. calkit/overleaf.py +89 -8
  27. calkit/pipeline.py +499 -15
  28. calkit/questions.py +30 -160
  29. calkit/tests/cli/main/test_core.py +548 -7
  30. calkit/tests/cli/main/test_xr.py +8 -0
  31. calkit/tests/cli/test_check.py +326 -0
  32. calkit/tests/cli/test_delete.py +53 -0
  33. calkit/tests/cli/test_describe.py +17 -0
  34. calkit/tests/cli/test_latex.py +80 -0
  35. calkit/tests/cli/test_list.py +61 -0
  36. calkit/tests/cli/test_new.py +60 -0
  37. calkit/tests/cli/test_overleaf.py +60 -1
  38. calkit/tests/cli/test_scheduler.py +146 -0
  39. calkit/tests/cli/test_update.py +162 -0
  40. calkit/tests/models/test_core.py +136 -1
  41. calkit/tests/test_conda.py +325 -0
  42. calkit/tests/test_conditions.py +114 -0
  43. calkit/tests/test_core.py +125 -3
  44. calkit/tests/test_docker.py +81 -0
  45. calkit/tests/test_environments.py +830 -1
  46. calkit/tests/test_overleaf.py +76 -0
  47. calkit/tests/test_pipeline.py +456 -3
  48. calkit/tests/test_questions.py +79 -127
  49. calkit/tests/test_workspace.py +40 -2
  50. calkit/upgrade.py +1 -1
  51. calkit/workspace.py +88 -26
  52. {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/METADATA +4 -3
  53. {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/RECORD +69 -66
  54. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
  55. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
  56. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
  57. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
  58. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +0 -0
  59. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
  60. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +0 -0
  61. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +0 -0
  62. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +0 -0
  63. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +0 -0
  64. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +0 -0
  65. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
  66. {calkit_python-0.48.3.data → calkit_python-0.49.0.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
  67. {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/WHEEL +0 -0
  68. {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/entry_points.txt +0 -0
  69. {calkit_python-0.48.3.dist-info → calkit_python-0.49.0.dist-info}/licenses/LICENSE +0 -0
calkit/cli/__init__.py CHANGED
@@ -1,7 +1,17 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING
4
+
1
5
  from .core import * # noqa: F403
2
6
 
7
+ if TYPE_CHECKING:
8
+ import typer
9
+
3
10
 
4
11
  def run() -> None:
12
+ import os
13
+ import sys
14
+
5
15
  # Intercept ``--use-version`` before Typer/Click parses argv: Click
6
16
  # processes eager options like ``--help`` and ``--version`` (and the
7
17
  # group's ``no_args_is_help``) ahead of the callback body, so an
@@ -11,7 +21,61 @@ def run() -> None:
11
21
  _maybe_exec_with_version()
12
22
  from .main import app
13
23
 
14
- app()
24
+ # Completion lists commands, so needs them all
25
+ if any(k.startswith("_") and k.endswith("_COMPLETE") for k in os.environ):
26
+ app()
27
+ else:
28
+ _only_invoked(app, sys.argv[1:])()
29
+
30
+
31
+ def _only_invoked(
32
+ app: typer.Typer, args: list[str], top: bool = True
33
+ ) -> typer.Typer:
34
+ """Narrow ``app`` to the command or group ``args`` invoke.
35
+
36
+ Typer builds every command's options before running one, which for all
37
+ of Calkit's takes a few tenths of a second, so this leaves out the ones
38
+ that won't run. Anything else, e.g., ``--help`` or a misspelled command,
39
+ keeps them all, so help and suggestions list everything.
40
+ decisions/calkit.yaml has a question on how much that saves.
41
+ """
42
+ import copy
43
+ import re
44
+
45
+ from typer.main import get_command_name, solve_typer_info_defaults
46
+
47
+ from .main.core import APP_OPTIONS_WITH_VALUES
48
+
49
+ # Find the command name, skipping the top level's options
50
+ takes_value = {
51
+ flag for flags in APP_OPTIONS_WITH_VALUES.values() for flag in flags
52
+ }
53
+ i = 0
54
+ while i < len(args) and args[i].startswith("-") and top:
55
+ i += 2 if args[i] in takes_value else 1
56
+ if i >= len(args) or args[i].startswith("-"):
57
+ return app
58
+ name, rest = args[i], args[i + 1 :]
59
+
60
+ def names(spec: str | None) -> list[str]:
61
+ return re.split(r" ?[,|] ?", spec) if spec else []
62
+
63
+ narrowed = copy.copy(app)
64
+ for cmd in app.registered_commands:
65
+ default = cmd.callback and get_command_name(cmd.callback.__name__)
66
+ if name in names(cmd.name or default):
67
+ narrowed.registered_commands = [cmd]
68
+ narrowed.registered_groups = []
69
+ return narrowed
70
+ for group in app.registered_groups:
71
+ sub_app = group.typer_instance
72
+ if sub_app and name in names(solve_typer_info_defaults(group).name):
73
+ group = copy.copy(group)
74
+ group.typer_instance = _only_invoked(sub_app, rest, top=False)
75
+ narrowed.registered_commands = []
76
+ narrowed.registered_groups = [group]
77
+ return narrowed
78
+ return app
15
79
 
16
80
 
17
81
  def _maybe_exec_with_version() -> None:
calkit/cli/check.py CHANGED
@@ -6,6 +6,7 @@ import functools
6
6
  import json
7
7
  import os
8
8
  import platform as _platform
9
+ import re
9
10
  import shutil
10
11
  import subprocess
11
12
  import textwrap
@@ -18,7 +19,15 @@ import calkit
18
19
  from calkit.cli import AliasGroup, raise_error, warn
19
20
  from calkit.core import get_md5
20
21
 
21
- check_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
22
+
23
+ def stage_calkit_files(*args: object, **kwargs: object) -> None:
24
+ """Stage what a check wrote, which stages depend on."""
25
+ calkit.environments.stage_calkit_files()
26
+
27
+
28
+ check_app = typer.Typer(
29
+ cls=AliasGroup, no_args_is_help=True, result_callback=stage_calkit_files
30
+ )
22
31
 
23
32
 
24
33
  def _juliaup_version_installed(julia_version: str) -> bool:
@@ -62,6 +71,7 @@ def _check_julia_env(
62
71
  julia_version: str | None = None,
63
72
  verbose: bool = False,
64
73
  cache_key: str | None = None,
74
+ force: bool = False,
65
75
  ) -> str:
66
76
  """Check a Julia environment and instantiate only when needed."""
67
77
  abs_env_path = os.path.abspath(env_path)
@@ -80,7 +90,7 @@ def _check_julia_env(
80
90
  cache_env_name = cache_key or (
81
91
  f"julia::{abs_env_path}::{julia_version or ''}"
82
92
  )
83
- if calkit.environments.check_cache(
93
+ if not force and calkit.environments.check_cache(
84
94
  env_name=cache_env_name,
85
95
  env=env,
86
96
  wdir=env_dir,
@@ -471,8 +481,20 @@ def check_environment(
471
481
  verbose: Annotated[
472
482
  bool, typer.Option("--verbose", help="Print verbose output.")
473
483
  ] = False,
484
+ force: Annotated[
485
+ bool,
486
+ typer.Option(
487
+ "--force",
488
+ "-f",
489
+ help="Rebuild the environment from scratch, e.g., after "
490
+ "something it was built from changed outside the project.",
491
+ ),
492
+ ] = False,
474
493
  ) -> str | None:
475
- """Check that an environment is up-to-date."""
494
+ """Check that an environment is up-to-date.
495
+
496
+ An environment is also rebuilt when one of its 'inputs' changes.
497
+ """
476
498
  from calkit.environments import (
477
499
  get_all_conda_lock_fpaths,
478
500
  get_all_docker_lock_fpaths,
@@ -480,7 +502,6 @@ def check_environment(
480
502
  get_default_venv_prefix,
481
503
  get_env_lock_fpath,
482
504
  write_scheduler_env_lock,
483
- write_system_env_lock,
484
505
  )
485
506
 
486
507
  dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
@@ -491,9 +512,28 @@ def check_environment(
491
512
  if isinstance(envs, list):
492
513
  raise_error("Error: Environments should be a dict, not a list")
493
514
  assert isinstance(envs, dict)
494
- if env_name not in envs:
515
+ found = calkit.environments.get_env(env_name, envs)
516
+ if found is None:
495
517
  raise_error(f"Environment '{env_name}' does not exist")
496
- env = envs[env_name]
518
+ env = found
519
+ # Docker records its inputs in its lock, which rebuilds it already
520
+ try:
521
+ rebuild = force or (
522
+ env["kind"] != "docker"
523
+ and calkit.environments.inputs_changed_since_build(env_name, env)
524
+ )
525
+ except ValueError as e:
526
+ raise_error(f"Environment '{env_name}': {e}")
527
+ if rebuild and env["kind"] not in (
528
+ "docker",
529
+ calkit.environments.SWITCH_KIND,
530
+ ):
531
+ try:
532
+ removed = calkit.environments.remove_built_env(env_name, envs)
533
+ except (ValueError, OSError, subprocess.CalledProcessError) as e:
534
+ raise_error(f"Failed to remove environment '{env_name}': {e}")
535
+ if removed and verbose:
536
+ typer.echo(f"Removed {removed} to rebuild it")
497
537
  if env["kind"] == "docker":
498
538
  lock_fpath = get_env_lock_fpath(
499
539
  env=env, env_name=env_name, as_posix=False
@@ -535,6 +575,7 @@ def check_environment(
535
575
  registry=env.get("registry"),
536
576
  lock_archs=calkit.docker.get_lock_archs(env),
537
577
  quiet=not verbose,
578
+ rebuild=rebuild,
538
579
  )
539
580
  elif env["kind"] == "conda":
540
581
  lock_fpath = get_env_lock_fpath(
@@ -615,6 +656,7 @@ def check_environment(
615
656
  output_fpath=get_env_lock_fpath(
616
657
  env=env, env_name=env_name, as_posix=False
617
658
  ), # type: ignore
659
+ rebuild=rebuild,
618
660
  )
619
661
  elif env["kind"] == "julia":
620
662
  env_path = env.get("path")
@@ -630,6 +672,7 @@ def check_environment(
630
672
  julia_version=julia_version,
631
673
  verbose=verbose,
632
674
  cache_key=env_name,
675
+ force=rebuild,
633
676
  )
634
677
  elif env["kind"] in ("slurm", "pbs"):
635
678
  # Job-scheduler envs have no external manifest to validate; the
@@ -637,6 +680,81 @@ def check_environment(
637
680
  # env config so DVC stages that depend on the env get invalidated
638
681
  # when the config changes.
639
682
  write_scheduler_env_lock(env_name=env_name, env=env)
683
+ elif env["kind"] == calkit.environments.SWITCH_KIND:
684
+ # Checking a switch means checking what it picks here. Until a stage
685
+ # has run with it, i.e., none depends on its lock, options on other
686
+ # machines are locked too, where they can be reached, and every
687
+ # option's lock is part of the switch's.
688
+ switch_lock = os.path.join(
689
+ calkit.environments.get_env_lock_dir(), env_name, "switch.json"
690
+ )
691
+ try:
692
+ with open("dvc.lock") as f:
693
+ ran = switch_lock.replace(os.sep, "/") in f.read()
694
+ except OSError:
695
+ ran = False
696
+ try:
697
+ options = calkit.environments.get_switch_options(env_name, envs)
698
+ except ValueError as e:
699
+ raise_error(str(e))
700
+ for option in [] if ran else options:
701
+ option_env = calkit.environments.get_env(option, envs) or {}
702
+ option_lock = get_env_lock_fpath(
703
+ env=option_env, env_name=option, as_posix=False
704
+ )
705
+ if (
706
+ option_env.get("kind") != "system"
707
+ or not option_env.get("lock")
708
+ or not option_env.get("host")
709
+ or calkit.environments.env_is_local(option_env)
710
+ or (option_lock is not None and os.path.isfile(option_lock))
711
+ ):
712
+ continue
713
+ try:
714
+ check_environment(env_name=option, verbose=verbose)
715
+ except typer.Exit:
716
+ warn(
717
+ f"Option '{option}' of '{env_name}' couldn't be locked "
718
+ "from here, so it will be on its first check there"
719
+ )
720
+ try:
721
+ calkit.environments.write_switch_env_lock(
722
+ env_name, envs, relocked=None if ran else options
723
+ )
724
+ picked = calkit.environments.resolve_switch(env_name, envs)
725
+ except ValueError as e:
726
+ raise_error(str(e))
727
+ if picked is None:
728
+ warn(calkit.environments.describe_switch_no_match(env_name, env))
729
+ return None
730
+ if verbose:
731
+ typer.echo(f"Environment '{env_name}' picks '{picked}' here")
732
+ if env.get("lock"):
733
+ machine_env = calkit.environments.switch_machine_lock_env(env)
734
+ try:
735
+ mismatch = calkit.environments.gate_system_env_lock(
736
+ env_name,
737
+ machine_env,
738
+ system_info=calkit.environments.picked_machine_info(
739
+ picked,
740
+ (calkit.environments.get_env(picked, envs) or {}),
741
+ ck_info,
742
+ lock=env.get("lock"),
743
+ ),
744
+ )
745
+ except ValueError as e:
746
+ raise_error(f"Environment '{env_name}': {e}")
747
+ if mismatch:
748
+ warn(
749
+ calkit.environments.describe_system_env_lock_mismatch(
750
+ env_name, mismatch
751
+ )
752
+ )
753
+ lock_fpath = check_environment(
754
+ env_name=picked, verbose=verbose, force=rebuild
755
+ )
756
+ calkit.environments.record_build_inputs(env_name, env)
757
+ return lock_fpath
640
758
  elif env["kind"] == "system":
641
759
  # Nothing is installed or built for a system env; checking it means
642
760
  # making sure the machine is as the project requires, then reading
@@ -647,7 +765,17 @@ def check_environment(
647
765
  calkit.check_requirements(
648
766
  requirements=env.get("requirements", [])
649
767
  )
650
- write_system_env_lock(env_name=env_name, env=env)
768
+ # A mismatch warns, and stages that need to run are
769
+ # skipped, or fail at the gate in 'xenv'
770
+ mismatch = calkit.environments.gate_system_env_lock(
771
+ env_name, env
772
+ )
773
+ if mismatch:
774
+ warn(
775
+ calkit.environments.describe_system_env_lock_mismatch(
776
+ env_name, mismatch
777
+ )
778
+ )
651
779
  except ValueError as e:
652
780
  # A requirement that isn't met, or a property that can't be
653
781
  # locked -- a misspelled one, or a tool that isn't
@@ -662,9 +790,9 @@ def check_environment(
662
790
  # from that machine, since those are the ones the results
663
791
  # depend on.
664
792
  import calkit.workspace as workspace
665
- from calkit.dependencies import _is_interactive
793
+ from calkit.core import is_tty
666
794
 
667
- interactive = _is_interactive()
795
+ interactive = is_tty()
668
796
  ck_info = calkit.load_calkit_info()
669
797
  try:
670
798
  # Anything written as ${CK_SSH_HOST} is the first thing to
@@ -713,7 +841,12 @@ def check_environment(
713
841
  )
714
842
  system_info = None
715
843
  if needs_system_info:
716
- system_info = workspace.remote_system_info(ws)
844
+ system_info = workspace.remote_system_info(
845
+ ws,
846
+ apps=calkit.environments.lock_app_names(
847
+ env.get("lock")
848
+ ),
849
+ )
717
850
  # Before anything else is decided from it, so a
718
851
  # mismatched machine is reported rather than measured
719
852
  # and recorded as what results depend on
@@ -727,19 +860,22 @@ def check_environment(
727
860
  system_info=system_info,
728
861
  verbose=verbose,
729
862
  )
730
- # Returns None when the env locks nothing, so nothing is
731
- # written for an env with nothing to record
732
- write_system_env_lock(
733
- env_name=env_name,
734
- env=env,
735
- system_info=system_info,
863
+ mismatch = calkit.environments.gate_system_env_lock(
864
+ env_name, env, system_info=system_info
736
865
  )
866
+ if mismatch:
867
+ warn(
868
+ calkit.environments.describe_system_env_lock_mismatch(
869
+ env_name, mismatch
870
+ )
871
+ )
737
872
  except ValueError as e:
738
873
  raise_error(f"Environment '{env_name}': {e}")
739
874
  elif env["kind"] == "nix":
740
875
  check_nix_env(env=env, verbose=verbose)
741
876
  else:
742
877
  raise_error(f"Environment kind '{env['kind']}' not supported")
878
+ calkit.environments.record_build_inputs(env_name, env)
743
879
  return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)
744
880
 
745
881
 
@@ -1116,15 +1252,28 @@ def check_docker_env(
1116
1252
  quiet: Annotated[
1117
1253
  bool, typer.Option("--quiet", "-q", help="Be quiet.")
1118
1254
  ] = False,
1255
+ rebuild: Annotated[
1256
+ bool,
1257
+ typer.Option(
1258
+ "--rebuild",
1259
+ help="Build the image from scratch, pulling its base image, "
1260
+ "rather than reusing a cached or locked one.",
1261
+ ),
1262
+ ] = False,
1119
1263
  ) -> None:
1120
1264
  """Check that Docker environment is up-to-date."""
1121
1265
  from calkit import docker as ck_docker
1266
+ from calkit.core import is_tty
1122
1267
  from calkit.environments import get_docker_arch
1123
1268
 
1124
1269
  if fpath is None and lock_fpath is None:
1125
1270
  raise_error(
1126
1271
  "Lock file output path must be provided if input Dockerfile is not"
1127
1272
  )
1273
+ try:
1274
+ ck_docker.ensure_ready(interactive=is_tty())
1275
+ except ck_docker.DockerNotReadyError as e:
1276
+ raise_error(str(e))
1128
1277
  outfile = open(os.devnull, "w") if quiet else None
1129
1278
  if fpath is not None:
1130
1279
  typer.echo(f"Reading Dockerfile from {fpath}", file=outfile)
@@ -1201,6 +1350,10 @@ def check_docker_env(
1201
1350
  )
1202
1351
  lock = None
1203
1352
  lock_is_current_arch = False
1353
+ # Without a Dockerfile, rebuilding means pulling the image again
1354
+ if rebuild:
1355
+ lock = None
1356
+ lock_is_current_arch = False
1204
1357
  # Work out where this image lives in a registry, so it can be pulled
1205
1358
  # instead of rebuilt, and pushed after being built. Only an image built
1206
1359
  # from a Dockerfile goes to the project's registry; one named directly
@@ -1350,10 +1503,14 @@ def check_docker_env(
1350
1503
  "--push",
1351
1504
  "-f",
1352
1505
  dockerfile_name,
1353
- ".",
1354
1506
  ]
1507
+ if rebuild:
1508
+ cmd += ["--no-cache", "--pull"]
1509
+ cmd.append(".")
1355
1510
  else:
1356
1511
  cmd = ["docker", "build", "-t", tag, "-f", dockerfile_name]
1512
+ if rebuild:
1513
+ cmd += ["--no-cache", "--pull"]
1357
1514
  build_platform = (
1358
1515
  build_platforms[0] if build_platforms else platform
1359
1516
  )
@@ -1792,21 +1949,26 @@ def check_venv(
1792
1949
  if lock_fpath is None:
1793
1950
  fname, ext = os.path.splitext(path)
1794
1951
  lock_fpath = fname + "-lock" + ext
1795
- lock_dir = os.path.dirname(lock_fpath)
1952
+
1953
+ def full(fpath: str) -> str:
1954
+ return os.path.join(wdir or "", fpath)
1955
+
1956
+ lock_fpath_full = full(lock_fpath)
1957
+ lock_dir = os.path.dirname(lock_fpath_full)
1796
1958
  if lock_dir:
1797
1959
  os.makedirs(lock_dir, exist_ok=True)
1798
1960
  # Use main lock file if exists, else try alternatives (including legacy)
1799
1961
  reqs_to_use = lock_fpath
1800
1962
  used_legacy_lock = None
1801
- if not os.path.isfile(lock_fpath):
1963
+ if not os.path.isfile(lock_fpath_full):
1802
1964
  for alt_fpath in alt_lock_fpaths:
1803
- if os.path.isfile(alt_fpath):
1965
+ if os.path.isfile(full(alt_fpath)):
1804
1966
  reqs_to_use = alt_fpath
1805
1967
  if verbose:
1806
1968
  typer.echo(f"Using alternative lock file: {alt_fpath}")
1807
1969
  break
1808
1970
  for legacy_fpath in alt_lock_fpaths_delete:
1809
- if os.path.isfile(legacy_fpath):
1971
+ if os.path.isfile(full(legacy_fpath)):
1810
1972
  reqs_to_use = legacy_fpath
1811
1973
  used_legacy_lock = legacy_fpath
1812
1974
  if verbose:
@@ -1814,20 +1976,89 @@ def check_venv(
1814
1976
  break
1815
1977
  activate_cmd = calkit.environments.get_venv_activate_cmd(prefix)
1816
1978
 
1979
+ def venv_python_version() -> str | None:
1980
+ """The Python version a plain venv was created with."""
1981
+ if _platform.system() == "Windows":
1982
+ exe = os.path.join(prefix_full_path, "Scripts", "python.exe")
1983
+ else:
1984
+ exe = os.path.join(prefix_full_path, "bin", "python")
1985
+ try:
1986
+ return subprocess.check_output(
1987
+ [
1988
+ exe,
1989
+ "-c",
1990
+ "import platform; print(platform.python_version())",
1991
+ ],
1992
+ text=True,
1993
+ ).strip()
1994
+ except (OSError, subprocess.CalledProcessError):
1995
+ return None
1996
+
1997
+ def is_local(line: str) -> bool:
1998
+ """Whether a frozen requirement names a path on this machine."""
1999
+ line = line.strip()
2000
+ if line.startswith(("-e ", "--editable")):
2001
+ return re.match(r"^(-e|--editable)[\s=]+\w+\+", line) is None
2002
+ return re.search(r"@\s*file:", line) is not None
2003
+
2004
+ recorded_hash = calkit.environments.read_env_spec_hash(lock_fpath, wdir)
2005
+ spec_hash = calkit.environments.env_spec_hash(full(path), python=python)
2006
+ # A lock resolved from the current spec is an input, not rewritten here
2007
+ lock_is_current = (
2008
+ os.path.isfile(lock_fpath_full) and recorded_hash == spec_hash
2009
+ )
2010
+
1817
2011
  def pip_install_and_freeze(reqs_arg: str) -> None:
1818
2012
  check_cmd = (
1819
- f"{activate_cmd} "
1820
- f"&& {pip_cmd} install {pip_install_args} {reqs_arg} "
1821
- f"&& {pip_freeze_cmd} > {lock_fpath} "
1822
- "&& deactivate"
2013
+ f"{activate_cmd} && {pip_cmd} install {pip_install_args} "
2014
+ f"{reqs_arg}"
1823
2015
  )
2016
+ if not lock_is_current:
2017
+ check_cmd += f" && {pip_freeze_cmd} > {lock_fpath}"
2018
+ check_cmd += " && deactivate"
1824
2019
  if verbose:
1825
2020
  typer.echo(f"Running command: {check_cmd}")
2021
+ try:
2022
+ with open(lock_fpath_full, encoding="utf-8") as f:
2023
+ lock_before = f.read()
2024
+ except OSError:
2025
+ lock_before = None
1826
2026
  subprocess.run(check_cmd, shell=True, cwd=wdir, check=True)
2027
+ if lock_is_current:
2028
+ return
2029
+ # Local packages are installed from the spec, since their paths
2030
+ # differ between machines
2031
+ with open(lock_fpath_full, encoding="utf-8") as f:
2032
+ frozen = f.read()
2033
+ lock = "".join(
2034
+ ln for ln in frozen.splitlines(keepends=True) if not is_local(ln)
2035
+ )
2036
+ if lock != frozen:
2037
+ with open(lock_fpath_full, "w", encoding="utf-8") as f:
2038
+ f.write(lock)
2039
+ lock_changed = lock != lock_before
2040
+ # All platforms are relocked together, so they agree; an existing
2041
+ # lock that matches is adopted as is, and one with no record is
2042
+ # only added to
2043
+ spec_changed = recorded_hash != spec_hash
2044
+ adopted = recorded_hash is None and not lock_changed
2045
+ if (lock_changed or spec_changed) and not adopted:
2046
+ calkit.environments.write_cross_platform_venv_locks(
2047
+ spec_fpath=path,
2048
+ lock_fpath=lock_fpath,
2049
+ python_version=python or venv_python_version(),
2050
+ wdir=wdir,
2051
+ verbose=verbose,
2052
+ relock=spec_changed and recorded_hash is not None,
2053
+ )
2054
+ # Last, so an interrupted relock isn't taken as current
2055
+ calkit.environments.write_env_spec_hash(
2056
+ lock_fpath, path, python=python, wdir=wdir
2057
+ )
1827
2058
  # Delete legacy lock file after use
1828
2059
  if used_legacy_lock:
1829
2060
  try:
1830
- os.remove(used_legacy_lock)
2061
+ os.remove(full(used_legacy_lock))
1831
2062
  if verbose:
1832
2063
  typer.echo(
1833
2064
  "Deleted legacy lock file after use: "
@@ -1840,9 +2071,19 @@ def check_venv(
1840
2071
  f"{used_legacy_lock}: {e}"
1841
2072
  )
1842
2073
 
1843
- # If the lock file exists, try to install with that
2074
+ # A lock resolved from an older spec would bring back what the spec no
2075
+ # longer has, as would the env built from it, so both start over
1844
2076
  dep_file_txt = f"-r {path}"
1845
- if os.path.isfile(reqs_to_use):
2077
+ if recorded_hash is not None and recorded_hash != spec_hash:
2078
+ if verbose:
2079
+ typer.echo(f"Rebuilding {kind} at {prefix} since its spec changed")
2080
+ try:
2081
+ shutil.rmtree(prefix_full_path)
2082
+ create_venv()
2083
+ except OSError as e:
2084
+ warn(f"Failed to remove {kind} at {prefix}: {e}")
2085
+ elif os.path.isfile(full(reqs_to_use)):
2086
+ # If the lock file exists, try to install with that
1846
2087
  dep_file_txt += f" -r {reqs_to_use}"
1847
2088
  try:
1848
2089
  pip_install_and_freeze(dep_file_txt)
@@ -1861,6 +2102,14 @@ def check_venv(
1861
2102
  create_venv()
1862
2103
  pip_install_and_freeze(dep_file_txt)
1863
2104
  except (subprocess.CalledProcessError, OSError):
2105
+ if lock_is_current:
2106
+ raise_error(
2107
+ f"Failed to create {kind} at {prefix} from its lock file "
2108
+ f"({lock_fpath}), which matches the spec ({path}). "
2109
+ "Rebuilding from the spec would leave the environment "
2110
+ "out of sync with its lock; delete the lock file to "
2111
+ "re-resolve it, which reruns stages that depend on it"
2112
+ )
1864
2113
  warn(
1865
2114
  f"Failed to create environment from lock file ({reqs_to_use}); "
1866
2115
  f"attempting rebuild from input file {path}"
@@ -1879,15 +2128,20 @@ def check_matlab_env(
1879
2128
  typer.Option("--name", "-n", help="Environment name in calkit.yaml."),
1880
2129
  ],
1881
2130
  output_fpath: Annotated[str, typer.Option("--output", "-o")],
2131
+ rebuild: Annotated[
2132
+ bool,
2133
+ typer.Option("--rebuild", help="Rebuild the image from scratch."),
2134
+ ] = False,
1882
2135
  ) -> None:
1883
2136
  """Check a MATLAB environment matches its spec and export a JSON lock
1884
2137
  file.
1885
2138
  """
1886
2139
  ck_info = calkit.load_calkit_info()
1887
2140
  environments = ck_info.get("environments", {})
1888
- if env_name not in environments:
2141
+ found = calkit.environments.get_env(env_name, environments)
2142
+ if found is None:
1889
2143
  raise_error(f"Environment '{env_name}' not found in calkit.yaml")
1890
- env = environments[env_name]
2144
+ env = found
1891
2145
  if env.get("kind") != "matlab":
1892
2146
  raise_error(f"Environment '{env_name}' is not a MATLAB environment")
1893
2147
  if "version" not in env:
@@ -1913,6 +2167,7 @@ def check_matlab_env(
1913
2167
  fpath=dockerfile_fpath,
1914
2168
  lock_fpath=output_fpath,
1915
2169
  platform="linux/amd64", # Only one available for now
2170
+ rebuild=rebuild,
1916
2171
  )
1917
2172
 
1918
2173
 
@@ -1953,6 +2208,54 @@ def check_project_requirements(
1953
2208
  calkit.echo(message)
1954
2209
 
1955
2210
 
2211
+ @check_app.command(name="stage-runs")
2212
+ def check_stage_runs() -> None:
2213
+ """Check each stage in dvc.lock matches the recorded run that made it.
2214
+
2215
+ A stage's entry in .calkit/stage-runs.json records a checksum of its
2216
+ dvc.lock entry after it ran, so one that disagrees got its dvc.lock
2217
+ entry some other way, e.g., 'dvc commit' after an output was edited, an
2218
+ edit or merge of dvc.lock, or running it outside Calkit. This catches
2219
+ mistakes and shortcuts, not forgery, since a matching entry can be
2220
+ written by anyone.
2221
+ """
2222
+ try:
2223
+ with open(calkit.STAGE_RUNS_FPATH) as f:
2224
+ stage_runs = json.load(f)
2225
+ except OSError:
2226
+ stage_runs = {}
2227
+ except json.JSONDecodeError as e:
2228
+ raise_error(f"Can't read {calkit.STAGE_RUNS_FPATH}: {e}")
2229
+ # By directory, for a subproject's stages
2230
+ lock_stages = {None: calkit.dvc.read_lock_stages()}
2231
+ problems = []
2232
+ for name in sorted(set(lock_stages[None]) | set(stage_runs)):
2233
+ lock_dir, lock_name = calkit.dvc.split_lock_stage_name(name)
2234
+ if lock_dir not in lock_stages:
2235
+ lock_stages[lock_dir] = calkit.dvc.read_lock_stages(lock_dir)
2236
+ locked = lock_stages[lock_dir].get(lock_name)
2237
+ # One no longer in dvc.lock has nothing to vouch for
2238
+ if not isinstance(locked, dict):
2239
+ continue
2240
+ recorded = stage_runs.get(name)
2241
+ if not isinstance(recorded, dict):
2242
+ problems.append(f"Stage '{name}' has no recorded run")
2243
+ elif recorded.get("dvc_lock") != calkit.dvc.stage_lock_md5(locked):
2244
+ problems.append(
2245
+ f"Stage '{name}' has changed in dvc.lock since its last "
2246
+ "recorded run"
2247
+ )
2248
+ for problem in problems:
2249
+ typer.echo(problem)
2250
+ if problems:
2251
+ raise_error(
2252
+ f"{len(problems)} stage(s) in dvc.lock don't match a recorded "
2253
+ f"run in {calkit.STAGE_RUNS_FPATH}; rerunning them with "
2254
+ "'calkit run --force <stage>' records them"
2255
+ )
2256
+ typer.echo("Every stage in dvc.lock matches its recorded run")
2257
+
2258
+
1956
2259
  @check_app.command(name="env-vars")
1957
2260
  def check_env_vars(
1958
2261
  verbose: Annotated[
calkit/cli/delete.py CHANGED
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import typer
6
+ from typing_extensions import Annotated
6
7
 
7
8
  import calkit
8
9
  from calkit.cli import AliasGroup, raise_error
@@ -31,3 +32,27 @@ def delete_question(
31
32
  with open("calkit.yaml", "w") as f:
32
33
  ryaml.dump(ck_info, f)
33
34
  typer.echo(f"Removed question: {removed}")
35
+
36
+
37
+ @delete_app.command(name="comment")
38
+ def delete_comment(
39
+ path: Annotated[
40
+ str | None,
41
+ typer.Argument(help="File the thread is in, if not found by --id."),
42
+ ] = None,
43
+ id: Annotated[
44
+ str | None,
45
+ typer.Option(
46
+ "--id",
47
+ help="ID of the thread, which finds it in any of the project's documents.",
48
+ ),
49
+ ] = None,
50
+ line: Annotated[
51
+ int | None,
52
+ typer.Option("--line", "-l", help="Number of any line in the thread."),
53
+ ] = None,
54
+ ) -> None:
55
+ """Delete a review comment thread and its replies."""
56
+ from calkit.cli.latex import edit_comment
57
+
58
+ edit_comment(path, id, line, None)