calkit-python 0.41.23__py3-none-any.whl → 0.41.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/main/core.py +96 -59
- calkit/cli/overleaf.py +2 -0
- calkit/cli/sync.py +80 -0
- calkit/tests/cli/main/test_core.py +47 -0
- calkit/tests/cli/main/test_subprojects.py +77 -0
- calkit/tests/cli/test_sync.py +116 -0
- {calkit_python-0.41.23.dist-info → calkit_python-0.41.24.dist-info}/METADATA +1 -1
- {calkit_python-0.41.23.dist-info → calkit_python-0.41.24.dist-info}/RECORD +25 -23
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.41.23.data → calkit_python-0.41.24.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.41.23.dist-info → calkit_python-0.41.24.dist-info}/WHEEL +0 -0
- {calkit_python-0.41.23.dist-info → calkit_python-0.41.24.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.41.23.dist-info → calkit_python-0.41.24.dist-info}/licenses/LICENSE +0 -0
calkit/cli/main/core.py
CHANGED
|
@@ -59,6 +59,7 @@ from calkit.cli.notebooks import notebooks_app
|
|
|
59
59
|
from calkit.cli.office import office_app
|
|
60
60
|
from calkit.cli.overleaf import overleaf_app
|
|
61
61
|
from calkit.cli.scheduler import scheduler_app
|
|
62
|
+
from calkit.cli.sync import sync_app
|
|
62
63
|
from calkit.cli.update import update_app
|
|
63
64
|
|
|
64
65
|
app = typer.Typer(
|
|
@@ -89,6 +90,7 @@ app.add_typer(
|
|
|
89
90
|
help="Work with a job scheduler (SLURM or PBS).",
|
|
90
91
|
)
|
|
91
92
|
app.add_typer(dev_app, name="dev", help="Developer tools.", hidden=True)
|
|
93
|
+
app.add_typer(sync_app, name="sync", help="Sync with external systems.")
|
|
92
94
|
|
|
93
95
|
|
|
94
96
|
def _to_shell_cmd(cmd: list[str]) -> str:
|
|
@@ -1157,6 +1159,12 @@ def save(
|
|
|
1157
1159
|
@app.command(name="pull")
|
|
1158
1160
|
def pull(
|
|
1159
1161
|
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
1162
|
+
no_dvc: Annotated[
|
|
1163
|
+
bool, typer.Option("--no-dvc", help="Do not pull from DVC.")
|
|
1164
|
+
] = False,
|
|
1165
|
+
no_git: Annotated[
|
|
1166
|
+
bool, typer.Option("--no-git", help="Do not pull from Git.")
|
|
1167
|
+
] = False,
|
|
1160
1168
|
git_args: Annotated[
|
|
1161
1169
|
list[str],
|
|
1162
1170
|
typer.Option("--git-arg", help="Additional Git args."),
|
|
@@ -1181,58 +1189,62 @@ def pull(
|
|
|
1181
1189
|
] = False,
|
|
1182
1190
|
):
|
|
1183
1191
|
"""Pull with both Git and DVC."""
|
|
1184
|
-
typer.echo("Git pulling")
|
|
1185
1192
|
if force:
|
|
1186
1193
|
if "-f" not in git_args and "--force" not in git_args:
|
|
1187
1194
|
git_args.append("-f")
|
|
1188
1195
|
if "-f" not in dvc_args and "--force" not in dvc_args:
|
|
1189
1196
|
dvc_args.append("-f")
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
git_cmd
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
["
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1197
|
+
if not no_git:
|
|
1198
|
+
typer.echo("Git pulling")
|
|
1199
|
+
try:
|
|
1200
|
+
git_cmd = ["git", "pull"]
|
|
1201
|
+
if not no_recursive and "--recurse-submodules" not in git_args:
|
|
1202
|
+
git_cmd.append("--recurse-submodules")
|
|
1203
|
+
subprocess.check_call(git_cmd + git_args)
|
|
1204
|
+
except subprocess.CalledProcessError:
|
|
1205
|
+
raise_error("Git pull failed")
|
|
1206
|
+
if not no_dvc:
|
|
1207
|
+
typer.echo("DVC pulling")
|
|
1208
|
+
if not no_check_auth:
|
|
1209
|
+
# Check that our dvc remotes all have our DVC token set for them
|
|
1210
|
+
remotes = calkit.dvc.get_remotes()
|
|
1211
|
+
for name, url in remotes.items():
|
|
1212
|
+
if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
|
|
1213
|
+
typer.echo(
|
|
1214
|
+
f"Checking authentication for DVC remote: {name}"
|
|
1215
|
+
)
|
|
1216
|
+
calkit.dvc.set_remote_auth(remote_name=name)
|
|
1217
|
+
if (
|
|
1218
|
+
not no_recursive
|
|
1219
|
+
and "--recursive" not in dvc_args
|
|
1220
|
+
and "-R" not in dvc_args
|
|
1221
|
+
):
|
|
1222
|
+
dvc_args.append("--recursive")
|
|
1223
|
+
result = calkit.dvc.run_dvc_command(
|
|
1224
|
+
["pull"] + dvc_args,
|
|
1225
|
+
lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
|
|
1226
|
+
)
|
|
1227
|
+
if result != 0:
|
|
1228
|
+
raise_error("DVC pull failed")
|
|
1229
|
+
calkit.dvc.zip.sync_all(direction="to-workspace")
|
|
1230
|
+
if not no_recursive:
|
|
1231
|
+
# Pull DVC in isolated subprojects (those with their own .dvc folder)
|
|
1232
|
+
ck_info = calkit.load_calkit_info()
|
|
1233
|
+
for sp in ck_info.get("subprojects", []):
|
|
1234
|
+
if not isinstance(sp, dict) or not sp.get("path"):
|
|
1235
|
+
continue
|
|
1236
|
+
sp_path = sp["path"]
|
|
1237
|
+
if not os.path.isdir(os.path.join(sp_path, ".dvc")):
|
|
1238
|
+
continue
|
|
1239
|
+
typer.echo(f"DVC pulling subproject: {sp_path}")
|
|
1240
|
+
sp_result = calkit.dvc.run_dvc_command(
|
|
1241
|
+
["pull"] + dvc_args,
|
|
1242
|
+
cwd=sp_path,
|
|
1243
|
+
lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
|
|
1244
|
+
)
|
|
1245
|
+
if sp_result != 0:
|
|
1246
|
+
raise_error(f"DVC pull failed in subproject: {sp_path}")
|
|
1247
|
+
calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
|
|
1236
1248
|
|
|
1237
1249
|
|
|
1238
1250
|
@app.command(name="push")
|
|
@@ -1284,16 +1296,6 @@ def push(
|
|
|
1284
1296
|
raise_error("Git push failed")
|
|
1285
1297
|
|
|
1286
1298
|
|
|
1287
|
-
@app.command(name="sync")
|
|
1288
|
-
def sync(
|
|
1289
|
-
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
1290
|
-
):
|
|
1291
|
-
"""Sync the project repo by pulling and then pushing."""
|
|
1292
|
-
# TODO: Walk users through merge conflicts if they arise
|
|
1293
|
-
pull(no_check_auth=no_check_auth)
|
|
1294
|
-
push(no_check_auth=no_check_auth)
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
1299
|
@app.command(name="ignore")
|
|
1298
1300
|
def ignore(
|
|
1299
1301
|
path: Annotated[str, typer.Argument(help="Path to ignore.")],
|
|
@@ -1685,6 +1687,29 @@ def _concurrent_scheduler_prepass(
|
|
|
1685
1687
|
)
|
|
1686
1688
|
|
|
1687
1689
|
|
|
1690
|
+
def _get_subproject_targets_for_run(
|
|
1691
|
+
subproject_path: str,
|
|
1692
|
+
targets: list[str] | None,
|
|
1693
|
+
include_dvc_yaml_targets: bool = False,
|
|
1694
|
+
) -> tuple[bool, list[str] | None]:
|
|
1695
|
+
"""Return whether and which subproject stages a run target selects."""
|
|
1696
|
+
if not targets:
|
|
1697
|
+
return True, None
|
|
1698
|
+
sp = Path(subproject_path).as_posix()
|
|
1699
|
+
target_prefixes = {sp, Path(sp).name}
|
|
1700
|
+
if include_dvc_yaml_targets:
|
|
1701
|
+
target_prefixes.add(f"{sp}/dvc.yaml")
|
|
1702
|
+
selected_stages = []
|
|
1703
|
+
for target in targets:
|
|
1704
|
+
target_prefix, separator, stage_name = target.partition(":")
|
|
1705
|
+
if target_prefix not in target_prefixes:
|
|
1706
|
+
continue
|
|
1707
|
+
if not separator or not stage_name:
|
|
1708
|
+
return True, None
|
|
1709
|
+
selected_stages.append(stage_name)
|
|
1710
|
+
return bool(selected_stages), selected_stages or None
|
|
1711
|
+
|
|
1712
|
+
|
|
1688
1713
|
@app.command(name="run")
|
|
1689
1714
|
def run(
|
|
1690
1715
|
targets: Annotated[
|
|
@@ -1942,6 +1967,18 @@ def run(
|
|
|
1942
1967
|
sp = Path(subproject["path"]).as_posix()
|
|
1943
1968
|
if not os.path.isdir(sp):
|
|
1944
1969
|
continue
|
|
1970
|
+
if single_item:
|
|
1971
|
+
sp_selected, sp_targets = _get_subproject_targets_for_run(
|
|
1972
|
+
subproject_path=sp,
|
|
1973
|
+
targets=targets,
|
|
1974
|
+
include_dvc_yaml_targets=not os.path.isdir(
|
|
1975
|
+
os.path.join(sp, ".dvc")
|
|
1976
|
+
),
|
|
1977
|
+
)
|
|
1978
|
+
if not sp_selected:
|
|
1979
|
+
continue
|
|
1980
|
+
else:
|
|
1981
|
+
sp_targets = None
|
|
1945
1982
|
os.chdir(sp)
|
|
1946
1983
|
try:
|
|
1947
1984
|
sp_ck_info = calkit.load_calkit_info()
|
|
@@ -1950,7 +1987,7 @@ def run(
|
|
|
1950
1987
|
f"📦 Checking environments for subproject: {sp}"
|
|
1951
1988
|
)
|
|
1952
1989
|
sp_env_results = calkit.environments.check_all_in_pipeline(
|
|
1953
|
-
ck_info=sp_ck_info, force=force
|
|
1990
|
+
ck_info=sp_ck_info, targets=sp_targets, force=force
|
|
1954
1991
|
)
|
|
1955
1992
|
for env_name, sp_result in sp_env_results.items():
|
|
1956
1993
|
if verbose:
|
|
@@ -2181,7 +2218,7 @@ def run(
|
|
|
2181
2218
|
file_handler.setFormatter(formatter)
|
|
2182
2219
|
dvc.log.logger.addHandler(file_handler)
|
|
2183
2220
|
# Remove newline logging in dvc.repo.reproduce
|
|
2184
|
-
dvc.repo.reproduce.logger.setLevel(logging.
|
|
2221
|
+
dvc.repo.reproduce.logger.setLevel(logging.WARNING)
|
|
2185
2222
|
# Disable other misc DVC output
|
|
2186
2223
|
dvc.ui.ui.write = lambda *args, **kwargs: None
|
|
2187
2224
|
# Tell `calkit scheduler batch` to resubmit completed jobs under --force;
|
calkit/cli/overleaf.py
CHANGED
|
@@ -13,6 +13,7 @@ from typing_extensions import Annotated
|
|
|
13
13
|
|
|
14
14
|
import calkit
|
|
15
15
|
from calkit.cli import AliasGroup, raise_error, warn
|
|
16
|
+
from calkit.cli.sync import sync_app
|
|
16
17
|
|
|
17
18
|
overleaf_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
|
|
18
19
|
|
|
@@ -362,6 +363,7 @@ def import_publication(
|
|
|
362
363
|
sync(paths=[dest_dir], no_commit=no_commit, push_only=push_only)
|
|
363
364
|
|
|
364
365
|
|
|
366
|
+
@sync_app.command(name="overleaf")
|
|
365
367
|
@overleaf_app.command(name="sync")
|
|
366
368
|
def sync(
|
|
367
369
|
paths: Annotated[
|
calkit/cli/sync.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""CLI for syncing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Callable
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
from typing_extensions import Annotated
|
|
9
|
+
|
|
10
|
+
import calkit
|
|
11
|
+
from calkit.cli import AliasGroup, raise_error
|
|
12
|
+
|
|
13
|
+
sync_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@sync_app.command(name="git")
|
|
17
|
+
def sync_git(
|
|
18
|
+
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
19
|
+
) -> None:
|
|
20
|
+
"""Sync the Git repository by pulling and then pushing."""
|
|
21
|
+
from calkit.cli.main.core import pull, push
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
repo = calkit.git.get_repo()
|
|
25
|
+
except Exception:
|
|
26
|
+
raise_error("No Git repository found. Run 'git init' first.")
|
|
27
|
+
if not repo.remotes:
|
|
28
|
+
raise_error(
|
|
29
|
+
"No Git remotes configured. Add a remote with "
|
|
30
|
+
"'git remote add <name> <url>'."
|
|
31
|
+
)
|
|
32
|
+
pull(no_dvc=True, no_check_auth=no_check_auth)
|
|
33
|
+
push(no_dvc=True, no_check_auth=no_check_auth)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@sync_app.command(name="dvc")
|
|
37
|
+
def sync_dvc(
|
|
38
|
+
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Sync the DVC repository by pulling and then pushing."""
|
|
41
|
+
from calkit.cli.main.core import pull, push
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
calkit.dvc.get_dvc_repo()
|
|
45
|
+
except Exception:
|
|
46
|
+
raise_error("No DVC repository found. Run 'calkit init' first.")
|
|
47
|
+
if not calkit.dvc.get_remotes():
|
|
48
|
+
raise_error(
|
|
49
|
+
"No DVC remotes configured. Add a remote with "
|
|
50
|
+
"'dvc remote add <name> <url>'."
|
|
51
|
+
)
|
|
52
|
+
pull(no_git=True, no_check_auth=no_check_auth)
|
|
53
|
+
push(no_git=True, no_check_auth=no_check_auth)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@sync_app.command(name="all")
|
|
57
|
+
def sync_all() -> None:
|
|
58
|
+
"""Sync all registered systems."""
|
|
59
|
+
from calkit.cli.overleaf import sync as overleaf_sync
|
|
60
|
+
|
|
61
|
+
# Run each known target in a stable order, reporting and collecting any
|
|
62
|
+
# failures. Each target is responsible for raising a clear error if it is
|
|
63
|
+
# not configured, so users calling 'calkit sync <target>' directly get a
|
|
64
|
+
# helpful message.
|
|
65
|
+
sync_funcs: list[tuple[str, Callable[[], None]]] = [
|
|
66
|
+
("git", sync_git),
|
|
67
|
+
("dvc", sync_dvc),
|
|
68
|
+
("overleaf", overleaf_sync),
|
|
69
|
+
]
|
|
70
|
+
failures = []
|
|
71
|
+
for target_name, sync_func in sync_funcs:
|
|
72
|
+
typer.echo(f"Syncing {target_name}...")
|
|
73
|
+
try:
|
|
74
|
+
sync_func()
|
|
75
|
+
except Exception as e:
|
|
76
|
+
typer.echo(f"Failed to sync {target_name}: {e}", err=True)
|
|
77
|
+
failures.append(target_name)
|
|
78
|
+
# Exit with an error if any target failed so callers can react.
|
|
79
|
+
if failures:
|
|
80
|
+
raise typer.Exit(1)
|
|
@@ -22,6 +22,7 @@ import calkit.cli.main
|
|
|
22
22
|
from calkit.cli.core import complete_stage_names
|
|
23
23
|
from calkit.cli.main.core import (
|
|
24
24
|
_get_running_pipeline_status,
|
|
25
|
+
_get_subproject_targets_for_run,
|
|
25
26
|
_prune_run_logs,
|
|
26
27
|
_run_dvc_repro,
|
|
27
28
|
_stage_run_info_from_log_content,
|
|
@@ -45,6 +46,52 @@ skipif_windows_mock_scheduler = pytest.mark.skipif(
|
|
|
45
46
|
)
|
|
46
47
|
|
|
47
48
|
|
|
49
|
+
@pytest.mark.parametrize(
|
|
50
|
+
("subproject_path", "targets", "include_dvc_yaml_targets", "expected"),
|
|
51
|
+
[
|
|
52
|
+
("sub1", None, False, (True, None)),
|
|
53
|
+
("sub1", ["parent-stage"], False, (False, None)),
|
|
54
|
+
("sub1", ["sub1"], False, (True, None)),
|
|
55
|
+
("nested/sub1", ["sub1:build"], False, (True, ["build"])),
|
|
56
|
+
(
|
|
57
|
+
"nested/sub1",
|
|
58
|
+
["nested/sub1:build"],
|
|
59
|
+
False,
|
|
60
|
+
(True, ["build"]),
|
|
61
|
+
),
|
|
62
|
+
(
|
|
63
|
+
"nested/sub1",
|
|
64
|
+
["nested/sub1/dvc.yaml"],
|
|
65
|
+
True,
|
|
66
|
+
(True, None),
|
|
67
|
+
),
|
|
68
|
+
(
|
|
69
|
+
"nested/sub1",
|
|
70
|
+
["nested/sub1/dvc.yaml:build"],
|
|
71
|
+
True,
|
|
72
|
+
(True, ["build"]),
|
|
73
|
+
),
|
|
74
|
+
(
|
|
75
|
+
"nested/sub1",
|
|
76
|
+
["nested/sub1/dvc.yaml:build"],
|
|
77
|
+
False,
|
|
78
|
+
(False, None),
|
|
79
|
+
),
|
|
80
|
+
],
|
|
81
|
+
)
|
|
82
|
+
def test_get_subproject_targets_for_run(
|
|
83
|
+
subproject_path, targets, include_dvc_yaml_targets, expected
|
|
84
|
+
):
|
|
85
|
+
assert (
|
|
86
|
+
_get_subproject_targets_for_run(
|
|
87
|
+
subproject_path,
|
|
88
|
+
targets,
|
|
89
|
+
include_dvc_yaml_targets=include_dvc_yaml_targets,
|
|
90
|
+
)
|
|
91
|
+
== expected
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
48
95
|
def _repo_test_file(name: str) -> Path:
|
|
49
96
|
"""Find a file in the repository-level ``test`` directory."""
|
|
50
97
|
for parent in Path(__file__).resolve().parents:
|
|
@@ -115,6 +115,83 @@ def test_inline_subproject(tmp_dir):
|
|
|
115
115
|
assert "v2" in f.read()
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
def test_single_item_skips_unrelated_subproject_preparation(tmp_dir):
|
|
119
|
+
subprocess.check_call(["calkit", "init"])
|
|
120
|
+
os.makedirs("sub1")
|
|
121
|
+
write_ck_info(
|
|
122
|
+
"sub1/calkit.yaml",
|
|
123
|
+
{
|
|
124
|
+
"pipeline": {
|
|
125
|
+
"stages": {
|
|
126
|
+
"subproject-only": make_stage(
|
|
127
|
+
'echo "subproject" > subproject.txt'
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
)
|
|
133
|
+
write_ck_info(
|
|
134
|
+
"calkit.yaml",
|
|
135
|
+
{
|
|
136
|
+
"subprojects": [{"path": "sub1"}],
|
|
137
|
+
"pipeline": {
|
|
138
|
+
"stages": {
|
|
139
|
+
"parent-only": make_stage('echo "parent" > parent.txt')
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
)
|
|
144
|
+
result = subprocess.run(
|
|
145
|
+
["calkit", "run", "--single-item", "parent-only"],
|
|
146
|
+
check=True,
|
|
147
|
+
capture_output=True,
|
|
148
|
+
text=True,
|
|
149
|
+
)
|
|
150
|
+
assert os.path.isfile("parent.txt")
|
|
151
|
+
assert "Checking environments for subproject" not in result.stdout
|
|
152
|
+
selected_subproject = subprocess.run(
|
|
153
|
+
["calkit", "run", "--single-item", "sub1:subproject-only"],
|
|
154
|
+
check=True,
|
|
155
|
+
capture_output=True,
|
|
156
|
+
text=True,
|
|
157
|
+
)
|
|
158
|
+
assert os.path.isfile("sub1/subproject.txt")
|
|
159
|
+
assert "Checking environments for subproject: sub1" in (
|
|
160
|
+
selected_subproject.stdout
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
# A DVC-style inline target should still prepare the selected subproject.
|
|
164
|
+
write_ck_info(
|
|
165
|
+
"sub1/calkit.yaml",
|
|
166
|
+
{
|
|
167
|
+
"pipeline": {
|
|
168
|
+
"stages": {
|
|
169
|
+
"subproject-only": make_stage(
|
|
170
|
+
'echo "dvc target" > subproject.txt'
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
)
|
|
176
|
+
calkit.pipeline.to_dvc(write=True, manage_gitignore=False)
|
|
177
|
+
selected_inline_dvc_target = subprocess.run(
|
|
178
|
+
[
|
|
179
|
+
"calkit",
|
|
180
|
+
"run",
|
|
181
|
+
"--single-item",
|
|
182
|
+
"sub1/dvc.yaml:subproject-only",
|
|
183
|
+
],
|
|
184
|
+
check=True,
|
|
185
|
+
capture_output=True,
|
|
186
|
+
text=True,
|
|
187
|
+
)
|
|
188
|
+
with open("sub1/subproject.txt") as f:
|
|
189
|
+
assert "dvc target" in f.read()
|
|
190
|
+
assert "Checking environments for subproject: sub1" in (
|
|
191
|
+
selected_inline_dvc_target.stdout
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
118
195
|
def init_isolated_subproject(path, stages):
|
|
119
196
|
os.makedirs(path, exist_ok=True)
|
|
120
197
|
subprocess.check_call(["git", "init"], cwd=path)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Tests for the sync CLI commands."""
|
|
2
|
+
|
|
3
|
+
from unittest.mock import MagicMock, patch
|
|
4
|
+
|
|
5
|
+
from typer.testing import CliRunner
|
|
6
|
+
|
|
7
|
+
from calkit.cli.main import app
|
|
8
|
+
|
|
9
|
+
runner = CliRunner()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_sync_help():
|
|
13
|
+
result = runner.invoke(app, ["sync", "--help"])
|
|
14
|
+
assert result.exit_code == 0
|
|
15
|
+
assert "Sync with external systems" in result.output
|
|
16
|
+
assert "git" in result.output
|
|
17
|
+
assert "dvc" in result.output
|
|
18
|
+
assert "overleaf" in result.output
|
|
19
|
+
assert "all" in result.output
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_sync_git_calls_pull_and_push():
|
|
23
|
+
mock_repo = MagicMock()
|
|
24
|
+
mock_repo.remotes = ["origin"]
|
|
25
|
+
with patch("calkit.git.get_repo", return_value=mock_repo):
|
|
26
|
+
with patch("calkit.cli.main.core.pull") as mock_pull:
|
|
27
|
+
with patch("calkit.cli.main.core.push") as mock_push:
|
|
28
|
+
result = runner.invoke(app, ["sync", "git"])
|
|
29
|
+
assert result.exit_code == 0
|
|
30
|
+
mock_pull.assert_called_once_with(
|
|
31
|
+
no_dvc=True, no_check_auth=False
|
|
32
|
+
)
|
|
33
|
+
mock_push.assert_called_once_with(
|
|
34
|
+
no_dvc=True, no_check_auth=False
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_sync_git_errors_when_not_initialized():
|
|
39
|
+
with patch("calkit.git.get_repo", side_effect=Exception("not a repo")):
|
|
40
|
+
result = runner.invoke(app, ["sync", "git"])
|
|
41
|
+
assert result.exit_code != 0
|
|
42
|
+
assert "No Git repository found" in result.output
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_sync_dvc_calls_pull_and_push():
|
|
46
|
+
with patch("calkit.dvc.get_dvc_repo"):
|
|
47
|
+
with patch("calkit.dvc.get_remotes", return_value={"origin": "url"}):
|
|
48
|
+
with patch("calkit.cli.main.core.pull") as mock_pull:
|
|
49
|
+
with patch("calkit.cli.main.core.push") as mock_push:
|
|
50
|
+
result = runner.invoke(app, ["sync", "dvc"])
|
|
51
|
+
assert result.exit_code == 0
|
|
52
|
+
mock_pull.assert_called_once_with(
|
|
53
|
+
no_git=True, no_check_auth=False
|
|
54
|
+
)
|
|
55
|
+
mock_push.assert_called_once_with(
|
|
56
|
+
no_git=True, no_check_auth=False
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_sync_dvc_errors_when_not_initialized():
|
|
61
|
+
with patch(
|
|
62
|
+
"calkit.dvc.get_dvc_repo", side_effect=Exception("not a dvc repo")
|
|
63
|
+
):
|
|
64
|
+
result = runner.invoke(app, ["sync", "dvc"])
|
|
65
|
+
assert result.exit_code != 0
|
|
66
|
+
assert "No DVC repository found" in result.output
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_sync_all_runs_all_targets():
|
|
70
|
+
def mock_sync_git():
|
|
71
|
+
print("Mock syncing git")
|
|
72
|
+
|
|
73
|
+
def mock_sync_dvc():
|
|
74
|
+
print("Mock syncing dvc")
|
|
75
|
+
|
|
76
|
+
def mock_sync_overleaf():
|
|
77
|
+
print("Mock syncing overleaf")
|
|
78
|
+
|
|
79
|
+
with patch("calkit.cli.sync.sync_git", mock_sync_git):
|
|
80
|
+
with patch("calkit.cli.sync.sync_dvc", mock_sync_dvc):
|
|
81
|
+
with patch("calkit.cli.overleaf.sync", mock_sync_overleaf):
|
|
82
|
+
result = runner.invoke(app, ["sync", "all"])
|
|
83
|
+
assert result.exit_code == 0
|
|
84
|
+
assert "Syncing git..." in result.output
|
|
85
|
+
assert "Mock syncing git" in result.output
|
|
86
|
+
assert "Syncing dvc..." in result.output
|
|
87
|
+
assert "Mock syncing dvc" in result.output
|
|
88
|
+
assert "Syncing overleaf..." in result.output
|
|
89
|
+
assert "Mock syncing overleaf" in result.output
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_sync_all_reports_target_failures():
|
|
93
|
+
def mock_sync_git():
|
|
94
|
+
print("Mock syncing git")
|
|
95
|
+
|
|
96
|
+
def mock_sync_dvc():
|
|
97
|
+
raise RuntimeError("dvc is broken")
|
|
98
|
+
|
|
99
|
+
with patch("calkit.cli.sync.sync_git", mock_sync_git):
|
|
100
|
+
with patch("calkit.cli.sync.sync_dvc", mock_sync_dvc):
|
|
101
|
+
with patch("calkit.cli.overleaf.sync"):
|
|
102
|
+
result = runner.invoke(app, ["sync", "all"])
|
|
103
|
+
assert result.exit_code == 1
|
|
104
|
+
assert "Syncing git..." in result.output
|
|
105
|
+
assert "Mock syncing git" in result.output
|
|
106
|
+
assert "Syncing dvc..." in result.output
|
|
107
|
+
assert "Failed to sync dvc: dvc is broken" in result.output
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_sync_overleaf_is_accessible():
|
|
111
|
+
# Since we can't easily mock the entire overleaf sync environment here
|
|
112
|
+
# without duplicating test_overleaf.py, we just test that the command
|
|
113
|
+
# is registered and displays its help correctly.
|
|
114
|
+
result = runner.invoke(app, ["sync", "overleaf", "--help"])
|
|
115
|
+
assert result.exit_code == 0
|
|
116
|
+
assert "Sync folders with Overleaf" in result.output
|
|
@@ -43,11 +43,12 @@ calkit/cli/list.py,sha256=jzkZC7dgbmYz8aCBfEWibu_ZRm6i0xwXmIgbUR4NZgA,12040
|
|
|
43
43
|
calkit/cli/new.py,sha256=UDFUi3_MvQcI7q5VWFnRRrX7pW6KCoTcssmJtxkfXpw,137435
|
|
44
44
|
calkit/cli/notebooks.py,sha256=mMNHE8yT0BDN0iJE07GCBFAseyDF0jSRshIm1CQjgrY,24645
|
|
45
45
|
calkit/cli/office.py,sha256=jVSTvbl91TCzlglST5O2b8hdApZA_QHw_UiEQFJqxdc,1754
|
|
46
|
-
calkit/cli/overleaf.py,sha256=
|
|
46
|
+
calkit/cli/overleaf.py,sha256=G4k4LdBDmuPvT3afVpb8QfWNYUSFiOhHc4vY7offDgA,24489
|
|
47
47
|
calkit/cli/scheduler.py,sha256=OWJpW57G6Ebs5FTZ_nf2PpEG9QNd6WPe0mb3TmbVTyE,42402
|
|
48
|
+
calkit/cli/sync.py,sha256=UGen0Yey6BARooI5K83eiArw4YN2ZQcRQjCocaGy6E4,2539
|
|
48
49
|
calkit/cli/update.py,sha256=5Jjo-5uRTxd5UWHvOETPYFWtJJRr81qmKmsibthkLPk,43501
|
|
49
50
|
calkit/cli/main/__init__.py,sha256=qu1POZPyqs33ZKfasOxv_Wc-EzVcEgK3Dt_vwFL8Bi8,65
|
|
50
|
-
calkit/cli/main/core.py,sha256=
|
|
51
|
+
calkit/cli/main/core.py,sha256=FMNgp7yAyLcUAaFvpyNiUbdhYu5579D6j_ic8FNIIqQ,129246
|
|
51
52
|
calkit/cli/main/xr.py,sha256=Bug5qJuiIq0tusmdu30t-ZdyaT_Sy5RwPSTiHvCtIqE,25600
|
|
52
53
|
calkit/dvc/__init__.py,sha256=-B6tilCb_jw7QFmJ2srILez24wDhNKUzIBhZUMt01dE,381
|
|
53
54
|
calkit/dvc/core.py,sha256=p42i_uCzgZD6ClGnLOn88zaiGTwrdSG09quL8Hb8Aas,30908
|
|
@@ -104,11 +105,12 @@ calkit/tests/cli/test_new.py,sha256=G1CpwlhV7a4oiGYep0DgolaAHOrdxQshxE7fE_EeD8M,
|
|
|
104
105
|
calkit/tests/cli/test_notebooks.py,sha256=2t1KiGhEz9H9LcIdWy4jGM05REUxJpWiIg6fSpX12ME,10678
|
|
105
106
|
calkit/tests/cli/test_overleaf.py,sha256=D1fMB68s4zNKNY_OkfDlb2FyOcGIle2UOItePCnlwTo,19578
|
|
106
107
|
calkit/tests/cli/test_scheduler.py,sha256=mE0KLwcb9gozOnylTLsaL_xYGAptlD9hB8ZiKpPhmcc,17934
|
|
108
|
+
calkit/tests/cli/test_sync.py,sha256=jmhfgoMIjdihIuu9UVyHSizBDTF0hlBt2VWqEdVkAKk,4375
|
|
107
109
|
calkit/tests/cli/test_update.py,sha256=bwRF0kpqbCVxNvGH777LHFX3oUP2DyN3XXUBDCEKNoc,6356
|
|
108
110
|
calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
109
|
-
calkit/tests/cli/main/test_core.py,sha256=
|
|
111
|
+
calkit/tests/cli/main/test_core.py,sha256=sIceEsGAHYSML9nPnIWdiMfw9d9sBvKU3TsQ2Z8np0g,73910
|
|
110
112
|
calkit/tests/cli/main/test_lock.py,sha256=1iX8iSbGtr4O9OkaAi4d3gP3_QzNl6Rr6HLtbEpS6iU,9847
|
|
111
|
-
calkit/tests/cli/main/test_subprojects.py,sha256=
|
|
113
|
+
calkit/tests/cli/main/test_subprojects.py,sha256=Hq_BsYdltXc3YaIMvRWGa8fbsMr1dojHj1dXz9WkaNo,14940
|
|
112
114
|
calkit/tests/cli/main/test_xr.py,sha256=shk8LHS33bZpr_iAW5eA5sMD3nQDKm8fIwXlE6dA2uI,22527
|
|
113
115
|
calkit/tests/dvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
114
116
|
calkit/tests/dvc/test_core.py,sha256=ywXBEyw1XpoSuu1EnMbFl-ebbdEJjpaBEPG157_3RBY,12914
|
|
@@ -123,22 +125,22 @@ calkit/tests/models/test_pipeline.py,sha256=9F0g6ij7vc8c2hIRIVTHpT0I-JHXC6i2NRB2
|
|
|
123
125
|
calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
|
|
124
126
|
calkit/agent_skills/conventions/SKILL.md,sha256=nAM2FjSMk6ED56Dr5zh2bL_dhfzDra-h2ZgzHU7ruS4,9524
|
|
125
127
|
calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
|
|
126
|
-
calkit_python-0.41.
|
|
127
|
-
calkit_python-0.41.
|
|
128
|
-
calkit_python-0.41.
|
|
129
|
-
calkit_python-0.41.
|
|
130
|
-
calkit_python-0.41.
|
|
131
|
-
calkit_python-0.41.
|
|
132
|
-
calkit_python-0.41.
|
|
133
|
-
calkit_python-0.41.
|
|
134
|
-
calkit_python-0.41.
|
|
135
|
-
calkit_python-0.41.
|
|
136
|
-
calkit_python-0.41.
|
|
137
|
-
calkit_python-0.41.
|
|
138
|
-
calkit_python-0.41.
|
|
139
|
-
calkit_python-0.41.
|
|
140
|
-
calkit_python-0.41.
|
|
141
|
-
calkit_python-0.41.
|
|
142
|
-
calkit_python-0.41.
|
|
143
|
-
calkit_python-0.41.
|
|
144
|
-
calkit_python-0.41.
|
|
128
|
+
calkit_python-0.41.24.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
|
|
129
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
|
|
130
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
|
|
131
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
|
|
132
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
|
|
133
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
|
|
134
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
|
|
135
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
|
|
136
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
|
|
137
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
|
|
138
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
|
|
139
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
|
|
140
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
|
|
141
|
+
calkit_python-0.41.24.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
|
|
142
|
+
calkit_python-0.41.24.dist-info/METADATA,sha256=QPiEwXWF7Hy-C7HxoWYBDX8jDzBC2MARv4V9n7q1Jp4,12567
|
|
143
|
+
calkit_python-0.41.24.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
144
|
+
calkit_python-0.41.24.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
|
|
145
|
+
calkit_python-0.41.24.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
|
|
146
|
+
calkit_python-0.41.24.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|