calkit-python 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {calkit_python-0.1.0 → calkit_python-0.2.0}/PKG-INFO +1 -1
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/__init__.py +1 -1
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/core.py +5 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/main.py +157 -41
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/new.py +45 -44
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/dvc.py +15 -3
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/models.py +17 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/cli/test_main.py +31 -9
- calkit_python-0.2.0/calkit/tests/test_dvc.py +15 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/.github/FUNDING.yml +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/.github/workflows/publish-test.yml +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/.github/workflows/publish.yml +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/.gitignore +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/LICENSE +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/README.md +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/__init__.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/config.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/import_.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/list.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cli/notebooks.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/cloud.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/config.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/core.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/data.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/docker.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/git.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/gui.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/jupyter.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/server.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/__init__.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/cli/__init__.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/cli/test_list.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/cli/test_new.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/test_core.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/calkit/tests/test_jupyter.py +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/cfd-study/README.md +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/cfd-study/calkit.yaml +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/cfd-study/config/simulations/runs.csv +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/cfd-study/notebook.ipynb +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/ms-office/.gitignore +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/ms-office/README.md +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/examples/ms-office/calkit.yaml +0 -0
- {calkit_python-0.1.0 → calkit_python-0.2.0}/pyproject.toml +0 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import json
|
|
5
6
|
import os
|
|
6
7
|
import subprocess
|
|
7
8
|
import sys
|
|
@@ -11,7 +12,7 @@ import typer
|
|
|
11
12
|
from typing_extensions import Annotated, Optional
|
|
12
13
|
|
|
13
14
|
import calkit
|
|
14
|
-
from calkit.cli import print_sep, run_cmd
|
|
15
|
+
from calkit.cli import print_sep, raise_error, run_cmd
|
|
15
16
|
from calkit.cli.config import config_app
|
|
16
17
|
from calkit.cli.import_ import import_app
|
|
17
18
|
from calkit.cli.list import list_app
|
|
@@ -25,8 +26,11 @@ app = typer.Typer(
|
|
|
25
26
|
pretty_exceptions_show_locals=False,
|
|
26
27
|
)
|
|
27
28
|
app.add_typer(config_app, name="config", help="Configure Calkit.")
|
|
29
|
+
app.add_typer(new_app, name="new", help="Create a new Calkit object.")
|
|
28
30
|
app.add_typer(
|
|
29
|
-
new_app,
|
|
31
|
+
new_app,
|
|
32
|
+
name="create",
|
|
33
|
+
help="Create a new Calkit object (alias for 'new').",
|
|
30
34
|
)
|
|
31
35
|
app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
|
|
32
36
|
app.add_typer(list_app, name="list", help="List Calkit objects.")
|
|
@@ -45,6 +49,53 @@ def main(
|
|
|
45
49
|
raise typer.Exit()
|
|
46
50
|
|
|
47
51
|
|
|
52
|
+
@app.command(name="clone")
|
|
53
|
+
def clone(
|
|
54
|
+
url: Annotated[str, typer.Argument(help="Repo URL.")],
|
|
55
|
+
location: Annotated[
|
|
56
|
+
str,
|
|
57
|
+
typer.Argument(
|
|
58
|
+
help="Location to clone to (default will be ./{repo_name})"
|
|
59
|
+
),
|
|
60
|
+
] = None,
|
|
61
|
+
no_config_remote: Annotated[
|
|
62
|
+
bool,
|
|
63
|
+
typer.Option(
|
|
64
|
+
"--no-config-remote",
|
|
65
|
+
help="Do not automatically configure Calkit DVC remote.",
|
|
66
|
+
),
|
|
67
|
+
] = False,
|
|
68
|
+
no_dvc_pull: Annotated[
|
|
69
|
+
bool, typer.Option("--no-dvc-pull", help="Do not pull DVC objects.")
|
|
70
|
+
] = False,
|
|
71
|
+
):
|
|
72
|
+
"""Clone a Git repo and by default configure and pull from the DVC
|
|
73
|
+
remote.
|
|
74
|
+
"""
|
|
75
|
+
# Git clone
|
|
76
|
+
cmd = ["git", "clone", url]
|
|
77
|
+
if location is not None:
|
|
78
|
+
cmd.append(location)
|
|
79
|
+
try:
|
|
80
|
+
subprocess.call(cmd)
|
|
81
|
+
except Exception as e:
|
|
82
|
+
raise_error(str(e))
|
|
83
|
+
if location is None:
|
|
84
|
+
location = url.split("/")[-1].removesuffix(".git")
|
|
85
|
+
typer.echo(f"Moving into repo dir: {location}")
|
|
86
|
+
os.chdir(location)
|
|
87
|
+
# Setup auth for any Calkit remotes
|
|
88
|
+
if not no_config_remote:
|
|
89
|
+
remotes = calkit.dvc.get_remotes()
|
|
90
|
+
for name, url in remotes.items():
|
|
91
|
+
if name == "calkit" or name.startswith("calkit:"):
|
|
92
|
+
typer.echo(f"Setting up authentication for DVC remote: {name}")
|
|
93
|
+
calkit.dvc.set_remote_auth(remote_name=name)
|
|
94
|
+
# DVC pull
|
|
95
|
+
if not no_dvc_pull:
|
|
96
|
+
subprocess.call(["dvc", "pull"])
|
|
97
|
+
|
|
98
|
+
|
|
48
99
|
@app.command(name="status")
|
|
49
100
|
def get_status():
|
|
50
101
|
"""Get a unified Git and DVC status."""
|
|
@@ -96,8 +147,7 @@ def add(
|
|
|
96
147
|
adding any .dvc files to Git when adding to DVC.
|
|
97
148
|
"""
|
|
98
149
|
if to is not None and to not in ["git", "dvc"]:
|
|
99
|
-
|
|
100
|
-
raise typer.Exit(1)
|
|
150
|
+
raise_error(f"Invalid option for 'to': {to}")
|
|
101
151
|
# Ensure autostage is enabled for DVC
|
|
102
152
|
subprocess.call(["dvc", "config", "core.autostage", "true"])
|
|
103
153
|
subprocess.call(["git", "add", ".dvc/config"])
|
|
@@ -116,13 +166,11 @@ def add(
|
|
|
116
166
|
]
|
|
117
167
|
dvc_size_thresh_bytes = 1_000_000
|
|
118
168
|
if "." in paths and to is None:
|
|
119
|
-
|
|
120
|
-
raise typer.Exit(1)
|
|
169
|
+
raise_error("Cannot add '.' with calkit; use git or dvc")
|
|
121
170
|
if to is None:
|
|
122
171
|
for path in paths:
|
|
123
172
|
if os.path.isdir(path):
|
|
124
|
-
|
|
125
|
-
raise typer.Exit(1)
|
|
173
|
+
raise_error("Cannot auto-add directories; use git or dvc")
|
|
126
174
|
repo = git.Repo()
|
|
127
175
|
for path in paths:
|
|
128
176
|
# Detect if this file should be tracked with Git or DVC
|
|
@@ -322,8 +370,7 @@ def run_dvc_repro(
|
|
|
322
370
|
subprocess.call(["dvc", "repro"] + args)
|
|
323
371
|
# Now parse stage metadata for calkit objects
|
|
324
372
|
if not os.path.isfile("dvc.yaml"):
|
|
325
|
-
|
|
326
|
-
raise typer.Exit(1)
|
|
373
|
+
raise_error("No dvc.yaml file found")
|
|
327
374
|
objects = []
|
|
328
375
|
with open("dvc.yaml") as f:
|
|
329
376
|
pipeline = calkit.ryaml.load(f)
|
|
@@ -331,21 +378,18 @@ def run_dvc_repro(
|
|
|
331
378
|
ckmeta = stage_info.get("meta", {}).get("calkit")
|
|
332
379
|
if ckmeta is not None:
|
|
333
380
|
if not isinstance(ckmeta, dict):
|
|
334
|
-
|
|
381
|
+
raise_error(
|
|
335
382
|
f"Calkit metadata for {stage_name} is not a dictionary"
|
|
336
383
|
)
|
|
337
|
-
typer.Exit(1)
|
|
338
384
|
# Stage must have a single output
|
|
339
385
|
outs = stage_info.get("outs", [])
|
|
340
386
|
if len(outs) != 1:
|
|
341
|
-
|
|
387
|
+
raise_error(
|
|
342
388
|
f"Stage {stage_name} does not have exactly one output"
|
|
343
389
|
)
|
|
344
|
-
raise typer.Exit(1)
|
|
345
390
|
cktype = ckmeta.get("type")
|
|
346
391
|
if cktype not in ["figure", "dataset", "publication"]:
|
|
347
|
-
|
|
348
|
-
raise typer.Exit(1)
|
|
392
|
+
raise_error(f"Invalid Calkit output type '{cktype}'")
|
|
349
393
|
objects.append(
|
|
350
394
|
dict(path=outs[0]) | ckmeta | dict(stage=stage_name)
|
|
351
395
|
)
|
|
@@ -386,13 +430,6 @@ def manual_step(
|
|
|
386
430
|
),
|
|
387
431
|
],
|
|
388
432
|
cmd: Annotated[str, typer.Option("--cmd", help="Command to run.")] = None,
|
|
389
|
-
shell: Annotated[
|
|
390
|
-
bool,
|
|
391
|
-
typer.Option(
|
|
392
|
-
"--shell",
|
|
393
|
-
help="Whether or not to execute the command in shell mode.",
|
|
394
|
-
),
|
|
395
|
-
] = False,
|
|
396
433
|
show_stdout: Annotated[
|
|
397
434
|
bool, typer.Option("--show-stdout", help="Show stdout.")
|
|
398
435
|
] = False,
|
|
@@ -403,17 +440,17 @@ def manual_step(
|
|
|
403
440
|
if cmd is not None:
|
|
404
441
|
typer.echo(f"Running command: {cmd}")
|
|
405
442
|
subprocess.Popen(
|
|
406
|
-
cmd
|
|
443
|
+
cmd,
|
|
407
444
|
stderr=subprocess.PIPE if not show_stderr else None,
|
|
408
445
|
stdout=subprocess.PIPE if not show_stdout else None,
|
|
409
|
-
shell=
|
|
446
|
+
shell=True,
|
|
410
447
|
)
|
|
411
448
|
input(message + " (press enter to confirm): ")
|
|
412
449
|
typer.echo("Done")
|
|
413
450
|
|
|
414
451
|
|
|
415
452
|
@app.command(
|
|
416
|
-
name="
|
|
453
|
+
name="runenv",
|
|
417
454
|
help="Run a command in an environment.",
|
|
418
455
|
context_settings={"ignore_unknown_options": True},
|
|
419
456
|
)
|
|
@@ -439,21 +476,24 @@ def run_in_env(
|
|
|
439
476
|
ck_info = calkit.load_calkit_info()
|
|
440
477
|
envs = ck_info.get("environments", {})
|
|
441
478
|
if not envs:
|
|
442
|
-
|
|
443
|
-
raise typer.Exit(1)
|
|
479
|
+
raise_error("No environments defined in calkit.yaml")
|
|
444
480
|
if isinstance(envs, list):
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
481
|
+
raise_error("Error: Environments should be a dict, not a list")
|
|
482
|
+
if env_name is None:
|
|
483
|
+
# See if there's a default env, or only one env defined
|
|
484
|
+
default_env_name = None
|
|
485
|
+
for n, e in envs.items():
|
|
486
|
+
if e.get("default"):
|
|
487
|
+
if default_env_name is not None:
|
|
488
|
+
raise_error(
|
|
489
|
+
"Only one default environment can be specified"
|
|
490
|
+
)
|
|
491
|
+
default_env_name = n
|
|
492
|
+
if default_env_name is None and len(envs) == 1:
|
|
493
|
+
default_env_name = list(envs.keys())[0]
|
|
494
|
+
env_name = default_env_name
|
|
455
495
|
if env_name is None:
|
|
456
|
-
|
|
496
|
+
raise_error("Environment must be specified if there are multiple")
|
|
457
497
|
env = envs[env_name]
|
|
458
498
|
cwd = os.getcwd()
|
|
459
499
|
image_name = env.get("image", env_name)
|
|
@@ -483,5 +523,81 @@ def run_in_env(
|
|
|
483
523
|
typer.echo(f"Running command: {cmd}")
|
|
484
524
|
subprocess.call(cmd)
|
|
485
525
|
else:
|
|
486
|
-
|
|
487
|
-
|
|
526
|
+
raise_error("Environment kind not supported")
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
@app.command(
|
|
530
|
+
name="check-call",
|
|
531
|
+
help=(
|
|
532
|
+
"Check that a call to a command succeeds and run another command "
|
|
533
|
+
"if there is an error."
|
|
534
|
+
),
|
|
535
|
+
)
|
|
536
|
+
def check_call(
|
|
537
|
+
cmd: Annotated[str, typer.Argument(help="Command to check.")],
|
|
538
|
+
if_error: Annotated[
|
|
539
|
+
str,
|
|
540
|
+
typer.Option(
|
|
541
|
+
"--if-error", help="Command to run if there is an error."
|
|
542
|
+
),
|
|
543
|
+
],
|
|
544
|
+
):
|
|
545
|
+
try:
|
|
546
|
+
subprocess.check_call(cmd, shell=True)
|
|
547
|
+
typer.echo("Command succeeded")
|
|
548
|
+
except subprocess.CalledProcessError:
|
|
549
|
+
typer.echo("Command failed")
|
|
550
|
+
try:
|
|
551
|
+
typer.echo("Attempting fallback call")
|
|
552
|
+
subprocess.check_call(if_error, shell=True)
|
|
553
|
+
typer.echo("Fallback call succeeded")
|
|
554
|
+
except subprocess.CalledProcessError:
|
|
555
|
+
raise_error("Fallback call failed")
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
@app.command(
|
|
559
|
+
name="build-docker",
|
|
560
|
+
help="Build Docker image if missing or different from lock file.",
|
|
561
|
+
)
|
|
562
|
+
def build_docker(
|
|
563
|
+
tag: Annotated[str, typer.Argument(help="Image tag.")],
|
|
564
|
+
fpath: Annotated[
|
|
565
|
+
str, typer.Option("-i", "--input", help="Path to input Dockerfile.")
|
|
566
|
+
] = "Dockerfile",
|
|
567
|
+
):
|
|
568
|
+
def get_docker_inspect():
|
|
569
|
+
out = json.loads(
|
|
570
|
+
subprocess.check_output(["docker", "inspect", tag]).decode()
|
|
571
|
+
)
|
|
572
|
+
# Remove some keys that can change without the important aspects of
|
|
573
|
+
# the image changing
|
|
574
|
+
_ = out[0].pop("Id")
|
|
575
|
+
_ = out[0].pop("RepoDigests")
|
|
576
|
+
_ = out[0].pop("Metadata")
|
|
577
|
+
return out
|
|
578
|
+
|
|
579
|
+
typer.echo(f"Checking for existing image with tag {tag}")
|
|
580
|
+
# First call Docker inspect
|
|
581
|
+
try:
|
|
582
|
+
inspect = get_docker_inspect()
|
|
583
|
+
except subprocess.CalledProcessError:
|
|
584
|
+
typer.echo(f"No image with tag {tag} found locally")
|
|
585
|
+
inspect = []
|
|
586
|
+
lock_fpath = fpath + "-lock.json"
|
|
587
|
+
rebuild = True
|
|
588
|
+
if os.path.isfile(lock_fpath):
|
|
589
|
+
typer.echo(f"Reading lock file: {lock_fpath}")
|
|
590
|
+
with open(lock_fpath) as f:
|
|
591
|
+
lock = json.load(f)
|
|
592
|
+
else:
|
|
593
|
+
typer.echo(f"Lock file ({lock_fpath}) does not exist")
|
|
594
|
+
lock = None
|
|
595
|
+
if inspect and lock:
|
|
596
|
+
typer.echo("Checking image against lock file")
|
|
597
|
+
rebuild = inspect[0]["RootFS"]["Layers"] != lock[0]["RootFS"]["Layers"]
|
|
598
|
+
if rebuild:
|
|
599
|
+
subprocess.check_call(["docker", "build", "-t", tag, "-f", fpath, "."])
|
|
600
|
+
# Write the lock file
|
|
601
|
+
inspect = get_docker_inspect()
|
|
602
|
+
with open(lock_fpath, "w") as f:
|
|
603
|
+
json.dump(inspect, f, indent=4)
|
|
@@ -10,17 +10,13 @@ import typer
|
|
|
10
10
|
from typing_extensions import Annotated
|
|
11
11
|
|
|
12
12
|
import calkit
|
|
13
|
+
from calkit.cli import raise_error
|
|
13
14
|
from calkit.core import ryaml
|
|
14
15
|
from calkit.docker import LAYERS
|
|
15
16
|
|
|
16
17
|
new_app = typer.Typer(no_args_is_help=True)
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
def _error(txt):
|
|
20
|
-
typer.echo(txt, err=txt)
|
|
21
|
-
raise typer.Exit(1)
|
|
22
|
-
|
|
23
|
-
|
|
24
20
|
@new_app.command(name="figure")
|
|
25
21
|
def new_figure(
|
|
26
22
|
path: str,
|
|
@@ -59,7 +55,7 @@ def new_figure(
|
|
|
59
55
|
help="Stage name from which to add outputs as dependencies.",
|
|
60
56
|
),
|
|
61
57
|
] = None,
|
|
62
|
-
|
|
58
|
+
no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
|
|
63
59
|
overwrite: Annotated[
|
|
64
60
|
bool,
|
|
65
61
|
typer.Option(
|
|
@@ -69,18 +65,18 @@ def new_figure(
|
|
|
69
65
|
),
|
|
70
66
|
] = False,
|
|
71
67
|
):
|
|
72
|
-
"""
|
|
68
|
+
"""Create a new figure."""
|
|
73
69
|
ck_info = calkit.load_calkit_info()
|
|
74
70
|
figures = ck_info.get("figures", [])
|
|
75
71
|
paths = [f.get("path") for f in figures]
|
|
76
72
|
if not overwrite and path in paths:
|
|
77
|
-
|
|
73
|
+
raise_error(f"Figure at path {path} already exists")
|
|
78
74
|
if cmd is not None and stage_name is None:
|
|
79
|
-
|
|
75
|
+
raise_error("Stage name must be provided if command is specified")
|
|
80
76
|
if (deps or outs or outs_from_stage) and not cmd:
|
|
81
|
-
|
|
77
|
+
raise_error("Command must be provided")
|
|
82
78
|
if (deps or outs or outs_from_stage) and not stage_name:
|
|
83
|
-
|
|
79
|
+
raise_error("Stage name must be provided")
|
|
84
80
|
obj = dict(path=path, title=title)
|
|
85
81
|
if description is not None:
|
|
86
82
|
obj["description"] = description
|
|
@@ -91,7 +87,7 @@ def new_figure(
|
|
|
91
87
|
pipeline = calkit.dvc.read_pipeline()
|
|
92
88
|
stages = pipeline.get("stages", {})
|
|
93
89
|
if outs_from_stage not in stages:
|
|
94
|
-
|
|
90
|
+
raise_error(f"Stage {outs_from_stage} does not exist")
|
|
95
91
|
deps += stages[outs_from_stage].get("outs", [])
|
|
96
92
|
if path not in outs:
|
|
97
93
|
outs.append(path)
|
|
@@ -111,7 +107,7 @@ def new_figure(
|
|
|
111
107
|
ck_info["figures"] = figures
|
|
112
108
|
with open("calkit.yaml", "w") as f:
|
|
113
109
|
ryaml.dump(ck_info, f)
|
|
114
|
-
if
|
|
110
|
+
if not no_commit:
|
|
115
111
|
repo = git.Repo()
|
|
116
112
|
repo.git.add("calkit.yaml")
|
|
117
113
|
if cmd:
|
|
@@ -183,9 +179,9 @@ def new_docker_env(
|
|
|
183
179
|
image_name: Annotated[
|
|
184
180
|
str,
|
|
185
181
|
typer.Option(
|
|
186
|
-
"--image
|
|
182
|
+
"--image",
|
|
187
183
|
help=(
|
|
188
|
-
"Image
|
|
184
|
+
"Image identifier. Should be unique and descriptive. "
|
|
189
185
|
"Will default to environment name if not specified."
|
|
190
186
|
),
|
|
191
187
|
),
|
|
@@ -200,10 +196,10 @@ def new_docker_env(
|
|
|
200
196
|
path: Annotated[
|
|
201
197
|
str, typer.Option("--path", help="Dockerfile path.")
|
|
202
198
|
] = "Dockerfile",
|
|
203
|
-
|
|
199
|
+
stage: Annotated[
|
|
204
200
|
str,
|
|
205
201
|
typer.Option(
|
|
206
|
-
"--
|
|
202
|
+
"--stage", help="DVC pipeline stage name, if built in one."
|
|
207
203
|
),
|
|
208
204
|
] = None,
|
|
209
205
|
layers: Annotated[
|
|
@@ -226,27 +222,24 @@ def new_docker_env(
|
|
|
226
222
|
help="Overwrite any existing environment with this name.",
|
|
227
223
|
),
|
|
228
224
|
] = False,
|
|
225
|
+
no_commit: Annotated[
|
|
226
|
+
bool, typer.Option("--no-commit", help="Do not commit changes.")
|
|
227
|
+
] = False,
|
|
229
228
|
):
|
|
230
229
|
"""Create a new Docker environment."""
|
|
231
230
|
if base and os.path.isfile(path) and not overwrite:
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
)
|
|
235
|
-
raise typer.Exit(1)
|
|
236
|
-
if create_stage and not base:
|
|
237
|
-
typer.echo(
|
|
238
|
-
"--from must be specified when creating a build stage", err=True
|
|
239
|
-
)
|
|
240
|
-
raise typer.Exit(1)
|
|
231
|
+
raise_error("Output path already exists (use -f to overwrite)")
|
|
232
|
+
if stage and not base:
|
|
233
|
+
raise_error("--from must be specified when creating a build stage")
|
|
241
234
|
if image_name is None:
|
|
242
235
|
typer.echo("No image name specified; using environment name")
|
|
243
236
|
image_name = name
|
|
237
|
+
repo = git.Repo()
|
|
244
238
|
if base:
|
|
245
239
|
txt = "FROM " + base + "\n\n"
|
|
246
240
|
for layer in layers:
|
|
247
241
|
if layer not in LAYERS:
|
|
248
|
-
|
|
249
|
-
raise typer.Exit(1)
|
|
242
|
+
raise_error(f"Unknown layer type '{layer}'")
|
|
250
243
|
txt += LAYERS[layer] + "\n\n"
|
|
251
244
|
txt += f"RUN mkdir {wdir}\n"
|
|
252
245
|
txt += f"WORKDIR {wdir}\n"
|
|
@@ -260,11 +253,12 @@ def new_docker_env(
|
|
|
260
253
|
typer.echo("Converting environments from list to dict")
|
|
261
254
|
envs = {env.pop("name"): env for env in envs}
|
|
262
255
|
if name in envs and not overwrite:
|
|
263
|
-
|
|
256
|
+
raise_error(
|
|
264
257
|
f"Environment with name {name} already exists "
|
|
265
258
|
"(use -f to overwrite)"
|
|
266
259
|
)
|
|
267
|
-
|
|
260
|
+
if base:
|
|
261
|
+
repo.git.add(path)
|
|
268
262
|
typer.echo("Adding environment to calkit.yaml")
|
|
269
263
|
env = dict(
|
|
270
264
|
kind="docker",
|
|
@@ -273,8 +267,8 @@ def new_docker_env(
|
|
|
273
267
|
)
|
|
274
268
|
if base is not None:
|
|
275
269
|
env["path"] = path
|
|
276
|
-
if
|
|
277
|
-
env["stage"] =
|
|
270
|
+
if stage is not None:
|
|
271
|
+
env["stage"] = stage
|
|
278
272
|
if description is not None:
|
|
279
273
|
env["description"] = description
|
|
280
274
|
if layers:
|
|
@@ -284,8 +278,8 @@ def new_docker_env(
|
|
|
284
278
|
with open("calkit.yaml", "w") as f:
|
|
285
279
|
ryaml.dump(ck_info, f)
|
|
286
280
|
# If we're creating a stage, do so with DVC
|
|
287
|
-
if
|
|
288
|
-
typer.echo(f"Creating DVC stage {
|
|
281
|
+
if stage:
|
|
282
|
+
typer.echo(f"Creating DVC stage {stage}")
|
|
289
283
|
subprocess.call(
|
|
290
284
|
[
|
|
291
285
|
"dvc",
|
|
@@ -293,18 +287,19 @@ def new_docker_env(
|
|
|
293
287
|
"add",
|
|
294
288
|
"-f",
|
|
295
289
|
"-n",
|
|
296
|
-
|
|
290
|
+
stage,
|
|
291
|
+
"--always-changed",
|
|
297
292
|
"-d",
|
|
298
293
|
path,
|
|
299
|
-
"--outs-no-cache",
|
|
300
|
-
path
|
|
301
|
-
|
|
302
|
-
f"docker build -t {image_name} -f {path} . "
|
|
303
|
-
"&& docker inspect --format "
|
|
304
|
-
f"'{{{{.Id}}}}' {image_name} > {path}.digest"
|
|
305
|
-
),
|
|
294
|
+
"--outs-persist-no-cache",
|
|
295
|
+
f"{path}-lock.json",
|
|
296
|
+
f"calkit build-docker {image_name} -i {path}",
|
|
306
297
|
]
|
|
307
298
|
)
|
|
299
|
+
repo.git.add("calkit.yaml")
|
|
300
|
+
repo.git.add("dvc.yaml")
|
|
301
|
+
if not no_commit and repo.git.diff("--staged"):
|
|
302
|
+
repo.git.commit(["-m", f"Add Docker environment {name}"])
|
|
308
303
|
|
|
309
304
|
|
|
310
305
|
@new_app.command(name="foreach-stage")
|
|
@@ -330,6 +325,9 @@ def new_foreach_stage(
|
|
|
330
325
|
"--overwrite", "-f", help="Overwrite stage if one already exists."
|
|
331
326
|
),
|
|
332
327
|
] = False,
|
|
328
|
+
no_commit: Annotated[
|
|
329
|
+
bool, typer.Option("--no-commit", help="Do not commit changes.")
|
|
330
|
+
] = False,
|
|
333
331
|
):
|
|
334
332
|
"""Create a new DVC 'foreach' stage.
|
|
335
333
|
|
|
@@ -338,8 +336,7 @@ def new_foreach_stage(
|
|
|
338
336
|
"""
|
|
339
337
|
pipeline = calkit.dvc.read_pipeline()
|
|
340
338
|
if name in pipeline and not overwrite:
|
|
341
|
-
|
|
342
|
-
raise typer.Exit(1)
|
|
339
|
+
raise_error("Stage already exists; use -f to overwrite")
|
|
343
340
|
if "stages" not in pipeline:
|
|
344
341
|
pipeline["stages"] = {}
|
|
345
342
|
pipeline["stages"][name] = dict(
|
|
@@ -352,3 +349,7 @@ def new_foreach_stage(
|
|
|
352
349
|
)
|
|
353
350
|
with open("dvc.yaml", "w") as f:
|
|
354
351
|
calkit.ryaml.dump(pipeline, f)
|
|
352
|
+
repo = git.Repo()
|
|
353
|
+
repo.git.add("dvc.yaml")
|
|
354
|
+
if not no_commit and repo.git.diff("--staged"):
|
|
355
|
+
repo.git.commit(["-m", f"Add foreach stage {name}"])
|
|
@@ -67,9 +67,7 @@ def add_external_remote(owner_name: str, project_name: str):
|
|
|
67
67
|
base_url = calkit.cloud.get_base_url()
|
|
68
68
|
remote_url = f"{base_url}/projects/{owner_name}/{project_name}/dvc"
|
|
69
69
|
remote_name = f"{get_app_name()}:{owner_name}/{project_name}"
|
|
70
|
-
subprocess.call(
|
|
71
|
-
["dvc", "remote", "add", "-f", remote_name, remote_url]
|
|
72
|
-
)
|
|
70
|
+
subprocess.call(["dvc", "remote", "add", "-f", remote_name, remote_url])
|
|
73
71
|
subprocess.call(["dvc", "remote", "modify", remote_name, "auth", "custom"])
|
|
74
72
|
set_remote_auth(remote_name)
|
|
75
73
|
|
|
@@ -79,3 +77,17 @@ def read_pipeline() -> dict:
|
|
|
79
77
|
return {}
|
|
80
78
|
with open("dvc.yaml") as f:
|
|
81
79
|
return calkit.ryaml.load(f)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_remotes() -> dict[str, str]:
|
|
83
|
+
"""Get a dictionary of DVC remotes, keyed by name, with URL as the
|
|
84
|
+
value.
|
|
85
|
+
"""
|
|
86
|
+
out = subprocess.check_output(["dvc", "remote", "list"]).decode().strip()
|
|
87
|
+
if not out:
|
|
88
|
+
return {}
|
|
89
|
+
resp = {}
|
|
90
|
+
for line in out.split("\n"):
|
|
91
|
+
name, url = line.split("\t")
|
|
92
|
+
resp[name] = url
|
|
93
|
+
return resp
|
|
@@ -66,6 +66,7 @@ class Environment(BaseModel):
|
|
|
66
66
|
path: str | None = None
|
|
67
67
|
description: str | None = None
|
|
68
68
|
stage: str | None = None
|
|
69
|
+
default: bool | None = None
|
|
69
70
|
|
|
70
71
|
|
|
71
72
|
class DockerEnvironment(Environment):
|
|
@@ -89,8 +90,24 @@ class Notebook(_CalkitObject):
|
|
|
89
90
|
class ProjectInfo(BaseModel):
|
|
90
91
|
"""All of the project's information or metadata, written to the
|
|
91
92
|
``calkit.yaml`` file.
|
|
93
|
+
|
|
94
|
+
Attributes
|
|
95
|
+
----------
|
|
96
|
+
parent : str
|
|
97
|
+
The project's parent project, if applicable. This should be set if
|
|
98
|
+
the project was created as a copy of another. This is similar to the
|
|
99
|
+
concept of forking, but unlike a fork, a child project's changes
|
|
100
|
+
are not meant to be merged back into the parent.
|
|
101
|
+
The format of this field should be something like
|
|
102
|
+
{owner_name}/{project_name}, e.g., 'someuser/some-project-name'.
|
|
103
|
+
Note that individual objects can be imported from other projects, but
|
|
104
|
+
that doesn't necessarily make them parent projects.
|
|
105
|
+
This is probably not that important of a distinction.
|
|
106
|
+
The real use case is being able to trace where things came from and
|
|
107
|
+
distinguish what has been newly created here.
|
|
92
108
|
"""
|
|
93
109
|
|
|
110
|
+
parent: str | None = None
|
|
94
111
|
questions: list[str] = []
|
|
95
112
|
datasets: list[Dataset] = []
|
|
96
113
|
figures: list[Figure] = []
|
|
@@ -15,7 +15,7 @@ def test_run_in_env(tmp_dir):
|
|
|
15
15
|
subprocess.check_call(
|
|
16
16
|
"calkit new docker-env "
|
|
17
17
|
"--name my-image "
|
|
18
|
-
"--
|
|
18
|
+
"--stage build-image "
|
|
19
19
|
"--from ubuntu "
|
|
20
20
|
"--add-layer mambaforge "
|
|
21
21
|
"--description 'This is a test image'",
|
|
@@ -23,7 +23,7 @@ def test_run_in_env(tmp_dir):
|
|
|
23
23
|
)
|
|
24
24
|
subprocess.check_call("calkit run", shell=True)
|
|
25
25
|
out = (
|
|
26
|
-
subprocess.check_output("calkit
|
|
26
|
+
subprocess.check_output("calkit runenv echo sup", shell=True)
|
|
27
27
|
.decode()
|
|
28
28
|
.strip()
|
|
29
29
|
)
|
|
@@ -33,8 +33,8 @@ def test_run_in_env(tmp_dir):
|
|
|
33
33
|
subprocess.check_call(
|
|
34
34
|
"calkit new docker-env "
|
|
35
35
|
"-n env2 "
|
|
36
|
-
"--image
|
|
37
|
-
"--
|
|
36
|
+
"--image my-image-2 "
|
|
37
|
+
"--stage build-image-2 "
|
|
38
38
|
"--path Dockerfile.2 "
|
|
39
39
|
"--from ubuntu "
|
|
40
40
|
"--add-layer mambaforge "
|
|
@@ -46,11 +46,11 @@ def test_run_in_env(tmp_dir):
|
|
|
46
46
|
pipeline = ryaml.load(f)
|
|
47
47
|
stg = pipeline["stages"]["build-image-2"]
|
|
48
48
|
cmd = stg["cmd"]
|
|
49
|
-
assert "-
|
|
49
|
+
assert "-i Dockerfile.2" in cmd
|
|
50
50
|
subprocess.check_call("calkit run", shell=True)
|
|
51
51
|
with pytest.raises(subprocess.CalledProcessError):
|
|
52
52
|
out = (
|
|
53
|
-
subprocess.check_output("calkit
|
|
53
|
+
subprocess.check_output("calkit runenv echo sup", shell=True)
|
|
54
54
|
.decode()
|
|
55
55
|
.strip()
|
|
56
56
|
)
|
|
@@ -58,7 +58,7 @@ def test_run_in_env(tmp_dir):
|
|
|
58
58
|
subprocess.check_output(
|
|
59
59
|
[
|
|
60
60
|
"calkit",
|
|
61
|
-
"
|
|
61
|
+
"runenv",
|
|
62
62
|
"-n",
|
|
63
63
|
"env2",
|
|
64
64
|
"python",
|
|
@@ -74,13 +74,13 @@ def test_run_in_env(tmp_dir):
|
|
|
74
74
|
subprocess.check_call(
|
|
75
75
|
"calkit new docker-env "
|
|
76
76
|
"--name py3.10 "
|
|
77
|
-
"--image
|
|
77
|
+
"--image python:3.10.15-bookworm "
|
|
78
78
|
"--description 'Just Python.'",
|
|
79
79
|
shell=True,
|
|
80
80
|
)
|
|
81
81
|
out = (
|
|
82
82
|
subprocess.check_output(
|
|
83
|
-
"calkit
|
|
83
|
+
"calkit runenv -n py3.10 python --version", shell=True
|
|
84
84
|
)
|
|
85
85
|
.decode()
|
|
86
86
|
.strip()
|
|
@@ -89,3 +89,25 @@ def test_run_in_env(tmp_dir):
|
|
|
89
89
|
ck_info = calkit.load_calkit_info()
|
|
90
90
|
env = ck_info["environments"]["py3.10"]
|
|
91
91
|
assert env.get("path") is None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_check_call():
|
|
95
|
+
out = (
|
|
96
|
+
subprocess.check_output(
|
|
97
|
+
["calkit", "check-call", "echo sup", "--if-error", "echo yo"]
|
|
98
|
+
)
|
|
99
|
+
.decode()
|
|
100
|
+
.strip()
|
|
101
|
+
.split("\n")
|
|
102
|
+
)
|
|
103
|
+
assert "sup" in out
|
|
104
|
+
assert "yo" not in out
|
|
105
|
+
out = (
|
|
106
|
+
subprocess.check_output(
|
|
107
|
+
["calkit", "check-call", "sup", "--if-error", "echo yo"]
|
|
108
|
+
)
|
|
109
|
+
.decode()
|
|
110
|
+
.strip()
|
|
111
|
+
.split("\n")
|
|
112
|
+
)
|
|
113
|
+
assert "yo" in out
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Tests for the ``dvc`` module."""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
import calkit
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_get_remotes(tmp_dir):
|
|
9
|
+
subprocess.call(["git", "init"])
|
|
10
|
+
assert not calkit.dvc.get_remotes()
|
|
11
|
+
subprocess.call(["dvc", "init"])
|
|
12
|
+
assert not calkit.dvc.get_remotes()
|
|
13
|
+
subprocess.call(["dvc", "remote", "add", "something", "https://sup.com"])
|
|
14
|
+
resp = calkit.dvc.get_remotes()
|
|
15
|
+
assert resp == {"something": "https://sup.com"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|