calkit-python 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/__init__.py +1 -1
- calkit/cli/__init__.py +6 -0
- calkit/cli/config.py +42 -0
- calkit/cli/core.py +22 -0
- calkit/cli/list.py +70 -0
- calkit/{cli.py → cli/main.py} +109 -221
- calkit/cli/new.py +96 -0
- calkit/cli/notebooks.py +76 -0
- calkit/core.py +7 -0
- {calkit_python-0.0.7.dist-info → calkit_python-0.0.9.dist-info}/METADATA +1 -1
- calkit_python-0.0.9.dist-info/RECORD +26 -0
- calkit_python-0.0.7.dist-info/RECORD +0 -20
- {calkit_python-0.0.7.dist-info → calkit_python-0.0.9.dist-info}/WHEEL +0 -0
- {calkit_python-0.0.7.dist-info → calkit_python-0.0.9.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.0.7.dist-info → calkit_python-0.0.9.dist-info}/licenses/LICENSE +0 -0
calkit/__init__.py
CHANGED
calkit/cli/__init__.py
ADDED
calkit/cli/config.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Config CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from calkit import config
|
|
8
|
+
from calkit.dvc import configure_remote, set_remote_auth
|
|
9
|
+
|
|
10
|
+
config_app = typer.Typer(no_args_is_help=True)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@config_app.command(name="set")
|
|
14
|
+
def set_config_value(key: str, value: str):
|
|
15
|
+
try:
|
|
16
|
+
cfg = config.read()
|
|
17
|
+
cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
|
|
18
|
+
# Kind of a hack for setting the password computed field
|
|
19
|
+
# Types have been validated above, so this won't hurt to do again
|
|
20
|
+
setattr(cfg, key, value)
|
|
21
|
+
except FileNotFoundError:
|
|
22
|
+
# TODO: This fails if we try to set password before any config has
|
|
23
|
+
# been written
|
|
24
|
+
# Username is fine
|
|
25
|
+
cfg = config.Settings.model_validate({key: value})
|
|
26
|
+
cfg.write()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@config_app.command(name="get")
|
|
30
|
+
def get_config_value(key: str) -> None:
|
|
31
|
+
cfg = config.read()
|
|
32
|
+
val = getattr(cfg, key)
|
|
33
|
+
if val is not None:
|
|
34
|
+
print(val)
|
|
35
|
+
else:
|
|
36
|
+
print()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@config_app.command(name="setup-remote")
|
|
40
|
+
def setup_remote():
|
|
41
|
+
configure_remote()
|
|
42
|
+
set_remote_auth()
|
calkit/cli/core.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Core CLI functionality."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import pty
|
|
5
|
+
import subprocess
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def print_sep(name: str):
|
|
11
|
+
width = 66
|
|
12
|
+
txt_width = len(name) + 2
|
|
13
|
+
buffer_width = (width - txt_width) // 2
|
|
14
|
+
buffer = "-" * buffer_width
|
|
15
|
+
typer.echo(f"{buffer} {name} {buffer}")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def run_cmd(cmd: list[str]):
|
|
19
|
+
if os.name == "nt":
|
|
20
|
+
subprocess.call(cmd)
|
|
21
|
+
else:
|
|
22
|
+
pty.spawn(cmd, lambda fd: os.read(fd, 1024))
|
calkit/cli/list.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""CLI for listing objects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Literal
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
import calkit
|
|
10
|
+
|
|
11
|
+
list_app = typer.Typer(no_args_is_help=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _list_objects(
|
|
15
|
+
kind: Literal[
|
|
16
|
+
"notebooks", "datasets", "figures", "references", "publications"
|
|
17
|
+
]
|
|
18
|
+
):
|
|
19
|
+
"""List objects.
|
|
20
|
+
|
|
21
|
+
TODO: This should probably just use some library to dump YAML to string.
|
|
22
|
+
"""
|
|
23
|
+
ck_info = calkit.load_calkit_info()
|
|
24
|
+
objects = ck_info.get(kind, [])
|
|
25
|
+
for obj in objects:
|
|
26
|
+
path = obj.pop("path")
|
|
27
|
+
typer.echo(f"- path: {path}")
|
|
28
|
+
for k, v in obj.items():
|
|
29
|
+
if isinstance(v, dict):
|
|
30
|
+
typer.echo(f" {k}:")
|
|
31
|
+
for k1, v1 in v.items():
|
|
32
|
+
typer.echo(f" {k1}: {v1}")
|
|
33
|
+
elif isinstance(v, list):
|
|
34
|
+
typer.echo(f" {k}:")
|
|
35
|
+
for item in v:
|
|
36
|
+
if isinstance(item, dict):
|
|
37
|
+
for n, (k1, v1) in enumerate(item.items()):
|
|
38
|
+
if n == 0:
|
|
39
|
+
typer.echo(f" - {k1}: {v1}")
|
|
40
|
+
else:
|
|
41
|
+
typer.echo(f" {k1}: {v1}")
|
|
42
|
+
else:
|
|
43
|
+
typer.echo(f" - {item}")
|
|
44
|
+
else:
|
|
45
|
+
typer.echo(f" {k}: {v}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@list_app.command(name="notebooks")
|
|
49
|
+
def list_notebooks():
|
|
50
|
+
_list_objects("notebooks")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@list_app.command(name="figures")
|
|
54
|
+
def list_figures():
|
|
55
|
+
_list_objects("figures")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@list_app.command(name="datasets")
|
|
59
|
+
def list_datasets():
|
|
60
|
+
_list_objects("datasets")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@list_app.command(name="publications")
|
|
64
|
+
def list_publications():
|
|
65
|
+
_list_objects("publications")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@list_app.command(name="references")
|
|
69
|
+
def list_references():
|
|
70
|
+
_list_objects("references")
|
calkit/{cli.py → cli/main.py}
RENAMED
|
@@ -1,21 +1,19 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Main CLI app."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
|
-
import pty
|
|
7
6
|
import subprocess
|
|
8
|
-
import sys
|
|
9
7
|
|
|
10
|
-
import git
|
|
11
8
|
import typer
|
|
12
9
|
from typing_extensions import Annotated, Optional
|
|
13
10
|
|
|
14
11
|
import calkit
|
|
15
|
-
from calkit.
|
|
16
|
-
|
|
17
|
-
from . import
|
|
18
|
-
from .
|
|
12
|
+
from calkit.cli import print_sep, run_cmd
|
|
13
|
+
from calkit.cli.config import config_app
|
|
14
|
+
from calkit.cli.list import list_app
|
|
15
|
+
from calkit.cli.new import new_app
|
|
16
|
+
from calkit.cli.notebooks import notebooks_app
|
|
19
17
|
|
|
20
18
|
app = typer.Typer(
|
|
21
19
|
invoke_without_command=True,
|
|
@@ -23,14 +21,12 @@ app = typer.Typer(
|
|
|
23
21
|
context_settings=dict(help_option_names=["-h", "--help"]),
|
|
24
22
|
pretty_exceptions_show_locals=False,
|
|
25
23
|
)
|
|
26
|
-
config_app = typer.Typer(no_args_is_help=True)
|
|
27
|
-
new_app = typer.Typer(no_args_is_help=True)
|
|
28
|
-
notebooks_app = typer.Typer(no_args_is_help=True)
|
|
29
24
|
app.add_typer(config_app, name="config", help="Configure Calkit.")
|
|
30
25
|
app.add_typer(
|
|
31
26
|
new_app, name="new", help="Add new Calkit object (to calkit.yaml)."
|
|
32
27
|
)
|
|
33
28
|
app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
|
|
29
|
+
app.add_typer(list_app, name="list", help="List Calkit objects.")
|
|
34
30
|
|
|
35
31
|
|
|
36
32
|
@app.callback()
|
|
@@ -45,55 +41,9 @@ def main(
|
|
|
45
41
|
raise typer.Exit()
|
|
46
42
|
|
|
47
43
|
|
|
48
|
-
@config_app.command(name="set")
|
|
49
|
-
def set_config_value(key: str, value: str):
|
|
50
|
-
try:
|
|
51
|
-
cfg = config.read()
|
|
52
|
-
cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
|
|
53
|
-
# Kind of a hack for setting the password computed field
|
|
54
|
-
# Types have been validated above, so this won't hurt to do again
|
|
55
|
-
setattr(cfg, key, value)
|
|
56
|
-
except FileNotFoundError:
|
|
57
|
-
# TODO: This fails if we try to set password before any config has
|
|
58
|
-
# been written
|
|
59
|
-
# Username is fine
|
|
60
|
-
cfg = config.Settings.model_validate({key: value})
|
|
61
|
-
cfg.write()
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
@config_app.command(name="get")
|
|
65
|
-
def get_config_value(key: str) -> None:
|
|
66
|
-
cfg = config.read()
|
|
67
|
-
val = getattr(cfg, key)
|
|
68
|
-
if val is not None:
|
|
69
|
-
print(val)
|
|
70
|
-
else:
|
|
71
|
-
print()
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
@config_app.command(name="setup-remote")
|
|
75
|
-
def setup_remote():
|
|
76
|
-
configure_remote()
|
|
77
|
-
set_remote_auth()
|
|
78
|
-
|
|
79
|
-
|
|
80
44
|
@app.command(name="status")
|
|
81
45
|
def get_status():
|
|
82
46
|
"""Get a unified Git and DVC status."""
|
|
83
|
-
|
|
84
|
-
def print_sep(name: str):
|
|
85
|
-
width = 66
|
|
86
|
-
txt_width = len(name) + 2
|
|
87
|
-
buffer_width = (width - txt_width) // 2
|
|
88
|
-
buffer = "-" * buffer_width
|
|
89
|
-
typer.echo(f"{buffer} {name} {buffer}")
|
|
90
|
-
|
|
91
|
-
def run_cmd(cmd: list[str]):
|
|
92
|
-
if os.name == "nt":
|
|
93
|
-
subprocess.call(cmd)
|
|
94
|
-
else:
|
|
95
|
-
pty.spawn(cmd, lambda fd: os.read(fd, 1024))
|
|
96
|
-
|
|
97
47
|
print_sep("Code (Git)")
|
|
98
48
|
run_cmd(["git", "status"])
|
|
99
49
|
typer.echo()
|
|
@@ -104,6 +54,15 @@ def get_status():
|
|
|
104
54
|
run_cmd(["dvc", "status"])
|
|
105
55
|
|
|
106
56
|
|
|
57
|
+
@app.command(name="diff")
|
|
58
|
+
def diff():
|
|
59
|
+
"""Get a unified Git and DVC diff."""
|
|
60
|
+
print_sep("Code (Git)")
|
|
61
|
+
run_cmd(["git", "diff"])
|
|
62
|
+
print_sep("Pipeline (DVC)")
|
|
63
|
+
run_cmd(["dvc", "diff"])
|
|
64
|
+
|
|
65
|
+
|
|
107
66
|
@app.command(name="add")
|
|
108
67
|
def add(
|
|
109
68
|
paths: list[str],
|
|
@@ -225,165 +184,6 @@ def push():
|
|
|
225
184
|
subprocess.call(["dvc", "push"])
|
|
226
185
|
|
|
227
186
|
|
|
228
|
-
@new_app.command(name="figure")
|
|
229
|
-
def new_figure(
|
|
230
|
-
path: str,
|
|
231
|
-
title: Annotated[str, typer.Option("--title")],
|
|
232
|
-
description: Annotated[str, typer.Option("--desc")] = None,
|
|
233
|
-
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
234
|
-
):
|
|
235
|
-
"""Add a new figure."""
|
|
236
|
-
if os.path.isfile("calkit.yaml"):
|
|
237
|
-
with open("calkit.yaml") as f:
|
|
238
|
-
ck_info = ryaml.load(f)
|
|
239
|
-
else:
|
|
240
|
-
ck_info = {}
|
|
241
|
-
figures = ck_info.get("figures", [])
|
|
242
|
-
paths = [f.get("path") for f in figures]
|
|
243
|
-
if path in paths:
|
|
244
|
-
raise ValueError(f"Figure at path {path} already exists")
|
|
245
|
-
obj = dict(path=path, title=title)
|
|
246
|
-
if description is not None:
|
|
247
|
-
obj["description"] = description
|
|
248
|
-
figures.append(obj)
|
|
249
|
-
ck_info["figures"] = figures
|
|
250
|
-
with open("calkit.yaml", "w") as f:
|
|
251
|
-
ryaml.dump(ck_info, f)
|
|
252
|
-
if commit:
|
|
253
|
-
repo = git.Repo()
|
|
254
|
-
repo.git.add("calkit.yaml")
|
|
255
|
-
repo.git.commit(["-m", f"Add figure {path}"])
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
@new_app.command("question")
|
|
259
|
-
def new_question(
|
|
260
|
-
question: str,
|
|
261
|
-
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
262
|
-
):
|
|
263
|
-
"""Add a new question."""
|
|
264
|
-
if os.path.isfile("calkit.yaml"):
|
|
265
|
-
with open("calkit.yaml") as f:
|
|
266
|
-
ck_info = ryaml.load(f)
|
|
267
|
-
else:
|
|
268
|
-
ck_info = {}
|
|
269
|
-
questions = ck_info.get("questions", [])
|
|
270
|
-
if question in questions:
|
|
271
|
-
raise ValueError("Question already exists")
|
|
272
|
-
if not question.endswith("?"):
|
|
273
|
-
raise ValueError("Questions must end with a question mark")
|
|
274
|
-
questions.append(question)
|
|
275
|
-
ck_info["questions"] = questions
|
|
276
|
-
with open("calkit.yaml", "w") as f:
|
|
277
|
-
ryaml.dump(ck_info, f)
|
|
278
|
-
if commit:
|
|
279
|
-
repo = git.Repo()
|
|
280
|
-
repo.git.add("calkit.yaml")
|
|
281
|
-
repo.git.commit(["-m", "Add question"])
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
@new_app.command("notebook")
|
|
285
|
-
def new_notebook(
|
|
286
|
-
path: Annotated[str, typer.Argument(help="Notebook path (relative)")],
|
|
287
|
-
title: Annotated[str, typer.Option("--title")],
|
|
288
|
-
description: Annotated[str, typer.Option("--desc")] = None,
|
|
289
|
-
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
290
|
-
):
|
|
291
|
-
"""Add a new notebook."""
|
|
292
|
-
if os.path.isabs(path):
|
|
293
|
-
raise ValueError("Path must be relative")
|
|
294
|
-
if not os.path.isfile(path):
|
|
295
|
-
raise ValueError("Path is not a file")
|
|
296
|
-
if not path.endswith(".ipynb"):
|
|
297
|
-
raise ValueError("Path does not have .ipynb extension")
|
|
298
|
-
# TODO: Add option to create stages that run `calkit nb clean` and
|
|
299
|
-
# `calkit nb execute`
|
|
300
|
-
if os.path.isfile("calkit.yaml"):
|
|
301
|
-
with open("calkit.yaml") as f:
|
|
302
|
-
ck_info = ryaml.load(f)
|
|
303
|
-
else:
|
|
304
|
-
ck_info = {}
|
|
305
|
-
notebooks = ck_info.get("notebooks", [])
|
|
306
|
-
paths = [f.get("path") for f in notebooks]
|
|
307
|
-
if path in paths:
|
|
308
|
-
raise ValueError(f"Notebook at path {path} already exists")
|
|
309
|
-
obj = dict(path=path, title=title)
|
|
310
|
-
if description is not None:
|
|
311
|
-
obj["description"] = description
|
|
312
|
-
notebooks.append(obj)
|
|
313
|
-
ck_info["notebooks"] = notebooks
|
|
314
|
-
with open("calkit.yaml", "w") as f:
|
|
315
|
-
ryaml.dump(ck_info, f)
|
|
316
|
-
if commit:
|
|
317
|
-
repo = git.Repo()
|
|
318
|
-
repo.git.add("calkit.yaml")
|
|
319
|
-
repo.git.commit(["-m", f"Add notebook {path}"])
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
@notebooks_app.command("clean")
|
|
323
|
-
def clean_notebook_outputs(path: str):
|
|
324
|
-
"""Clean notebook and place a copy in the cleaned notebooks directory.
|
|
325
|
-
|
|
326
|
-
This can be useful to use as a preprocessing DVC stage to use a clean
|
|
327
|
-
notebook as a dependency for a stage that caches and executed notebook.
|
|
328
|
-
"""
|
|
329
|
-
if os.path.isabs(path):
|
|
330
|
-
raise ValueError("Path must be relative")
|
|
331
|
-
fpath_out = os.path.join(".calkit", "notebooks", "cleaned", path)
|
|
332
|
-
folder = os.path.dirname(fpath_out)
|
|
333
|
-
os.makedirs(folder, exist_ok=True)
|
|
334
|
-
subprocess.call(
|
|
335
|
-
[
|
|
336
|
-
"jupyter",
|
|
337
|
-
"nbconvert",
|
|
338
|
-
path,
|
|
339
|
-
"--clear-output",
|
|
340
|
-
"--to",
|
|
341
|
-
"notebook",
|
|
342
|
-
"--output",
|
|
343
|
-
fpath_out,
|
|
344
|
-
]
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
@notebooks_app.command("execute")
|
|
349
|
-
def execute_notebook(
|
|
350
|
-
path: str,
|
|
351
|
-
to: Annotated[
|
|
352
|
-
str, typer.Option("--to", help="Output format ('html' or 'notebook').")
|
|
353
|
-
] = "notebook",
|
|
354
|
-
):
|
|
355
|
-
"""Execute notebook and place a copy in the relevant directory.
|
|
356
|
-
|
|
357
|
-
This can be useful to use as a preprocessing DVC stage to use a clean
|
|
358
|
-
notebook as a dependency for a stage that caches and executed notebook.
|
|
359
|
-
"""
|
|
360
|
-
if os.path.isabs(path):
|
|
361
|
-
raise ValueError("Path must be relative")
|
|
362
|
-
if to == "html":
|
|
363
|
-
subdir = "html"
|
|
364
|
-
fname_out = path.removesuffix(".ipynb") + ".html"
|
|
365
|
-
elif to == "notebook":
|
|
366
|
-
subdir = "executed"
|
|
367
|
-
fname_out = path
|
|
368
|
-
else:
|
|
369
|
-
raise ValueError(f"Invalid output format: '{to}'")
|
|
370
|
-
fpath_out = os.path.join(".calkit", "notebooks", subdir, fname_out)
|
|
371
|
-
folder = os.path.dirname(fpath_out)
|
|
372
|
-
os.makedirs(folder, exist_ok=True)
|
|
373
|
-
subprocess.call(
|
|
374
|
-
[
|
|
375
|
-
"jupyter",
|
|
376
|
-
"nbconvert",
|
|
377
|
-
path,
|
|
378
|
-
"--execute",
|
|
379
|
-
"--to",
|
|
380
|
-
to,
|
|
381
|
-
"--output",
|
|
382
|
-
fpath_out,
|
|
383
|
-
]
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
|
|
387
187
|
@app.command(name="server", help="Run the local server.")
|
|
388
188
|
def run_server():
|
|
389
189
|
import uvicorn
|
|
@@ -432,7 +232,9 @@ def run_dvc_repro(
|
|
|
432
232
|
no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
|
|
433
233
|
no_run_cache: Annotated[bool, typer.Option("--no-run-cache")] = False,
|
|
434
234
|
):
|
|
435
|
-
"""A simple wrapper for ``dvc repro
|
|
235
|
+
"""A simple wrapper for ``dvc repro`` that will automatically create any
|
|
236
|
+
necessary Calkit objects from stage metadata.
|
|
237
|
+
"""
|
|
436
238
|
if targets is None:
|
|
437
239
|
targets = []
|
|
438
240
|
args = targets
|
|
@@ -462,7 +264,93 @@ def run_dvc_repro(
|
|
|
462
264
|
if downstream is not None:
|
|
463
265
|
args += downstream
|
|
464
266
|
subprocess.call(["dvc", "repro"] + args)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
267
|
+
# Now parse stage metadata for calkit objects
|
|
268
|
+
if not os.path.isfile("dvc.yaml"):
|
|
269
|
+
typer.echo("No dvc.yaml file found")
|
|
270
|
+
raise typer.Exit(1)
|
|
271
|
+
objects = []
|
|
272
|
+
with open("dvc.yaml") as f:
|
|
273
|
+
pipeline = calkit.ryaml.load(f)
|
|
274
|
+
for stage_name, stage_info in pipeline.get("stages", {}).items():
|
|
275
|
+
ckmeta = stage_info.get("meta", {}).get("calkit")
|
|
276
|
+
if ckmeta is not None:
|
|
277
|
+
if not isinstance(ckmeta, dict):
|
|
278
|
+
typer.echo(
|
|
279
|
+
f"Calkit metadata for {stage_name} is not a dictionary"
|
|
280
|
+
)
|
|
281
|
+
typer.Exit(1)
|
|
282
|
+
# Stage must have a single output
|
|
283
|
+
outs = stage_info.get("outs", [])
|
|
284
|
+
if len(outs) != 1:
|
|
285
|
+
typer.echo(
|
|
286
|
+
f"Stage {stage_name} does not have exactly one output"
|
|
287
|
+
)
|
|
288
|
+
raise typer.Exit(1)
|
|
289
|
+
cktype = ckmeta.get("type")
|
|
290
|
+
if cktype not in ["figure", "dataset", "publication"]:
|
|
291
|
+
typer.echo(f"Invalid Calkit output type '{cktype}'")
|
|
292
|
+
raise typer.Exit(1)
|
|
293
|
+
objects.append(
|
|
294
|
+
dict(path=outs[0]) | ckmeta | dict(stage=stage_name)
|
|
295
|
+
)
|
|
296
|
+
# Now that we've extracted Calkit objects from stage metadata, we can put
|
|
297
|
+
# them into the calkit.yaml file, overwriting objects with the same path
|
|
298
|
+
ck_info = calkit.load_calkit_info()
|
|
299
|
+
for obj in objects:
|
|
300
|
+
cktype = obj.pop("type")
|
|
301
|
+
cktype_plural = cktype + "s"
|
|
302
|
+
existing = ck_info.get(cktype_plural, [])
|
|
303
|
+
new = []
|
|
304
|
+
added = False
|
|
305
|
+
for ex_obj in existing:
|
|
306
|
+
if ex_obj.get("path") == obj["path"]:
|
|
307
|
+
typer.echo(f"Updating {cktype} {ex_obj['path']}")
|
|
308
|
+
new.append(obj)
|
|
309
|
+
added = True
|
|
310
|
+
else:
|
|
311
|
+
new.append(ex_obj)
|
|
312
|
+
if not added:
|
|
313
|
+
typer.echo(f"Adding new {cktype} {obj['path']}")
|
|
314
|
+
new.append(obj)
|
|
315
|
+
ck_info[cktype_plural] = new
|
|
316
|
+
if not dry:
|
|
317
|
+
with open("calkit.yaml", "w") as f:
|
|
318
|
+
calkit.ryaml.dump(ck_info, f)
|
|
319
|
+
run_cmd(["git", "add", "calkit.yaml"])
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@app.command(name="manual-step", help="Execute a manual step.")
|
|
323
|
+
def manual_step(
|
|
324
|
+
message: Annotated[
|
|
325
|
+
str,
|
|
326
|
+
typer.Option(
|
|
327
|
+
"--message",
|
|
328
|
+
"-m",
|
|
329
|
+
help="Message to display as a prompt.",
|
|
330
|
+
),
|
|
331
|
+
],
|
|
332
|
+
cmd: Annotated[str, typer.Option("--cmd", help="Command to run.")] = None,
|
|
333
|
+
shell: Annotated[
|
|
334
|
+
bool,
|
|
335
|
+
typer.Option(
|
|
336
|
+
"--shell",
|
|
337
|
+
help="Whether or not to execute the command in shell mode.",
|
|
338
|
+
),
|
|
339
|
+
] = False,
|
|
340
|
+
show_stdout: Annotated[
|
|
341
|
+
bool, typer.Option("--show-stdout", help="Show stdout.")
|
|
342
|
+
] = False,
|
|
343
|
+
show_stderr: Annotated[
|
|
344
|
+
bool, typer.Option("--show-stderr", help="Show stderr.")
|
|
345
|
+
] = False,
|
|
346
|
+
) -> None:
|
|
347
|
+
if cmd is not None:
|
|
348
|
+
typer.echo(f"Running command: {cmd}")
|
|
349
|
+
subprocess.Popen(
|
|
350
|
+
cmd.split() if not shell else cmd,
|
|
351
|
+
stderr=subprocess.PIPE if not show_stderr else None,
|
|
352
|
+
stdout=subprocess.PIPE if not show_stdout else None,
|
|
353
|
+
shell=shell,
|
|
354
|
+
)
|
|
355
|
+
input(message + " (press enter to confirm): ")
|
|
356
|
+
typer.echo("Done")
|
calkit/cli/new.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""CLI for creating new objects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
import git
|
|
8
|
+
import typer
|
|
9
|
+
from typing_extensions import Annotated
|
|
10
|
+
|
|
11
|
+
import calkit
|
|
12
|
+
from calkit.core import ryaml
|
|
13
|
+
|
|
14
|
+
new_app = typer.Typer(no_args_is_help=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@new_app.command(name="figure")
|
|
18
|
+
def new_figure(
|
|
19
|
+
path: str,
|
|
20
|
+
title: Annotated[str, typer.Option("--title")],
|
|
21
|
+
description: Annotated[str, typer.Option("--desc")] = None,
|
|
22
|
+
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
23
|
+
):
|
|
24
|
+
"""Add a new figure."""
|
|
25
|
+
ck_info = calkit.load_calkit_info()
|
|
26
|
+
figures = ck_info.get("figures", [])
|
|
27
|
+
paths = [f.get("path") for f in figures]
|
|
28
|
+
if path in paths:
|
|
29
|
+
raise ValueError(f"Figure at path {path} already exists")
|
|
30
|
+
obj = dict(path=path, title=title)
|
|
31
|
+
if description is not None:
|
|
32
|
+
obj["description"] = description
|
|
33
|
+
figures.append(obj)
|
|
34
|
+
ck_info["figures"] = figures
|
|
35
|
+
with open("calkit.yaml", "w") as f:
|
|
36
|
+
ryaml.dump(ck_info, f)
|
|
37
|
+
if commit:
|
|
38
|
+
repo = git.Repo()
|
|
39
|
+
repo.git.add("calkit.yaml")
|
|
40
|
+
repo.git.commit(["-m", f"Add figure {path}"])
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@new_app.command("question")
|
|
44
|
+
def new_question(
|
|
45
|
+
question: str,
|
|
46
|
+
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
47
|
+
):
|
|
48
|
+
"""Add a new question."""
|
|
49
|
+
ck_info = calkit.load_calkit_info()
|
|
50
|
+
questions = ck_info.get("questions", [])
|
|
51
|
+
if question in questions:
|
|
52
|
+
raise ValueError("Question already exists")
|
|
53
|
+
if not question.endswith("?"):
|
|
54
|
+
raise ValueError("Questions must end with a question mark")
|
|
55
|
+
questions.append(question)
|
|
56
|
+
ck_info["questions"] = questions
|
|
57
|
+
with open("calkit.yaml", "w") as f:
|
|
58
|
+
ryaml.dump(ck_info, f)
|
|
59
|
+
if commit:
|
|
60
|
+
repo = git.Repo()
|
|
61
|
+
repo.git.add("calkit.yaml")
|
|
62
|
+
repo.git.commit(["-m", "Add question"])
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@new_app.command("notebook")
|
|
66
|
+
def new_notebook(
|
|
67
|
+
path: Annotated[str, typer.Argument(help="Notebook path (relative)")],
|
|
68
|
+
title: Annotated[str, typer.Option("--title")],
|
|
69
|
+
description: Annotated[str, typer.Option("--desc")] = None,
|
|
70
|
+
commit: Annotated[bool, typer.Option("--commit")] = False,
|
|
71
|
+
):
|
|
72
|
+
"""Add a new notebook."""
|
|
73
|
+
if os.path.isabs(path):
|
|
74
|
+
raise ValueError("Path must be relative")
|
|
75
|
+
if not os.path.isfile(path):
|
|
76
|
+
raise ValueError("Path is not a file")
|
|
77
|
+
if not path.endswith(".ipynb"):
|
|
78
|
+
raise ValueError("Path does not have .ipynb extension")
|
|
79
|
+
# TODO: Add option to create stages that run `calkit nb clean` and
|
|
80
|
+
# `calkit nb execute`
|
|
81
|
+
ck_info = calkit.load_calkit_info()
|
|
82
|
+
notebooks = ck_info.get("notebooks", [])
|
|
83
|
+
paths = [f.get("path") for f in notebooks]
|
|
84
|
+
if path in paths:
|
|
85
|
+
raise ValueError(f"Notebook at path {path} already exists")
|
|
86
|
+
obj = dict(path=path, title=title)
|
|
87
|
+
if description is not None:
|
|
88
|
+
obj["description"] = description
|
|
89
|
+
notebooks.append(obj)
|
|
90
|
+
ck_info["notebooks"] = notebooks
|
|
91
|
+
with open("calkit.yaml", "w") as f:
|
|
92
|
+
ryaml.dump(ck_info, f)
|
|
93
|
+
if commit:
|
|
94
|
+
repo = git.Repo()
|
|
95
|
+
repo.git.add("calkit.yaml")
|
|
96
|
+
repo.git.commit(["-m", f"Add notebook {path}"])
|
calkit/cli/notebooks.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Notebooks CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from typing_extensions import Annotated
|
|
10
|
+
|
|
11
|
+
notebooks_app = typer.Typer(no_args_is_help=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@notebooks_app.command("clean")
|
|
15
|
+
def clean_notebook_outputs(path: str):
|
|
16
|
+
"""Clean notebook and place a copy in the cleaned notebooks directory.
|
|
17
|
+
|
|
18
|
+
This can be useful to use as a preprocessing DVC stage to use a clean
|
|
19
|
+
notebook as a dependency for a stage that caches and executed notebook.
|
|
20
|
+
"""
|
|
21
|
+
if os.path.isabs(path):
|
|
22
|
+
raise ValueError("Path must be relative")
|
|
23
|
+
fpath_out = os.path.join(".calkit", "notebooks", "cleaned", path)
|
|
24
|
+
folder = os.path.dirname(fpath_out)
|
|
25
|
+
os.makedirs(folder, exist_ok=True)
|
|
26
|
+
subprocess.call(
|
|
27
|
+
[
|
|
28
|
+
"jupyter",
|
|
29
|
+
"nbconvert",
|
|
30
|
+
path,
|
|
31
|
+
"--clear-output",
|
|
32
|
+
"--to",
|
|
33
|
+
"notebook",
|
|
34
|
+
"--output",
|
|
35
|
+
fpath_out,
|
|
36
|
+
]
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@notebooks_app.command("execute")
|
|
41
|
+
def execute_notebook(
|
|
42
|
+
path: str,
|
|
43
|
+
to: Annotated[
|
|
44
|
+
str, typer.Option("--to", help="Output format ('html' or 'notebook').")
|
|
45
|
+
] = "notebook",
|
|
46
|
+
):
|
|
47
|
+
"""Execute notebook and place a copy in the relevant directory.
|
|
48
|
+
|
|
49
|
+
This can be useful to use as a preprocessing DVC stage to use a clean
|
|
50
|
+
notebook as a dependency for a stage that caches and executed notebook.
|
|
51
|
+
"""
|
|
52
|
+
if os.path.isabs(path):
|
|
53
|
+
raise ValueError("Path must be relative")
|
|
54
|
+
if to == "html":
|
|
55
|
+
subdir = "html"
|
|
56
|
+
fname_out = path.removesuffix(".ipynb") + ".html"
|
|
57
|
+
elif to == "notebook":
|
|
58
|
+
subdir = "executed"
|
|
59
|
+
fname_out = path
|
|
60
|
+
else:
|
|
61
|
+
raise ValueError(f"Invalid output format: '{to}'")
|
|
62
|
+
fpath_out = os.path.join(".calkit", "notebooks", subdir, fname_out)
|
|
63
|
+
folder = os.path.dirname(fpath_out)
|
|
64
|
+
os.makedirs(folder, exist_ok=True)
|
|
65
|
+
subprocess.call(
|
|
66
|
+
[
|
|
67
|
+
"jupyter",
|
|
68
|
+
"nbconvert",
|
|
69
|
+
path,
|
|
70
|
+
"--execute",
|
|
71
|
+
"--to",
|
|
72
|
+
to,
|
|
73
|
+
"--output",
|
|
74
|
+
fpath_out,
|
|
75
|
+
]
|
|
76
|
+
)
|
calkit/core.py
CHANGED
|
@@ -44,3 +44,10 @@ def find_project_dirs(relative=False, max_depth=3) -> list[str]:
|
|
|
44
44
|
continue
|
|
45
45
|
final_res.append(path)
|
|
46
46
|
return final_res
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def load_calkit_info() -> dict:
|
|
50
|
+
if os.path.isfile("calkit.yaml"):
|
|
51
|
+
with open("calkit.yaml") as f:
|
|
52
|
+
return ryaml.load(f)
|
|
53
|
+
return {}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
calkit/__init__.py,sha256=4Jd2HCpYelv6y6AHN-hOa90tNl4CxtZTsdHOUbwtufg,142
|
|
2
|
+
calkit/cloud.py,sha256=CnQKms4mBQo4X8jVLIa-70-3jPDl38zRNFyK5TjiaHM,2209
|
|
3
|
+
calkit/config.py,sha256=NPFRk5inn66Wy44pyx8WppyqvIzkW1sO6VjXwIcCacE,2044
|
|
4
|
+
calkit/core.py,sha256=P9QOEkCIyBr5Xfig_rUeEj02ZZIyBicj3P6K4rad1tU,1383
|
|
5
|
+
calkit/data.py,sha256=DZDkj-BLFWXz56R-A9XZeSKSgXpm3aKtiqD6JstfmnY,1631
|
|
6
|
+
calkit/dvc.py,sha256=tX5172Ahw0mHcuJIrMenz1oGYxCrMzMURJJMHEHenbg,1506
|
|
7
|
+
calkit/git.py,sha256=NLd7A32dPbfVnZLa8xbmviJ-ChGZcdkcd6wcOOnsSKQ,350
|
|
8
|
+
calkit/gui.py,sha256=2UCrMyosc5v4CLlDcHO7oDwW9eLn5_WbNaLaG5bjMTc,23
|
|
9
|
+
calkit/jupyter.py,sha256=S6HUOVafaLQJ7_ZfC1ddBwF3fT5q3GcbouLHdlH-oSo,1034
|
|
10
|
+
calkit/models.py,sha256=ejl032XIrbaCFkO-4VbCaKVQyHqNl7N4WWU20RjeAfc,1447
|
|
11
|
+
calkit/server.py,sha256=gq3643ABKTQu1eVR_OqJ-SdQh2dPLHSFsipmmAK5ttA,5959
|
|
12
|
+
calkit/cli/__init__.py,sha256=2Id-_5dsHbL29qqScLkRUhWXzTgxsZ_A_lMPs9rf5uY,77
|
|
13
|
+
calkit/cli/config.py,sha256=y-TWv150qWvUtlMfySgCyMyk8n5SNL-8hNRm9NycJ98,1098
|
|
14
|
+
calkit/cli/core.py,sha256=N-I2Yw1STgEMN5xLmfOuVq1od1EJx2oMCyVQtNa8Nwg,424
|
|
15
|
+
calkit/cli/list.py,sha256=kxQsq1SQIZEDdWHIvakzdcMhayqJlfiCmWkj09dhXuA,1799
|
|
16
|
+
calkit/cli/main.py,sha256=bn8DpUFw0neBJ4-THTjH-mcULnEJrD8Etx_6S5kdGDA,11370
|
|
17
|
+
calkit/cli/new.py,sha256=WVg9bEfGO8QXJj4fajSrbmQTUJMLAgHsNRTxkFZl8Xo,3088
|
|
18
|
+
calkit/cli/notebooks.py,sha256=GsNmm8pdFkbs_KyZxClfIUThD78GCdu0R5udCXhXc9E,2083
|
|
19
|
+
calkit/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
calkit/tests/test_core.py,sha256=CdRqvYnq3MidyTknvfIbkmmhMfydpyKtTW9hakAvzsc,167
|
|
21
|
+
calkit/tests/test_jupyter.py,sha256=YTL6zI740UM2KUjskSipzvSKxsyQ8rVPFQIX5cb2tMQ,114
|
|
22
|
+
calkit_python-0.0.9.dist-info/METADATA,sha256=NQau3E3P1uQPAAG2v5pxk-CF8sRcbQG931r8Rtxfp4Q,4004
|
|
23
|
+
calkit_python-0.0.9.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
24
|
+
calkit_python-0.0.9.dist-info/entry_points.txt,sha256=tiPfbDDZNB-YStw5YVO_4Gww0W3Dcdy8Mh6muX_GWnY,42
|
|
25
|
+
calkit_python-0.0.9.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
|
|
26
|
+
calkit_python-0.0.9.dist-info/RECORD,,
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
calkit/__init__.py,sha256=uoXIE8mOCRYee4G20-SznuFNUiUdjdR8jJpkpbrYY-c,142
|
|
2
|
-
calkit/cli.py,sha256=IxF2P_sO2J9MiO8mIxYedVfjWWTTKlLYFGZqYXjPIxI,14112
|
|
3
|
-
calkit/cloud.py,sha256=CnQKms4mBQo4X8jVLIa-70-3jPDl38zRNFyK5TjiaHM,2209
|
|
4
|
-
calkit/config.py,sha256=NPFRk5inn66Wy44pyx8WppyqvIzkW1sO6VjXwIcCacE,2044
|
|
5
|
-
calkit/core.py,sha256=qLNjBwlECzMitL85pMGIHzne4j7FpQAR9-ukYtdIMig,1225
|
|
6
|
-
calkit/data.py,sha256=DZDkj-BLFWXz56R-A9XZeSKSgXpm3aKtiqD6JstfmnY,1631
|
|
7
|
-
calkit/dvc.py,sha256=tX5172Ahw0mHcuJIrMenz1oGYxCrMzMURJJMHEHenbg,1506
|
|
8
|
-
calkit/git.py,sha256=NLd7A32dPbfVnZLa8xbmviJ-ChGZcdkcd6wcOOnsSKQ,350
|
|
9
|
-
calkit/gui.py,sha256=2UCrMyosc5v4CLlDcHO7oDwW9eLn5_WbNaLaG5bjMTc,23
|
|
10
|
-
calkit/jupyter.py,sha256=S6HUOVafaLQJ7_ZfC1ddBwF3fT5q3GcbouLHdlH-oSo,1034
|
|
11
|
-
calkit/models.py,sha256=ejl032XIrbaCFkO-4VbCaKVQyHqNl7N4WWU20RjeAfc,1447
|
|
12
|
-
calkit/server.py,sha256=gq3643ABKTQu1eVR_OqJ-SdQh2dPLHSFsipmmAK5ttA,5959
|
|
13
|
-
calkit/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
-
calkit/tests/test_core.py,sha256=CdRqvYnq3MidyTknvfIbkmmhMfydpyKtTW9hakAvzsc,167
|
|
15
|
-
calkit/tests/test_jupyter.py,sha256=YTL6zI740UM2KUjskSipzvSKxsyQ8rVPFQIX5cb2tMQ,114
|
|
16
|
-
calkit_python-0.0.7.dist-info/METADATA,sha256=IrdVfoHNAjkHwlKb3s_SsjMujO9QTKLi9lzY4Q3n9kE,4004
|
|
17
|
-
calkit_python-0.0.7.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
18
|
-
calkit_python-0.0.7.dist-info/entry_points.txt,sha256=tiPfbDDZNB-YStw5YVO_4Gww0W3Dcdy8Mh6muX_GWnY,42
|
|
19
|
-
calkit_python-0.0.7.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
|
|
20
|
-
calkit_python-0.0.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|