calkit-python 0.0.7__tar.gz → 0.0.9__tar.gz

{calkit_python-0.0.7 → calkit_python-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.0.7
+Version: 0.0.9
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.0.7 → calkit_python-0.0.9}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.7"
+__version__ = "0.0.9"
 
 from .core import *
 from . import git

calkit_python-0.0.9/calkit/cli/__init__.py

@@ -0,0 +1,6 @@
+from .core import *
+
+
+def run() -> None:
+    from .main import app
+    app()

calkit_python-0.0.9/calkit/cli/config.py

@@ -0,0 +1,42 @@
+"""Config CLI."""
+
+from __future__ import annotations
+
+import typer
+
+from calkit import config
+from calkit.dvc import configure_remote, set_remote_auth
+
+config_app = typer.Typer(no_args_is_help=True)
+
+
+@config_app.command(name="set")
+def set_config_value(key: str, value: str):
+    try:
+        cfg = config.read()
+        cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
+        # Kind of a hack for setting the password computed field
+        # Types have been validated above, so this won't hurt to do again
+        setattr(cfg, key, value)
+    except FileNotFoundError:
+        # TODO: This fails if we try to set password before any config has
+        # been written
+        # Username is fine
+        cfg = config.Settings.model_validate({key: value})
+    cfg.write()
+
+
+@config_app.command(name="get")
+def get_config_value(key: str) -> None:
+    cfg = config.read()
+    val = getattr(cfg, key)
+    if val is not None:
+        print(val)
+    else:
+        print()
+
+
+@config_app.command(name="setup-remote")
+def setup_remote():
+    configure_remote()
+    set_remote_auth()

calkit_python-0.0.9/calkit/cli/core.py

@@ -0,0 +1,22 @@
+"""Core CLI functionality."""
+
+import os
+import pty
+import subprocess
+
+import typer
+
+
+def print_sep(name: str):
+    width = 66
+    txt_width = len(name) + 2
+    buffer_width = (width - txt_width) // 2
+    buffer = "-" * buffer_width
+    typer.echo(f"{buffer} {name} {buffer}")
+
+
+def run_cmd(cmd: list[str]):
+    if os.name == "nt":
+        subprocess.call(cmd)
+    else:
+        pty.spawn(cmd, lambda fd: os.read(fd, 1024))

calkit_python-0.0.9/calkit/cli/list.py

@@ -0,0 +1,70 @@
+"""CLI for listing objects."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import typer
+
+import calkit
+
+list_app = typer.Typer(no_args_is_help=True)
+
+
+def _list_objects(
+    kind: Literal[
+        "notebooks", "datasets", "figures", "references", "publications"
+    ]
+):
+    """List objects.
+
+    TODO: This should probably just use some library to dump YAML to string.
+    """
+    ck_info = calkit.load_calkit_info()
+    objects = ck_info.get(kind, [])
+    for obj in objects:
+        path = obj.pop("path")
+        typer.echo(f"- path: {path}")
+        for k, v in obj.items():
+            if isinstance(v, dict):
+                typer.echo(f"    {k}:")
+                for k1, v1 in v.items():
+                    typer.echo(f"      {k1}: {v1}")
+            elif isinstance(v, list):
+                typer.echo(f"    {k}:")
+                for item in v:
+                    if isinstance(item, dict):
+                        for n, (k1, v1) in enumerate(item.items()):
+                            if n == 0:
+                                typer.echo(f"      - {k1}: {v1}")
+                            else:
+                                typer.echo(f"        {k1}: {v1}")
+                    else:
+                        typer.echo(f"        - {item}")
+            else:
+                typer.echo(f"  {k}: {v}")
+
+
+@list_app.command(name="notebooks")
+def list_notebooks():
+    _list_objects("notebooks")
+
+
+@list_app.command(name="figures")
+def list_figures():
+    _list_objects("figures")
+
+
+@list_app.command(name="datasets")
+def list_datasets():
+    _list_objects("datasets")
+
+
+@list_app.command(name="publications")
+def list_publications():
+    _list_objects("publications")
+
+
+@list_app.command(name="references")
+def list_references():
+    _list_objects("references")

calkit_python-0.0.7/calkit/cli.py → calkit_python-0.0.9/calkit/cli/main.py

@@ -1,21 +1,19 @@
-"""The command line interface."""
+"""Main CLI app."""
 
 from __future__ import annotations
 
 import os
-import pty
 import subprocess
-import sys
 
-import git
 import typer
 from typing_extensions import Annotated, Optional
 
 import calkit
-from calkit.core import ryaml
-
-from . import config
-from .dvc import configure_remote, set_remote_auth
+from calkit.cli import print_sep, run_cmd
+from calkit.cli.config import config_app
+from calkit.cli.list import list_app
+from calkit.cli.new import new_app
+from calkit.cli.notebooks import notebooks_app
 
 app = typer.Typer(
     invoke_without_command=True,
@@ -23,14 +21,12 @@ app = typer.Typer(
     context_settings=dict(help_option_names=["-h", "--help"]),
     pretty_exceptions_show_locals=False,
 )
-config_app = typer.Typer(no_args_is_help=True)
-new_app = typer.Typer(no_args_is_help=True)
-notebooks_app = typer.Typer(no_args_is_help=True)
 app.add_typer(config_app, name="config", help="Configure Calkit.")
 app.add_typer(
     new_app, name="new", help="Add new Calkit object (to calkit.yaml)."
 )
 app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
+app.add_typer(list_app, name="list", help="List Calkit objects.")
 
 
 @app.callback()
@@ -45,55 +41,9 @@ def main(
         raise typer.Exit()
 
 
-@config_app.command(name="set")
-def set_config_value(key: str, value: str):
-    try:
-        cfg = config.read()
-        cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
-        # Kind of a hack for setting the password computed field
-        # Types have been validated above, so this won't hurt to do again
-        setattr(cfg, key, value)
-    except FileNotFoundError:
-        # TODO: This fails if we try to set password before any config has
-        # been written
-        # Username is fine
-        cfg = config.Settings.model_validate({key: value})
-    cfg.write()
-
-
-@config_app.command(name="get")
-def get_config_value(key: str) -> None:
-    cfg = config.read()
-    val = getattr(cfg, key)
-    if val is not None:
-        print(val)
-    else:
-        print()
-
-
-@config_app.command(name="setup-remote")
-def setup_remote():
-    configure_remote()
-    set_remote_auth()
-
-
 @app.command(name="status")
 def get_status():
     """Get a unified Git and DVC status."""
-
-    def print_sep(name: str):
-        width = 66
-        txt_width = len(name) + 2
-        buffer_width = (width - txt_width) // 2
-        buffer = "-" * buffer_width
-        typer.echo(f"{buffer} {name} {buffer}")
-
-    def run_cmd(cmd: list[str]):
-        if os.name == "nt":
-            subprocess.call(cmd)
-        else:
-            pty.spawn(cmd, lambda fd: os.read(fd, 1024))
-
     print_sep("Code (Git)")
     run_cmd(["git", "status"])
     typer.echo()
@@ -104,6 +54,15 @@ def get_status():
     run_cmd(["dvc", "status"])
 
 
+@app.command(name="diff")
+def diff():
+    """Get a unified Git and DVC diff."""
+    print_sep("Code (Git)")
+    run_cmd(["git", "diff"])
+    print_sep("Pipeline (DVC)")
+    run_cmd(["dvc", "diff"])
+
+
 @app.command(name="add")
 def add(
     paths: list[str],
@@ -225,165 +184,6 @@ def push():
     subprocess.call(["dvc", "push"])
 
 
-@new_app.command(name="figure")
-def new_figure(
-    path: str,
-    title: Annotated[str, typer.Option("--title")],
-    description: Annotated[str, typer.Option("--desc")] = None,
-    commit: Annotated[bool, typer.Option("--commit")] = False,
-):
-    """Add a new figure."""
-    if os.path.isfile("calkit.yaml"):
-        with open("calkit.yaml") as f:
-            ck_info = ryaml.load(f)
-    else:
-        ck_info = {}
-    figures = ck_info.get("figures", [])
-    paths = [f.get("path") for f in figures]
-    if path in paths:
-        raise ValueError(f"Figure at path {path} already exists")
-    obj = dict(path=path, title=title)
-    if description is not None:
-        obj["description"] = description
-    figures.append(obj)
-    ck_info["figures"] = figures
-    with open("calkit.yaml", "w") as f:
-        ryaml.dump(ck_info, f)
-    if commit:
-        repo = git.Repo()
-        repo.git.add("calkit.yaml")
-        repo.git.commit(["-m", f"Add figure {path}"])
-
-
-@new_app.command("question")
-def new_question(
-    question: str,
-    commit: Annotated[bool, typer.Option("--commit")] = False,
-):
-    """Add a new question."""
-    if os.path.isfile("calkit.yaml"):
-        with open("calkit.yaml") as f:
-            ck_info = ryaml.load(f)
-    else:
-        ck_info = {}
-    questions = ck_info.get("questions", [])
-    if question in questions:
-        raise ValueError("Question already exists")
-    if not question.endswith("?"):
-        raise ValueError("Questions must end with a question mark")
-    questions.append(question)
-    ck_info["questions"] = questions
-    with open("calkit.yaml", "w") as f:
-        ryaml.dump(ck_info, f)
-    if commit:
-        repo = git.Repo()
-        repo.git.add("calkit.yaml")
-        repo.git.commit(["-m", "Add question"])
-
-
-@new_app.command("notebook")
-def new_notebook(
-    path: Annotated[str, typer.Argument(help="Notebook path (relative)")],
-    title: Annotated[str, typer.Option("--title")],
-    description: Annotated[str, typer.Option("--desc")] = None,
-    commit: Annotated[bool, typer.Option("--commit")] = False,
-):
-    """Add a new notebook."""
-    if os.path.isabs(path):
-        raise ValueError("Path must be relative")
-    if not os.path.isfile(path):
-        raise ValueError("Path is not a file")
-    if not path.endswith(".ipynb"):
-        raise ValueError("Path does not have .ipynb extension")
-    # TODO: Add option to create stages that run `calkit nb clean` and
-    # `calkit nb execute`
-    if os.path.isfile("calkit.yaml"):
-        with open("calkit.yaml") as f:
-            ck_info = ryaml.load(f)
-    else:
-        ck_info = {}
-    notebooks = ck_info.get("notebooks", [])
-    paths = [f.get("path") for f in notebooks]
-    if path in paths:
-        raise ValueError(f"Notebook at path {path} already exists")
-    obj = dict(path=path, title=title)
-    if description is not None:
-        obj["description"] = description
-    notebooks.append(obj)
-    ck_info["notebooks"] = notebooks
-    with open("calkit.yaml", "w") as f:
-        ryaml.dump(ck_info, f)
-    if commit:
-        repo = git.Repo()
-        repo.git.add("calkit.yaml")
-        repo.git.commit(["-m", f"Add notebook {path}"])
-
-
-@notebooks_app.command("clean")
-def clean_notebook_outputs(path: str):
-    """Clean notebook and place a copy in the cleaned notebooks directory.
-
-    This can be useful to use as a preprocessing DVC stage to use a clean
-    notebook as a dependency for a stage that caches and executed notebook.
-    """
-    if os.path.isabs(path):
-        raise ValueError("Path must be relative")
-    fpath_out = os.path.join(".calkit", "notebooks", "cleaned", path)
-    folder = os.path.dirname(fpath_out)
-    os.makedirs(folder, exist_ok=True)
-    subprocess.call(
-        [
-            "jupyter",
-            "nbconvert",
-            path,
-            "--clear-output",
-            "--to",
-            "notebook",
-            "--output",
-            fpath_out,
-        ]
-    )
-
-
-@notebooks_app.command("execute")
-def execute_notebook(
-    path: str,
-    to: Annotated[
-        str, typer.Option("--to", help="Output format ('html' or 'notebook').")
-    ] = "notebook",
-):
-    """Execute notebook and place a copy in the relevant directory.
-
-    This can be useful to use as a preprocessing DVC stage to use a clean
-    notebook as a dependency for a stage that caches and executed notebook.
-    """
-    if os.path.isabs(path):
-        raise ValueError("Path must be relative")
-    if to == "html":
-        subdir = "html"
-        fname_out = path.removesuffix(".ipynb") + ".html"
-    elif to == "notebook":
-        subdir = "executed"
-        fname_out = path
-    else:
-        raise ValueError(f"Invalid output format: '{to}'")
-    fpath_out = os.path.join(".calkit", "notebooks", subdir, fname_out)
-    folder = os.path.dirname(fpath_out)
-    os.makedirs(folder, exist_ok=True)
-    subprocess.call(
-        [
-            "jupyter",
-            "nbconvert",
-            path,
-            "--execute",
-            "--to",
-            to,
-            "--output",
-            fpath_out,
-        ]
-    )
-
-
 @app.command(name="server", help="Run the local server.")
 def run_server():
     import uvicorn
@@ -432,7 +232,9 @@ def run_dvc_repro(
     no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
     no_run_cache: Annotated[bool, typer.Option("--no-run-cache")] = False,
 ):
-    """A simple wrapper for ``dvc repro``."""
+    """A simple wrapper for ``dvc repro`` that will automatically create any
+    necessary Calkit objects from stage metadata.
+    """
     if targets is None:
         targets = []
     args = targets
@@ -462,7 +264,93 @@ def run_dvc_repro(
     if downstream is not None:
         args += downstream
     subprocess.call(["dvc", "repro"] + args)
-
-
-def run() -> None:
-    app()
+    # Now parse stage metadata for calkit objects
+    if not os.path.isfile("dvc.yaml"):
+        typer.echo("No dvc.yaml file found")
+        raise typer.Exit(1)
+    objects = []
+    with open("dvc.yaml") as f:
+        pipeline = calkit.ryaml.load(f)
+        for stage_name, stage_info in pipeline.get("stages", {}).items():
+            ckmeta = stage_info.get("meta", {}).get("calkit")
+            if ckmeta is not None:
+                if not isinstance(ckmeta, dict):
+                    typer.echo(
+                        f"Calkit metadata for {stage_name} is not a dictionary"
+                    )
+                    typer.Exit(1)
+                # Stage must have a single output
+                outs = stage_info.get("outs", [])
+                if len(outs) != 1:
+                    typer.echo(
+                        f"Stage {stage_name} does not have exactly one output"
+                    )
+                    raise typer.Exit(1)
+                cktype = ckmeta.get("type")
+                if cktype not in ["figure", "dataset", "publication"]:
+                    typer.echo(f"Invalid Calkit output type '{cktype}'")
+                    raise typer.Exit(1)
+                objects.append(
+                    dict(path=outs[0]) | ckmeta | dict(stage=stage_name)
+                )
+    # Now that we've extracted Calkit objects from stage metadata, we can put
+    # them into the calkit.yaml file, overwriting objects with the same path
+    ck_info = calkit.load_calkit_info()
+    for obj in objects:
+        cktype = obj.pop("type")
+        cktype_plural = cktype + "s"
+        existing = ck_info.get(cktype_plural, [])
+        new = []
+        added = False
+        for ex_obj in existing:
+            if ex_obj.get("path") == obj["path"]:
+                typer.echo(f"Updating {cktype} {ex_obj['path']}")
+                new.append(obj)
+                added = True
+            else:
+                new.append(ex_obj)
+        if not added:
+            typer.echo(f"Adding new {cktype} {obj['path']}")
+            new.append(obj)
+        ck_info[cktype_plural] = new
+    if not dry:
+        with open("calkit.yaml", "w") as f:
+            calkit.ryaml.dump(ck_info, f)
+        run_cmd(["git", "add", "calkit.yaml"])
+
+
+@app.command(name="manual-step", help="Execute a manual step.")
+def manual_step(
+    message: Annotated[
+        str,
+        typer.Option(
+            "--message",
+            "-m",
+            help="Message to display as a prompt.",
+        ),
+    ],
+    cmd: Annotated[str, typer.Option("--cmd", help="Command to run.")] = None,
+    shell: Annotated[
+        bool,
+        typer.Option(
+            "--shell",
+            help="Whether or not to execute the command in shell mode.",
+        ),
+    ] = False,
+    show_stdout: Annotated[
+        bool, typer.Option("--show-stdout", help="Show stdout.")
+    ] = False,
+    show_stderr: Annotated[
+        bool, typer.Option("--show-stderr", help="Show stderr.")
+    ] = False,
+) -> None:
+    if cmd is not None:
+        typer.echo(f"Running command: {cmd}")
+        subprocess.Popen(
+            cmd.split() if not shell else cmd,
+            stderr=subprocess.PIPE if not show_stderr else None,
+            stdout=subprocess.PIPE if not show_stdout else None,
+            shell=shell,
+        )
+    input(message + " (press enter to confirm): ")
+    typer.echo("Done")

calkit_python-0.0.9/calkit/cli/new.py

@@ -0,0 +1,96 @@
+"""CLI for creating new objects."""
+
+from __future__ import annotations
+
+import os
+
+import git
+import typer
+from typing_extensions import Annotated
+
+import calkit
+from calkit.core import ryaml
+
+new_app = typer.Typer(no_args_is_help=True)
+
+
+@new_app.command(name="figure")
+def new_figure(
+    path: str,
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str, typer.Option("--desc")] = None,
+    commit: Annotated[bool, typer.Option("--commit")] = False,
+):
+    """Add a new figure."""
+    ck_info = calkit.load_calkit_info()
+    figures = ck_info.get("figures", [])
+    paths = [f.get("path") for f in figures]
+    if path in paths:
+        raise ValueError(f"Figure at path {path} already exists")
+    obj = dict(path=path, title=title)
+    if description is not None:
+        obj["description"] = description
+    figures.append(obj)
+    ck_info["figures"] = figures
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if commit:
+        repo = git.Repo()
+        repo.git.add("calkit.yaml")
+        repo.git.commit(["-m", f"Add figure {path}"])
+
+
+@new_app.command("question")
+def new_question(
+    question: str,
+    commit: Annotated[bool, typer.Option("--commit")] = False,
+):
+    """Add a new question."""
+    ck_info = calkit.load_calkit_info()
+    questions = ck_info.get("questions", [])
+    if question in questions:
+        raise ValueError("Question already exists")
+    if not question.endswith("?"):
+        raise ValueError("Questions must end with a question mark")
+    questions.append(question)
+    ck_info["questions"] = questions
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if commit:
+        repo = git.Repo()
+        repo.git.add("calkit.yaml")
+        repo.git.commit(["-m", "Add question"])
+
+
+@new_app.command("notebook")
+def new_notebook(
+    path: Annotated[str, typer.Argument(help="Notebook path (relative)")],
+    title: Annotated[str, typer.Option("--title")],
+    description: Annotated[str, typer.Option("--desc")] = None,
+    commit: Annotated[bool, typer.Option("--commit")] = False,
+):
+    """Add a new notebook."""
+    if os.path.isabs(path):
+        raise ValueError("Path must be relative")
+    if not os.path.isfile(path):
+        raise ValueError("Path is not a file")
+    if not path.endswith(".ipynb"):
+        raise ValueError("Path does not have .ipynb extension")
+    # TODO: Add option to create stages that run `calkit nb clean` and
+    # `calkit nb execute`
+    ck_info = calkit.load_calkit_info()
+    notebooks = ck_info.get("notebooks", [])
+    paths = [f.get("path") for f in notebooks]
+    if path in paths:
+        raise ValueError(f"Notebook at path {path} already exists")
+    obj = dict(path=path, title=title)
+    if description is not None:
+        obj["description"] = description
+    notebooks.append(obj)
+    ck_info["notebooks"] = notebooks
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if commit:
+        repo = git.Repo()
+        repo.git.add("calkit.yaml")
+        repo.git.commit(["-m", f"Add notebook {path}"])

calkit_python-0.0.9/calkit/cli/notebooks.py

@@ -0,0 +1,76 @@
+"""Notebooks CLI."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+
+import typer
+from typing_extensions import Annotated
+
+notebooks_app = typer.Typer(no_args_is_help=True)
+
+
+@notebooks_app.command("clean")
+def clean_notebook_outputs(path: str):
+    """Clean notebook and place a copy in the cleaned notebooks directory.
+
+    This can be useful to use as a preprocessing DVC stage to use a clean
+    notebook as a dependency for a stage that caches and executed notebook.
+    """
+    if os.path.isabs(path):
+        raise ValueError("Path must be relative")
+    fpath_out = os.path.join(".calkit", "notebooks", "cleaned", path)
+    folder = os.path.dirname(fpath_out)
+    os.makedirs(folder, exist_ok=True)
+    subprocess.call(
+        [
+            "jupyter",
+            "nbconvert",
+            path,
+            "--clear-output",
+            "--to",
+            "notebook",
+            "--output",
+            fpath_out,
+        ]
+    )
+
+
+@notebooks_app.command("execute")
+def execute_notebook(
+    path: str,
+    to: Annotated[
+        str, typer.Option("--to", help="Output format ('html' or 'notebook').")
+    ] = "notebook",
+):
+    """Execute notebook and place a copy in the relevant directory.
+
+    This can be useful to use as a preprocessing DVC stage to use a clean
+    notebook as a dependency for a stage that caches and executed notebook.
+    """
+    if os.path.isabs(path):
+        raise ValueError("Path must be relative")
+    if to == "html":
+        subdir = "html"
+        fname_out = path.removesuffix(".ipynb") + ".html"
+    elif to == "notebook":
+        subdir = "executed"
+        fname_out = path
+    else:
+        raise ValueError(f"Invalid output format: '{to}'")
+    fpath_out = os.path.join(".calkit", "notebooks", subdir, fname_out)
+    folder = os.path.dirname(fpath_out)
+    os.makedirs(folder, exist_ok=True)
+    subprocess.call(
+        [
+            "jupyter",
+            "nbconvert",
+            path,
+            "--execute",
+            "--to",
+            to,
+            "--output",
+            fpath_out,
+        ]
+    )

{calkit_python-0.0.7 → calkit_python-0.0.9}/calkit/core.py

@@ -44,3 +44,10 @@ def find_project_dirs(relative=False, max_depth=3) -> list[str]:
             continue
         final_res.append(path)
     return final_res
+
+
+def load_calkit_info() -> dict:
+    if os.path.isfile("calkit.yaml"):
+        with open("calkit.yaml") as f:
+            return ryaml.load(f)
+    return {}