calkit-python 0.17.6__tar.gz → 0.19.0__tar.gz

{calkit_python-0.17.6 → calkit_python-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.17.6
+Version: 0.19.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.17.6"
+__version__ = "0.19.0"
 
 from .core import *
 from . import git

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/cli/config.py

@@ -65,13 +65,14 @@ def setup_remote(
         raise_error("DVC remote config failed; have you run `dvc init`?")
     except InvalidGitRepositoryError:
         raise_error("Current directory is not a Git repository")
+    except ValueError as e:
+        raise_error(e)
     if not no_commit:
         repo = git.Repo()
-        repo.git.reset()
         repo.git.add(".dvc/config")
-        if calkit.git.get_staged_files():
+        if ".dvc/config" in calkit.git.get_staged_files():
             typer.echo("Committing changes to DVC config")
-            repo.git.commit(["-m", "Set DVC remote"])
+            repo.git.commit([".dvc/config", "-m", "Set DVC remote"])
 
 
 @config_app.command(name="setup-remote-auth", help="Alias for 'remote-auth'.")

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/cli/core.py

@@ -11,7 +11,10 @@ def print_sep(name: str):
     txt_width = len(name) + 2
     buffer_width = (width - txt_width) // 2
     buffer = "-" * buffer_width
-    typer.echo(f"{buffer} {name} {buffer}")
+    line = f"{buffer} {name} {buffer}"
+    if len(line) == (width - 1):
+        line += "-"
+    typer.echo(line)
 
 
 def run_cmd(cmd: list[str]):

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/cli/main.py

@@ -208,6 +208,23 @@ def clone(
 @app.command(name="status")
 def get_status():
     """Get a unified Git and DVC status."""
+    print_sep("Project")
+    # Print latest status
+    status = calkit.get_latest_project_status()
+    if status is not None:
+        ts = status.timestamp.strftime("%Y-%m-%d %H:%M:%S")
+        colors = {
+            "in-progress": "blue",
+            "on-hold": "yellow",
+            "completed": "green",
+        }
+        status_txt = typer.style(status.status, fg=colors.get(status.status))
+        typer.echo(f"Current status: {status_txt} (updated {ts} UTC)")
+    else:
+        typer.echo(
+            'Project status not set. Use "calkit new status" to update.'
+        )
+    typer.echo()
     print_sep("Code (Git)")
     run_cmd(["git", "status"])
     typer.echo()
@@ -1047,6 +1064,17 @@ def run_in_env(
         jobs[job_key] = job
         with open(jobs_fpath, "w") as f:
             calkit.ryaml.dump(jobs, f)
+    elif env["kind"] == "renv":
+        try:
+            subprocess.check_call(
+                ["Rscript", "-e", "'renv::restore()'"], cwd=wdir
+            )
+        except subprocess.CalledProcessError:
+            raise_error("Failed to check renv")
+        try:
+            subprocess.check_call(cmd, cwd=wdir)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in renv")
     else:
         raise_error("Environment kind not supported")
 

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/cli/new.py

@@ -2,13 +2,15 @@
 
 from __future__ import annotations
 
+import csv
 import os
 import shutil
 import subprocess
+from enum import Enum
 
 import git
 import typer
-from git.exc import InvalidGitRepositoryError
+from git.exc import GitCommandError, InvalidGitRepositoryError
 from typing_extensions import Annotated
 
 import calkit
@@ -95,10 +97,10 @@ def new_project(
         f"Check out the docs at {docs_url}."
     )
     abs_path = os.path.abspath(path)
-    if (cloud or template) and os.path.exists(abs_path):
-        raise_error(
-            "Must specify a new directory if using --cloud or --template"
-        )
+    if template and os.path.exists(abs_path):
+        raise_error("Must specify a new directory if using --template")
+    if cloud and os.path.isdir(os.path.join(abs_path, ".git")):
+        raise_error("Must not already be a Git repo to use --cloud")
     ck_info_fpath = os.path.join(abs_path, "calkit.yaml")
     if os.path.isfile(ck_info_fpath) and not overwrite:
         raise_error(
@@ -118,8 +120,8 @@ def new_project(
     typer.echo(f"Using title: {title}")
     if cloud:
         # Cloud should allow None, which will allow us to post just the name
-        # NOTE: This will fail if the user hasn't logged into GitHub in a
-        # while, and their token stored in the Calkit cloud is expired
+        # NOTE: This will fail if the user hasn't logged into the Calkit Cloud
+        # in 6 months, since their GitHub refresh token stored is expired
         try:
             resp = calkit.cloud.post(
                 "/projects",
@@ -134,8 +136,21 @@ def new_project(
             )
         except Exception as e:
             raise_error(f"Posting new project to cloud failed: {e}")
-        # Now clone here and that's about
-        subprocess.run(["git", "clone", resp["git_repo_url"], abs_path])
+        # Now clone here
+        if not os.path.isdir(abs_path):
+            subprocess.run(["git", "clone", resp["git_repo_url"], abs_path])
+        else:
+            typer.echo("Fetching from newly create Git repo")
+            repo = git.Repo.init(abs_path, initial_branch="main")
+            repo.git.remote(["add", "origin", resp["git_repo_url"]])
+            repo.git.fetch()
+            checkout_cmd = ["-t", "origin/main"]
+            if overwrite:
+                checkout_cmd.append("--force")
+            try:
+                repo.git.checkout(checkout_cmd)
+            except GitCommandError as e:
+                raise_error(f"Failed to check out main branch: {e}")
         try:
             calkit.dvc.set_remote_auth(wdir=abs_path)
         except Exception:
@@ -1231,3 +1246,191 @@ def new_pixi_env(
     repo.git.add("calkit.yaml")
     if not no_commit and repo.git.diff("--staged"):
         repo.git.commit(["-m", f"Add pixi env {name}"])
+
+
+class Status(str, Enum):
+    in_progress = "in-progress"
+    on_hold = "on-hold"
+    completed = "completed"
+
+
+@new_app.command(name="status")
+def new_status(
+    status: Annotated[
+        Status,
+        typer.Argument(help="Current status of the project."),
+    ],
+    message: Annotated[
+        str,
+        typer.Option(
+            "--message",
+            "-m",
+            help="Optional message describing the status.",
+        ),
+    ] = "",
+    no_commit: Annotated[
+        bool,
+        typer.Option(
+            "--no-commit", help="Do not commit changes to the status log."
+        ),
+    ] = False,
+):
+    """Add a new project status to the log."""
+    typer.echo(f"Adding {status.value} status log entry")
+    fpath = os.path.join(".calkit", "status.csv")
+    os.makedirs(".calkit", exist_ok=True)
+    now = calkit.utcnow(remove_tz=False)
+    # Append to end of CSV
+    write_header = not os.path.isfile(fpath)
+    with open(fpath, "a") as f:
+        writer = csv.writer(f)
+        if write_header:
+            writer.writerow(["timestamp", "status", "message"])
+        writer.writerow([now.isoformat(), status.value, message])
+    if not no_commit:
+        typer.echo("Committing")
+        repo = git.Repo()
+        repo.git.add(fpath)
+        repo.git.commit([fpath, "-m", f"Add {status.value} status log entry"])
+
+
+class StageKind(str, Enum):
+    python_script = "python-script"
+    latex = "latex"
+    r_script = "r-script"
+    sh_script = "sh-script"
+    bash_script = "bash-script"
+    zsh_script = "zsh-script"
+    matlab_script = "matlab-script"
+
+
+@new_app.command(name="stage")
+def new_stage(
+    name: Annotated[
+        str,
+        typer.Option("--name", "-n", help="Stage name, typically kebab-case."),
+    ],
+    kind: Annotated[
+        StageKind, typer.Option("--kind", help="What kind of stage to create.")
+    ],
+    target: Annotated[
+        str,
+        typer.Option(
+            "--target", "-t", help="Target file, e.g., the script to run."
+        ),
+    ],
+    environment: Annotated[
+        str,
+        typer.Option(
+            "--environment", "-e", help="Environment to use to run the stage."
+        ),
+    ] = None,
+    deps: Annotated[
+        list[str],
+        typer.Option("--dep", "-d", help="A path on which the stage depends."),
+    ] = [],
+    outs: Annotated[
+        list[str],
+        typer.Option(
+            "--out", "-o", help="A path that is produced by the stage."
+        ),
+    ] = [],
+    outs_persist: Annotated[
+        list[str],
+        typer.Option(
+            "--out-persist",
+            help="An output that should not be deleted before running.",
+        ),
+    ] = [],
+    outs_no_cache: Annotated[
+        list[str],
+        typer.Option(
+            "--out-git",
+            help="An output that should be tracked with Git instead of DVC.",
+        ),
+    ] = [],
+    outs_persist_no_cache: Annotated[
+        list[str],
+        typer.Option(
+            "--out-git-persist",
+            help=(
+                "An output that should be tracked with Git instead of DVC, "
+                "and also should not be deleted before running stage."
+            ),
+        ),
+    ] = [],
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "--force",
+            "-f",
+            help="Overwrite an existing stage with this name if necessary.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes to Git.")
+    ] = False,
+):
+    """Create a new pipeline stage."""
+    ck_info = calkit.load_calkit_info()
+    if environment is None:
+        warn("No environment is specified")
+        cmd = ""
+    else:
+        if environment not in ck_info["environments"]:
+            raise_error(f"Environment '{environment}' does not exist")
+        cmd = f"calkit xenv -n {environment} -- "
+        # Add environment path as a dependency
+        env_path = ck_info["environments"][environment].get("path")
+        if env_path is not None:
+            deps = [env_path] + deps
+    if not os.path.exists(target):
+        raise_error(f"Target '{target}' does not exist")
+    if kind.value == "python-script":
+        cmd += f"python {target}"
+    elif kind.value == "latex":
+        cmd += f"latexmk -cd -interaction=nonstopmode -pdf {target}"
+        out_target = target.removesuffix(".tex") + ".pdf"
+        if out_target not in (
+            outs + outs_no_cache + outs_persist + outs_persist_no_cache
+        ):
+            outs = [out_target] + outs
+    elif kind.value == "matlab-script":
+        cmd += f"matlab -noFigureWindows -batch \"run('{target}');\""
+    elif kind.value == "sh-script":
+        cmd += f"sh {target}"
+    elif kind.value == "bash-script":
+        cmd += f"bash {target}"
+    elif kind.value == "zsh-script":
+        cmd += f"zsh {target}"
+    elif kind.value == "r-script":
+        cmd += f"Rscript {target}"
+    add_cmd = ["dvc", "stage", "add", "-n", name]
+    for dep in [target] + deps:
+        add_cmd += ["-d", dep]
+    for out in outs:
+        add_cmd += ["-o", out]
+    for out in outs_no_cache:
+        add_cmd += ["--outs-no-cache", out]
+    for out in outs_persist:
+        add_cmd += ["--outs-persist", out]
+    for out in outs_persist_no_cache:
+        add_cmd += ["--outs-persist-no-cache", out]
+    if overwrite:
+        add_cmd.append("-f")
+    add_cmd.append(cmd)
+    try:
+        subprocess.check_call(add_cmd)
+    except subprocess.CalledProcessError:
+        raise_error("Failed to create stage")
+    if not no_commit:
+        try:
+            repo = git.Repo()
+        except InvalidGitRepositoryError:
+            raise_error("Can't commit because this is not a Git repo")
+        repo.git.add("dvc.yaml")
+        if "dvc.yaml" in calkit.git.get_staged_files():
+            repo.git.commit(
+                ["dvc.yaml", "-m", f"Add {kind.value} pipeline stage '{name}'"]
+            )

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/core.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import base64
+import csv
 import glob
 import json
 import logging
@@ -11,6 +12,8 @@ import pickle
 import re
 import subprocess
 
+from calkit.models import ProjectStatus
+
 import requests
 
 try:
@@ -385,3 +388,32 @@ def get_size(path: str):
 def to_kebab_case(str) -> str:
     """Convert a string to kebab-case."""
     return re.sub(r"[-_,\.\ ]", "-", str.lower())
+
+
+def get_project_status_history(
+    wdir: str = None, as_pydantic=True
+) -> list[ProjectStatus] | list[dict]:
+    statuses = []
+    fpath = os.path.join(".calkit", "status.csv")
+    if wdir is not None:
+        fpath = os.path.join(wdir, fpath)
+    if os.path.isfile(fpath):
+        with open(fpath) as f:
+            reader = csv.reader(f)
+            next(reader, None)  # Skip header row
+            for line in reader:
+                ts, status, message = line
+                ts = datetime.fromisoformat(ts)
+                obj = ProjectStatus(
+                    timestamp=ts, status=status, message=message
+                )
+                if not as_pydantic:
+                    obj = obj.model_dump()
+                statuses.append(obj)
+    return statuses
+
+
+def get_latest_project_status(wdir: str = None) -> ProjectStatus | None:
+    statuses = get_project_status_history(wdir=wdir)
+    if statuses:
+        return statuses[-1]

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/dvc.py

@@ -7,8 +7,10 @@ import os
 import subprocess
 
 import dvc.repo
+import git
 
 import calkit
+from calkit.cli import warn
 from calkit.config import get_app_name
 
 logger = logging.getLogger(__package__)
@@ -16,7 +18,25 @@ logger.setLevel(logging.INFO)
 
 
 def configure_remote(wdir: str = None):
-    project_name = calkit.git.detect_project_name(path=wdir)
+    try:
+        project_name = calkit.git.detect_project_name(path=wdir)
+    except ValueError as e:
+        raise ValueError(f"Can't detect project name: {e}")
+    # If Git origin is not set, set that
+    repo = git.Repo(wdir)
+    try:
+        repo.remote()
+    except ValueError:
+        warn("No Git remote defined; querying Calkit Cloud")
+        # Try to fetch Git repo URL from Calkit cloud
+        try:
+            project = calkit.cloud.get(f"/projects/{project_name}")
+            url = project["git_repo_url"]
+        except Exception as e:
+            raise ValueError(f"Could not fetch project info: {e}")
+        if not url.endswith(".git"):
+            url += ".git"
+        repo.git.remote(["add", "origin", url])
     base_url = calkit.cloud.get_base_url()
     remote_url = f"{base_url}/projects/{project_name}/dvc"
     subprocess.check_call(
@@ -110,3 +130,8 @@ def list_paths(wdir: str = None) -> list[str]:
     """List paths tracked with DVC."""
     dvc_repo = dvc.repo.Repo(wdir)
     return [p.get("path") for p in dvc_repo.ls(".", dvc_only=True)]
+
+
+def get_output_revisions(path: str):
+    """Get all revisions of a pipeline output."""
+    pass

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/git.py

@@ -12,9 +12,13 @@ def detect_project_name(path: str = None) -> str:
     ck_info = calkit.load_calkit_info(wdir=path)
     name = ck_info.get("name")
     owner = ck_info.get("owner")
-    url = git.Repo(path=path).remote().url
-    from_url = url.split("github.com")[-1][1:].removesuffix(".git")
-    owner_name, project_name = from_url.split("/")
+    if name is None or owner is None:
+        try:
+            url = git.Repo(path=path).remote().url
+        except ValueError:
+            raise ValueError("No Git remote set with name 'origin'")
+        from_url = url.split("github.com")[-1][1:].removesuffix(".git")
+        owner_name, project_name = from_url.split("/")
     if name is None:
         name = project_name
     if owner is None:

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/models.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import Literal
 
 from pydantic import BaseModel
@@ -225,6 +225,12 @@ class DerivedFromProject(BaseModel):
     git_rev: str
 
 
+class ProjectStatus(BaseModel):
+    timestamp: datetime
+    status: Literal["in-progress", "on-hold", "completed"]
+    message: str | None = None
+
+
 class ProjectInfo(BaseModel):
     """All of the project's information or metadata, written to the
     ``calkit.yaml`` file.

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/tests/cli/test_main.py

@@ -296,3 +296,25 @@ def test_add(tmp_dir):
         f.write(os.urandom(2_000_000))
     subprocess.check_call(["calkit", "add", "data2", "-M"])
     assert repo.head.commit.message.strip() == "Add data2"
+
+
+def test_status(tmp_dir):
+    subprocess.check_call(["calkit", "status"])
+    subprocess.check_call(["calkit", "init"])
+    subprocess.check_call(["calkit", "new", "status", "in-progress"])
+    subprocess.check_call(["calkit", "status"])
+    status = calkit.get_latest_project_status()
+    assert status.status == "in-progress"
+    assert not status.message
+    subprocess.check_call(
+        ["calkit", "new", "status", "completed", "-m", "We're done."]
+    )
+    subprocess.check_call(["calkit", "status"])
+    status = calkit.get_latest_project_status()
+    assert status.status == "completed"
+    assert status.message == "We're done."
+    history = calkit.get_project_status_history()
+    assert history[-1] == status
+    calkit.get_project_status_history(as_pydantic=False)
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(["calkit", "new", "status", "very-cool"])

{calkit_python-0.17.6 → calkit_python-0.19.0}/calkit/tests/cli/test_new.py

@@ -228,3 +228,178 @@ def test_new_project(tmp_dir):
     assert repo.git.ls_files(".devcontainer")
     ck_info = calkit.load_calkit_info()
     assert ck_info["title"] == "My new project"
+
+
+def test_new_stage(tmp_dir):
+    subprocess.check_call(["calkit", "init"])
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "docker-env",
+            "--name",
+            "tex",
+            "--image",
+            "texlive/texlive:latest-full",
+        ]
+    )
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "uv-venv",
+            "--name",
+            "py",
+            "requests",
+        ]
+    )
+    with open("plot.py", "w") as f:
+        f.write("print('hi')")
+    with open("paper.tex", "w") as f:
+        f.write("Hello")
+    with open("data.csv", "w") as f:
+        f.write("data")
+    # Create a Python script stage
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "stage",
+            "--name",
+            "plot",
+            "--kind",
+            "python-script",
+            "--environment",
+            "py",
+            "--target",
+            "plot.py",
+            "--dep",
+            "data.csv",
+            "--out",
+            "plot1.png",
+            "-o",
+            "plot2.png",
+        ]
+    )
+    pipeline = calkit.dvc.read_pipeline()
+    assert (
+        pipeline["stages"]["plot"]["cmd"]
+        == "calkit xenv -n py -- python plot.py"
+    )
+    assert set(pipeline["stages"]["plot"]["deps"]) == set(
+        ["plot.py", "data.csv", "requirements.txt"]
+    )
+    assert set(pipeline["stages"]["plot"]["outs"]) == set(
+        ["plot1.png", "plot2.png"]
+    )
+    # Create a LaTeX stage
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "stage",
+            "--name",
+            "build-paper",
+            "--kind",
+            "latex",
+            "--environment",
+            "tex",
+            "--target",
+            "paper.tex",
+            "--dep",
+            "plot1.png",
+            "-d",
+            "plot2.png",
+            "--out",
+            "paper.pdf",
+        ]
+    )
+    pipeline = calkit.dvc.read_pipeline()
+    assert pipeline["stages"]["build-paper"]["cmd"] == (
+        "calkit xenv -n tex -- "
+        "latexmk -cd -interaction=nonstopmode -pdf paper.tex"
+    )
+    assert set(pipeline["stages"]["build-paper"]["deps"]) == set(
+        [
+            "paper.tex",
+            "plot1.png",
+            "plot2.png",
+        ]
+    )
+    assert pipeline["stages"]["build-paper"]["outs"] == ["paper.pdf"]
+    # Check that we can create a MATLAB script with no environment
+    with open("script.m", "w") as f:
+        f.write("script")
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "stage",
+            "--name",
+            "plot",
+            "-f",
+            "--kind",
+            "matlab-script",
+            "--target",
+            "script.m",
+            "--dep",
+            "data.csv",
+            "--out",
+            "plot1.png",
+            "-o",
+            "plot2.png",
+        ]
+    )
+    pipeline = calkit.dvc.read_pipeline()
+    assert (
+        pipeline["stages"]["plot"]["cmd"]
+        == "matlab -noFigureWindows -batch \"run('script.m');\""
+    )
+    assert set(pipeline["stages"]["plot"]["deps"]) == set(
+        ["script.m", "data.csv"]
+    )
+    assert set(pipeline["stages"]["plot"]["outs"]) == set(
+        ["plot1.png", "plot2.png"]
+    )
+    # Check that we fail for a nonexistent target
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(
+            [
+                "calkit",
+                "new",
+                "stage",
+                "--name",
+                "plot",
+                "-f",
+                "--kind",
+                "matlab-script",
+                "--target",
+                "script2.m",
+                "--out",
+                "plot1.png",
+                "-o",
+                "plot2.png",
+            ]
+        )
+    # Check that we fail to create a stage with a non-existent environment
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(
+            [
+                "calkit",
+                "new",
+                "stage",
+                "--name",
+                "plot",
+                "-f",
+                "--kind",
+                "python-script",
+                "--target",
+                "plot.py",
+                "--out",
+                "plot1.png",
+                "-o",
+                "plot2.png",
+                "-e",
+                "nonexistent-env",
+            ]
+        )