calkit-python 0.35.7__tar.gz → 0.37.0__tar.gz

{calkit_python-0.35.7 → calkit_python-0.37.0}/.gitignore

@@ -1,7 +1,7 @@
 dev.ipynb
 *.pyc
 .DS_Store
-.vscode
+.vscode/settings.json
 site
 .env
 .venv

calkit_python-0.37.0/.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Run Calkit VS Code Extension",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceFolder}/vscode-ext",
+        "--enable-proposed-api=calkit.calkit-vscode"
+      ],
+      "outFiles": ["${workspaceFolder}/vscode-ext/out/**/*.js"],
+      "preLaunchTask": "npm: compile (vscode-ext)"
+    }
+  ]
+}

calkit_python-0.37.0/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "shell",
+      "command": "npm",
+      "args": ["run", "dev:compile"],
+      "options": {
+        "cwd": "${workspaceFolder}/vscode-ext"
+      },
+      "label": "npm: compile (vscode-ext)",
+      "problemMatcher": ["$tsc"]
+    }
+  ]
+}

{calkit_python-0.35.7 → calkit_python-0.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.7
+Version: 0.37.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.35.7 → calkit_python-0.37.0}/calkit/cli/main/core.py

@@ -25,6 +25,7 @@ from git.exc import InvalidGitRepositoryError
 from typing_extensions import Annotated, Optional
 
 import calkit
+import calkit.dvc.zip
 import calkit.matlab
 import calkit.pipeline
 from calkit import (
@@ -225,6 +226,69 @@ def clone(
                 raise_error("Failed to pull from DVC remote(s)")
         except Exception as e:
             raise_error(f"Failed to pull from DVC remote(s): {e}")
+        calkit.dvc.zip.sync_all(direction="to-workspace")
+
+
+def _format_dvc_data_status(status: dict, zip_path_map: dict) -> str:
+    """Format DVC data status, substituting zip paths with workspace paths."""
+    reverse_zip = {zip_p: ws for ws, zip_p in zip_path_map.items()}
+    color_map = {
+        "added": typer.colors.GREEN,
+        "modified": typer.colors.YELLOW,
+        "deleted": typer.colors.RED,
+        "renamed": typer.colors.CYAN,
+    }
+
+    def transform(path: str) -> str | None:
+        if path in reverse_zip:
+            return reverse_zip[path] + " (zipped)"
+        return path
+
+    def format_section(changes: dict, title: str) -> list[str]:
+        entries = []
+        for change_type, color in color_map.items():
+            for item in changes.get(change_type, []):
+                if isinstance(item, dict):
+                    old = transform(item.get("old", ""))
+                    new = transform(item.get("new", ""))
+                    if old is not None and new is not None:
+                        line = typer.style(
+                            f"        {change_type}: {old} -> {new}", fg=color
+                        )
+                        entries.append(line)
+                else:
+                    display = transform(item)
+                    if display is not None:
+                        line = typer.style(
+                            f"        {change_type}: {display}", fg=color
+                        )
+                        entries.append(line)
+        if not entries:
+            return []
+        return [title] + entries
+
+    lines = []
+    lines += format_section(
+        status.get("committed", {}), "DVC committed changes:"
+    )
+    lines += format_section(
+        status.get("uncommitted", {}), "DVC uncommitted changes:"
+    )
+    not_in_cache = [transform(p) for p in status.get("not_in_cache", [])]
+    not_in_cache = [p for p in not_in_cache if p is not None]
+    if not_in_cache:
+        lines.append(typer.style("Files not in cache:", bold=True))
+        for p in not_in_cache:
+            lines.append(f"        {p}")
+    not_in_remote = [transform(p) for p in status.get("not_in_remote", [])]
+    not_in_remote = [p for p in not_in_remote if p is not None]
+    if not_in_remote:
+        lines.append(typer.style("Files not in remote:", bold=True))
+        for p in not_in_remote:
+            lines.append(f"        {p}")
+    if not lines:
+        return "No changes.\n"
+    return "\n".join(lines) + "\n"
 
 
 @app.command(name="status")
@@ -249,6 +313,9 @@ def get_status(
             ),
         ),
     ] = None,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Output status as JSON.")
+    ] = False,
 ):
     """View status (project, version control, and/or pipeline)."""
     ck_info = calkit.load_calkit_info()
@@ -270,6 +337,9 @@ def get_status(
         categories = valid_categories
     pipeline_status = None
     if "pipeline" in categories or "dvc" in categories:
+        # Sync zips so the zip files reflect current workspace state before
+        # reporting status
+        calkit.dvc.zip.sync_all(direction="to-zip")
         pipeline_status = calkit.pipeline.get_status(
             ck_info=ck_info,
             targets=targets,
@@ -284,6 +354,78 @@ def get_status(
                 "Failed pipeline environment checks for: "
                 + ", ".join(pipeline_status.failed_environment_checks)
             )
+    if as_json:
+        status_dict = {}
+        if "project" in categories:
+            status = calkit.get_latest_project_status()
+            status_dict["project"] = (
+                None
+                if status is None
+                else {
+                    "status": status.status,
+                    "message": status.message,
+                    "timestamp": status.timestamp.isoformat(),
+                }
+            )
+        if "git" in categories:
+            try:
+                repo = git.Repo()
+                changed_files = calkit.git.get_changed_files(repo=repo)
+                staged_files = calkit.git.get_staged_files(repo=repo)
+                untracked_files = calkit.git.get_untracked_files(repo=repo)
+                if targets:
+                    target_prefixes = [
+                        Path(target).as_posix().rstrip("/")
+                        for target in targets
+                    ]
+
+                    def _matches_target(path: str) -> bool:
+                        path = Path(path).as_posix()
+                        return any(
+                            path == prefix or path.startswith(prefix + "/")
+                            for prefix in target_prefixes
+                        )
+
+                    changed_files = [
+                        path for path in changed_files if _matches_target(path)
+                    ]
+                    staged_files = [
+                        path for path in staged_files if _matches_target(path)
+                    ]
+                    untracked_files = [
+                        path
+                        for path in untracked_files
+                        if _matches_target(path)
+                    ]
+                status_dict["git"] = {
+                    "branch": None
+                    if repo.head.is_detached
+                    else repo.active_branch.name,
+                    "is_dirty": repo.is_dirty(untracked_files=True),
+                    "changed_files": changed_files,
+                    "staged_files": staged_files,
+                    "untracked_files": untracked_files,
+                }
+            except InvalidGitRepositoryError:
+                status_dict["git"] = {"error": "Not a Git repository"}
+        if "dvc" in categories:
+            try:
+                dvc_repo = calkit.dvc.get_dvc_repo()
+                data_status = dvc_repo.data_status()
+                if isinstance(data_status, dict):
+                    data_status.pop("git", None)
+                status_dict["dvc"] = data_status
+            except Exception as e:
+                status_dict["dvc"] = {"error": f"{e.__class__.__name__}: {e}"}
+        if "pipeline" in categories or "dvc" in categories:
+            if pipeline_status is None:
+                status_dict["pipeline"] = None
+            else:
+                status_dict["pipeline"] = pipeline_status.model_dump(
+                    mode="json"
+                )
+        print(json.dumps(status_dict, indent=2, default=str))
+        return
     if "project" in categories:
         print_sep("Project")
         # Print latest status
@@ -313,37 +455,49 @@ def get_status(
         typer.echo()
     if "dvc" in categories:
         print_sep("DVC")
-        dvc_data_cmd = ["data", "status"]
-        if targets:
-            dvc_data_cmd += targets
-        run_dvc_command(dvc_data_cmd)
-        typer.echo()
+        try:
+            get_dvc_repo()
+        except Exception:
+            typer.echo("This is not a DVC repository.\n")
+        else:
+            zip_path_map = calkit.dvc.zip.get_zip_path_map()
+            dvc_repo = get_dvc_repo()
+            raw = dict(dvc_repo.data_status())
+            raw.pop("git", None)
+            typer.echo(_format_dvc_data_status(raw, zip_path_map))
     if "pipeline" in categories or "dvc" in categories:
         print_sep("Pipeline")
         # Nicely format the results from pipeline status
         if pipeline_status and pipeline_status.errors:
-            typer.echo("Pipeline status unavailable due to errors.")
+            warn("Pipeline status unavailable due to errors.")
+            return
+        if pipeline_status and not pipeline_status.has_pipeline:
+            typer.echo("This project has no pipeline.")
+            return
         elif pipeline_status and pipeline_status.is_stale:
+            typer.echo("Stale stages:")
             for stage_name in pipeline_status.stale_stage_names:
                 stale_stage = pipeline_status.stale_stages.get(stage_name)
                 if stale_stage is None:
                     continue
-                typer.echo(f"{typer.style(stage_name, fg='yellow')}:")
+                typer.echo(f"        {typer.style(stage_name, fg='yellow')}:")
+                if stale_stage.modified_command:
+                    typer.echo("            modified command")
                 # Show stale outputs for this stage
                 if stale_stage.stale_outputs:
-                    typer.echo("  stale outputs:")
+                    typer.echo("          stale outputs:")
                     for output_path in stale_stage.stale_outputs:
-                        typer.echo(f"    - {output_path}")
+                        typer.echo(f"            {output_path}")
                 # Show modified outputs from this stage
                 if stale_stage.modified_outputs:
-                    typer.echo("  modified outputs:")
+                    typer.echo("          modified outputs:")
                     for output_path in stale_stage.modified_outputs:
-                        typer.echo(f"    - {output_path}")
+                        typer.echo(f"            {output_path}")
                 # Show modified inputs making the stage stale
                 if stale_stage.modified_inputs:
-                    typer.echo("  modified inputs:")
+                    typer.echo("          modified inputs:")
                     for input_path in stale_stage.modified_inputs:
-                        typer.echo(f"    - {input_path}")
+                        typer.echo(f"            {input_path}")
         elif pipeline_status:
             typer.echo("Pipeline is up to date.")
 
@@ -395,7 +549,9 @@ def add(
     to: Annotated[
         str | None,
         typer.Option(
-            "--to", "-t", help="System with which to add (git or dvc)."
+            "--to",
+            "-t",
+            help="System with which to add (git, dvc, or dvc-zip).",
         ),
     ] = None,
 ):
@@ -421,7 +577,7 @@ def add(
             raise_error(
                 "Cannot auto-generate commit message for more than one path"
             )
-    if to is not None and to not in ["git", "dvc"]:
+    if to is not None and to not in ["git", "dvc", "dvc-zip"]:
         raise_error(f"Invalid option for 'to': {to}")
     try:
         repo = git.Repo()
@@ -464,6 +620,10 @@ def add(
             subprocess.call(["git", "add"] + paths)
         elif to == "dvc":
             run_dvc_command(["add"] + paths)
+        elif to == "dvc-zip":
+            for path in paths:
+                typer.echo(f"Adding {path} as a DVC zip")
+                calkit.dvc.zip.add(path)
         else:
             raise_error(f"Invalid option for 'to': {to}")
     else:
@@ -512,7 +672,14 @@ def add(
                 d.a_path for d in repo.index.diff(None) if d.a_path is not None
             ]:
                 paths.append(changed_file)
+        zip_path_map = calkit.dvc.zip.get_zip_path_map()
         for path in paths:
+            # Check if this path is already registered as a zip
+            posix_path = Path(path).as_posix()
+            if posix_path in zip_path_map:
+                typer.echo(f"Adding {path} via DVC zip")
+                calkit.dvc.zip.add(path)
+                continue
             # Detect if this file should be tracked with Git or DVC
             # First see if it's in Git
             if repo.git.ls_files(path):
@@ -528,6 +695,12 @@ def add(
             elif os.path.splitext(path)[-1] in DVC_EXTENSIONS:
                 typer.echo(f"Adding {path} to DVC per its extension")
                 run_dvc_command(["add", path])
+            elif calkit.dvc.zip.is_zip_candidate(path):
+                typer.echo(
+                    f"Adding {path} as a DVC zip "
+                    "(large directory of small files)"
+                )
+                calkit.dvc.zip.add(path)
             elif calkit.get_size(path) > DVC_SIZE_THRESH_BYTES:
                 typer.echo(
                     f"Adding {path} to DVC since it's greater than 1 MB"
@@ -763,6 +936,7 @@ def pull(
     result = run_dvc_command(["pull"] + dvc_args)
     if result != 0:
         raise_error("DVC pull failed")
+    calkit.dvc.zip.sync_all(direction="to-workspace")
 
 
 @app.command(name="push")
@@ -1098,6 +1272,7 @@ def run(
     import dvc.ui
     from dvc.cli import main as dvc_cli_main
 
+    import calkit.dvc.zip
     import calkit.environments
     import calkit.pipeline
     from calkit.cli.overleaf import sync as overleaf_sync
@@ -1297,10 +1472,38 @@ def run(
     dvc.ui.ui.write = lambda *args, **kwargs: None
     res = dvc_cli_main(["repro"] + args)
     failed = res != 0
-    # Parse log to get timing
+    # Parse log to get timing and which stages ran
     with open(log_fpath, "r") as f:
         log_content = f.read()
         stage_run_info = _stage_run_info_from_log_content(log_content)
+    # Zip dvc-zip outputs for stages that actually ran
+    if stage_run_info:
+        from calkit.models.io import PathOutput
+        from calkit.models.pipeline import Pipeline
+
+        ck_info = calkit.load_calkit_info()
+        pipeline_cfg = ck_info.get("pipeline", {})
+        if pipeline_cfg:
+            try:
+                ck_pipeline = Pipeline.model_validate(pipeline_cfg)
+                zip_input_paths = []
+                for stage_name in stage_run_info:
+                    stage = ck_pipeline.stages.get(stage_name)
+                    if stage is None:
+                        continue
+                    for out in stage.outputs:
+                        if (
+                            isinstance(out, PathOutput)
+                            and out.storage == "dvc-zip"
+                        ):
+                            zip_input_paths.append(out.path)
+                if zip_input_paths:
+                    calkit.dvc.zip.sync_some(
+                        zip_input_paths, direction="to-zip"
+                    )
+            except Exception:
+                # Fall back to syncing all zips if pipeline parsing fails
+                calkit.dvc.zip.sync_all(direction="to-zip")
     # Close logger file handler to prevent permissions issues if deleting
     dvc.log.logger.removeHandler(file_handler)
     file_handler.close()
@@ -2166,6 +2369,39 @@ def switch_branch(name: Annotated[str, typer.Argument(help="Branch name.")]):
     else:
         cmd = [name]
     repo.git.checkout(cmd)
+    # After switching branches, DVC-tracked zips may have changed; check out
+    # the new branch's DVC data and unzip to the workspace
+    run_dvc_command(["checkout"])
+    calkit.dvc.zip.sync_all(direction="to-workspace")
+
+
+@app.command(name="stash")
+def stash(
+    pop: Annotated[
+        bool, typer.Option("--pop", help="Pop the most recent stash.")
+    ] = False,
+):
+    """Stash or restore workspace changes including dvc-zip tracked dirs.
+
+    Without --pop: zips any modified workspace dirs into the DVC cache, then
+    git-stashes (saving the updated .dvc files), checks out the committed DVC
+    state, and unzips it to the workspace.
+
+    With --pop: pops the git stash (restoring the saved .dvc files), checks
+    out the stashed DVC state, and unzips it to the workspace.
+    """
+    if pop:
+        subprocess.check_call(["git", "stash", "pop"])
+        run_dvc_command(["checkout"])
+        calkit.dvc.zip.sync_all(direction="to-workspace")
+    else:
+        # Zip any modified workspace dirs so their current state is in the DVC
+        # cache (the updated .dvc file will be captured by git stash)
+        calkit.dvc.zip.sync_all(direction="to-zip")
+        subprocess.check_call(["git", "stash"])
+        # Restore the committed zip versions and unzip them
+        run_dvc_command(["checkout"])
+        calkit.dvc.zip.sync_all(direction="to-workspace")
 
 
 @app.command(

{calkit_python-0.35.7 → calkit_python-0.37.0}/calkit/cli/new.py

@@ -1333,6 +1333,75 @@ def new_uv_env(
             repo.git.commit(["-m", f"Add uv environment {name}"])
 
 
+@new_app.command("slurm-env")
+def new_slurm_env(
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    host: Annotated[
+        str,
+        typer.Option("--host", help="Host where SLURM commands should run."),
+    ] = "localhost",
+    default_options: Annotated[
+        list[str],
+        typer.Option(
+            "--default-option",
+            help=(
+                "Default sbatch/srun option string (for example --gpus=1). "
+                "Repeat for multiple options."
+            ),
+        ),
+    ] = [],
+    description: Annotated[
+        str | None, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new SLURM environment."""
+    host = host.strip()
+    if not host:
+        raise_error("Host is required")
+    ck_info = calkit.load_calkit_info()
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    normalized_default_options = [
+        opt.strip() for opt in default_options if opt.strip()
+    ]
+    env = {"kind": "slurm", "host": host}
+    if normalized_default_options:
+        env["default_options"] = normalized_default_options  # type: ignore
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if not no_commit:
+        repo = git.Repo()
+        repo.git.add("calkit.yaml")
+        if repo.git.diff(["--staged", "calkit.yaml"]):
+            repo.git.commit(
+                ["calkit.yaml", "-m", f"Add SLURM environment {name}"]
+            )
+
+
 @new_app.command("uv-venv")
 def new_uv_venv(
     name: Annotated[
@@ -1615,21 +1684,29 @@ def new_pixi_env(
 @new_app.command("julia-env")
 def new_julia_env(
     packages: Annotated[
-        list[str],
-        typer.Argument(help="Packages to include in the environment."),
-    ],
+        list[str] | None,
+        typer.Argument(
+            help="Optional packages to include in the environment."
+        ),
+    ] = None,
     name: Annotated[
         str, typer.Option("--name", "-n", help="Environment name.")
-    ],
+    ] = "main",
     path: Annotated[
-        str, typer.Option("--path", help="Path for Project.toml file.")
-    ] = "Project.toml",
+        str | None,
+        typer.Option("--path", help="Path for Project.toml file."),
+    ] = None,
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
     julia_version: Annotated[
-        str, typer.Option("--julia", "-j", help="Julia version.")
-    ] = "1.11",
+        str | None,
+        typer.Option(
+            "--julia",
+            "-j",
+            help="Julia version. Auto-detected if not supplied.",
+        ),
+    ] = None,
     overwrite: Annotated[
         bool,
         typer.Option(
@@ -1641,8 +1718,17 @@ def new_julia_env(
     no_commit: Annotated[
         bool, typer.Option("--no-commit", help="Do not commit changes.")
     ] = False,
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check",
+            help="Do not check environment is up-to-date after creation.",
+        ),
+    ] = False,
 ):
-    """Create a new Julia environment."""
+    """Create a new Julia environment or add an existing one to calkit.yaml."""
+    if path is None:
+        path = "Project.toml"
     if not os.path.basename(path) == "Project.toml":
         raise_error(
             "Julia environment paths must point to a Project.toml file"
@@ -1665,27 +1751,57 @@ def new_julia_env(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Create the environment now
+    # Detect Julia version if not supplied
+    if julia_version is None:
+        try:
+            julia_version = calkit.julia.get_version()
+        except ValueError as e:
+            raise_error(
+                f"Failed to detect Julia version: {e}. "
+                "Use --julia to specify a version."
+            )
+    # Check for duplicate Julia environments with same path and version
+    for env_name, env_config in envs.items():
+        if (
+            env_config.get("kind") == "julia"
+            and env_config.get("path") == path
+            and env_config.get("julia") == julia_version
+        ):
+            raise_error(
+                f"Julia environment '{env_name}' already exists with path "
+                f"'{path}' and version '{julia_version}'; "
+                f"Use a different path, version, or --overwrite to replace it"
+            )
     env_dir = os.path.dirname(path)
     if env_dir:
         os.makedirs(env_dir, exist_ok=True)
     else:
         env_dir = "."
-    # First make sure the Julia version is installed
-    cmd = ["juliaup", "add", julia_version]
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError:
-        raise_error(f"Failed to install Julia version {julia_version}")
-    cmd = ["julia", f"+{julia_version}", f"--project={env_dir}", "-e"]
-    install_cmd = "using Pkg;"
-    for package in packages:
-        install_cmd += f' Pkg.add("{package}");'
-    cmd.append(install_cmd)
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError:
-        raise_error("Failed to create new Julia environment")
+    project_exists = os.path.isfile(path)
+    created_project_file = False
+    # If packages are provided, initialize via Julia/Pkg.
+    if packages:
+        # First make sure the Julia version is installed
+        cmd = ["juliaup", "add", julia_version]
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError:
+            raise_error(f"Failed to install Julia version {julia_version}")
+        cmd = ["julia", f"+{julia_version}", f"--project={env_dir}", "-e"]
+        install_cmd = f'using Pkg; Pkg.activate("{env_dir}");'
+        for package in packages or []:
+            install_cmd += f' Pkg.add("{package}");'
+        cmd.append(install_cmd)
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to create new Julia environment")
+    elif not project_exists:
+        # Allow creating a Julia environment scaffold without requiring Julia
+        # tooling at creation time. IJulia or other packages can be added later.
+        with open(path, "w") as f:
+            f.write("[deps]\n")
+        created_project_file = True
     typer.echo("Adding environment to calkit.yaml")
     env = dict(kind="julia", path=path, julia=julia_version)
     if description is not None:
@@ -1694,11 +1810,18 @@ def new_julia_env(
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
         ryaml.dump(ck_info, f)
-    repo.git.add(path)
-    repo.git.add(os.path.join(env_dir, "Manifest.toml"))
-    repo.git.add("calkit.yaml")
-    if not no_commit and repo.git.diff("--staged"):
-        repo.git.commit(["-m", f"Add Julia env {name}"])
+    if not no_commit:
+        repo.git.add("calkit.yaml")
+        if packages or created_project_file:
+            repo.git.add(path)
+        if packages:
+            repo.git.add(os.path.join(env_dir, "Manifest.toml"))
+        if not no_check and packages:
+            env_lock_fpath = check_environment(env_name=name)
+            if env_lock_fpath is not None:
+                repo.git.add(env_lock_fpath)
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add Julia env {name}"])
 
 
 @new_app.command("renv")