calkit-python 0.35.6__tar.gz → 0.36.0__tar.gz

{calkit_python-0.35.6 → calkit_python-0.36.0}/.gitignore

@@ -1,7 +1,7 @@
 dev.ipynb
 *.pyc
 .DS_Store
-.vscode
+.vscode/settings.json
 site
 .env
 .venv

calkit_python-0.36.0/.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Run Calkit VS Code Extension",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceFolder}/vscode-ext",
+        "--enable-proposed-api=calkit.calkit-vscode"
+      ],
+      "outFiles": ["${workspaceFolder}/vscode-ext/out/**/*.js"],
+      "preLaunchTask": "npm: compile (vscode-ext)"
+    }
+  ]
+}

calkit_python-0.36.0/.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "shell",
+      "command": "npm",
+      "args": ["run", "dev:compile"],
+      "options": {
+        "cwd": "${workspaceFolder}/vscode-ext"
+      },
+      "label": "npm: compile (vscode-ext)",
+      "problemMatcher": ["$tsc"]
+    }
+  ]
+}

{calkit_python-0.35.6 → calkit_python-0.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.6
+Version: 0.36.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.35.6 → calkit_python-0.36.0}/calkit/cli/main/core.py

@@ -55,7 +55,7 @@ from calkit.cli.office import office_app
 from calkit.cli.overleaf import overleaf_app
 from calkit.cli.slurm import slurm_app
 from calkit.cli.update import update_app
-from calkit.dvc import register_ck_scheme, run_dvc_command
+from calkit.dvc import get_dvc_repo, run_dvc_command
 from calkit.environments import get_env_lock_fpath
 from calkit.models import Procedure
 
@@ -249,13 +249,18 @@ def get_status(
             ),
         ),
     ] = None,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Output status as JSON.")
+    ] = False,
 ):
     """View status (project, version control, and/or pipeline)."""
     ck_info = calkit.load_calkit_info()
-    try:
-        calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
-    except Exception as e:
-        warn(f"Failed to compile pipeline: {e.__class__.__name__}: {e}")
+    # If there's anything in ck_info and this isn't a Git repo, initialize one
+    if ck_info:
+        try:
+            git.Repo()
+        except InvalidGitRepositoryError:
+            git.Repo.init()
     valid_categories = ["project", "git", "dvc", "pipeline"]
     if categories is not None:
         for category in categories:
@@ -266,11 +271,94 @@ def get_status(
                 )
     else:
         categories = valid_categories
-    # Clean all notebooks in the pipeline
-    try:
-        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
-    except Exception as e:
-        warn(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
+    pipeline_status = None
+    if "pipeline" in categories or "dvc" in categories:
+        pipeline_status = calkit.pipeline.get_status(
+            ck_info=ck_info,
+            targets=targets,
+            check_environments=True,
+            clean_notebooks=True,
+            compile_to_dvc=True,
+        )
+        for error in pipeline_status.errors:
+            warn(error)
+        if pipeline_status.failed_environment_checks:
+            warn(
+                "Failed pipeline environment checks for: "
+                + ", ".join(pipeline_status.failed_environment_checks)
+            )
+    if as_json:
+        status_dict = {}
+        if "project" in categories:
+            status = calkit.get_latest_project_status()
+            status_dict["project"] = (
+                None
+                if status is None
+                else {
+                    "status": status.status,
+                    "message": status.message,
+                    "timestamp": status.timestamp.isoformat(),
+                }
+            )
+        if "git" in categories:
+            try:
+                repo = git.Repo()
+                changed_files = calkit.git.get_changed_files(repo=repo)
+                staged_files = calkit.git.get_staged_files(repo=repo)
+                untracked_files = calkit.git.get_untracked_files(repo=repo)
+                if targets:
+                    target_prefixes = [
+                        Path(target).as_posix().rstrip("/")
+                        for target in targets
+                    ]
+
+                    def _matches_target(path: str) -> bool:
+                        path = Path(path).as_posix()
+                        return any(
+                            path == prefix or path.startswith(prefix + "/")
+                            for prefix in target_prefixes
+                        )
+
+                    changed_files = [
+                        path for path in changed_files if _matches_target(path)
+                    ]
+                    staged_files = [
+                        path for path in staged_files if _matches_target(path)
+                    ]
+                    untracked_files = [
+                        path
+                        for path in untracked_files
+                        if _matches_target(path)
+                    ]
+                status_dict["git"] = {
+                    "branch": None
+                    if repo.head.is_detached
+                    else repo.active_branch.name,
+                    "is_dirty": repo.is_dirty(untracked_files=True),
+                    "changed_files": changed_files,
+                    "staged_files": staged_files,
+                    "untracked_files": untracked_files,
+                }
+            except InvalidGitRepositoryError:
+                status_dict["git"] = {"error": "Not a Git repository"}
+        if "dvc" in categories:
+            try:
+                dvc_repo = calkit.dvc.get_dvc_repo()
+                data_status = dvc_repo.data_status()
+                if isinstance(data_status, dict):
+                    data_status.pop("git", None)
+                status_dict["dvc"] = data_status
+            except Exception as e:
+                status_dict["dvc"] = {"error": f"{e.__class__.__name__}: {e}"}
+        if "pipeline" in categories or "dvc" in categories:
+            if pipeline_status is None:
+                status_dict["pipeline"] = None
+            else:
+                status_dict["pipeline"] = pipeline_status.model_dump(
+                    mode="json"
+                )
+        print(json.dumps(status_dict, indent=2, default=str))
+        return
     if "project" in categories:
         print_sep("Project")
         # Print latest status
@@ -307,10 +395,36 @@ def get_status(
         typer.echo()
     if "pipeline" in categories or "dvc" in categories:
         print_sep("Pipeline")
-        dvc_pipeline_cmd = ["status"]
-        if targets:
-            dvc_pipeline_cmd += targets
-        run_dvc_command(dvc_pipeline_cmd)
+        # Nicely format the results from pipeline status
+        if pipeline_status and pipeline_status.errors:
+            warn("Pipeline status unavailable due to errors.")
+            return
+        if pipeline_status and not pipeline_status.has_pipeline:
+            typer.echo("This project has no pipeline.")
+            return
+        elif pipeline_status and pipeline_status.is_stale:
+            for stage_name in pipeline_status.stale_stage_names:
+                stale_stage = pipeline_status.stale_stages.get(stage_name)
+                if stale_stage is None:
+                    continue
+                typer.echo(f"{typer.style(stage_name, fg='yellow')}:")
+                # Show stale outputs for this stage
+                if stale_stage.stale_outputs:
+                    typer.echo("  stale outputs:")
+                    for output_path in stale_stage.stale_outputs:
+                        typer.echo(f"    {output_path}")
+                # Show modified outputs from this stage
+                if stale_stage.modified_outputs:
+                    typer.echo("  modified outputs:")
+                    for output_path in stale_stage.modified_outputs:
+                        typer.echo(f"    {output_path}")
+                # Show modified inputs making the stage stale
+                if stale_stage.modified_inputs:
+                    typer.echo("  modified inputs:")
+                    for input_path in stale_stage.modified_inputs:
+                        typer.echo(f"    {input_path}")
+        elif pipeline_status:
+            typer.echo("Pipeline is up to date.")
 
 
 @app.command(name="diff")
@@ -403,8 +517,7 @@ def add(
             raise_error("Not currently in a Git repo; run `calkit init` first")
         repo = git.Repo()
     try:
-        register_ck_scheme()
-        dvc_repo = dvc.repo.Repo()
+        dvc_repo = get_dvc_repo()
     except NotDvcRepoError:
         warn("DVC not initialized yet; initializing")
         dvc_repo = dvc.repo.Repo.init()
@@ -1139,10 +1252,8 @@ def run(
             os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
             raise_error(f"Pipeline compilation failed: {e}")
     # Initialize DVC repo if necessary
-    # Register the ck:// scheme before accessing DVC repo
-    register_ck_scheme()
     try:
-        dvc.repo.Repo()
+        get_dvc_repo()
     except Exception:
         if not quiet:
             typer.echo("Initializing DVC repo")
@@ -1203,8 +1314,7 @@ def run(
         git_staged_files_before = calkit.git.get_staged_files(repo=repo)
         git_untracked_files_before = calkit.git.get_untracked_files(repo=repo)
         # Get status of DVC repo before running
-        register_ck_scheme()
-        dvc_repo = dvc.repo.Repo()
+        dvc_repo = get_dvc_repo()
         dvc_status_before = dvc_repo.status()
         dvc_data_status_before = dvc_repo.data_status()
         dvc_data_status_before.pop("git", None)  # Remove git status

{calkit_python-0.35.6 → calkit_python-0.36.0}/calkit/cli/new.py

@@ -17,6 +17,7 @@ from git.exc import GitCommandError, InvalidGitRepositoryError, NoSuchPathError
 from typing_extensions import Annotated
 
 import calkit
+import calkit.pipeline
 from calkit.cli import raise_error, warn
 from calkit.cli.check import check_environment
 from calkit.cli.update import update_devcontainer
@@ -1332,6 +1333,75 @@ def new_uv_env(
             repo.git.commit(["-m", f"Add uv environment {name}"])
 
 
+@new_app.command("slurm-env")
+def new_slurm_env(
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    host: Annotated[
+        str,
+        typer.Option("--host", help="Host where SLURM commands should run."),
+    ] = "localhost",
+    default_options: Annotated[
+        list[str],
+        typer.Option(
+            "--default-option",
+            help=(
+                "Default sbatch/srun option string (for example --gpus=1). "
+                "Repeat for multiple options."
+            ),
+        ),
+    ] = [],
+    description: Annotated[
+        str | None, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new SLURM environment."""
+    host = host.strip()
+    if not host:
+        raise_error("Host is required")
+    ck_info = calkit.load_calkit_info()
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    normalized_default_options = [
+        opt.strip() for opt in default_options if opt.strip()
+    ]
+    env = {"kind": "slurm", "host": host}
+    if normalized_default_options:
+        env["default_options"] = normalized_default_options  # type: ignore
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    if not no_commit:
+        repo = git.Repo()
+        repo.git.add("calkit.yaml")
+        if repo.git.diff(["--staged", "calkit.yaml"]):
+            repo.git.commit(
+                ["calkit.yaml", "-m", f"Add SLURM environment {name}"]
+            )
+
+
 @new_app.command("uv-venv")
 def new_uv_venv(
     name: Annotated[
@@ -1614,21 +1684,29 @@ def new_pixi_env(
 @new_app.command("julia-env")
 def new_julia_env(
     packages: Annotated[
-        list[str],
-        typer.Argument(help="Packages to include in the environment."),
-    ],
+        list[str] | None,
+        typer.Argument(
+            help="Optional packages to include in the environment."
+        ),
+    ] = None,
     name: Annotated[
         str, typer.Option("--name", "-n", help="Environment name.")
-    ],
+    ] = "main",
     path: Annotated[
-        str, typer.Option("--path", help="Path for Project.toml file.")
-    ] = "Project.toml",
+        str | None,
+        typer.Option("--path", help="Path for Project.toml file."),
+    ] = None,
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
     julia_version: Annotated[
-        str, typer.Option("--julia", "-j", help="Julia version.")
-    ] = "1.11",
+        str | None,
+        typer.Option(
+            "--julia",
+            "-j",
+            help="Julia version. Auto-detected if not supplied.",
+        ),
+    ] = None,
     overwrite: Annotated[
         bool,
         typer.Option(
@@ -1640,8 +1718,17 @@ def new_julia_env(
     no_commit: Annotated[
         bool, typer.Option("--no-commit", help="Do not commit changes.")
     ] = False,
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check",
+            help="Do not check environment is up-to-date after creation.",
+        ),
+    ] = False,
 ):
-    """Create a new Julia environment."""
+    """Create a new Julia environment or add an existing one to calkit.yaml."""
+    if path is None:
+        path = "Project.toml"
     if not os.path.basename(path) == "Project.toml":
         raise_error(
             "Julia environment paths must point to a Project.toml file"
@@ -1664,27 +1751,57 @@ def new_julia_env(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Create the environment now
+    # Detect Julia version if not supplied
+    if julia_version is None:
+        try:
+            julia_version = calkit.julia.get_version()
+        except ValueError as e:
+            raise_error(
+                f"Failed to detect Julia version: {e}. "
+                "Use --julia to specify a version."
+            )
+    # Check for duplicate Julia environments with same path and version
+    for env_name, env_config in envs.items():
+        if (
+            env_config.get("kind") == "julia"
+            and env_config.get("path") == path
+            and env_config.get("julia") == julia_version
+        ):
+            raise_error(
+                f"Julia environment '{env_name}' already exists with path "
+                f"'{path}' and version '{julia_version}'; "
+                f"Use a different path, version, or --overwrite to replace it"
+            )
     env_dir = os.path.dirname(path)
     if env_dir:
         os.makedirs(env_dir, exist_ok=True)
     else:
         env_dir = "."
-    # First make sure the Julia version is installed
-    cmd = ["juliaup", "add", julia_version]
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError:
-        raise_error(f"Failed to install Julia version {julia_version}")
-    cmd = ["julia", f"+{julia_version}", f"--project={env_dir}", "-e"]
-    install_cmd = "using Pkg;"
-    for package in packages:
-        install_cmd += f' Pkg.add("{package}");'
-    cmd.append(install_cmd)
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError:
-        raise_error("Failed to create new Julia environment")
+    project_exists = os.path.isfile(path)
+    created_project_file = False
+    # If packages are provided, initialize via Julia/Pkg.
+    if packages:
+        # First make sure the Julia version is installed
+        cmd = ["juliaup", "add", julia_version]
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError:
+            raise_error(f"Failed to install Julia version {julia_version}")
+        cmd = ["julia", f"+{julia_version}", f"--project={env_dir}", "-e"]
+        install_cmd = f'using Pkg; Pkg.activate("{env_dir}");'
+        for package in packages or []:
+            install_cmd += f' Pkg.add("{package}");'
+        cmd.append(install_cmd)
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError:
+            raise_error("Failed to create new Julia environment")
+    elif not project_exists:
+        # Allow creating a Julia environment scaffold without requiring Julia
+        # tooling at creation time. IJulia or other packages can be added later.
+        with open(path, "w") as f:
+            f.write("[deps]\n")
+        created_project_file = True
     typer.echo("Adding environment to calkit.yaml")
     env = dict(kind="julia", path=path, julia=julia_version)
     if description is not None:
@@ -1693,11 +1810,18 @@ def new_julia_env(
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
         ryaml.dump(ck_info, f)
-    repo.git.add(path)
-    repo.git.add(os.path.join(env_dir, "Manifest.toml"))
-    repo.git.add("calkit.yaml")
-    if not no_commit and repo.git.diff("--staged"):
-        repo.git.commit(["-m", f"Add Julia env {name}"])
+    if not no_commit:
+        repo.git.add("calkit.yaml")
+        if packages or created_project_file:
+            repo.git.add(path)
+        if packages:
+            repo.git.add(os.path.join(env_dir, "Manifest.toml"))
+        if not no_check and packages:
+            env_lock_fpath = check_environment(env_name=name)
+            if env_lock_fpath is not None:
+                repo.git.add(env_lock_fpath)
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add Julia env {name}"])
 
 
 @new_app.command("renv")
@@ -2523,6 +2647,26 @@ def new_release(
     repo = git.Repo()
     if name in repo.tags:
         raise_error(f"Git tag with name '{name}' already exists")
+    # Check that the pipeline is up-to-date
+    typer.echo("Checking pipeline is up-to-date for release")
+    status = calkit.pipeline.get_status(
+        ck_info=ck_info,
+        check_environments=True,
+        clean_notebooks=True,
+        compile_to_dvc=True,
+    )
+    if status.errors:
+        raise_error("Pipeline checks failed: " + "; ".join(status.errors))
+    if status.failed_environment_checks:
+        raise_error(
+            "Pipeline environment checks failed for: "
+            + ", ".join(status.failed_environment_checks)
+        )
+    if status.is_stale:
+        raise_error(
+            "Pipeline is not up-to-date; out-of-date stages: "
+            + ", ".join(status.stale_stage_names)
+        )
     release_dir = f".calkit/releases/{name}"
     release_files_dir = release_dir + "/files"
     os.makedirs(release_files_dir, exist_ok=True)
@@ -2543,8 +2687,14 @@ def new_release(
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
         with zipfile.ZipFile(zip_path, "w") as zipf:
-            for fpath in all_paths:
-                zipf.write(fpath)
+            calkit.releases.add_paths_to_zip(zipf, all_paths)
+        typer.echo("Checking extracted project release archive")
+        try:
+            calkit.releases.check_project_release_archive(
+                zip_path, verbose=verbose
+            )
+        except Exception as e:
+            raise_error(str(e))
         title = ck_info.get("title")
         if title is None:
             warn("Project has no title")
@@ -2888,10 +3038,12 @@ def new_release(
             + new_lines[first_content_line_index:]
         )
         readme_txt = "\n".join(new_lines)
-        with open("README.md", "w") as f:
-            f.write(readme_txt)
         if not dry_run:
-            repo.git.add("README.md")
+            with open("README.md", "w") as f:
+                f.write(readme_txt)
+                repo.git.add("README.md")
+        else:
+            typer.echo(f"Would have updated README.md to:\n{readme_txt}")
     # Create Git tag
     git_tag_message = release_description
     if git_tag_message is None:
@@ -2919,13 +3071,15 @@ def new_release(
     ck_info["releases"] = releases
     # Create CITATION.cff file
     if release_type == "project":
-        typer.echo("Writing CITATION.cff")
         cff = calkit.releases.create_citation_cff(
             ck_info=ck_info, release_name=name, release_date=release_date
         )
-        with open("CITATION.cff", "w") as f:
-            calkit.ryaml.dump(cff, f)
-        if not dry_run:
+        if dry_run:
+            typer.echo(f"Would write CITATION.cff:\n{cff}")
+        else:
+            typer.echo("Writing CITATION.cff")
+            with open("CITATION.cff", "w") as f:
+                calkit.ryaml.dump(cff, f)
             repo.git.add("CITATION.cff")
     # Add to references so it can be cited
     typer.echo("Adding BibTeX entry to references")
@@ -2944,28 +3098,50 @@ def new_release(
                 reflib = bibtexparser.load(f)
         else:
             reflib = bibtexparser.bibdatabase.BibDatabase()
+        bibtex_doi = doi
+        if bibtex_doi is None and dry_run:
+            mock_suffix = "".join(
+                ch if (ch.isalnum() or ch in ".-_") else "-"
+                for ch in name.lower()
+            )
+            bibtex_doi = f"10.0000/calkit-dry-run.{mock_suffix}"
         invenio_bibtex = calkit.releases.create_bibtex(
             authors=authors,
             release_date=release_date,
             title=title,  # type: ignore
-            doi=doi,  # type: ignore
+            doi=bibtex_doi,
             record_id=record_id,  # type: ignore
             service=to,  # type: ignore
         )
-        new_entry = bibtexparser.loads(invenio_bibtex).entries[0]
+        new_entries = bibtexparser.loads(invenio_bibtex).entries
+        if not new_entries:
+            raise ValueError("Failed to parse generated BibTeX entry")
+        new_entry = new_entries[0]
         # Search through entries for one with the same DOI, and replace if
         # there is a match
-        existing_index = None
-        for n, entry in enumerate(reflib.entries):
-            if entry.get("doi") == doi:
-                typer.echo("Found matching DOI in existing references")
-                existing_index = n
-        if existing_index is not None:
-            _ = reflib.entries.pop(existing_index)
+        new_doi = new_entry.get("doi")
+        matching_indices = []
+        if new_doi:
+            matching_indices = [
+                n
+                for n, entry in enumerate(reflib.entries)
+                if entry.get("doi") == new_doi
+            ]
+        if matching_indices:
+            typer.echo(
+                "Found matching DOI in existing references "
+                f"({len(matching_indices)} entr"
+                f"{'y' if len(matching_indices) == 1 else 'ies'}); "
+                "replacing"
+            )
+            for n in reversed(matching_indices):
+                _ = reflib.entries.pop(n)
         reflib.entries.append(new_entry)
-        with open(ref_path, "w") as f:
-            bibtexparser.dump(reflib, f)
-        if not dry_run:
+        if dry_run:
+            typer.echo(f"Would write updated references to {ref_path}")
+        else:
+            with open(ref_path, "w") as f:
+                bibtexparser.dump(reflib, f)
             repo.git.add(ref_path)
     except Exception as e:
         warn(f"Failed to add to references: {e}")