calkit-python 0.32.11__tar.gz → 0.33.1__tar.gz

{calkit_python-0.32.11 → calkit_python-0.33.1}/.gitignore

@@ -8,3 +8,10 @@ site
 .coverage*
 coverage.xml
 import.log
+tsconfig.tsbuildinfo
+lib
+node_modules
+.yarn
+calkit/labextension
+coverage
+junit.xml

{calkit_python-0.32.11 → calkit_python-0.33.1}/.pre-commit-config.yaml

@@ -20,3 +20,5 @@ repos:
     hooks:
       - id: prettier
         exclude: calkit
+        args:
+          - --config=package.json

calkit_python-0.33.1/.prettierignore

@@ -0,0 +1,10 @@
+node_modules
+**/node_modules
+**/lib
+**/package.json
+!/package.json
+calkit
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv

calkit_python-0.33.1/.yarnrc.yml

@@ -0,0 +1,2 @@
+nodeLinker: node-modules
+enableScripts: false

{calkit_python-0.32.11 → calkit_python-0.33.1}/CONTRIBUTING.md

@@ -31,6 +31,24 @@ This guide will help you get started.
    ```bash
    uv run pytest
    ```
+1. Install the Calkit CLI in dev mode:
+   ```sh
+   uv tool install -e .
+   ```
+1. Install the JupyterLab extension in development mode
+   (if working on the JupyterLab extension):
+   ```sh
+   uv run jupyter labextension develop . --overwrite
+   ```
+1. Start the JupyterLab extension TypeScript compiler in watch mode
+   (if working on the JupyterLab extension):
+   ```sh
+   make jlab-dev
+   ```
+1. Start JupyterLab:
+   ```sh
+   uv run jupyter lab
+   ```
 
 ### 3. Make your changes
 

{calkit_python-0.32.11 → calkit_python-0.33.1}/Makefile

@@ -32,14 +32,33 @@ test-cov: ## Test the code coverage with pytest.
 	@echo "🚀 Testing code coverage with pytest"
 	@uv run pytest --cov --cov-config=pyproject.toml
 
-.PHONY: docs-test
-docs-test: ## Test if documentation can be built without warnings or errors.
+.PHONY: test-docs
+test-docs: ## Test if documentation can be built without warnings or errors.
 	@uv run mkdocs build -s
 
 .PHONY: docs
 docs: ## Build and serve the documentation.
-	@uv run mkdocs serve
+	@uv run mkdocs serve --livereload
 
 .PHONY: import-profile
 import-profile: ## Profile the import time of the CLI.
 	uv run python -X importtime -m calkit --help 2> import.log && uvx tuna import.log
+
+.PHONY: jlab-dev
+jlab-dev: ## Develop the JupyterLab extension.
+	uv run jlpm run watch
+
+.PHONY: jlab
+jlab: ## Build the JupyterLab extension.
+	uv run jlpm run build:prod
+
+.PHONY: test-frontend
+test-frontend: ## Run frontend unit tests with Jest.
+	@echo "🚀 Running frontend unit tests"
+	@uv run jlpm test
+
+.PHONY: test-ui
+test-ui: ## Run the JupyterLab UI integration tests.
+	@echo "🚀 Running JupyterLab UI tests with Playwright"
+	@uv run npm run build
+	@uv run --directory=ui-tests jlpm playwright test -u --reporter=list

{calkit_python-0.32.11 → calkit_python-0.33.1}/PKG-INFO

@@ -1,15 +1,26 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.32.11
+Version: 0.33.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
 Project-URL: Repository, https://github.com/calkit/calkit
 Author-email: Pete Bachant <petebachant@gmail.com>
 License-File: LICENSE
+Classifier: Framework :: Jupyter
+Classifier: Framework :: Jupyter :: JupyterLab
+Classifier: Framework :: Jupyter :: JupyterLab :: 4
+Classifier: Framework :: Jupyter :: JupyterLab :: Extensions
+Classifier: Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.10
 Requires-Dist: arithmeval
 Requires-Dist: bibtexparser
@@ -34,7 +45,7 @@ Requires-Dist: pywin32; platform_system == 'Windows'
 Requires-Dist: requests
 Requires-Dist: sqlitedict>=2.1.0
 Requires-Dist: tqdm>=4.67.1
-Requires-Dist: typer==0.20.0
+Requires-Dist: typer==0.21.1
 Requires-Dist: uvicorn
 Provides-Extra: data
 Requires-Dist: pandas>=2.2.3; extra == 'data'

calkit_python-0.33.1/babel.config.js

@@ -0,0 +1 @@
+module.exports = require("@jupyterlab/testutils/lib/babel.config");

calkit_python-0.33.1/calkit/__init__.py

@@ -0,0 +1,49 @@
+__version__ = "0.33.1"
+
+from .core import *  # noqa: F403, I001
+from . import git  # noqa: F401
+from . import dvc  # noqa: F401
+from . import cloud  # noqa: F401
+from . import jupyter  # noqa: F401
+from . import config  # noqa: F401
+from . import models  # noqa: F401
+from . import office  # noqa: F401
+from . import templates  # noqa: F401
+from . import conda  # noqa: F401
+from . import calc  # noqa: F401
+from . import check  # noqa: F401
+from . import github  # noqa: F401
+from . import invenio  # noqa: F401
+from . import releases  # noqa: F401
+from . import licenses  # noqa: F401
+from . import overleaf  # noqa: F401
+from .notebooks import declare_notebook  # noqa: F401
+from .jupyterlab.routes import setup_route_handlers
+
+
+def _jupyter_labextension_paths():
+    return [{"src": "labextension", "dest": "calkit"}]
+
+
+def _jupyter_server_extension_points():
+    return [{"module": "calkit"}]
+
+
+def _load_jupyter_server_extension(server_app):
+    """Registers the API handler to receive HTTP requests from the frontend
+    extension.
+
+    Parameters
+    ----------
+    server_app: jupyterlab.labapp.LabApp
+        JupyterLab application instance
+    """
+    import os
+
+    # Change to root_dir so all handlers work in the correct directory context
+    root_dir = server_app.root_dir
+    os.chdir(root_dir)
+    server_app.log.info(f"Changed working directory to {root_dir}")
+    setup_route_handlers(server_app.web_app)
+    name = "calkit"
+    server_app.log.info(f"Registered {name} server extension")

{calkit_python-0.32.11 → calkit_python-0.33.1}/calkit/cli/import_.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import base64
+import fnmatch
+import json
 import os
 import subprocess
 import sys
@@ -15,7 +17,9 @@ import typer
 from tqdm import tqdm
 
 import calkit
+import calkit.invenio
 from calkit.cli import raise_error
+from calkit.models.core import _ImportedFromProject
 
 import_app = typer.Typer(no_args_is_help=True)
 
@@ -32,11 +36,11 @@ def import_dataset(
         ),
     ],
     dest_path: Annotated[
-        str,
+        str | None,
         typer.Argument(help="Output path at which to save."),
     ] = None,
     filter_paths: Annotated[
-        list[str],
+        list[str] | None,
         typer.Option(
             "--filter-paths",
             help="Filter paths in target dataset if it's a folder.",
@@ -186,7 +190,7 @@ def import_dataset(
         title=resp.get("title"),
         description=resp.get("description"),
         stage=None,
-        imported_from=calkit.models._ImportedFromProject(
+        imported_from=_ImportedFromProject(
             project=f"{owner_name}/{project_name}",
             path=path,
             git_rev=resp.get("git_rev"),
@@ -301,3 +305,196 @@ def import_environment(
     repo.git.add("calkit.yaml")
     if not no_commit and calkit.git.get_staged_files():
         repo.git.commit(["-m", f"Import environment {src}"])
+
+
+@import_app.command(name="zenodo")
+def import_from_zenodo(
+    src: Annotated[str, typer.Argument(help="Source URL or DOI.")],
+    dest_dir: Annotated[
+        str,
+        typer.Argument(
+            help="Destination folder. Will be created if it doesn't exist"
+        ),
+    ],
+    kind: Annotated[
+        str | None,
+        typer.Option(
+            "--kind",
+            "-k",
+            help=(
+                "What kind of artifact is being imported, "
+                "e.g., a figure, dataset, publication."
+            ),
+        ),
+    ] = None,
+    name_like: Annotated[
+        list[str],
+        typer.Option(
+            "--name-like",
+            help="Filter for file names like this. Glob patterns accepted.",
+        ),
+    ] = [],
+    name_not_like: Annotated[
+        list[str],
+        typer.Option(
+            "--name-not-like",
+            help="Exclude names matching pattern.",
+        ),
+    ] = [],
+    storage: Annotated[
+        str | None,
+        typer.Option(
+            "--storage",
+            help="Storage backend to use (Git or DVC). "
+            "If not specified, will be chosen based on size.",
+        ),
+    ] = None,
+    no_commit: Annotated[
+        bool,
+        typer.Option("--no-commit", help="Do not commit changes to project."),
+    ] = False,
+):
+    """Import files from a Zenodo record."""
+    # Ensure destination directory either doesn't exist or is empty
+    if os.path.isdir(dest_dir):
+        if os.listdir(dest_dir):
+            raise_error("Destination directory exists and is not empty")
+    if kind is not None and kind not in ["figure", "dataset", "publication"]:
+        raise_error(
+            "Invalid kind; must be one of 'figure', 'dataset', or "
+            "'publication'"
+        )
+    # Fetch information about the record
+    # First, extract the record ID from the src
+    if "zenodo.org/record/" in src:
+        try:
+            record_id = int(src.split("zenodo.org/record/")[1].split("/")[0])
+        except Exception:
+            raise_error("Could not parse Zenodo record ID from URL")
+    elif "10.5281/zenodo." in src:
+        try:
+            record_id = int(src.split("10.5281/zenodo.")[1])
+        except Exception:
+            raise_error("Could not parse Zenodo record ID from DOI")
+    else:
+        raise_error("Source must be a Zenodo URL or DOI")
+    typer.echo(f"Fetching info for Zenodo record {record_id}")
+    try:
+        record = calkit.invenio.get(
+            f"/records/{record_id}", service="zenodo", auth=False
+        )
+    except Exception as e:
+        raise_error(f"Could not fetch record info from Zenodo: {e}")
+    # Get download URLs
+    download_urls = calkit.invenio.get_download_urls(
+        record_id, service="zenodo", auth=False
+    )
+    # Filter download URLs
+    filtered_urls = {}
+    for fname, url in download_urls.items():
+        if name_like:
+            if not any(
+                fnmatch.fnmatch(fname, pattern) for pattern in name_like
+            ):
+                continue
+        if name_not_like:
+            if any(
+                fnmatch.fnmatch(fname, pattern) for pattern in name_not_like
+            ):
+                continue
+        filtered_urls[fname] = url
+    if not filtered_urls:
+        raise_error("No files matched the specified name filters")
+    # Download files
+    for fname, url in filtered_urls.items():
+        out_path = os.path.join(dest_dir, fname)  # fname may include subdirs
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+        typer.echo(f"Downloading {fname} to {out_path}")
+        resp = requests.get(url, stream=True)
+        try:
+            resp.raise_for_status()
+        except Exception as e:
+            raise_error(f"Failed to download {fname} from {url}: {e}")
+        with open(out_path, "wb") as f:
+            for chunk in resp.iter_content(chunk_size=8192):
+                f.write(chunk)
+    # Decide if this should be stored in Git or DVC
+    repo = git.Repo()
+    commit_paths = []
+    if storage is None:
+        if calkit.get_size(dest_dir) > calkit.DVC_SIZE_THRESH_BYTES:
+            res = subprocess.run(
+                [sys.executable, "-m", "dvc", "add"], check=False
+            )
+            if res.returncode != 0:
+                raise_error("Failed to DVC add")
+            # Git add the .dvc file
+            dvc_file = dest_dir + ".dvc"
+            repo.git.add(dvc_file)
+            commit_paths.append(dvc_file)
+        else:
+            repo.git.add(dest_dir)
+            commit_paths.append(dest_dir)
+    elif storage.lower() == "dvc":
+        res = subprocess.run(
+            [sys.executable, "-m", "dvc", "add", dest_dir], check=False
+        )
+        if res.returncode != 0:
+            raise_error("Failed to DVC add")
+        # Git add the .dvc file
+        dvc_file = dest_dir + ".dvc"
+        repo.git.add(dvc_file)
+        commit_paths.append(dvc_file)
+    elif storage.lower() == "git":
+        repo.git.add(dest_dir)
+        commit_paths.append(dest_dir)
+    else:
+        raise_error("Invalid storage option; must be 'git' or 'dvc'")
+    # Determine if we should put this in one of the calkit.yaml categories
+    if kind is not None:
+        ck_info = calkit.load_calkit_info()
+        items = ck_info.get(kind + "s", [])
+        item_record = {
+            "path": dest_dir,
+            "imported_from": {
+                "doi": record.get("doi"),
+            },
+            "title": record.get("metadata", {}).get("title"),
+        }
+        items.append(item_record)
+        ck_info[kind + "s"] = items
+        with open("calkit.yaml", "w") as f:
+            calkit.ryaml.dump(ck_info, f)
+        commit_paths.append("calkit.yaml")
+    # Save a record of this import in .calkit/imports.json
+    import_record = {
+        "from": "zenodo",
+        "record_id": record_id,
+        "src": src,
+        "dest_dir": dest_dir,
+        "timestamp": calkit.utcnow(),
+        "kind": kind,
+        "storage": storage,
+        "name_like": name_like,
+        "name_not_like": name_not_like,
+    }
+    os.makedirs(".calkit", exist_ok=True)
+    imports_fpath = os.path.join(".calkit", "imports.json")
+    commit_paths.append(imports_fpath)
+    if os.path.isfile(imports_fpath):
+        with open(imports_fpath) as f:
+            imports = json.load(f)
+    else:
+        imports = []
+    imports.append(import_record)
+    with open(imports_fpath, "w") as f:
+        json.dump(imports, f, indent=2)
+    # Commit if desired
+    if not no_commit:
+        repo = git.Repo()
+        repo.git.add(commit_paths)
+        if repo.git.diff(commit_paths + ["--staged"]):
+            repo.git.commit(
+                commit_paths
+                + ["-m", f"Import {src} from Zenodo to {dest_dir}"]
+            )

{calkit_python-0.32.11 → calkit_python-0.33.1}/calkit/cli/main.py

@@ -27,6 +27,13 @@ from typing_extensions import Annotated, Optional
 import calkit
 import calkit.matlab
 import calkit.pipeline
+from calkit import (
+    AUTO_IGNORE_PATHS,
+    AUTO_IGNORE_PREFIXES,
+    AUTO_IGNORE_SUFFIXES,
+    DVC_EXTENSIONS,
+    DVC_SIZE_THRESH_BYTES,
+)
 from calkit.cli import print_sep, raise_error, run_cmd, warn
 from calkit.cli.check import (
     check_app,
@@ -76,46 +83,6 @@ app.add_typer(overleaf_app, name="overleaf", help="Interact with Overleaf.")
 app.add_typer(cloud_app, name="cloud", help="Interact with a Calkit Cloud.")
 app.add_typer(slurm_app, name="slurm", help="Work with SLURM.")
 
-# Constants for version control auto-ignore
-AUTO_IGNORE_SUFFIXES = [
-    ".DS_Store",
-    ".env",
-    ".pyc",
-    ".synctex.gz",
-    ".ipynb_checkpoints",
-]
-AUTO_IGNORE_PATHS = [os.path.join(".dvc", "config.local")]
-AUTO_IGNORE_PREFIXES = [
-    ".venv",
-    "__pycache__",
-    ".ipynb_checkpoints",
-    ".calkit/overleaf/",
-]
-# Constants for version control auto-add to DVC
-DVC_EXTENSIONS = [
-    ".png",
-    ".jpeg",
-    ".jpg",
-    ".gif",
-    ".h5",
-    ".parquet",
-    ".pickle",
-    ".mp4",
-    ".avi",
-    ".webm",
-    ".pdf",
-    ".xlsx",
-    ".docx",
-    ".pptx",
-    ".xls",
-    ".doc",
-    ".ppt",
-    ".nc",
-    ".nc4",
-    ".zarr",
-]
-DVC_SIZE_THRESH_BYTES = 1_000_000
-
 
 def _to_shell_cmd(cmd: list[str]) -> str:
     """Join a command to be compatible with running at the shell.
@@ -891,9 +858,9 @@ def _stage_run_info_from_log_content(log_content: str) -> dict:
 
 @app.command(name="run")
 def run(
-    targets: Optional[list[str]] = typer.Argument(
-        default=None, help="Stages to run."
-    ),
+    targets: Annotated[
+        list[str] | None, typer.Argument(help="Stages to run.")
+    ] = None,
     quiet: Annotated[
         bool, typer.Option("-q", "--quiet", help="Be quiet.")
     ] = False,
@@ -993,9 +960,11 @@ def run(
     no_run_cache: Annotated[
         bool, typer.Option("--no-run-cache", help="Ignore the run cache.")
     ] = False,
-    save_log: Annotated[
+    save_logs: Annotated[
         bool,
-        typer.Option("--log", "-l", help="Log the run."),
+        typer.Option(
+            "--log", "-l", help="Log the run and system information."
+        ),
     ] = False,
     save_after_run: Annotated[
         bool,
@@ -1031,7 +1000,7 @@ def run(
             help="Sync with Overleaf before and after running.",
         ),
     ] = False,
-):
+) -> dict:
     """Check dependencies and run the pipeline."""
     import dvc.log
     import dvc.repo
@@ -1059,7 +1028,7 @@ def run(
         typer.echo("Getting system information")
     # Get system information
     system_info = calkit.get_system_info()
-    if save_log:
+    if save_logs:
         # Save the system to .calkit/systems
         if verbose:
             typer.echo("Saving system information:")
@@ -1144,7 +1113,7 @@ def run(
                         targets.append(dvc_stage_name)
         if not targets:
             raise_error("No stages found to run")
-    if save_log:
+    if save_logs:
         # Get status of Git repo before running
         repo = git.Repo()
         git_rev = repo.head.commit.hexsha
@@ -1191,35 +1160,37 @@ def run(
     start_time_no_tz = calkit.utcnow(remove_tz=True)
     start_time = calkit.utcnow(remove_tz=False)
     run_id = uuid.uuid4().hex
-    if save_log:
-        log_fpath = os.path.join(
-            ".calkit",
-            "logs",
-            start_time_no_tz.isoformat(timespec="seconds").replace(":", "-")
-            + "-"
-            + run_id
-            + ".log",
-        )
-        if verbose:
-            typer.echo(f"Starting run ID: {run_id}")
-            typer.echo(f"Saving logs to {log_fpath}")
-        os.makedirs(os.path.dirname(log_fpath), exist_ok=True)
-        # Create a file handler for dvc.stage.run logger
-        file_handler = logging.FileHandler(log_fpath, mode="w")
-        file_handler.setLevel(logging.DEBUG)
-        formatter = logging.Formatter(
-            "%(asctime)s - %(levelname)s - %(message)s"
-        )
-        formatter.converter = time.gmtime  # Use UTC time for asctime
-        file_handler.setFormatter(formatter)
-        dvc.log.logger.addHandler(file_handler)
+    # Always log output, but only save systems/run data if specified
+    log_fpath = os.path.join(
+        ".calkit",
+        "logs",
+        start_time_no_tz.isoformat(timespec="seconds").replace(":", "-")
+        + "-"
+        + run_id
+        + ".log",
+    )
+    if verbose:
+        typer.echo(f"Starting run ID: {run_id}")
+        typer.echo(f"Saving logs to {log_fpath}")
+    os.makedirs(os.path.dirname(log_fpath), exist_ok=True)
+    # Create a file handler for dvc.stage.run logger
+    file_handler = logging.FileHandler(log_fpath, mode="w")
+    file_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    formatter.converter = time.gmtime  # Use UTC time for asctime
+    file_handler.setFormatter(formatter)
+    dvc.log.logger.addHandler(file_handler)
     # Remove newline logging in dvc.repo.reproduce
     dvc.repo.reproduce.logger.setLevel(logging.ERROR)
     # Disable other misc DVC output
     dvc.ui.ui.write = lambda *args, **kwargs: None
     res = dvc_cli_main(["repro"] + args)
     failed = res != 0
-    if save_log:
+    # Parse log to get timing
+    with open(log_fpath, "r") as f:
+        log_content = f.read()
+        stage_run_info = _stage_run_info_from_log_content(log_content)
+    if save_logs:
         # Get Git status after running
         git_changed_files_after = calkit.git.get_changed_files(repo=repo)
         git_staged_files_after = calkit.git.get_staged_files(repo=repo)
@@ -1228,10 +1199,6 @@ def run(
         dvc_status_after = dvc_repo.status()
         dvc_data_status_after = dvc_repo.data_status()
         dvc_data_status_after.pop("git", None)  # Remove git status
-        # Parse log to get timing
-        with open(log_fpath, "r") as f:
-            log_content = f.read()
-            stage_run_info = _stage_run_info_from_log_content(log_content)
         # Save run information to a file
         if verbose:
             typer.echo("Saving run info")
@@ -1269,6 +1236,8 @@ def run(
         os.makedirs(os.path.dirname(run_info_fpath), exist_ok=True)
         with open(run_info_fpath, "w") as f:
             json.dump(run_info, f, indent=2)
+    else:
+        os.remove(log_fpath)
     os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
     if failed:
         raise_error("Pipeline failed")
@@ -1291,6 +1260,7 @@ def run(
             no_commit=False,
             no_push=not save_after_run,
         )
+    return {"dvc_stages": dvc_stages, "stage_run_info": stage_run_info}
 
 
 @app.command(name="manual-step", help="Execute a manual step.")

{calkit_python-0.32.11 → calkit_python-0.33.1}/calkit/cli/new.py

@@ -1246,7 +1246,7 @@ def new_uv_venv(
     ] = ".venv",
     python_version: Annotated[
         str | None, typer.Option("--python", "-p", help="Python version.")
-    ] = "3.13",
+    ] = "3.14",
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1274,7 +1274,6 @@ def new_uv_venv(
         raise_error("Output path already exists (use -f to overwrite)")
     elif packages is None and not os.path.isfile(path):
         raise_error("If path doesn't exist, packages must be specified")
-    repo = git.Repo()
     # Add environment to Calkit info
     ck_info = calkit.load_calkit_info()
     # If environments is a list instead of a dict, reformulate it
@@ -1301,7 +1300,6 @@ def new_uv_venv(
         _check_path_dir(path)
         with open(path, "w") as f:
             f.write(packages_txt)
-    repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
     env = dict(path=path, kind="uv-venv", prefix=prefix)
     if python_version is not None:
@@ -1312,10 +1310,13 @@ def new_uv_venv(
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
         ryaml.dump(ck_info, f)
-    repo.git.add("calkit.yaml")
+    if not no_commit:
+        repo = git.Repo()
+        repo.git.add(path)
+        repo.git.add("calkit.yaml")
     if not no_check:
         env_lock_fpath = check_environment(env_name=name)
-        if env_lock_fpath is not None:
+        if env_lock_fpath is not None and not no_commit:
             repo.git.add(env_lock_fpath)
     if not no_commit and repo.git.diff("--staged"):
         repo.git.commit(["-m", f"Add uv venv {name}"])