calkit-python 0.47.12__tar.gz → 0.48.0__tar.gz

{calkit_python-0.47.12 → calkit_python-0.48.0}/.pre-commit-config.yaml

@@ -2,6 +2,8 @@
 # with the root pyproject config, so the version and rules can't drift
 # between packages. The hub frontend is formatted by its own pinned Biome
 # to match hub/frontend/biome.json exactly.
+# DVC writes dvc.lock and compares it as written, so no hook touches one.
+exclude: (^|/)dvc\.lock$
 repos:
   # renv generates renv/activate.R and rewrites it whenever it activates,
   # trailing whitespace and all. Trimming it here would only make the
@@ -79,14 +81,13 @@ repos:
     rev: "v3.0.3"
     hooks:
       - id: prettier
-        # dvc.yaml and dvc.lock are generated---by calkit and DVC
-        # respectively---and are rewritten wholesale whenever the pipeline
-        # changes. Prettier and ruamel disagree about where a long 'desc:'
-        # value goes, so formatting them here just means the two take turns
-        # reformatting the same file.
+        # dvc.yaml is generated by calkit and rewritten wholesale whenever
+        # the pipeline changes. Prettier and ruamel disagree about where a
+        # long 'desc:' value goes, so formatting it here just means the two
+        # take turns reformatting the same file.
         # Likewise the example projects' results files, which are stage
         # outputs and have to match what the stage wrote byte for byte.
-        exclude: calkit|^hub/|(^|/)dvc\.(yaml|lock)$|^examples/.*/results/
+        exclude: calkit|^hub/|(^|/)dvc\.yaml$|^examples/.*/results/
 
   - repo: local
     hooks:

{calkit_python-0.47.12 → calkit_python-0.48.0}/CONTRIBUTING.md

@@ -44,6 +44,10 @@ This guide will help you get started.
    ```sh
    uv tool install -e .
    ```
+   Dev installs aren't upgraded automatically.
+   To pull the latest changes on the current branch and reinstall,
+   picking up any new dependencies, run `calkit upgrade`.
+   To install from another branch, run `calkit dev upgrade <branch>`.
 1. Install the JupyterLab extension in development mode
    (if working on the JupyterLab extension):
    ```sh

{calkit_python-0.47.12 → calkit_python-0.48.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: calkit-python
-Version: 0.47.12
+Version: 0.48.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -290,6 +290,33 @@ If you use AI agents like Claude, Copilot, or Codex,
 see [AI tools](https://docs.calkit.org/ai-tools)
 to learn how to install agent skills for working with Calkit.
 
+### Upgrading
+
+Calkit checks for a new version once a day.
+If it was installed with uv, as above, or pipx,
+it upgrades itself in the background,
+so your next command runs on the new version
+and starts by telling you which version that is.
+Otherwise, it tells you a new version is available.
+
+To be told about new versions but not upgrade automatically, run:
+
+```sh
+calkit config set auto_upgrade false
+```
+
+To stop checking for new versions altogether, run:
+
+```sh
+calkit config set auto_upgrade no-check
+```
+
+To upgrade by hand at any time, run:
+
+```sh
+calkit upgrade
+```
+
 ### Use without installing
 
 If you want to use Calkit without installing it,

{calkit_python-0.47.12 → calkit_python-0.48.0}/README.md

@@ -228,6 +228,33 @@ If you use AI agents like Claude, Copilot, or Codex,
 see [AI tools](https://docs.calkit.org/ai-tools)
 to learn how to install agent skills for working with Calkit.
 
+### Upgrading
+
+Calkit checks for a new version once a day.
+If it was installed with uv, as above, or pipx,
+it upgrades itself in the background,
+so your next command runs on the new version
+and starts by telling you which version that is.
+Otherwise, it tells you a new version is available.
+
+To be told about new versions but not upgrade automatically, run:
+
+```sh
+calkit config set auto_upgrade false
+```
+
+To stop checking for new versions altogether, run:
+
+```sh
+calkit config set auto_upgrade no-check
+```
+
+To upgrade by hand at any time, run:
+
+```sh
+calkit upgrade
+```
+
 ### Use without installing
 
 If you want to use Calkit without installing it,

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/__init__.py

@@ -48,6 +48,7 @@ if TYPE_CHECKING:
         schema,
         server,
         templates,
+        upgrade,
         workspace,
     )
 
@@ -94,6 +95,7 @@ _SUBMODULES = {
     "schema",
     "server",
     "templates",
+    "upgrade",
     "workspace",
 }
 

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/cli/dev.py

@@ -45,3 +45,26 @@ def run_ipython(
     from IPython import start_ipython
 
     start_ipython(argv=sys.argv[3:], user_ns={"calkit": calkit})
+
+
+@dev_app.command(name="upgrade")
+def upgrade(
+    branch: Annotated[
+        str | None,
+        typer.Argument(
+            help=(
+                "Branch to install from: the worktree that has it checked "
+                "out, or else the install's own, if it has no changes. "
+                "Defaults to the current branch."
+            )
+        ),
+    ] = None,
+) -> None:
+    """Update a dev (editable) install: pull the branch and reinstall."""
+    from calkit.cli.core import raise_error
+    from calkit.upgrade import get_dev_upgrade_cmds, run_upgrade_cmds
+
+    try:
+        run_upgrade_cmds(get_dev_upgrade_cmds(branch))
+    except Exception as e:
+        raise_error(str(e))

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/cli/main/core.py

@@ -125,6 +125,7 @@ def _to_shell_cmd(cmd: list[str]) -> str:
 
 @app.callback()
 def main(
+    ctx: typer.Context,
     version: Annotated[
         bool,
         typer.Option("--version", help="Show version and exit."),
@@ -169,6 +170,7 @@ def main(
     # didn't, so a CALKIT_HUB in .env sent the two to different hubs.
     dotenv.load_dotenv(dotenv_path=".env")
     _warn_on_stale_calkit_env()
+    calkit.upgrade.check(ctx.invoked_subcommand)
 
 
 def _warn_on_stale_calkit_env() -> None:
@@ -510,8 +512,18 @@ def get_status(
     import git
     from git.exc import InvalidGitRepositoryError
 
+    def get_dvc_data_status() -> dict:
+        # Reuse what the pipeline status computed under the same DVC lock
+        if (
+            pipeline_status is not None
+            and pipeline_status.dvc_data_status is not None
+        ):
+            return dict(pipeline_status.dvc_data_status)
+        return dict(calkit.dvc.get_dvc_repo().data_status())
+
     dotenv.load_dotenv(dotenv_path=".env")
-    ck_info = calkit.load_calkit_info()
+    # Status only reads calkit.yaml, so skip the slower round-trip parser
+    ck_info = calkit.load_calkit_info(read_only=True)
     # Status is usually the first command someone runs on a project that
     # isn't theirs, which is exactly when the variables it declares are
     # missing. Ask while there's somebody to answer -- the environment
@@ -576,6 +588,7 @@ def get_status(
             check_environments=not no_check_envs,
             clean_notebooks=True,
             compile_to_dvc=True,
+            with_data_status="dvc" in categories,
         )
         if pipeline_status.failed_environment_checks:
             warn(
@@ -608,10 +621,13 @@ def get_status(
                 frozen_stages = frozen_tainted_stage_names(ck_info=ck_info)
             except Exception:
                 frozen_stages = set()
+        # The summary doesn't say what changed since an answer was edited,
+        # which is the only part that needs history, so skip reading it
         questions_status = check_questions(
             ck_info=ck_info,
             stale_stages=stale_stages,
             frozen_stages=frozen_stages,
+            check_history=False,
         )
     if as_json:
         status_dict: dict[str, Any] = {}
@@ -671,8 +687,7 @@ def get_status(
                 status_dict["git"] = {"error": "Not a Git repository"}
         if "dvc" in categories:
             try:
-                dvc_repo = calkit.dvc.get_dvc_repo()
-                data_status = dict(dvc_repo.data_status())
+                data_status = get_dvc_data_status()
                 data_status.pop("git", None)
                 status_dict["dvc"] = calkit.dvc.data_status_as_posix(
                     data_status
@@ -741,8 +756,7 @@ def get_status(
             )
         else:
             zip_path_map = calkit.dvc.zip.get_zip_path_map()
-            dvc_repo = calkit.dvc.get_dvc_repo()
-            raw = dict(dvc_repo.data_status())
+            raw = get_dvc_data_status()
             raw.pop("git", None)
             raw = calkit.dvc.data_status_as_posix(raw)
             typer.echo(_format_dvc_data_status(raw, zip_path_map))
@@ -4327,42 +4341,26 @@ def upgrade(
         bool, typer.Option("--skills", help="Upgrade agent skills as well.")
     ] = False,
 ) -> None:
-    """Upgrade Calkit."""
-    # First detect how Calkit is installed
-    # If installed with uv tool, calkit will be located at something like
-    # ~/.local/bin/calkit
-    which_calkit = shutil.which("calkit")
-    if which_calkit is None:
-        raise_error("Calkit is not installed")
-    split_path = os.path.normpath(str(which_calkit)).split(os.sep)
-    if (
-        ".local" in split_path
-        and "bin" in split_path
-        and calkit.check_dep_exists("uv")
-    ):
-        # This is a uv tool install
-        cmd = [
-            "uv",
-            "tool",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    elif "pipx" in split_path and calkit.check_dep_exists("pipx"):
-        cmd = ["pipx", "upgrade", "calkit-python"]
-    else:
-        cmd = [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    res = subprocess.run(cmd)
-    if res.returncode != 0:
-        raise_error("Upgrade failed")
-    typer.echo("Success!")
+    """Upgrade Calkit.
+
+    A dev (editable) install is updated from the branch it's on, as with
+    'calkit dev upgrade'.
+    """
+    from calkit.upgrade import (
+        get_dev_upgrade_cmds,
+        get_editable_path,
+        get_upgrade_cmd,
+        run_upgrade_cmds,
+    )
+
+    try:
+        if get_editable_path() is not None:
+            cmds = get_dev_upgrade_cmds()
+        else:
+            cmds = [get_upgrade_cmd()]
+        run_upgrade_cmds(cmds)
+    except Exception as e:
+        raise_error(str(e))
     if skills:
         from calkit.cli.update import update_agent_skills
 

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/config.py

@@ -439,6 +439,9 @@ class Settings(BaseSettings):
     # 'calkit describe system' reports the ID actually in effect.
     machine_id: str | None = None
     dataframe_engine: Literal["pandas", "polars"] = "pandas"
+    # Upgrade Calkit when a daily check finds a new version; when false,
+    # the check only notifies, and 'no-check' turns the check off
+    auto_upgrade: bool | Literal["no-check"] = True
     run_history_length: int = 10
     github_token: KeyringOptionalSecret | None = None
     # Kept apart from github_token, since pushing to the GitHub Container

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/core.py

@@ -125,9 +125,65 @@ try:
     # PyYAML ships prebuilt wheels with libyaml on every platform we target,
     # so this is the normal path; the fallback keeps a source install or an
     # exotic build working, just slower.
-    from yaml import CSafeLoader as _YamlLoader
+    from yaml import CSafeLoader as _BaseYamlLoader
 except ImportError:  # pragma: no cover - depends on the libyaml build
-    from yaml import SafeLoader as _YamlLoader
+    from yaml import SafeLoader as _BaseYamlLoader
+
+
+class _YamlLoader(_BaseYamlLoader):  # type: ignore[no-any-unimported]
+    """The C loader, reading scalars as YAML 1.2 like ``ryaml`` does.
+
+    PyYAML follows YAML 1.1, where ``no`` and ``on`` are booleans, ``1:30``
+    is 90, and ``010`` is octal, so the same file would load differently
+    depending on which parser read it.
+    """
+
+
+def _construct_yaml12_int(loader: Any, node: Any) -> int:
+    value = str(loader.construct_scalar(node)).replace("_", "")
+    sign = -1 if value.startswith("-") else 1
+    value = value.lstrip("+-")
+    for prefix, base in (("0b", 2), ("0o", 8), ("0x", 16)):
+        if value.startswith(prefix):
+            return sign * int(value[2:], base)
+    return sign * int(value)
+
+
+# ruamel.yaml's YAML 1.2 resolvers for the scalars that differ from 1.1
+_YAML12_RESOLVERS = {
+    "tag:yaml.org,2002:bool": (
+        r"^(?:true|True|TRUE|false|False|FALSE)$",
+        "tTfF",
+    ),
+    "tag:yaml.org,2002:float": (
+        r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
+        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
+        |[-+]?\.[0-9_]+(?:[eE][-+][0-9]+)?
+        |[-+]?\.(?:inf|Inf|INF)
+        |\.(?:nan|NaN|NAN))$""",
+        "-+0123456789.",
+    ),
+    "tag:yaml.org,2002:int": (
+        r"""^(?:[-+]?0b[0-1_]+
+        |[-+]?0o?[0-7_]+
+        |[-+]?[0-9_]+
+        |[-+]?0x[0-9a-fA-F_]+)$""",
+        "-+0123456789",
+    ),
+}
+_YamlLoader.yaml_implicit_resolvers = {
+    first: [
+        (tag, regexp)
+        for tag, regexp in resolvers
+        if tag not in _YAML12_RESOLVERS
+    ]
+    for first, resolvers in _BaseYamlLoader.yaml_implicit_resolvers.items()
+}
+for _tag, (_pattern, _first) in _YAML12_RESOLVERS.items():
+    _YamlLoader.add_implicit_resolver(
+        _tag, re.compile(_pattern, re.X), list(_first)
+    )
+_YamlLoader.add_constructor("tag:yaml.org,2002:int", _construct_yaml12_int)
 
 
 def _load_yaml_readonly(stream: Any) -> Any:
@@ -1157,7 +1213,7 @@ def detect_project_name(
     If ``prepend_owner`` is False, fall back to the working directory name
     if there is no Git remote or name specified in ``calkit.yaml``.
     """
-    ck_info = load_calkit_info(wdir=wdir)
+    ck_info = load_calkit_info(wdir=wdir, read_only=True)
     name = ck_info.get("name")
     if name is not None and not prepend_owner:
         return name

{calkit_python-0.47.12 → calkit_python-0.48.0}/calkit/dvc/core.py

@@ -8,6 +8,8 @@ import json
 import logging
 import os
 import sys
+import threading
+from collections.abc import Iterator
 from itertools import groupby
 from pathlib import Path
 from typing import Any, Literal
@@ -209,6 +211,117 @@ def dvc_lock_timeout(seconds: float):
         dvc.lock.DEFAULT_TIMEOUT = previous
 
 
+_memoized_hashes_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def memoized_hashes() -> Iterator[None]:
+    """Hash each output or dependency path only once while active.
+
+    DVC hashes a directory again for every stage that depends on it, walking
+    every file each time, so a dataset that feeds twenty stages is walked
+    twenty times by one ``status``. Its ``data_status`` then walks the outputs
+    once more, for nothing but their directory hashes unless asked for a
+    granular status. Nothing on disk changes during a status check, so the
+    first answer stands for the rest.
+    """
+    import importlib
+
+    import dvc.repo.data
+    import dvc.repo.index
+    from dvc.output import Output
+
+    # By name, since dvc_data.index exports a build function that shadows it
+    index_build: Any = importlib.import_module("dvc_data.index.build")
+
+    # The patches are process-wide, so threads take turns rather than
+    # capturing and restoring each other's
+    with _memoized_hashes_lock:
+        original_get_hash_meta = Output._get_hash_meta
+        original_diff = dvc.repo.data._diff_index_to_wtree
+        original_build_index = dvc.repo.index.build_data_index
+        original_build_entries = index_build.build_entries
+        memo: dict[tuple, Any] = {}
+        # Local directories already hashed, as (meta, hash_info) by repo,
+        # path, and hash name, and whether the walk in progress only needs
+        # those
+        dirs: dict[tuple[int | None, str, str], Any] = {}
+        shallow = False
+        walking: int | None = None
+        skipped: dict[str, Any] = {}
+
+        def _get_hash_meta(self: Any) -> Any:
+            key = (
+                id(self.repo),
+                self.fs.protocol,
+                self.fs_path,
+                self.hash_name,
+                self.use_cache,
+            )
+            if key not in memo:
+                memo[key] = original_get_hash_meta(self)
+                meta, hash_info = memo[key]
+                if (
+                    self.fs.protocol == "local"
+                    and hash_info
+                    and hash_info.isdir
+                ):
+                    dirs[(id(self.repo), self.fs_path, self.hash_name)] = memo[
+                        key
+                    ]
+            return memo[key]
+
+        def _diff_index_to_wtree(*args: Any, **kwargs: Any) -> Any:
+            nonlocal shallow
+            shallow = not kwargs.get("granular", False)
+            try:
+                return original_diff(*args, **kwargs)
+            finally:
+                shallow = False
+
+        def build_entries(
+            path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            key = (walking, path, kwargs.get("hash_name", "md5"))
+            if key in dirs:
+                skipped[path] = dirs[key]
+                return iter(())
+            return original_build_entries(path, fs, *args, **kwargs)
+
+        def build_data_index(
+            index: Any, path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            nonlocal walking
+            if not (shallow and kwargs.get("compute_hash")):
+                return original_build_index(index, path, fs, *args, **kwargs)
+            walking = id(index.repo)
+            skipped.clear()
+            try:
+                data = original_build_index(index, path, fs, *args, **kwargs)
+            finally:
+                walking = None
+            # Outputs whose walk was skipped get the hashes already computed
+            # rather than those of their now-empty listing
+            for dir_path, (meta, hash_info) in skipped.items():
+                entry = data.get(fs.relparts(dir_path, path))
+                if entry is not None:
+                    entry.meta = meta
+                    entry.hash_info = hash_info
+            return data
+
+        Output._get_hash_meta = _get_hash_meta
+        dvc.repo.data._diff_index_to_wtree = _diff_index_to_wtree
+        dvc.repo.index.build_data_index = build_data_index
+        index_build.build_entries = build_entries
+        try:
+            yield
+        finally:
+            Output._get_hash_meta = original_get_hash_meta
+            dvc.repo.data._diff_index_to_wtree = original_diff
+            dvc.repo.index.build_data_index = original_build_index
+            index_build.build_entries = original_build_entries
+
+
 class CalkitDVCFileSystem(ObjectFileSystem):
     """DVC-facing filesystem wrapper for the ``ck://`` scheme."""