calkit-python 0.47.12__py3-none-any.whl → 0.48.0__py3-none-any.whl

calkit/__init__.py

@@ -48,6 +48,7 @@ if TYPE_CHECKING:
         schema,
         server,
         templates,
+        upgrade,
         workspace,
     )
 
@@ -94,6 +95,7 @@ _SUBMODULES = {
     "schema",
     "server",
     "templates",
+    "upgrade",
     "workspace",
 }
 

calkit/cli/dev.py

@@ -45,3 +45,26 @@ def run_ipython(
     from IPython import start_ipython
 
     start_ipython(argv=sys.argv[3:], user_ns={"calkit": calkit})
+
+
+@dev_app.command(name="upgrade")
+def upgrade(
+    branch: Annotated[
+        str | None,
+        typer.Argument(
+            help=(
+                "Branch to install from: the worktree that has it checked "
+                "out, or else the install's own, if it has no changes. "
+                "Defaults to the current branch."
+            )
+        ),
+    ] = None,
+) -> None:
+    """Update a dev (editable) install: pull the branch and reinstall."""
+    from calkit.cli.core import raise_error
+    from calkit.upgrade import get_dev_upgrade_cmds, run_upgrade_cmds
+
+    try:
+        run_upgrade_cmds(get_dev_upgrade_cmds(branch))
+    except Exception as e:
+        raise_error(str(e))

calkit/cli/main/core.py

@@ -125,6 +125,7 @@ def _to_shell_cmd(cmd: list[str]) -> str:
 
 @app.callback()
 def main(
+    ctx: typer.Context,
     version: Annotated[
         bool,
         typer.Option("--version", help="Show version and exit."),
@@ -169,6 +170,7 @@ def main(
     # didn't, so a CALKIT_HUB in .env sent the two to different hubs.
     dotenv.load_dotenv(dotenv_path=".env")
     _warn_on_stale_calkit_env()
+    calkit.upgrade.check(ctx.invoked_subcommand)
 
 
 def _warn_on_stale_calkit_env() -> None:
@@ -510,8 +512,18 @@ def get_status(
     import git
     from git.exc import InvalidGitRepositoryError
 
+    def get_dvc_data_status() -> dict:
+        # Reuse what the pipeline status computed under the same DVC lock
+        if (
+            pipeline_status is not None
+            and pipeline_status.dvc_data_status is not None
+        ):
+            return dict(pipeline_status.dvc_data_status)
+        return dict(calkit.dvc.get_dvc_repo().data_status())
+
     dotenv.load_dotenv(dotenv_path=".env")
-    ck_info = calkit.load_calkit_info()
+    # Status only reads calkit.yaml, so skip the slower round-trip parser
+    ck_info = calkit.load_calkit_info(read_only=True)
     # Status is usually the first command someone runs on a project that
     # isn't theirs, which is exactly when the variables it declares are
     # missing. Ask while there's somebody to answer -- the environment
@@ -576,6 +588,7 @@ def get_status(
             check_environments=not no_check_envs,
             clean_notebooks=True,
             compile_to_dvc=True,
+            with_data_status="dvc" in categories,
         )
         if pipeline_status.failed_environment_checks:
             warn(
@@ -608,10 +621,13 @@ def get_status(
                 frozen_stages = frozen_tainted_stage_names(ck_info=ck_info)
             except Exception:
                 frozen_stages = set()
+        # The summary doesn't say what changed since an answer was edited,
+        # which is the only part that needs history, so skip reading it
         questions_status = check_questions(
             ck_info=ck_info,
             stale_stages=stale_stages,
             frozen_stages=frozen_stages,
+            check_history=False,
         )
     if as_json:
         status_dict: dict[str, Any] = {}
@@ -671,8 +687,7 @@ def get_status(
                 status_dict["git"] = {"error": "Not a Git repository"}
         if "dvc" in categories:
             try:
-                dvc_repo = calkit.dvc.get_dvc_repo()
-                data_status = dict(dvc_repo.data_status())
+                data_status = get_dvc_data_status()
                 data_status.pop("git", None)
                 status_dict["dvc"] = calkit.dvc.data_status_as_posix(
                     data_status
@@ -741,8 +756,7 @@ def get_status(
             )
         else:
             zip_path_map = calkit.dvc.zip.get_zip_path_map()
-            dvc_repo = calkit.dvc.get_dvc_repo()
-            raw = dict(dvc_repo.data_status())
+            raw = get_dvc_data_status()
             raw.pop("git", None)
             raw = calkit.dvc.data_status_as_posix(raw)
             typer.echo(_format_dvc_data_status(raw, zip_path_map))
@@ -4327,42 +4341,26 @@ def upgrade(
         bool, typer.Option("--skills", help="Upgrade agent skills as well.")
     ] = False,
 ) -> None:
-    """Upgrade Calkit."""
-    # First detect how Calkit is installed
-    # If installed with uv tool, calkit will be located at something like
-    # ~/.local/bin/calkit
-    which_calkit = shutil.which("calkit")
-    if which_calkit is None:
-        raise_error("Calkit is not installed")
-    split_path = os.path.normpath(str(which_calkit)).split(os.sep)
-    if (
-        ".local" in split_path
-        and "bin" in split_path
-        and calkit.check_dep_exists("uv")
-    ):
-        # This is a uv tool install
-        cmd = [
-            "uv",
-            "tool",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    elif "pipx" in split_path and calkit.check_dep_exists("pipx"):
-        cmd = ["pipx", "upgrade", "calkit-python"]
-    else:
-        cmd = [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    res = subprocess.run(cmd)
-    if res.returncode != 0:
-        raise_error("Upgrade failed")
-    typer.echo("Success!")
+    """Upgrade Calkit.
+
+    A dev (editable) install is updated from the branch it's on, as with
+    'calkit dev upgrade'.
+    """
+    from calkit.upgrade import (
+        get_dev_upgrade_cmds,
+        get_editable_path,
+        get_upgrade_cmd,
+        run_upgrade_cmds,
+    )
+
+    try:
+        if get_editable_path() is not None:
+            cmds = get_dev_upgrade_cmds()
+        else:
+            cmds = [get_upgrade_cmd()]
+        run_upgrade_cmds(cmds)
+    except Exception as e:
+        raise_error(str(e))
     if skills:
         from calkit.cli.update import update_agent_skills
 

calkit/config.py

@@ -439,6 +439,9 @@ class Settings(BaseSettings):
     # 'calkit describe system' reports the ID actually in effect.
     machine_id: str | None = None
     dataframe_engine: Literal["pandas", "polars"] = "pandas"
+    # Upgrade Calkit when a daily check finds a new version; when false,
+    # the check only notifies, and 'no-check' turns the check off
+    auto_upgrade: bool | Literal["no-check"] = True
     run_history_length: int = 10
     github_token: KeyringOptionalSecret | None = None
     # Kept apart from github_token, since pushing to the GitHub Container

calkit/core.py

@@ -125,9 +125,65 @@ try:
     # PyYAML ships prebuilt wheels with libyaml on every platform we target,
     # so this is the normal path; the fallback keeps a source install or an
     # exotic build working, just slower.
-    from yaml import CSafeLoader as _YamlLoader
+    from yaml import CSafeLoader as _BaseYamlLoader
 except ImportError:  # pragma: no cover - depends on the libyaml build
-    from yaml import SafeLoader as _YamlLoader
+    from yaml import SafeLoader as _BaseYamlLoader
+
+
+class _YamlLoader(_BaseYamlLoader):  # type: ignore[no-any-unimported]
+    """The C loader, reading scalars as YAML 1.2 like ``ryaml`` does.
+
+    PyYAML follows YAML 1.1, where ``no`` and ``on`` are booleans, ``1:30``
+    is 90, and ``010`` is octal, so the same file would load differently
+    depending on which parser read it.
+    """
+
+
+def _construct_yaml12_int(loader: Any, node: Any) -> int:
+    value = str(loader.construct_scalar(node)).replace("_", "")
+    sign = -1 if value.startswith("-") else 1
+    value = value.lstrip("+-")
+    for prefix, base in (("0b", 2), ("0o", 8), ("0x", 16)):
+        if value.startswith(prefix):
+            return sign * int(value[2:], base)
+    return sign * int(value)
+
+
+# ruamel.yaml's YAML 1.2 resolvers for the scalars that differ from 1.1
+_YAML12_RESOLVERS = {
+    "tag:yaml.org,2002:bool": (
+        r"^(?:true|True|TRUE|false|False|FALSE)$",
+        "tTfF",
+    ),
+    "tag:yaml.org,2002:float": (
+        r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
+        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
+        |[-+]?\.[0-9_]+(?:[eE][-+][0-9]+)?
+        |[-+]?\.(?:inf|Inf|INF)
+        |\.(?:nan|NaN|NAN))$""",
+        "-+0123456789.",
+    ),
+    "tag:yaml.org,2002:int": (
+        r"""^(?:[-+]?0b[0-1_]+
+        |[-+]?0o?[0-7_]+
+        |[-+]?[0-9_]+
+        |[-+]?0x[0-9a-fA-F_]+)$""",
+        "-+0123456789",
+    ),
+}
+_YamlLoader.yaml_implicit_resolvers = {
+    first: [
+        (tag, regexp)
+        for tag, regexp in resolvers
+        if tag not in _YAML12_RESOLVERS
+    ]
+    for first, resolvers in _BaseYamlLoader.yaml_implicit_resolvers.items()
+}
+for _tag, (_pattern, _first) in _YAML12_RESOLVERS.items():
+    _YamlLoader.add_implicit_resolver(
+        _tag, re.compile(_pattern, re.X), list(_first)
+    )
+_YamlLoader.add_constructor("tag:yaml.org,2002:int", _construct_yaml12_int)
 
 
 def _load_yaml_readonly(stream: Any) -> Any:
@@ -1157,7 +1213,7 @@ def detect_project_name(
     If ``prepend_owner`` is False, fall back to the working directory name
     if there is no Git remote or name specified in ``calkit.yaml``.
     """
-    ck_info = load_calkit_info(wdir=wdir)
+    ck_info = load_calkit_info(wdir=wdir, read_only=True)
     name = ck_info.get("name")
     if name is not None and not prepend_owner:
         return name

calkit/dvc/core.py

@@ -8,6 +8,8 @@ import json
 import logging
 import os
 import sys
+import threading
+from collections.abc import Iterator
 from itertools import groupby
 from pathlib import Path
 from typing import Any, Literal
@@ -209,6 +211,117 @@ def dvc_lock_timeout(seconds: float):
         dvc.lock.DEFAULT_TIMEOUT = previous
 
 
+_memoized_hashes_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def memoized_hashes() -> Iterator[None]:
+    """Hash each output or dependency path only once while active.
+
+    DVC hashes a directory again for every stage that depends on it, walking
+    every file each time, so a dataset that feeds twenty stages is walked
+    twenty times by one ``status``. Its ``data_status`` then walks the outputs
+    once more, for nothing but their directory hashes unless asked for a
+    granular status. Nothing on disk changes during a status check, so the
+    first answer stands for the rest.
+    """
+    import importlib
+
+    import dvc.repo.data
+    import dvc.repo.index
+    from dvc.output import Output
+
+    # By name, since dvc_data.index exports a build function that shadows it
+    index_build: Any = importlib.import_module("dvc_data.index.build")
+
+    # The patches are process-wide, so threads take turns rather than
+    # capturing and restoring each other's
+    with _memoized_hashes_lock:
+        original_get_hash_meta = Output._get_hash_meta
+        original_diff = dvc.repo.data._diff_index_to_wtree
+        original_build_index = dvc.repo.index.build_data_index
+        original_build_entries = index_build.build_entries
+        memo: dict[tuple, Any] = {}
+        # Local directories already hashed, as (meta, hash_info) by repo,
+        # path, and hash name, and whether the walk in progress only needs
+        # those
+        dirs: dict[tuple[int | None, str, str], Any] = {}
+        shallow = False
+        walking: int | None = None
+        skipped: dict[str, Any] = {}
+
+        def _get_hash_meta(self: Any) -> Any:
+            key = (
+                id(self.repo),
+                self.fs.protocol,
+                self.fs_path,
+                self.hash_name,
+                self.use_cache,
+            )
+            if key not in memo:
+                memo[key] = original_get_hash_meta(self)
+                meta, hash_info = memo[key]
+                if (
+                    self.fs.protocol == "local"
+                    and hash_info
+                    and hash_info.isdir
+                ):
+                    dirs[(id(self.repo), self.fs_path, self.hash_name)] = memo[
+                        key
+                    ]
+            return memo[key]
+
+        def _diff_index_to_wtree(*args: Any, **kwargs: Any) -> Any:
+            nonlocal shallow
+            shallow = not kwargs.get("granular", False)
+            try:
+                return original_diff(*args, **kwargs)
+            finally:
+                shallow = False
+
+        def build_entries(
+            path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            key = (walking, path, kwargs.get("hash_name", "md5"))
+            if key in dirs:
+                skipped[path] = dirs[key]
+                return iter(())
+            return original_build_entries(path, fs, *args, **kwargs)
+
+        def build_data_index(
+            index: Any, path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            nonlocal walking
+            if not (shallow and kwargs.get("compute_hash")):
+                return original_build_index(index, path, fs, *args, **kwargs)
+            walking = id(index.repo)
+            skipped.clear()
+            try:
+                data = original_build_index(index, path, fs, *args, **kwargs)
+            finally:
+                walking = None
+            # Outputs whose walk was skipped get the hashes already computed
+            # rather than those of their now-empty listing
+            for dir_path, (meta, hash_info) in skipped.items():
+                entry = data.get(fs.relparts(dir_path, path))
+                if entry is not None:
+                    entry.meta = meta
+                    entry.hash_info = hash_info
+            return data
+
+        Output._get_hash_meta = _get_hash_meta
+        dvc.repo.data._diff_index_to_wtree = _diff_index_to_wtree
+        dvc.repo.index.build_data_index = build_data_index
+        index_build.build_entries = build_entries
+        try:
+            yield
+        finally:
+            Output._get_hash_meta = original_get_hash_meta
+            dvc.repo.data._diff_index_to_wtree = original_diff
+            dvc.repo.index.build_data_index = original_build_index
+            index_build.build_entries = original_build_entries
+
+
 class CalkitDVCFileSystem(ObjectFileSystem):
     """DVC-facing filesystem wrapper for the ``ck://`` scheme."""
 

calkit/pipeline.py

@@ -151,6 +151,9 @@ class PipelineStatus(BaseModel):
     ignored_files_in_inputs: dict[str, dict[str, list[str]]] = Field(
         default_factory=dict
     )
+    # DVC's data status, when asked for, computed alongside the stage status
+    # so it reuses that index and those hashes; not part of the output
+    dvc_data_status: dict | None = Field(default=None, exclude=True)
 
     @field_validator("stale_stages", mode="before")
     @classmethod
@@ -853,11 +856,14 @@ def get_status(
     clean_notebooks: bool = True,
     compile_to_dvc: bool = True,
     force_env_check: bool = False,
+    with_data_status: bool = False,
 ) -> PipelineStatus:
     """Get pipeline status after optional prep checks.
 
     This can compile the Calkit pipeline to DVC, clean notebook outputs,
     check pipeline environments, then query DVC for out-of-date stages.
+    With ``with_data_status``, DVC's data status is attached as well, left
+    None if it fails so the caller can ask again and report why.
     """
     import calkit.environments
     import calkit.markdown
@@ -934,6 +940,7 @@ def get_status(
                     f"{e.__class__.__name__}: {e}"
                 )
                 return PipelineStatus.model_validate(result)
+        import dvc.repo
         from dvc.lock import LockError
 
         try:
@@ -944,7 +951,23 @@ def get_status(
             # DVC's own target filtering can miss stale propagation from
             # isolated subprojects, which makes targeted status disagree with
             # the full-project status view.
-            raw_status = dvc_repo.status()
+            # Held across all of these so DVC builds its index and hashes
+            # each path once, rather than dropping them between calls
+            dvc_data_status = None
+            with (
+                dvc.repo.lock_repo(dvc_repo),
+                calkit.dvc.memoized_hashes(),
+            ):
+                raw_status = dvc_repo.status()
+                dvc_out_paths = [
+                    Path(os.path.relpath(out.fs_path)).as_posix()
+                    for out in dvc_repo.index.outs
+                ]
+                if with_data_status:
+                    try:
+                        dvc_data_status = dict(dvc_repo.data_status())
+                    except Exception:
+                        pass
             raw_status = calkit.dvc.status_as_posix(raw_status)
         except LockError:
             # Another DVC process is holding the repo lock---most often a
@@ -1207,9 +1230,7 @@ def get_status(
         # actually needs listing, so a pipeline without directory inputs
         # never pays for them.
         ignored_files_by_dir: dict[str, list[str]] = {}
-        git_repo = None
-        git_root = ""
-        dvc_out_paths: list[str] = []
+        stage_dirs: list[tuple[str, str]] = []
         for name, stage_def in dvc_yaml_stage_defs.items():
             if not isinstance(stage_def, dict):
                 continue
@@ -1230,71 +1251,70 @@ def get_status(
                     for target in targets
                 ):
                     continue
-                if dep_path not in ignored_files_by_dir:
-                    if git_repo is None:
-                        git_repo = calkit.git.get_repo()
-                        git_root = str(git_repo.working_tree_dir)
-                        dvc_out_paths = [
-                            Path(os.path.relpath(out.fs_path)).as_posix()
-                            for out in dvc_repo.index.outs
-                        ]
-                    # Git prints paths relative to the repo root, which may be
-                    # above the project, so make them project-relative; -z
-                    # keeps unusual file names from being quoted
-                    listing = git_repo.git.ls_files(
-                        "-z",
-                        "--others",
-                        "--ignored",
-                        "--exclude-standard",
-                        "--",
-                        os.path.abspath(dep_path),
-                    )
-                    files: list[str] = []
-                    for ignored_path in listing.split("\0"):
-                        if not ignored_path:
-                            continue
-                        ignored_path = Path(
-                            os.path.relpath(
-                                os.path.join(git_root, ignored_path)
-                            )
-                        ).as_posix()
-                        if any(
-                            ignored_path == out
-                            or ignored_path.startswith(out + "/")
-                            for out in dvc_out_paths
-                        ):
-                            continue
-                        if dvc_repo.dvcignore.is_ignored_file(
-                            os.path.abspath(ignored_path)
-                        ):
-                            continue
-                        files.append(ignored_path)
-                    ignored_files_by_dir[dep_path] = files
-                files = ignored_files_by_dir[dep_path]
-                if not files:
+                stage_dirs.append((name, dep_path))
+        if stage_dirs:
+            dirs = list(dict.fromkeys(d for _, d in stage_dirs))
+            ignored_files_by_dir = {d: [] for d in dirs}
+            git_repo = calkit.git.get_repo()
+            git_root = str(git_repo.working_tree_dir)
+            # One listing for every directory, chunked to stay under
+            # command line length limits, rather than a subprocess each
+            listing = ""
+            for i in range(0, len(dirs), 100):
+                # Git prints paths relative to the repo root, which may be
+                # above the project, so make them project-relative; -z
+                # keeps unusual file names from being quoted
+                listing += git_repo.git.ls_files(
+                    "-z",
+                    "--others",
+                    "--ignored",
+                    "--exclude-standard",
+                    "--",
+                    *[os.path.abspath(d) for d in dirs[i : i + 100]],
+                )
+                listing += "\0"
+            for ignored_path in dict.fromkeys(listing.split("\0")):
+                if not ignored_path:
                     continue
-                # When DVC reports the directory (or, if the dep was expanded
-                # around a subproject, a path inside it) as modified, annotate
-                # that input on the stale stage; otherwise it goes in the
-                # status-wide list as something that will bite elsewhere
-                annotated = False
-                for stale_stage in stale_by_base.get(name, []):
-                    for modified_input in stale_stage.modified_inputs:
-                        inside = [
-                            p
-                            for p in files
-                            if p == modified_input
-                            or p.startswith(modified_input.rstrip("/") + "/")
-                        ]
-                        if inside:
-                            stale_stage.ignored_files_in_inputs[
-                                modified_input
-                            ] = inside
-                            annotated = True
-                if not annotated:
-                    ignored_files_in_inputs.setdefault(name, {})[dep_path] = (
-                        files
-                    )
+                ignored_path = Path(
+                    os.path.relpath(os.path.join(git_root, ignored_path))
+                ).as_posix()
+                if any(
+                    ignored_path == out or ignored_path.startswith(out + "/")
+                    for out in dvc_out_paths
+                ):
+                    continue
+                if dvc_repo.dvcignore.is_ignored_file(
+                    os.path.abspath(ignored_path)
+                ):
+                    continue
+                for d in dirs:
+                    if ignored_path.startswith(d + "/") or d == ".":
+                        ignored_files_by_dir[d].append(ignored_path)
+        for name, dep_path in stage_dirs:
+            files = ignored_files_by_dir[dep_path]
+            if not files:
+                continue
+            # When DVC reports the directory (or, if the dep was expanded
+            # around a subproject, a path inside it) as modified, annotate
+            # that input on the stale stage; otherwise it goes in the
+            # status-wide list as something that will bite elsewhere
+            annotated = False
+            for stale_stage in stale_by_base.get(name, []):
+                for modified_input in stale_stage.modified_inputs:
+                    inside = [
+                        p
+                        for p in files
+                        if p == modified_input
+                        or p.startswith(modified_input.rstrip("/") + "/")
+                    ]
+                    if inside:
+                        stale_stage.ignored_files_in_inputs[modified_input] = (
+                            inside
+                        )
+                        annotated = True
+            if not annotated:
+                ignored_files_in_inputs.setdefault(name, {})[dep_path] = files
         result["stale_stages"] = ordered_stale_stages
         return PipelineStatus(
             has_pipeline=result["has_pipeline"],
@@ -1303,6 +1323,7 @@ def get_status(
             stale_stages=result["stale_stages"],
             errors=result["errors"],
             ignored_files_in_inputs=ignored_files_in_inputs,
+            dvc_data_status=dvc_data_status,
         )
     finally:
         if wdir is not None: