calkit-python 0.47.11__py3-none-any.whl → 0.48.0__py3-none-any.whl

calkit/__init__.py

@@ -48,6 +48,7 @@ if TYPE_CHECKING:
         schema,
         server,
         templates,
+        upgrade,
         workspace,
     )
 
@@ -94,6 +95,7 @@ _SUBMODULES = {
     "schema",
     "server",
     "templates",
+    "upgrade",
     "workspace",
 }
 

calkit/cli/check.py

@@ -1458,12 +1458,16 @@ def check_docker_env(
                     # Leaving the tag would fake a registry digest on the
                     # image, making every later push look unnecessary
                     ck_docker.untag_image(remote_ref)
+                    hint = ck_docker.registry_login_hint(
+                        remote_ref, push_output
+                    )
                     warn(
                         f"Failed to push image to {remote_ref}; its lock "
                         "file will record no digest, so anyone else who "
                         "uses this project will rebuild this image rather "
                         "than pull it\n"
                         + textwrap.indent(push_output.strip()[-500:], "    ")
+                        + (f"\n{hint}" if hint else "")
                     )
         identity = dict(identity, RepoDigests=remote_digests)
     elif fpath is None:

calkit/cli/dev.py

@@ -45,3 +45,26 @@ def run_ipython(
     from IPython import start_ipython
 
     start_ipython(argv=sys.argv[3:], user_ns={"calkit": calkit})
+
+
+@dev_app.command(name="upgrade")
+def upgrade(
+    branch: Annotated[
+        str | None,
+        typer.Argument(
+            help=(
+                "Branch to install from: the worktree that has it checked "
+                "out, or else the install's own, if it has no changes. "
+                "Defaults to the current branch."
+            )
+        ),
+    ] = None,
+) -> None:
+    """Update a dev (editable) install: pull the branch and reinstall."""
+    from calkit.cli.core import raise_error
+    from calkit.upgrade import get_dev_upgrade_cmds, run_upgrade_cmds
+
+    try:
+        run_upgrade_cmds(get_dev_upgrade_cmds(branch))
+    except Exception as e:
+        raise_error(str(e))

calkit/cli/main/core.py

@@ -125,6 +125,7 @@ def _to_shell_cmd(cmd: list[str]) -> str:
 
 @app.callback()
 def main(
+    ctx: typer.Context,
     version: Annotated[
         bool,
         typer.Option("--version", help="Show version and exit."),
@@ -169,6 +170,7 @@ def main(
     # didn't, so a CALKIT_HUB in .env sent the two to different hubs.
     dotenv.load_dotenv(dotenv_path=".env")
     _warn_on_stale_calkit_env()
+    calkit.upgrade.check(ctx.invoked_subcommand)
 
 
 def _warn_on_stale_calkit_env() -> None:
@@ -510,8 +512,18 @@ def get_status(
     import git
     from git.exc import InvalidGitRepositoryError
 
+    def get_dvc_data_status() -> dict:
+        # Reuse what the pipeline status computed under the same DVC lock
+        if (
+            pipeline_status is not None
+            and pipeline_status.dvc_data_status is not None
+        ):
+            return dict(pipeline_status.dvc_data_status)
+        return dict(calkit.dvc.get_dvc_repo().data_status())
+
     dotenv.load_dotenv(dotenv_path=".env")
-    ck_info = calkit.load_calkit_info()
+    # Status only reads calkit.yaml, so skip the slower round-trip parser
+    ck_info = calkit.load_calkit_info(read_only=True)
     # Status is usually the first command someone runs on a project that
     # isn't theirs, which is exactly when the variables it declares are
     # missing. Ask while there's somebody to answer -- the environment
@@ -576,6 +588,7 @@ def get_status(
             check_environments=not no_check_envs,
             clean_notebooks=True,
             compile_to_dvc=True,
+            with_data_status="dvc" in categories,
         )
         if pipeline_status.failed_environment_checks:
             warn(
@@ -608,10 +621,13 @@ def get_status(
                 frozen_stages = frozen_tainted_stage_names(ck_info=ck_info)
             except Exception:
                 frozen_stages = set()
+        # The summary doesn't say what changed since an answer was edited,
+        # which is the only part that needs history, so skip reading it
         questions_status = check_questions(
             ck_info=ck_info,
             stale_stages=stale_stages,
             frozen_stages=frozen_stages,
+            check_history=False,
         )
     if as_json:
         status_dict: dict[str, Any] = {}
@@ -671,8 +687,7 @@ def get_status(
                 status_dict["git"] = {"error": "Not a Git repository"}
         if "dvc" in categories:
             try:
-                dvc_repo = calkit.dvc.get_dvc_repo()
-                data_status = dict(dvc_repo.data_status())
+                data_status = get_dvc_data_status()
                 data_status.pop("git", None)
                 status_dict["dvc"] = calkit.dvc.data_status_as_posix(
                     data_status
@@ -741,8 +756,7 @@ def get_status(
             )
         else:
             zip_path_map = calkit.dvc.zip.get_zip_path_map()
-            dvc_repo = calkit.dvc.get_dvc_repo()
-            raw = dict(dvc_repo.data_status())
+            raw = get_dvc_data_status()
             raw.pop("git", None)
             raw = calkit.dvc.data_status_as_posix(raw)
             typer.echo(_format_dvc_data_status(raw, zip_path_map))
@@ -1780,9 +1794,13 @@ def push(
             if not success:
                 # Leaving the tag would fake a registry digest on the image
                 calkit.docker.untag_image(remote_ref)
+                hint = calkit.docker.registry_login_hint(
+                    remote_ref, push_output
+                )
                 warn(
                     f"Failed to push image to {remote_ref}\n"
                     + textwrap.indent(push_output.strip()[-500:], "    ")
+                    + (f"\n{hint}" if hint else "")
                 )
     if "git" in selected:
         typer.echo("Pushing to Git remote")
@@ -4323,42 +4341,26 @@ def upgrade(
         bool, typer.Option("--skills", help="Upgrade agent skills as well.")
     ] = False,
 ) -> None:
-    """Upgrade Calkit."""
-    # First detect how Calkit is installed
-    # If installed with uv tool, calkit will be located at something like
-    # ~/.local/bin/calkit
-    which_calkit = shutil.which("calkit")
-    if which_calkit is None:
-        raise_error("Calkit is not installed")
-    split_path = os.path.normpath(str(which_calkit)).split(os.sep)
-    if (
-        ".local" in split_path
-        and "bin" in split_path
-        and calkit.check_dep_exists("uv")
-    ):
-        # This is a uv tool install
-        cmd = [
-            "uv",
-            "tool",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    elif "pipx" in split_path and calkit.check_dep_exists("pipx"):
-        cmd = ["pipx", "upgrade", "calkit-python"]
-    else:
-        cmd = [
-            sys.executable,
-            "-m",
-            "pip",
-            "install",
-            "--upgrade",
-            "calkit-python",
-        ]
-    res = subprocess.run(cmd)
-    if res.returncode != 0:
-        raise_error("Upgrade failed")
-    typer.echo("Success!")
+    """Upgrade Calkit.
+
+    A dev (editable) install is updated from the branch it's on, as with
+    'calkit dev upgrade'.
+    """
+    from calkit.upgrade import (
+        get_dev_upgrade_cmds,
+        get_editable_path,
+        get_upgrade_cmd,
+        run_upgrade_cmds,
+    )
+
+    try:
+        if get_editable_path() is not None:
+            cmds = get_dev_upgrade_cmds()
+        else:
+            cmds = [get_upgrade_cmd()]
+        run_upgrade_cmds(cmds)
+    except Exception as e:
+        raise_error(str(e))
     if skills:
         from calkit.cli.update import update_agent_skills
 

calkit/config.py

@@ -439,6 +439,9 @@ class Settings(BaseSettings):
     # 'calkit describe system' reports the ID actually in effect.
     machine_id: str | None = None
     dataframe_engine: Literal["pandas", "polars"] = "pandas"
+    # Upgrade Calkit when a daily check finds a new version; when false,
+    # the check only notifies, and 'no-check' turns the check off
+    auto_upgrade: bool | Literal["no-check"] = True
     run_history_length: int = 10
     github_token: KeyringOptionalSecret | None = None
     # Kept apart from github_token, since pushing to the GitHub Container

calkit/core.py

@@ -125,9 +125,65 @@ try:
     # PyYAML ships prebuilt wheels with libyaml on every platform we target,
     # so this is the normal path; the fallback keeps a source install or an
     # exotic build working, just slower.
-    from yaml import CSafeLoader as _YamlLoader
+    from yaml import CSafeLoader as _BaseYamlLoader
 except ImportError:  # pragma: no cover - depends on the libyaml build
-    from yaml import SafeLoader as _YamlLoader
+    from yaml import SafeLoader as _BaseYamlLoader
+
+
+class _YamlLoader(_BaseYamlLoader):  # type: ignore[no-any-unimported]
+    """The C loader, reading scalars as YAML 1.2 like ``ryaml`` does.
+
+    PyYAML follows YAML 1.1, where ``no`` and ``on`` are booleans, ``1:30``
+    is 90, and ``010`` is octal, so the same file would load differently
+    depending on which parser read it.
+    """
+
+
+def _construct_yaml12_int(loader: Any, node: Any) -> int:
+    value = str(loader.construct_scalar(node)).replace("_", "")
+    sign = -1 if value.startswith("-") else 1
+    value = value.lstrip("+-")
+    for prefix, base in (("0b", 2), ("0o", 8), ("0x", 16)):
+        if value.startswith(prefix):
+            return sign * int(value[2:], base)
+    return sign * int(value)
+
+
+# ruamel.yaml's YAML 1.2 resolvers for the scalars that differ from 1.1
+_YAML12_RESOLVERS = {
+    "tag:yaml.org,2002:bool": (
+        r"^(?:true|True|TRUE|false|False|FALSE)$",
+        "tTfF",
+    ),
+    "tag:yaml.org,2002:float": (
+        r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
+        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
+        |[-+]?\.[0-9_]+(?:[eE][-+][0-9]+)?
+        |[-+]?\.(?:inf|Inf|INF)
+        |\.(?:nan|NaN|NAN))$""",
+        "-+0123456789.",
+    ),
+    "tag:yaml.org,2002:int": (
+        r"""^(?:[-+]?0b[0-1_]+
+        |[-+]?0o?[0-7_]+
+        |[-+]?[0-9_]+
+        |[-+]?0x[0-9a-fA-F_]+)$""",
+        "-+0123456789",
+    ),
+}
+_YamlLoader.yaml_implicit_resolvers = {
+    first: [
+        (tag, regexp)
+        for tag, regexp in resolvers
+        if tag not in _YAML12_RESOLVERS
+    ]
+    for first, resolvers in _BaseYamlLoader.yaml_implicit_resolvers.items()
+}
+for _tag, (_pattern, _first) in _YAML12_RESOLVERS.items():
+    _YamlLoader.add_implicit_resolver(
+        _tag, re.compile(_pattern, re.X), list(_first)
+    )
+_YamlLoader.add_constructor("tag:yaml.org,2002:int", _construct_yaml12_int)
 
 
 def _load_yaml_readonly(stream: Any) -> Any:
@@ -1157,7 +1213,7 @@ def detect_project_name(
     If ``prepend_owner`` is False, fall back to the working directory name
     if there is no Git remote or name specified in ``calkit.yaml``.
     """
-    ck_info = load_calkit_info(wdir=wdir)
+    ck_info = load_calkit_info(wdir=wdir, read_only=True)
     name = ck_info.get("name")
     if name is not None and not prepend_owner:
         return name

calkit/docker.py

@@ -1144,6 +1144,35 @@ def is_auth_error(output: str) -> bool:
     return any(marker in lowered for marker in _AUTH_ERROR_MARKERS)
 
 
+def registry_login_hint(ref: str, output: str) -> str:
+    """Return how to log in to a registry that refused a push, or ''.
+
+    Calkit walks the user through creating a token itself when it can
+    prompt, so this is for when it can't: a push from a pipeline, a script
+    or an agent would otherwise end on the registry's bare "unauthorized",
+    which says nothing about what to do next.
+    """
+    if not is_auth_error(output):
+        return ""
+    host = ref.split("/", 1)[0]
+    if host != "ghcr.io":
+        return (
+            f"{host} refused the push; log Docker in to it with "
+            f"'docker login {host}' and push again"
+        )
+    return (
+        "The GitHub Container Registry refused the push: no credential "
+        "Calkit has can write packages. To fix this, either:\n"
+        "  - run 'calkit push docker' in a terminal, which walks you "
+        "through creating a token and remembers it; or\n"
+        "  - create a token with the 'write:packages' scope at\n"
+        f"      {GITHUB_PACKAGES_TOKEN_URL}\n"
+        "    log Docker in with it, pasting the token as the password:\n"
+        f"      docker login ghcr.io -u {get_github_username()}\n"
+        "    and push again with 'calkit push docker'"
+    )
+
+
 def get_github_token_scopes(token: str) -> set[str]:
     """Return the OAuth scopes a GitHub token carries.
 

calkit/dvc/core.py

@@ -8,6 +8,8 @@ import json
 import logging
 import os
 import sys
+import threading
+from collections.abc import Iterator
 from itertools import groupby
 from pathlib import Path
 from typing import Any, Literal
@@ -209,6 +211,117 @@ def dvc_lock_timeout(seconds: float):
         dvc.lock.DEFAULT_TIMEOUT = previous
 
 
+_memoized_hashes_lock = threading.RLock()
+
+
+@contextlib.contextmanager
+def memoized_hashes() -> Iterator[None]:
+    """Hash each output or dependency path only once while active.
+
+    DVC hashes a directory again for every stage that depends on it, walking
+    every file each time, so a dataset that feeds twenty stages is walked
+    twenty times by one ``status``. Its ``data_status`` then walks the outputs
+    once more, for nothing but their directory hashes unless asked for a
+    granular status. Nothing on disk changes during a status check, so the
+    first answer stands for the rest.
+    """
+    import importlib
+
+    import dvc.repo.data
+    import dvc.repo.index
+    from dvc.output import Output
+
+    # By name, since dvc_data.index exports a build function that shadows it
+    index_build: Any = importlib.import_module("dvc_data.index.build")
+
+    # The patches are process-wide, so threads take turns rather than
+    # capturing and restoring each other's
+    with _memoized_hashes_lock:
+        original_get_hash_meta = Output._get_hash_meta
+        original_diff = dvc.repo.data._diff_index_to_wtree
+        original_build_index = dvc.repo.index.build_data_index
+        original_build_entries = index_build.build_entries
+        memo: dict[tuple, Any] = {}
+        # Local directories already hashed, as (meta, hash_info) by repo,
+        # path, and hash name, and whether the walk in progress only needs
+        # those
+        dirs: dict[tuple[int | None, str, str], Any] = {}
+        shallow = False
+        walking: int | None = None
+        skipped: dict[str, Any] = {}
+
+        def _get_hash_meta(self: Any) -> Any:
+            key = (
+                id(self.repo),
+                self.fs.protocol,
+                self.fs_path,
+                self.hash_name,
+                self.use_cache,
+            )
+            if key not in memo:
+                memo[key] = original_get_hash_meta(self)
+                meta, hash_info = memo[key]
+                if (
+                    self.fs.protocol == "local"
+                    and hash_info
+                    and hash_info.isdir
+                ):
+                    dirs[(id(self.repo), self.fs_path, self.hash_name)] = memo[
+                        key
+                    ]
+            return memo[key]
+
+        def _diff_index_to_wtree(*args: Any, **kwargs: Any) -> Any:
+            nonlocal shallow
+            shallow = not kwargs.get("granular", False)
+            try:
+                return original_diff(*args, **kwargs)
+            finally:
+                shallow = False
+
+        def build_entries(
+            path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            key = (walking, path, kwargs.get("hash_name", "md5"))
+            if key in dirs:
+                skipped[path] = dirs[key]
+                return iter(())
+            return original_build_entries(path, fs, *args, **kwargs)
+
+        def build_data_index(
+            index: Any, path: str, fs: Any, *args: Any, **kwargs: Any
+        ) -> Any:
+            nonlocal walking
+            if not (shallow and kwargs.get("compute_hash")):
+                return original_build_index(index, path, fs, *args, **kwargs)
+            walking = id(index.repo)
+            skipped.clear()
+            try:
+                data = original_build_index(index, path, fs, *args, **kwargs)
+            finally:
+                walking = None
+            # Outputs whose walk was skipped get the hashes already computed
+            # rather than those of their now-empty listing
+            for dir_path, (meta, hash_info) in skipped.items():
+                entry = data.get(fs.relparts(dir_path, path))
+                if entry is not None:
+                    entry.meta = meta
+                    entry.hash_info = hash_info
+            return data
+
+        Output._get_hash_meta = _get_hash_meta
+        dvc.repo.data._diff_index_to_wtree = _diff_index_to_wtree
+        dvc.repo.index.build_data_index = build_data_index
+        index_build.build_entries = build_entries
+        try:
+            yield
+        finally:
+            Output._get_hash_meta = original_get_hash_meta
+            dvc.repo.data._diff_index_to_wtree = original_diff
+            dvc.repo.index.build_data_index = original_build_index
+            index_build.build_entries = original_build_entries
+
+
 class CalkitDVCFileSystem(ObjectFileSystem):
     """DVC-facing filesystem wrapper for the ``ck://`` scheme."""
 

calkit/latex.py

@@ -55,7 +55,7 @@ WORKING_NAME = "working"
 # of a full TeX Live image. Pinned to an exact tag rather than :latest so
 # a document keeps building against the same TeX until this is moved
 # deliberately; what it carries is recorded in images/latex/README.md.
-DEFAULT_LATEX_IMAGE = "ghcr.io/calkit/latex:0.1.4"
+DEFAULT_LATEX_IMAGE = "ghcr.io/calkit/latex:0.1.5"
 # The environment created for a document that doesn't have one, wherever
 # that happens: a new publication, an Overleaf import, or a stage whose
 # environment is worked out from what it runs. Copied where it's used,