py-wlcli 0.8.0__py3-none-any.whl

wl_cli/utils/cache_store.py

@@ -0,0 +1,546 @@
+"""Generic keyed store with atomic writes.
+
+Extracted from ``CacheStorage`` in
+:mod:`wl_cli.commands.testcommands.cache_manager` -- the storage mechanics
+(temp file + rename, including the Windows retry loop) -- so they can be reused
+and tested on their own, without the test-cache specifics (default cache root,
+dependency analysis, statistics).
+
+Lives in ``utils`` (not ``commands/testcommands``) so every cache adapter can
+reach it without importing the test subsystem (#213, ADR-0011). The ledger in
+:mod:`wl_cli.utils.cache_ledger` is the production path to it; the format
+adapter goes through the ledger, never through this module.
+
+The interface is deliberately narrow:
+
+- :meth:`KeyedStore.put` -- write a JSON dict under a key, atomically
+- :meth:`KeyedStore.get` -- read it back, or ``None`` on miss
+- :meth:`KeyedStore.invalidate` -- drop it
+- :meth:`KeyedStore.invalidate_file` -- drop an entry by its file-name-derived
+  key, including one whose name is not a legal key
+- :meth:`KeyedStore.iter_entries` -- enumerate the readable entries
+- :meth:`KeyedStore.iter_entry_files` -- enumerate every entry file, unusable
+  ones included
+- :meth:`KeyedStore.clear` -- drop every entry
+- :meth:`KeyedStore.remove_temp_files` -- drop leftovers from interrupted writes
+- :meth:`KeyedStore.total_size_bytes` -- the store's on-disk footprint
+
+The store does not judge freshness. Entries carry their own ``timestamp`` field,
+and :func:`is_fresh` -- defined here, next to the store -- is the single
+freshness judgement in the cache stack: ``CacheLedger`` (in
+:mod:`wl_cli.utils.cache_ledger`) owns the clock and passes the TTL its adapter
+chose (24 hours for test results, 7 days for coverage and format). Nothing
+expires by file mtime.
+
+Keys become file names, so an illegal key is rejected rather than sanitized --
+sanitizing would alias distinct keys onto one file. A key is illegal when it is
+not a ``str``, is the empty string, contains a path separator (``/``, ``\\``)
+or a C0 control character (including NUL) or DEL, is one of the directory
+references ``.`` and ``..``, or holds a character that cannot be encoded as a
+file name (a lone surrogate, which no file-system encoding accepts). These are
+pure string checks: POSIX rejects a NUL in a path and Windows rejects several
+other control characters, so rejecting C0 and DEL on every platform keeps that
+part of the naming rule identical across platforms. Rejection never raises out
+of the public interface: like every other unusable state, it degrades to the
+documented miss/failure return -- :meth:`KeyedStore.put` and
+:meth:`KeyedStore.invalidate` report ``False``, :meth:`KeyedStore.get` reports
+``None``. Only the private :meth:`KeyedStore._entry_path` raises, and the
+public methods go through :meth:`KeyedStore._safe_entry_path` to turn that into
+the documented return.
+"""
+
+import json
+import os
+import time
+import uuid
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from .platform_adapter import PlatformAdapter
+
+_ENTRY_SUFFIX = ".json"
+_TEMP_SUFFIX = ".tmp"
+
+# Keys become file names, so anything that could escape the store directory
+# (or land outside the entry-file naming scheme) is rejected rather than
+# silently sanitized -- silent sanitizing would alias distinct keys together.
+_KEY_PATH_SEPARATORS = ("/", "\\")
+_KEY_DIRECTORY_REFERENCES = (".", "..")
+
+# C0 control characters (including NUL) and DEL are rejected on every platform:
+# a NUL in a path is rejected by POSIX and Windows alike, and Windows rejects
+# several of the others, so applying the same rule everywhere keeps that part of
+# the naming rule platform-independent. (Windows-only restrictions such as
+# reserved device names are deliberately out of scope.)
+_FIRST_PRINTABLE_CODEPOINT = 0x20
+_DELETE_CODEPOINT = 0x7F
+
+_WINDOWS_RENAME_ATTEMPTS = 5
+_WINDOWS_RENAME_DELAY_SECONDS = 0.1
+
+
+def is_fresh(entry: Any, max_age_seconds: float, *, now: float | None = None) -> bool:
+    """Judge whether an entry's payload timestamp is within ``max_age_seconds``.
+
+    This is the single freshness judgement in the cache stack. Every adapter
+    reaches it through ``CacheLedger``, which passes the age limit the adapter
+    owns: 24 hours for the test-result cache, 7 days for coverage and format.
+
+    Only the entry's own ``timestamp`` field is consulted -- never the file
+    mtime. An entry that is not a dict, carries no timestamp, or carries a
+    non-numeric one counts as stale, so a purge removes it.
+
+    A timestamp in the future (clock skew between writers, or a value stored in
+    milliseconds rather than seconds) yields a negative age and therefore
+    counts as fresh. That is deliberate and pinned by tests: the predicate has
+    no upper bound, so such an entry stays cached until it is invalidated or a
+    purge is asked to drop it by other means.
+
+    Args:
+        entry: The stored entry, as returned by :meth:`KeyedStore.get`.
+        max_age_seconds: Maximum age of a fresh entry, in seconds.
+        now: Current epoch time; ``time.time()`` when omitted. Injectable so
+            the boundary can be tested exactly.
+
+    Returns:
+        True when ``now - entry["timestamp"] < max_age_seconds``; False
+        otherwise, including for a malformed or timestamp-less entry.
+    """
+    if not isinstance(entry, dict):
+        return False
+
+    timestamp = entry.get("timestamp")
+    if not isinstance(timestamp, (int, float)):
+        return False
+
+    current_time = time.time() if now is None else now
+    return current_time - timestamp < max_age_seconds
+
+
+class KeyedStore:
+    """A directory of JSON entries addressed by key, written atomically.
+
+    Responsibilities:
+    - Create the store directory if it doesn't exist
+    - Read an entry (``None`` on miss)
+    - Write an entry atomically, with a retry loop on Windows file locking
+    - Delete an entry
+    - Delete an entry by its file-name-derived key, even when that name is not
+      a legal key (housekeeping over files this store would never have written)
+    - Enumerate the readable entries, and report their count and on-disk size
+    - Reject an illegal key by degrading to the documented miss/failure return,
+      never by raising (see the module docstring)
+    """
+
+    def __init__(self, store_dir: Path) -> None:
+        """Initialize the store.
+
+        Args:
+            store_dir: Directory holding the entry files. Created lazily on
+                first write, not here.
+        """
+        self._store_dir = store_dir
+
+    @property
+    def store_dir(self) -> Path:
+        """Get the directory holding the entry files."""
+        return self._store_dir
+
+    def ensure_store_dir_exists(self) -> None:
+        """Ensure the store directory exists, create it if not."""
+        if not self._store_dir.exists():
+            self._store_dir.mkdir(parents=True, exist_ok=True)
+
+    def put(self, key: str, value: dict[str, Any]) -> bool:
+        """Write an entry atomically.
+
+        Writes to a temporary file first and renames it into place, so a reader
+        never observes a half-written entry. Includes a retry loop for Windows
+        rename failures caused by file locking. The temporary name is unique
+        per write (see :meth:`_temp_path`), so two processes writing the same
+        key do not stage into -- and steal -- one another's file.
+
+        Args:
+            key: Entry key.
+            value: JSON-serializable dict to store.
+
+        Returns:
+            True if the write succeeded; False if the key is illegal (see the
+            module docstring) or the write failed, in which case the temporary
+            file has been cleaned up and no half-written entry is visible. The
+            previous entry file is removed just before the rename, so a write
+            that fails at the rename stage can leave the entry absent rather
+            than stale.
+        """
+        entry_file = self._safe_entry_path(key)
+        if entry_file is None:
+            return False
+
+        self.ensure_store_dir_exists()
+        temp_file = self._temp_path(entry_file)
+
+        try:
+            # Write to a temporary file first for atomicity
+            with open(temp_file, "w", encoding="utf-8") as f:
+                json.dump(value, f, indent=2, ensure_ascii=False)
+                f.write("\n")
+        except OSError:
+            self._discard(temp_file)
+            return False
+
+        try:
+            return self._rename_into_place(temp_file, entry_file)
+        except OSError:
+            self._discard(temp_file)
+            return False
+
+    def get(self, key: str) -> dict[str, Any] | None:
+        """Read an entry.
+
+        Args:
+            key: Entry key.
+
+        Returns:
+            The stored dict, or None if the key is illegal (see the module
+            docstring), or the entry is missing, unreadable, or holds something
+            other than a JSON object. Freshness is the caller's business --
+            see :func:`is_fresh`.
+        """
+        entry_file = self._safe_entry_path(key)
+        if entry_file is None or not entry_file.exists():
+            return None
+
+        try:
+            with open(entry_file, encoding="utf-8") as f:
+                data = json.load(f)
+        except (OSError, ValueError, RecursionError):
+            # ValueError covers both malformed JSON and bytes that are not
+            # valid UTF-8; RecursionError covers a pathologically deep document.
+            # A hand-written or corrupted entry must degrade to a miss, never
+            # escape as an exception.
+            return None
+
+        # Only ``put`` writes dicts, but an entry can also be hand-written or
+        # left by another tool; anything that is not a JSON object is a miss
+        # rather than a value that violates the declared return type.
+        return data if isinstance(data, dict) else None
+
+    def invalidate(self, key: str) -> bool:
+        """Delete an entry.
+
+        Args:
+            key: Entry key.
+
+        Returns:
+            True if an entry was deleted, False if the key is illegal (see the
+            module docstring), there was nothing to delete, or the deletion
+            failed.
+        """
+        entry_file = self._safe_entry_path(key)
+        if entry_file is None or not entry_file.exists():
+            return False
+
+        try:
+            entry_file.unlink()
+            return True
+        except OSError:
+            return False
+
+    def invalidate_file(self, key: object) -> bool:
+        """Delete an entry by the key encoded in its file name.
+
+        :meth:`iter_entry_files` yields every ``*.json`` file's stem, including
+        names that are not legal keys -- an entry file literally named
+        ``.json`` yields the empty key, and ``a<TAB>b.json`` yields a key
+        rejected for its control character. :meth:`invalidate` refuses those
+        keys by design (it is the key-addressed interface), so maintenance
+        could count such a file in its statistics but never reclaim it
+        (#172 wave 9). This is the housekeeping entry point: it accepts any
+        name that cannot escape the store directory, so every file
+        :meth:`iter_entry_files` reports is deletable.
+
+        The name is *not* held to the key-naming rule: a file whose name this
+        store would never have written still deserves deletion. Only path
+        traversal is refused (a separator, or NUL) -- the store appends the
+        ``.json`` suffix itself, so ``..`` cannot escape either.
+
+        Args:
+            key: Entry-file stem (a ``str``), as yielded by
+                :meth:`iter_entry_files`. Typed ``object`` because a caller
+                that breaks the declared type must still degrade, not crash.
+
+        Returns:
+            True if a file was deleted, False if the name is unusable, there
+            was nothing to delete, or the deletion failed.
+        """
+        if not isinstance(key, str):
+            return False
+        if "\x00" in key or any(separator in key for separator in _KEY_PATH_SEPARATORS):
+            return False
+
+        entry_file = self._store_dir / f"{key}{_ENTRY_SUFFIX}"
+        try:
+            entry_file.unlink()
+            return True
+        except (OSError, UnicodeError):
+            # Missing, a directory, or an unremovable file: all the same
+            # documented failure return, like every other unusable state.
+            # ``UnicodeError`` covers a lone surrogate, which reaches the
+            # filesystem encoding as a ``UnicodeEncodeError`` at ``unlink`` --
+            # a ``str`` that passed the path checks above but cannot name a
+            # file must still degrade, not crash (#172 review, MINOR-2).
+            return False
+
+    def iter_entry_files(self) -> Iterator[tuple[str, dict[str, Any] | None]]:
+        """Iterate over every entry file, with its payload when readable.
+
+        Unlike :meth:`iter_entries`, this does not hide unusable entries: a file
+        that cannot be read (unreadable, bytes that are not valid UTF-8, a
+        pathologically deep document) or that holds something other than a JSON
+        object is yielded with ``None`` as its payload. That lets a purge see --
+        and delete -- the same set of files the statistics count and measure.
+
+        The key is the entry file's name, so a file whose name is not a legal
+        key (see the module docstring) is still yielded but cannot be addressed
+        by :meth:`invalidate`.
+
+        Yields:
+            ``(key, entry-or-None)`` pairs in file-name order.
+        """
+        if not self._store_dir.exists():
+            return
+
+        for entry_file in sorted(self._store_dir.glob(f"*{_ENTRY_SUFFIX}")):
+            if not entry_file.is_file():
+                continue
+
+            try:
+                with open(entry_file, encoding="utf-8") as f:
+                    data = json.load(f)
+            except (OSError, ValueError, RecursionError):
+                data = None
+
+            key = entry_file.name[: -len(_ENTRY_SUFFIX)]
+            yield key, data if isinstance(data, dict) else None
+
+    def iter_entries(self) -> Iterator[tuple[str, dict[str, Any]]]:
+        """Iterate over the readable entries in the store.
+
+        Entries are addressed by the key encoded in their file name. Files that
+        are not entry files (leftover ``.tmp`` files), entries that cannot be
+        read, and entries holding something other than a JSON object are
+        skipped -- they are not values this store could ever have written. Use
+        :meth:`iter_entry_files` when the unusable ones matter (purging).
+
+        Yields:
+            ``(key, entry)`` pairs in file-name order.
+        """
+        for key, entry in self.iter_entry_files():
+            if entry is not None:
+                yield key, entry
+
+    def remove_temp_files(self) -> int:
+        """Delete leftover temporary files from interrupted writes.
+
+        A write stages its entry in ``<key>.<pid>.<random>.tmp`` and renames it
+        into place; a process killed between those steps leaves the temporary
+        file behind.
+
+        Returns:
+            Number of temporary files deleted.
+        """
+        if not self._store_dir.exists():
+            return 0
+
+        deleted = 0
+        for temp_file in self._store_dir.glob(f"*{_TEMP_SUFFIX}"):
+            try:
+                if temp_file.is_file():
+                    temp_file.unlink()
+                    deleted += 1
+            except OSError:
+                continue
+        return deleted
+
+    def clear(self) -> int:
+        """Delete every file in the store directory.
+
+        Returns:
+            Number of files deleted. ``0`` when the store directory does not
+            exist or nothing could be removed.
+        """
+        if not self._store_dir.exists():
+            return 0
+
+        deleted = 0
+        for item in self._store_dir.glob("*"):
+            try:
+                if item.is_file():
+                    item.unlink()
+                    deleted += 1
+            except OSError:
+                continue
+        return deleted
+
+    def total_size_bytes(self) -> int:
+        """Get the total size of the entry files in bytes.
+
+        Returns:
+            Total size in bytes; ``0`` when the store directory does not exist.
+        """
+        if not self._store_dir.exists():
+            return 0
+
+        total = 0
+        for entry_file in self._store_dir.glob(f"*{_ENTRY_SUFFIX}"):
+            if entry_file.is_file():
+                try:
+                    total += entry_file.stat().st_size
+                except OSError:
+                    continue
+        return total
+
+    def _safe_entry_path(self, key: object) -> Path | None:
+        """Get the entry file path for a key, or ``None`` when the key is illegal.
+
+        The public methods go through here so that an illegal key degrades to
+        their documented miss/failure return instead of raising -- see the
+        module docstring. The key is typed ``object`` because a caller that
+        breaks the declared ``str`` type must still degrade, not crash.
+
+        Args:
+            key: Entry key.
+
+        Returns:
+            Full Path to the entry file, or ``None`` if the key is illegal.
+        """
+        try:
+            return self._entry_path(key)
+        except ValueError:
+            return None
+
+    def _entry_path(self, key: object) -> Path:
+        """Get the entry file path for a key.
+
+        Args:
+            key: Entry key, which must be a ``str``, must not be empty, must not
+                contain a path separator or a control character (C0 or DEL),
+                must not be a directory reference, and must be encodable as a
+                file name.
+
+        Returns:
+            Full Path to the entry file.
+
+        Raises:
+            ValueError: If the key is not a ``str``, is empty, contains a path
+                separator or a control character, is a directory reference, or
+                cannot be encoded as a file name (a lone surrogate).
+        """
+        if not isinstance(key, str):
+            raise ValueError(f"Store key must be a str, not {type(key).__name__}")
+        if not key:
+            raise ValueError("Store key must not be empty")
+        if any(separator in key for separator in _KEY_PATH_SEPARATORS):
+            raise ValueError(f"Store key must not contain a path separator: {key!r}")
+        if key in _KEY_DIRECTORY_REFERENCES:
+            raise ValueError(f"Store key must not be a directory reference: {key!r}")
+        if any(
+            ord(character) < _FIRST_PRINTABLE_CODEPOINT
+            or ord(character) == _DELETE_CODEPOINT
+            for character in key
+        ):
+            raise ValueError(f"Store key must not contain control characters: {key!r}")
+        # A lone surrogate cannot be encoded by any file-system encoding (POSIX
+        # paths are UTF-8 bytes, Windows paths are UTF-16), so it can never name
+        # a file; reject it here rather than let the file-system layer raise
+        # ``UnicodeEncodeError`` -- a ``ValueError`` that would slip past the
+        # ``except OSError`` guards in ``put``.
+        try:
+            key.encode("utf-8")
+        except UnicodeEncodeError:
+            raise ValueError(
+                f"Store key must be encodable as a file name: {key!r}"
+            ) from None
+
+        return self._store_dir / f"{key}{_ENTRY_SUFFIX}"
+
+    @staticmethod
+    def _temp_path(entry_file: Path) -> Path:
+        """A temporary path next to *entry_file*, unique to this write.
+
+        Staging under the fixed ``<key>.tmp`` made two writers of one key share
+        a file: whichever renamed first took it away, and the other's rename
+        failed with ``ENOENT`` -- reported as a cache write failure even though
+        the entry was fine (#213 review NIT-6). The process id and a random
+        suffix keep concurrent writers apart; ``_TEMP_SUFFIX`` stays last so
+        :meth:`remove_temp_files` still finds every leftover.
+
+        Args:
+            entry_file: The entry file this write is staging.
+
+        Returns:
+            Path of the temporary file to write.
+        """
+        return entry_file.with_name(
+            f"{entry_file.stem}.{os.getpid()}.{uuid.uuid4().hex[:8]}{_TEMP_SUFFIX}"
+        )
+
+    @staticmethod
+    def _is_windows() -> bool:
+        """Report whether the rename needs the Windows file-locking retry loop.
+
+        Delegates to :class:`PlatformAdapter` so platform detection lives on a
+        single seam rather than as a bare ``sys.platform`` check here.
+
+        Returns:
+            True on Windows, as reported by :meth:`PlatformAdapter.is_windows`
+            -- platform detection is never read from ``sys.platform`` here
+            (review !159 minor).
+        """
+        return PlatformAdapter.is_windows()
+
+    def _rename_into_place(self, temp_file: Path, entry_file: Path) -> bool:
+        """Move the temporary file onto the entry file.
+
+        Args:
+            temp_file: Fully written temporary file.
+            entry_file: Destination entry file.
+
+        Returns:
+            True if the rename succeeded, False if every Windows retry failed.
+        """
+        if self._is_windows():
+            # On Windows, rename might fail due to file locking.
+            # Retry a few times with short delays.
+            for attempt in range(_WINDOWS_RENAME_ATTEMPTS):
+                try:
+                    if entry_file.exists():
+                        entry_file.unlink()
+                    temp_file.rename(entry_file)
+                    return True
+                except OSError:
+                    if attempt < _WINDOWS_RENAME_ATTEMPTS - 1:
+                        time.sleep(_WINDOWS_RENAME_DELAY_SECONDS)
+
+            # All retries failed
+            self._discard(temp_file)
+            return False
+
+        # On Unix, direct rename is atomic. A concurrent writer of the same key
+        # may have removed the entry between the existence check and the
+        # unlink; that is not a failure -- the rename publishes our copy.
+        try:
+            entry_file.unlink()
+        except FileNotFoundError:
+            pass
+        temp_file.rename(entry_file)
+        return True
+
+    def _discard(self, temp_file: Path) -> None:
+        """Remove a temporary file, ignoring failures."""
+        try:
+            if temp_file.exists():
+                temp_file.unlink()
+        except OSError:
+            pass

wl_cli/utils/config/__init__.py

@@ -0,0 +1,27 @@
+"""Configuration utilities for WL Commands."""
+
+from .core import (
+    ConfigContext,
+    ConfigManager,
+    ConfigProvider,
+    EnvExporter,
+    get_config,
+    get_config_manager,
+    get_message,
+    reload_config,
+    reset_config_manager,
+    set_config,
+)
+
+__all__ = [
+    "ConfigContext",
+    "ConfigManager",
+    "ConfigProvider",
+    "EnvExporter",
+    "get_config",
+    "get_config_manager",
+    "reload_config",
+    "reset_config_manager",
+    "set_config",
+    "get_message",
+]