tether-vcs 0.1.0__py3-none-any.whl

tether/__init__.py

@@ -0,0 +1,95 @@
+"""tether: jj-style version control for heterogeneous datasets.
+
+The primary entry point is `Repo`: open (or initialize) a dataset inside an
+existing git/jj repository, register objects with `Repo.add`, then `commit`,
+`new`, `open`, `verify`, `diff`, and `gc`. Everything the CLI does is a thin
+wrapper over it.
+
+Supporting modules:
+
+- `tether.handles`: the typed native handles `Repo.open` returns.
+- `tether.backends`: the `ObjectBackend` protocol, capability tiers, reports,
+  and the backend registry (one module per system under `tether.backends.*`).
+- `tether.testing`: the conformance suite for backend authors.
+- `tether.vcs`: the jj/git adapters tether stores its history through.
+
+Example:
+    >>> from tether import Repo
+    >>> repo = Repo.find(".")
+    >>> repo.commit("baseline")          # pin every object, commit the manifests
+    >>> repo.new()                       # fork writable branches off the pins
+    >>> handle = repo.open("zarr/imaging")
+"""
+
+from __future__ import annotations
+
+from tether import backends, handles, testing, vcs
+from tether.errors import (
+    BackendError,
+    CapabilityError,
+    ConfigError,
+    ImmutableObjectModified,
+    MultiObjectError,
+    PinDriftError,
+    StaleWorkingCopyError,
+    TetherError,
+    UnpinnedStateError,
+    VcsError,
+)
+from tether.manifest import (
+    ObjectManifest,
+    Pin,
+    Policy,
+    RepoConfig,
+    WorkspaceState,
+    compute_pin_id,
+    listing_name,
+    manifest_hash,
+    ref_for_pin,
+    working_ref_name,
+)
+from tether.repo import (
+    TETHER_REV_ENV,
+    CommitResult,
+    DiffEntry,
+    GcReport,
+    ObjectStatus,
+    Repo,
+    StatusReport,
+)
+
+__all__ = [
+    "TETHER_REV_ENV",
+    "BackendError",
+    "CapabilityError",
+    "CommitResult",
+    "ConfigError",
+    "DiffEntry",
+    "GcReport",
+    "ImmutableObjectModified",
+    "MultiObjectError",
+    "ObjectManifest",
+    "ObjectStatus",
+    "Pin",
+    "PinDriftError",
+    "Policy",
+    "Repo",
+    "RepoConfig",
+    "StaleWorkingCopyError",
+    "StatusReport",
+    "TetherError",
+    "UnpinnedStateError",
+    "VcsError",
+    "WorkspaceState",
+    "backends",
+    "compute_pin_id",
+    "handles",
+    "listing_name",
+    "manifest_hash",
+    "ref_for_pin",
+    "testing",
+    "vcs",
+    "working_ref_name",
+]
+
+__version__ = "0.1.0"

tether/_clickdoc.py

@@ -0,0 +1,97 @@
+"""Mirror the Typer CLI onto real Click objects for documentation tooling.
+
+Typer vendors its own copy of Click, so the command tree
+``typer.main.get_command(app)`` returns is not an instance of the ``click``
+package's classes and tools that introspect Click CLIs (Great Docs) see a single
+opaque command. This module rebuilds the tree with the installed ``click`` so
+every subcommand, argument, and option is discoverable.
+
+Documentation-only: importing it requires ``click`` (a Great Docs dependency),
+and nothing in tether imports it at runtime.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import click
+import typer
+
+from tether.cli import app
+
+# Typer's vendored param types report Python-ish names ("str", "int", ...).
+_TYPES: dict[str, click.ParamType] = {
+    "str": click.STRING,
+    "text": click.STRING,
+    "int": click.INT,
+    "integer": click.INT,
+    "float": click.FLOAT,
+    "bool": click.BOOL,
+    "boolean": click.BOOL,
+    "path": click.Path(),
+}
+
+
+def _param_type(param: Any) -> click.ParamType:
+    kind = str(getattr(param.type, "name", "text")).lower()
+    choices = getattr(param.type, "choices", None)
+    if choices:
+        return click.Choice([str(c) for c in choices])
+    return _TYPES.get(kind, click.STRING)
+
+
+def _default(param: Any) -> Any:
+    default = getattr(param, "default", None)
+    return None if callable(default) else default
+
+
+def _convert_param(param: Any) -> click.Parameter:
+    if getattr(param, "param_type_name", "") == "argument":
+        return click.Argument(
+            [param.name],
+            required=param.required,
+            type=_param_type(param),
+            default=_default(param),
+            nargs=getattr(param, "nargs", 1),
+        )
+    decls = list(param.opts)
+    if param.secondary_opts:
+        decls = [f"{param.opts[0]}/{param.secondary_opts[0]}", *param.opts[1:]]
+    return click.Option(
+        decls,
+        help=getattr(param, "help", None),
+        required=param.required,
+        type=None if param.is_flag else _param_type(param),
+        is_flag=bool(param.is_flag),
+        default=_default(param),
+        multiple=bool(getattr(param, "multiple", False)),
+        show_default=bool(getattr(param, "show_default", False)),
+    )
+
+
+def _convert_command(name: str, command: Any) -> click.Command:
+    params = [_convert_param(p) for p in command.params if p.name != "help"]
+    return click.Command(
+        name=name,
+        help=command.help,
+        short_help=getattr(command, "short_help", None),
+        epilog=getattr(command, "epilog", None),
+        params=params,
+    )
+
+
+def build_click_group() -> click.Group:
+    """Return a real ``click.Group`` mirroring the ``tether`` Typer app."""
+    typer_group: Any = typer.main.get_command(app)  # a TyperGroup at runtime
+    subcommands: dict[str, Any] = dict(getattr(typer_group, "commands", {}))
+    return click.Group(
+        name="tether",
+        help=typer_group.help,
+        commands={
+            name: _convert_command(name, cmd)
+            for name, cmd in sorted(subcommands.items())
+        },
+    )
+
+
+click_app = build_click_group()

tether/backends/__init__.py

@@ -0,0 +1,45 @@
+"""Backend protocol, capability tiers, reports, and the registry.
+
+A backend maps tether's operations onto one class of system. Built-in kinds
+(`file`, `icechunk`, `neon`, `git`, `iceberg`, `delta`, `lance`, `lakefs`,
+`ducklake`, `dolt`, `memory`) live in `tether.backends.<kind>` and register
+themselves on import; `build_backend` imports them lazily so importing this
+package does not pull in optional third-party dependencies. Third-party
+backends implement `ObjectBackend` and call `register_backend`.
+"""
+
+from __future__ import annotations
+
+from tether.backends.base import (
+    MAX_DIFF_ENTRIES,
+    Capability,
+    ChangeEntry,
+    Listings,
+    ObjectBackend,
+    ObjectDiff,
+    Tier,
+    VerifyReport,
+    VerifyStatus,
+    build_backend,
+    effective_capabilities,
+    known_kinds,
+    register_backend,
+    tier_of,
+)
+
+__all__ = [
+    "MAX_DIFF_ENTRIES",
+    "Capability",
+    "ChangeEntry",
+    "Listings",
+    "ObjectBackend",
+    "ObjectDiff",
+    "Tier",
+    "VerifyReport",
+    "VerifyStatus",
+    "build_backend",
+    "effective_capabilities",
+    "known_kinds",
+    "register_backend",
+    "tier_of",
+]

tether/backends/base.py

@@ -0,0 +1,348 @@
+"""Backend protocol, capability tiers, and a small registry.
+
+A *backend* maps tether operations onto one class of system (files, Icechunk,
+Neon, ...). Backends declare their capabilities; the engine (``repo.py``)
+enforces them per command and degrades explicitly, and the conformance suite
+(``testing.py``) runs the tier-appropriate checks against any backend.
+
+One backend instance serves every object of its kind; per-object addressing is
+carried in the ``locator`` argument to each method, so backends are cheap,
+stateless coordinators over user-supplied resources.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import Enum, Flag, auto
+from typing import Any, Protocol, runtime_checkable
+
+from tether.errors import CapabilityError
+from tether.handles import Handle
+from tether.manifest import Locator, Pin, State
+
+
+class Capability(Flag):
+    """What a backend can do.
+
+    The first four flags are cumulative tiers (see `Tier` and `tier_of`); the
+    rest are orthogonal refinements the engine and the docs use to explain
+    behavior.
+    """
+
+    NONE = 0
+    """No capabilities."""
+    FINGERPRINT = auto()
+    """Can read the current state (drift detection). Every backend has this."""
+    ADDRESSABLE = auto()
+    """A recorded state can be read back later without a native ref."""
+    PIN = auto()
+    """Can create and delete a durable, GC-proof native ref for a state."""
+    FORK = auto()
+    """Can create a writable branch off a pin."""
+    CHEAP_FINGERPRINT = auto()
+    """Fingerprints are metadata-only; no heavy connection is opened."""
+    RETENTION_BOUND = auto()
+    """Recorded/pinned states expire with the system's retention window."""
+    NEEDS_QUIESCENCE = auto()
+    """`commit` should check for active writers first."""
+    ATOMIC_REF = auto()
+    """Pins have create-if-absent semantics."""
+    DIFF = auto()
+    """Can describe what changed between two recorded states."""
+
+
+class Tier(Enum):
+    """The cumulative capability tier of a backend (or of one object)."""
+
+    OBSERVED = "observed"
+    """Drift detection only; committed state is not recoverable."""
+    ADDRESSABLE = "addressable"
+    """Committed state can be read back; nothing to create or GC."""
+    PINNABLE = "pinnable"
+    """Commit creates a durable native ref; `verify` and `gc` apply."""
+    FORKABLE = "forkable"
+    """`new` forks writable branches; `open` returns writable handles."""
+
+
+def tier_of(caps: Capability) -> Tier:
+    if Capability.FORK in caps:
+        return Tier.FORKABLE
+    if Capability.PIN in caps:
+        return Tier.PINNABLE
+    if Capability.ADDRESSABLE in caps:
+        return Tier.ADDRESSABLE
+    return Tier.OBSERVED
+
+
+class VerifyStatus(Enum):
+    """Outcome of `ObjectBackend.verify`."""
+
+    OK = "ok"
+    """The pin (or recorded state) resolves to what the manifest says."""
+    DRIFTED = "drifted"
+    """The pin exists but points at a different state."""
+    MISSING = "missing"
+    """The pin or recorded state is gone (deleted, expired)."""
+    UNKNOWN = "unknown"
+    """Cannot tell cheaply; verify with `deep=True`."""
+
+
+@dataclass
+class VerifyReport:
+    """Result of `ObjectBackend.verify` for one object."""
+
+    status: VerifyStatus
+    """The outcome."""
+    message: str = ""
+    """Human-readable detail (what it points at, why it is unknown, ...)."""
+
+    @property
+    def ok(self) -> bool:
+        """`True` when `status` is `VerifyStatus.OK`."""
+        return self.status is VerifyStatus.OK
+
+
+# --------------------------------------------------------------------------- #
+# Content diffs
+# --------------------------------------------------------------------------- #
+MAX_DIFF_ENTRIES = 2000
+
+
+@dataclass
+class ChangeEntry:
+    """One changed thing inside an object: a file, table, array, key, ..."""
+
+    path: str
+    """What changed (file path, table name, array path, ...)."""
+    change: str
+    """`"added"`, `"removed"`, `"modified"`, or `"renamed"`."""
+    detail: str = ""
+    """Backend-specific detail (`"+2 rows"`, `"+1 -1"`, `"12 chunks"`)."""
+
+    def to_dict(self) -> dict[str, str]:
+        d = {"path": self.path, "change": self.change}
+        if self.detail:
+            d["detail"] = self.detail
+        return d
+
+
+@dataclass
+class ObjectDiff:
+    """What changed inside one object between two recorded states.
+
+    Backends describe changes at whatever granularity their system exposes
+    natively and cheaply (files, tables, arrays, fragments, versions); ``unit``
+    names it. Entries are capped at :data:`MAX_DIFF_ENTRIES`; counts are not.
+    """
+
+    unit: str = "entries"
+    """What is being counted: files, tables, arrays, fragments, commits, ..."""
+    added: int = 0
+    """Exact count of added units."""
+    removed: int = 0
+    """Exact count of removed units."""
+    modified: int = 0
+    """Exact count of modified (or renamed) units."""
+    entries: list[ChangeEntry] = field(default_factory=list)
+    """Per-unit entries, capped at `MAX_DIFF_ENTRIES`."""
+    truncated: bool = False
+    """Whether entries were dropped because of the cap."""
+    note: str = ""
+    """Context the counts alone do not convey (e.g. a missing listing)."""
+
+    def add(self, path: str, change: str, detail: str = "") -> None:
+        """Record one change: bump the matching counter and append an entry."""
+        if change == "added":
+            self.added += 1
+        elif change == "removed":
+            self.removed += 1
+        else:
+            self.modified += 1
+        if len(self.entries) < MAX_DIFF_ENTRIES:
+            self.entries.append(ChangeEntry(path, change, detail))
+        else:
+            self.truncated = True
+
+    @property
+    def is_empty(self) -> bool:
+        """`True` when nothing changed."""
+        return not (self.added or self.removed or self.modified or self.entries)
+
+    @property
+    def summary(self) -> str:
+        """One line: `"+A -R ~M <unit>"` plus truncation and note."""
+        text = f"+{self.added} -{self.removed} ~{self.modified} {self.unit}"
+        if self.truncated:
+            text += f" (first {len(self.entries)} shown)"
+        if self.note:
+            text += f"; {self.note}"
+        return text
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "summary": self.summary,
+            "unit": self.unit,
+            "added": self.added,
+            "removed": self.removed,
+            "modified": self.modified,
+            "truncated": self.truncated,
+            "note": self.note,
+            "entries": [e.to_dict() for e in self.entries],
+        }
+
+
+Listings = tuple[str | None, str | None]
+
+
+@runtime_checkable
+class ObjectBackend(Protocol):
+    """The operations tether needs from one class of system.
+
+    ``listing`` and ``diff`` have default implementations (no listing; diff is a
+    capability error) so backends without ``DIFF`` need not define them.
+    """
+
+    kind: str
+    capabilities: Capability
+
+    def identity(self, locator: Locator) -> Locator:
+        """Locator subset that participates in the content-addressed pin id.
+
+        Defaults to the full locator; override to exclude non-identity fields
+        (region, credential references, source branch, ...).
+        """
+
+    def fingerprint(self, locator: Locator, working_ref: str | None) -> State:
+        """Read the current state at ``working_ref`` (or the locator's base)."""
+
+    def pin(self, locator: Locator, state: State, pin_id: str) -> Pin:
+        """Create a durable native ref for ``state``. Requires ``PIN``."""
+
+    def unpin(self, locator: Locator, pin: Pin) -> None:
+        """Release a pin. Requires ``PIN``."""
+
+    def list_pins(self, locator: Locator) -> set[str]:
+        """Return pin ids that currently exist natively. Requires ``PIN``."""
+
+    def verify(
+        self,
+        locator: Locator,
+        state: State,
+        pin: Pin | None,
+        deep: bool,
+    ) -> VerifyReport:
+        """Check that ``state``/``pin`` still hold."""
+
+    def fork(self, locator: Locator, pin: Pin, name: str) -> str:
+        """Create a writable branch ``name`` off ``pin``. Requires ``FORK``."""
+
+    def delete_working_ref(self, locator: Locator, ref: str) -> None:
+        """Delete a working ref created by :meth:`fork`. Requires ``FORK``."""
+
+    def open(
+        self,
+        locator: Locator,
+        target: str | Pin | State | None,
+        read_only: bool,
+    ) -> Handle:
+        """Return a native handle for a target.
+
+        ``target`` is one of: ``None`` (the base/working ref), a ``str`` working
+        ref, a :class:`~tether.manifest.Pin` (read a pinned state), or a
+        recorded ``State`` mapping (read an addressable state without a pin).
+        """
+
+    def listing(self, locator: Locator, state: State) -> str | None:
+        """Optional detailed description of ``state`` to store alongside it.
+
+        The engine writes the text content-addressed under ``.tether/listings/``
+        at commit time and hands both sides back to :meth:`diff` later. Used by
+        backends whose state is a digest (the ``file`` backend's directory and
+        prefix listings) so Observed states can still be diffed.
+        """
+        return None
+
+    def diff(
+        self,
+        locator: Locator,
+        a: State,
+        b: State,
+        *,
+        listings: Listings = (None, None),
+    ) -> ObjectDiff:
+        """Describe what changed from state ``a`` to state ``b``. Requires ``DIFF``."""
+        raise CapabilityError(f"{self.kind} backend cannot diff", kind=self.kind)
+
+
+# --------------------------------------------------------------------------- #
+# Registry
+# --------------------------------------------------------------------------- #
+BackendFactory = Callable[[dict], ObjectBackend]
+_REGISTRY: dict[str, BackendFactory] = {}
+
+# Built-in kinds and the module that registers them. Imported lazily so optional
+# third-party dependencies (icechunk, pyiceberg, ...) are only loaded on demand.
+_BUILTIN_MODULES: dict[str, str] = {
+    "memory": "tether.backends.memory",
+    "file": "tether.backends.file",
+    "icechunk": "tether.backends.icechunk",
+    "neon": "tether.backends.neon",
+    "git": "tether.backends.git",
+    "iceberg": "tether.backends.iceberg",
+    "delta": "tether.backends.delta",
+    "lance": "tether.backends.lance",
+    "lakefs": "tether.backends.lakefs",
+    "ducklake": "tether.backends.ducklake",
+    "dolt": "tether.backends.dolt",
+}
+
+
+def register_backend(kind: str, factory: BackendFactory) -> None:
+    _REGISTRY[kind] = factory
+
+
+def build_backend(kind: str, config: dict | None = None) -> ObjectBackend:
+    from importlib import import_module
+
+    from tether.errors import ConfigError
+
+    if kind not in _REGISTRY and kind in _BUILTIN_MODULES:
+        try:
+            import_module(_BUILTIN_MODULES[kind])
+        except ImportError as exc:
+            raise ConfigError(
+                f"backend {kind!r} needs an optional dependency: {exc}. "
+                f"Install the matching extra (e.g. `pip install tether-vcs[{kind}]`)."
+            ) from exc
+    try:
+        factory = _REGISTRY[kind]
+    except KeyError as exc:
+        raise ConfigError(
+            f"unknown backend kind: {kind!r} "
+            f"(known: {', '.join(known_kinds()) or 'none'})"
+        ) from exc
+    return factory(config or {})
+
+
+def known_kinds() -> list[str]:
+    return sorted(set(_REGISTRY) | set(_BUILTIN_MODULES))
+
+
+def effective_capabilities(
+    backend: ObjectBackend,
+    locator: Locator,
+    policy: object,
+) -> Capability:
+    """Capabilities for a specific object.
+
+    Some backends (notably ``file``) span tiers depending on the locator and
+    policy: a local file is Observed while an S3 object in a versioned bucket is
+    Addressable. Backends may implement ``effective_capabilities(locator, policy)``
+    to refine their class-level :attr:`capabilities`; otherwise the class value
+    is used.
+    """
+    fn = getattr(backend, "effective_capabilities", None)
+    if callable(fn):
+        return fn(locator, policy)
+    return backend.capabilities