txstore 0.1.0__py3-none-any.whl

txstore/__init__.py

@@ -0,0 +1,108 @@
+"""Public façade of the `txstore` package.
+
+Import from `txstore` only; submodules are not a supported surface.
+
+Concrete backends and helpers:
+
+    - StorageFS   : Filesystem storage with full transaction support.
+    - StorageS3   : S3 object storage with full transaction support.
+    - StorageHTTP : Read-only HTTP storage (no write/transaction support).
+    - StorageNull : Write-only /dev/null-like storage (no read/transaction support).
+
+Transactions are exposed through `FootmarkMixin` (mixed into FS/S3).
+Backends that do not support them still expose the `transaction_*` names
+as `NotImplementedError` stubs; use `is_transactional()` as the capability
+test.
+"""
+
+from importlib.metadata import version as _pkg_version
+
+from txstore._s3_client import BucketCannedACL, ObjectCannedACL
+from txstore._timestamp import Timestamp
+from txstore.base import (
+    CHUNK_SIZE,
+    REQUEST_TIMEOUT,
+    Storage,
+    StorageException,
+    StorageNotFoundException,
+    StorageType,
+)
+from txstore.factory import NO_ACL, make_storage
+from txstore.footmark import ObjectOperation
+from txstore.hexdigest import (
+    COMPARE_HASH_LIST,
+    SUPPORT_HASH_LIST,
+    ChecksumOpen,
+    calc_digests,
+    compare_digests,
+    normalize_digests,
+)
+from txstore.history import (
+    HISTORY_KEEP_DEFAULT,
+    HISTORY_MAX_AGE_DAYS_DEFAULT,
+    HistoryRetention,
+    PruneResult,
+    TransactionHistory,
+    history_lock,
+    maybe_transaction,
+)
+from txstore.metrics import StorageMetrics
+from txstore.storage_fs import StorageFS
+from txstore.storage_http import StorageHTTP
+from txstore.storage_null import StorageNull
+from txstore.storage_s3 import StorageS3, parse_bucket_acl, parse_object_acl
+from txstore.transaction import (
+    FootmarkMixin,
+    TransactionModel,
+    is_transactional,
+)
+from txstore.url import (
+    get_supported_schemes,
+    is_unsafe_path,
+    join_url,
+    parse_storage_url,
+)
+
+__version__ = _pkg_version("txstore")
+
+__all__ = [
+    "CHUNK_SIZE",
+    "COMPARE_HASH_LIST",
+    "HISTORY_KEEP_DEFAULT",
+    "HISTORY_MAX_AGE_DAYS_DEFAULT",
+    "NO_ACL",
+    "REQUEST_TIMEOUT",
+    "SUPPORT_HASH_LIST",
+    "BucketCannedACL",
+    "ChecksumOpen",
+    "FootmarkMixin",
+    "is_transactional",
+    "maybe_transaction",
+    "HistoryRetention",
+    "ObjectCannedACL",
+    "ObjectOperation",
+    "PruneResult",
+    "Storage",
+    "StorageException",
+    "StorageFS",
+    "StorageHTTP",
+    "StorageMetrics",
+    "StorageNotFoundException",
+    "StorageNull",
+    "StorageS3",
+    "StorageType",
+    "Timestamp",
+    "TransactionHistory",
+    "TransactionModel",
+    "calc_digests",
+    "compare_digests",
+    "get_supported_schemes",
+    "history_lock",
+    "is_unsafe_path",
+    "join_url",
+    "make_storage",
+    "normalize_digests",
+    "parse_bucket_acl",
+    "parse_object_acl",
+    "parse_storage_url",
+]

txstore/_capabilities.py

@@ -0,0 +1,185 @@
+"""Capability mixins for storage backends.
+
+`ReadOnlyStorageMixin` and `WriteOnlyStorageMixin` supply default
+``NotImplementedError`` stubs for the `Storage` ABC methods that a
+read-only or write-only backend cannot fulfil. Concrete backends
+inherit the mixin first (MRO) and then override only the methods they
+actually implement.
+"""
+
+from __future__ import annotations
+
+import builtins
+from typing import IO, Any, NoReturn
+
+READ_ONLY_MSG = "This operation is not supported by read-only storage."
+WRITE_ONLY_MSG = "This operation is not supported by write-only storage."
+NO_TRANSACTIONS_MSG = "Transactions are not supported by this storage backend."
+
+
+class TransactionNotSupportedMixin:
+    """Explicit error stubs for transaction methods on non-transactional backends.
+
+    Backends without transaction support (e.g. `StorageHTTP`, `StorageNull`)
+    inherit this to raise a clear `NotImplementedError` instead of a
+    confusing `AttributeError`, for the lifecycle and the reclaim methods alike.
+    """
+
+    def in_transaction(self) -> bool:
+        """Always False for non-transactional backends."""
+        return False
+
+    def transaction_begin(self, transaction_id: int = 0) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del transaction_id
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def transaction_commit(self) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def transaction_revert(self, transaction: Any) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del transaction
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def transaction_discard(self) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def hidden_objects(self) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def find_transaction_footmarks(self, models: Any, *, listing: Any = None) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del models, listing
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def transaction_forget(self, models: Any, *, paths: Any = None) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del models, paths
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def find_orphaned_footmarks(self, known: Any, *, listing: Any = None) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del known, listing
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+    def transaction_forget_orphans(self, known: Any, *, orphans: Any = None) -> NoReturn:
+        """Not supported (non-transactional backend)."""
+        del known, orphans
+        raise NotImplementedError(NO_TRANSACTIONS_MSG)
+
+
+class ReadOnlyStorageMixin:
+    """Default `NotImplementedError` stubs for write/delete/metadata/copy/move.
+
+    Backends inheriting this mixin (e.g. `StorageHTTP`) still define the
+    read-side methods (`list`, `get`, `get_stream`, `get_metadata`,
+    `exists`); a backend without a listing API makes `list` raise
+    `NotImplementedError` → [STORAGE.md](STORAGE.md#backends).
+    """
+
+    def put(
+        self,
+        path: str,
+        data: bytes,
+        metadata: dict[str, str] | None = None,
+        acl: str | None = None,
+    ) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def put_stream(
+        self,
+        path: str,
+        stream: IO[bytes],
+        metadata: dict[str, str] | None = None,
+        acl: str | None = None,
+    ) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def add_metadata(
+        self,
+        path: str,
+        metadata: dict[str, str],
+        acl: str | None = None,
+    ) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def delete(self, path: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def delete_metadata(self, path: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def delete_metadata_fields(self, path: str, keys: builtins.list[str]) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def copy(self, path_src: str, path_dst: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def move(self, path_src: str, path_dst: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def format(self) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def mkdir(self, path: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+    def rmdir(self, path: str) -> None:
+        """Not supported (read-only storage)."""
+        raise NotImplementedError(READ_ONLY_MSG)
+
+
+class WriteOnlyStorageMixin:
+    """Default `NotImplementedError` stubs for list/get/exists/copy/move/mkdir.
+
+    Backends inheriting this mixin (e.g. `StorageNull`) only need to
+    implement the write-side methods (`put`, `put_stream`, `delete`,
+    `add_metadata`, `delete_metadata`, `delete_metadata_fields`,
+    `format`, `rmdir`).
+    """
+
+    def list(self) -> builtins.list[str]:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def get(self, path: str) -> bytes:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def get_metadata(self, path: str) -> dict[str, Any]:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def exists(self, path: str) -> bool:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def get_stream(self, path: str) -> IO[bytes]:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def copy(self, path_src: str, path_dst: str) -> None:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def move(self, path_src: str, path_dst: str) -> None:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)
+
+    def mkdir(self, path: str) -> None:
+        """Not supported (write-only storage)."""
+        raise NotImplementedError(WRITE_ONLY_MSG)

txstore/_env.py

@@ -0,0 +1,70 @@
+"""Read ``TX_STORE_*`` environment variables.
+
+The only module that reads `os.environ`. Each helper returns one variable's parsed value, or None
+when it is unset or empty; callers resolve an explicit argument first, then the variable, then
+their default. Names → [CONFIG.md](CONFIG.md#environment-variables); grammar →
+[VALIDATION.md](VALIDATION.md#environment-values).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+ENV_PREFIX = "TX_STORE_"
+
+_TRUE = frozenset({"1", "true", "yes", "on"})
+_FALSE = frozenset({"0", "false", "no", "off"})
+# `int()` also takes `1_0` and non-ASCII digits; the grammar is plain decimal → VALIDATION.md.
+_INTEGER_RE = re.compile(r"[+-]?[0-9]+", re.ASCII)
+
+
+def _raw(name: str) -> str | None:
+    """Return the value of variable `name`, or None when unset or empty.
+
+    Raises:
+        ValueError: If `name` lacks the ``TX_STORE_`` prefix (a programming error).
+    """
+    if not name.startswith(ENV_PREFIX):
+        raise ValueError(f"environment variable {name!r} lacks the {ENV_PREFIX} prefix")
+    value = os.environ.get(name)
+    return value if value else None
+
+
+def env_str(name: str) -> str | None:
+    """Return the string value of `name`, or None when unset or empty."""
+    return _raw(name)
+
+
+def env_bool(name: str) -> bool | None:
+    """Return the boolean value of `name`, or None when unset or empty.
+
+    Accepts ``1/0``, ``true/false``, ``yes/no``, and ``on/off`` in any case.
+
+    Raises:
+        ValueError: If the value is outside that vocabulary; the message names the variable.
+    """
+    value = _raw(name)
+    if value is None:
+        return None
+    folded = value.strip().lower()
+    if folded in _TRUE:
+        return True
+    if folded in _FALSE:
+        return False
+    raise ValueError(f"{name}: expected a boolean (1/0, true/false, yes/no, on/off), got {value!r}")
+
+
+def env_int(name: str) -> int | None:
+    """Return the integer value of `name`, or None when unset or empty.
+
+    Raises:
+        ValueError: If the value is not a decimal integer; the message names the variable.
+    """
+    value = _raw(name)
+    if value is None:
+        return None
+    stripped = value.strip()
+    if not _INTEGER_RE.fullmatch(stripped):
+        raise ValueError(f"{name}: expected an integer, got {value!r}")
+    return int(stripped, 10)

txstore/_footmark_wire.py

@@ -0,0 +1,182 @@
+"""Binary encoding of a footmark.
+
+A footmark is the protocol-buffers message in
+[docs/reference/footmark.proto](docs/reference/footmark.proto); this module encodes and
+decodes it in the protobuf wire format without a protobuf runtime, so footmark objects stay
+byte-compatible with any protobuf implementation → [STORAGE.md](STORAGE.md#footmarks).
+
+Decoding follows proto3: absent fields are zero, the last scalar occurrence wins, repeated
+timestamp fields merge, unknown fields and fields of an unexpected wire type are skipped. Group
+wire types are rejected.
+"""
+
+from __future__ import annotations
+
+from typing import NamedTuple, cast
+
+_VARINT = 0
+_FIXED64 = 1
+_LENGTH = 2
+_FIXED32 = 5
+
+_HEADER = 1
+_TIMESTAMP = 2
+_TRANSACTION_ID = 3
+_OPERATION = 4
+
+_TS_SECONDS = 1
+_TS_NANOS = 2
+
+_MASK64 = (1 << 64) - 1
+_MAX_VARINT_BYTES = 10
+# A tag is a uint32 in at most 5 varint bytes.
+_MAX_TAG_BYTES = 5
+_MAX_TAG = 0xFFFF_FFFF
+
+
+class FootmarkFields(NamedTuple):
+    """Decoded footmark fields; absent fields are zero.
+
+    Attributes:
+        header: Format marker.
+        seconds: Begin timestamp seconds since the Unix epoch.
+        nanos: Begin timestamp nanoseconds.
+        transaction_id: Transaction identifier.
+        operation: Operation code.
+    """
+
+    header: int = 0
+    seconds: int = 0
+    nanos: int = 0
+    transaction_id: int = 0
+    operation: int = 0
+
+
+def encode(fields: FootmarkFields) -> bytes:
+    """Encode `fields`, always emitting the timestamp and omitting other zero fields.
+
+    Args:
+        fields: Footmark values; integers must fit their protobuf types.
+
+    Returns:
+        The serialized message.
+    """
+    timestamp = _scalar(_TS_SECONDS, fields.seconds) + _scalar(_TS_NANOS, fields.nanos)
+    return b"".join(
+        (
+            _scalar(_HEADER, fields.header),
+            _tag(_TIMESTAMP, _LENGTH) + _varint(len(timestamp)) + timestamp,
+            _scalar(_TRANSACTION_ID, fields.transaction_id),
+            _scalar(_OPERATION, fields.operation),
+        )
+    )
+
+
+def decode(data: bytes) -> FootmarkFields:
+    """Decode a serialized footmark.
+
+    Args:
+        data: The serialized message.
+
+    Returns:
+        The decoded fields.
+
+    Raises:
+        ValueError: If `data` is truncated, has a tag above 32 bits or field number 0, uses a
+            group or an invalid wire type, or has a value varint longer than 10 bytes.
+    """
+    header = transaction_id = operation = seconds = nanos = 0
+    for field, wire_type, value in _fields(data):
+        if wire_type == _VARINT and field == _HEADER:
+            header = _int32(cast(int, value))
+        elif wire_type == _VARINT and field == _TRANSACTION_ID:
+            transaction_id = _int32(cast(int, value))
+        elif wire_type == _VARINT and field == _OPERATION:
+            operation = _int32(cast(int, value))
+        elif wire_type == _LENGTH and field == _TIMESTAMP:
+            # `_fields` yields bytes for a length-delimited value and an int for a varint.
+            for ts_field, ts_wire_type, ts_value in _fields(cast(bytes, value)):
+                if ts_wire_type == _VARINT and ts_field == _TS_SECONDS:
+                    seconds = _int64(cast(int, ts_value))
+                elif ts_wire_type == _VARINT and ts_field == _TS_NANOS:
+                    nanos = _int32(cast(int, ts_value))
+    return FootmarkFields(header, seconds, nanos, transaction_id, operation)
+
+
+def _fields(data: bytes) -> list[tuple[int, int, int | bytes]]:
+    """Split `data` into ``(field, wire_type, value)``; varints stay unsigned 64-bit."""
+    out: list[tuple[int, int, int | bytes]] = []
+    pos = 0
+    while pos < len(data):
+        key, pos = _read_varint(data, pos, _MAX_TAG_BYTES)
+        if key > _MAX_TAG:
+            raise ValueError("footmark tag above 32 bits")
+        field, wire_type = key >> 3, key & 7
+        if field == 0:
+            raise ValueError("footmark field number 0")
+        value: int | bytes
+        if wire_type == _VARINT:
+            value, pos = _read_varint(data, pos)
+        elif wire_type == _LENGTH:
+            size, pos = _read_varint(data, pos)
+            value, pos = _take(data, pos, size)
+        elif wire_type == _FIXED64:
+            value, pos = _take(data, pos, 8)
+        elif wire_type == _FIXED32:
+            value, pos = _take(data, pos, 4)
+        else:
+            raise ValueError(f"footmark wire type {wire_type} not supported")
+        out.append((field, wire_type, value))
+    return out
+
+
+def _take(data: bytes, pos: int, size: int) -> tuple[bytes, int]:
+    end = pos + size
+    if end > len(data):
+        raise ValueError("footmark truncated")
+    return data[pos:end], end
+
+
+def _read_varint(data: bytes, pos: int, max_bytes: int = _MAX_VARINT_BYTES) -> tuple[int, int]:
+    result = 0
+    for index in range(max_bytes):
+        if pos >= len(data):
+            raise ValueError("footmark truncated inside a varint")
+        byte = data[pos]
+        pos += 1
+        result |= (byte & 0x7F) << (7 * index)
+        if not byte & 0x80:
+            return result & _MASK64, pos
+    raise ValueError(f"footmark varint longer than {max_bytes} bytes")
+
+
+def _int32(value: int) -> int:
+    value &= 0xFFFF_FFFF
+    return value - (1 << 32) if value >= 1 << 31 else value
+
+
+def _int64(value: int) -> int:
+    return value - (1 << 64) if value >= 1 << 63 else value
+
+
+def _tag(field: int, wire_type: int) -> bytes:
+    return _varint(field << 3 | wire_type)
+
+
+def _scalar(field: int, value: int) -> bytes:
+    """Encode a proto3 integer field; zero is omitted, negatives take 10 bytes."""
+    if value == 0:
+        return b""
+    return _tag(field, _VARINT) + _varint(value & _MASK64)
+
+
+def _varint(value: int) -> bytes:
+    out = bytearray()
+    while True:
+        byte = value & 0x7F
+        value >>= 7
+        if value:
+            out.append(byte | 0x80)
+        else:
+            out.append(byte)
+            return bytes(out)