tketool.storage 1.3.5__tar.gz

tketool_storage-1.3.5/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: tketool.storage
+Version: 1.3.5
+Summary: Extensible data structures over pluggable ordered storage backends
+Author-email: Ke <jiangke1207@icloud.com>
+License-Expression: MIT
+Project-URL: Homepage, https://pypi.org/project/tketool.storage/
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: sql
+Requires-Dist: SQLAlchemy<3,>=2.0; extra == "sql"
+Provides-Extra: postgresql
+Requires-Dist: SQLAlchemy<3,>=2.0; extra == "postgresql"
+Requires-Dist: psycopg[binary]<4,>=3.2; extra == "postgresql"
+Provides-Extra: redis
+Requires-Dist: redis<9,>=6; extra == "redis"
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Requires-Dist: SQLAlchemy<3,>=2.0; extra == "test"
+Requires-Dist: redis<9,>=6; extra == "test"
+
+# tketool.storage
+
+Extensible graphs, trees, maps, lists, and linked lists over replaceable
+ordered storage backends.
+
+```bash
+pip install tketool.storage
+```
+
+```python
+from tketool.storage import Graph, MemoryBackend, PersistentMap, Tree
+
+backend = MemoryBackend()
+settings = PersistentMap(backend, "settings")
+settings.put("language", "zh-CN")
+
+graph = Graph(backend, "knowledge")
+person = graph.add_node("alice", {"name": "Alice"}, type="person")
+```
+
+Optional adapters:
+
+```bash
+pip install "tketool.storage[sql]"
+pip install "tketool.storage[postgresql]"
+pip install "tketool.storage[redis]"
+```
+
+The dependency-free adapters are `MemoryBackend` and `SQLiteBackend`.
+`SQLAlchemyBackend` uses SQLAlchemy Core and database-specific drivers;
+`RedisBackend` uses redis-py and an atomic Lua batch.
+
+The old `tketool.ml`/`tketool.ml2` and domain-specific `StorageBase` APIs are
+not retained. New backends only
+implement point read, ordered scan, atomic batch, and transaction primitives;
+new data structures compose those primitives without changing adapters.

tketool_storage-1.3.5/README.md

@@ -0,0 +1,36 @@
+# tketool.storage
+
+Extensible graphs, trees, maps, lists, and linked lists over replaceable
+ordered storage backends.
+
+```bash
+pip install tketool.storage
+```
+
+```python
+from tketool.storage import Graph, MemoryBackend, PersistentMap, Tree
+
+backend = MemoryBackend()
+settings = PersistentMap(backend, "settings")
+settings.put("language", "zh-CN")
+
+graph = Graph(backend, "knowledge")
+person = graph.add_node("alice", {"name": "Alice"}, type="person")
+```
+
+Optional adapters:
+
+```bash
+pip install "tketool.storage[sql]"
+pip install "tketool.storage[postgresql]"
+pip install "tketool.storage[redis]"
+```
+
+The dependency-free adapters are `MemoryBackend` and `SQLiteBackend`.
+`SQLAlchemyBackend` uses SQLAlchemy Core and database-specific drivers;
+`RedisBackend` uses redis-py and an atomic Lua batch.
+
+The old `tketool.ml`/`tketool.ml2` and domain-specific `StorageBase` APIs are
+not retained. New backends only
+implement point read, ordered scan, atomic batch, and transaction primitives;
+new data structures compose those primitives without changing adapters.

tketool_storage-1.3.5/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tketool.storage"
+version = "1.3.5"
+description = "Extensible data structures over pluggable ordered storage backends"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Ke", email = "jiangke1207@icloud.com" }]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Operating System :: OS Independent",
+]
+
+[project.optional-dependencies]
+sql = ["SQLAlchemy>=2.0,<3"]
+postgresql = ["SQLAlchemy>=2.0,<3", "psycopg[binary]>=3.2,<4"]
+redis = ["redis>=6,<9"]
+test = ["pytest>=8,<9", "SQLAlchemy>=2.0,<3", "redis>=6,<9"]
+
+[project.urls]
+Homepage = "https://pypi.org/project/tketool.storage/"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["tketool*"]
+namespaces = true

tketool_storage-1.3.5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tketool_storage-1.3.5/src/tketool/storage/__init__.py

@@ -0,0 +1,72 @@
+"""Extensible data structures over a small ordered-storage contract."""
+
+from .abstractions import (
+    BackendCapabilities,
+    BytesCodec,
+    Codec,
+    CorruptDataError,
+    Delete,
+    DuplicateBatchKeyError,
+    JsonCodec,
+    KeyValue,
+    Put,
+    StorageBackend,
+    StorageClosedError,
+    StorageConflictError,
+    StorageError,
+    StorageTransaction,
+    StringCodec,
+    Subspace,
+    SupportLevel,
+    TupleKeyCodec,
+    WriteBatch,
+)
+from .adapters import MemoryBackend, RedisBackend, SQLAlchemyBackend, SQLiteBackend
+from .bootstrap import create_backend
+from .collections import (
+    LinkedEntry,
+    MultiMapEntry,
+    PersistentLinkedList,
+    PersistentList,
+    PersistentMap,
+    PersistentMultiMap,
+)
+from .structures import Graph, GraphEdge, GraphNode, Tree, TreeNode
+
+__all__ = [
+    "BackendCapabilities",
+    "BytesCodec",
+    "Codec",
+    "CorruptDataError",
+    "Delete",
+    "DuplicateBatchKeyError",
+    "Graph",
+    "GraphEdge",
+    "GraphNode",
+    "JsonCodec",
+    "KeyValue",
+    "LinkedEntry",
+    "MemoryBackend",
+    "MultiMapEntry",
+    "PersistentLinkedList",
+    "PersistentList",
+    "PersistentMap",
+    "PersistentMultiMap",
+    "Put",
+    "RedisBackend",
+    "SQLAlchemyBackend",
+    "SQLiteBackend",
+    "StorageBackend",
+    "StorageClosedError",
+    "StorageConflictError",
+    "StorageError",
+    "StorageTransaction",
+    "StringCodec",
+    "Subspace",
+    "SupportLevel",
+    "Tree",
+    "TreeNode",
+    "TupleKeyCodec",
+    "WriteBatch",
+    "create_backend",
+]

tketool_storage-1.3.5/src/tketool/storage/abstractions/__init__.py

@@ -0,0 +1,37 @@
+from .backend import StorageBackend, StorageTransaction
+from .codec import BytesCodec, Codec, JsonCodec, KeyPart, StringCodec, Subspace, TupleKeyCodec, prefix_end
+from .errors import (
+    CorruptDataError,
+    DuplicateBatchKeyError,
+    StorageClosedError,
+    StorageConflictError,
+    StorageError,
+    UnsupportedCapabilityError,
+)
+from .types import BackendCapabilities, Delete, KeyValue, Put, SupportLevel, WriteBatch, WriteOperation
+
+__all__ = [
+    "BackendCapabilities",
+    "BytesCodec",
+    "Codec",
+    "CorruptDataError",
+    "Delete",
+    "DuplicateBatchKeyError",
+    "JsonCodec",
+    "KeyPart",
+    "KeyValue",
+    "Put",
+    "StorageBackend",
+    "StorageClosedError",
+    "StorageConflictError",
+    "StorageError",
+    "StorageTransaction",
+    "StringCodec",
+    "Subspace",
+    "SupportLevel",
+    "TupleKeyCodec",
+    "UnsupportedCapabilityError",
+    "WriteBatch",
+    "WriteOperation",
+    "prefix_end",
+]

tketool_storage-1.3.5/src/tketool/storage/abstractions/backend.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import abc
+from contextlib import contextmanager
+from typing import Iterable, Iterator
+
+from .errors import StorageClosedError
+from .types import BackendCapabilities, KeyValue, WriteBatch, WriteOperation
+
+
+class StorageTransaction(abc.ABC):
+    """One explicit transaction over byte keys and values."""
+
+    @abc.abstractmethod
+    def get(self, key: bytes) -> KeyValue | None:
+        raise NotImplementedError
+
+    def get_many(self, keys: Iterable[bytes]) -> list[KeyValue | None]:
+        return [self.get(key) for key in keys]
+
+    @abc.abstractmethod
+    def scan(
+        self,
+        start: bytes,
+        end: bytes | None,
+        *,
+        reverse: bool = False,
+        limit: int | None = None,
+    ) -> list[KeyValue]:
+        """Return records in the half-open range ``[start, end)``."""
+
+    @abc.abstractmethod
+    def apply(self, batch: WriteBatch) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def commit(self) -> None:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def rollback(self) -> None:
+        raise NotImplementedError
+
+    def __enter__(self) -> "StorageTransaction":
+        return self
+
+    def __exit__(self, exc_type, exc, traceback) -> bool:
+        if exc_type is None:
+            self.commit()
+        else:
+            self.rollback()
+        return False
+
+
+class StorageBackend(abc.ABC):
+    """Minimal replaceable storage boundary used by all storage collections."""
+
+    capabilities: BackendCapabilities
+
+    @abc.abstractmethod
+    def begin(self) -> StorageTransaction:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def close(self) -> None:
+        raise NotImplementedError
+
+    @contextmanager
+    def transaction(self) -> Iterator[StorageTransaction]:
+        tx = self.begin()
+        try:
+            yield tx
+        except BaseException:
+            tx.rollback()
+            raise
+        else:
+            tx.commit()
+
+    def get(self, key: bytes) -> KeyValue | None:
+        with self.transaction() as tx:
+            return tx.get(key)
+
+    def get_many(self, keys: Iterable[bytes]) -> list[KeyValue | None]:
+        with self.transaction() as tx:
+            return tx.get_many(keys)
+
+    def scan(
+        self,
+        start: bytes,
+        end: bytes | None,
+        *,
+        reverse: bool = False,
+        limit: int | None = None,
+    ) -> list[KeyValue]:
+        with self.transaction() as tx:
+            return tx.scan(start, end, reverse=reverse, limit=limit)
+
+    def apply(self, operations: WriteBatch | Iterable[WriteOperation]) -> None:
+        batch = operations if isinstance(operations, WriteBatch) else WriteBatch(operations)
+        if not batch:
+            return
+        with self.transaction() as tx:
+            tx.apply(batch)
+
+
+class TransactionState:
+    """Small helper shared by concrete transactions."""
+
+    def __init__(self) -> None:
+        self._finished = False
+
+    @property
+    def finished(self) -> bool:
+        return self._finished
+
+    def ensure_active(self) -> None:
+        if self._finished:
+            raise StorageClosedError("transaction is already finished")
+
+    def finish(self) -> None:
+        self.ensure_active()
+        self._finished = True

tketool_storage-1.3.5/src/tketool/storage/abstractions/codec.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import abc
+import json
+import math
+import struct
+import uuid
+from typing import Generic, TypeVar, cast
+
+from .errors import CorruptDataError
+
+
+T = TypeVar("T")
+KeyPart = None | bool | int | float | str | bytes | uuid.UUID
+
+
+class Codec(abc.ABC, Generic[T]):
+    @abc.abstractmethod
+    def encode(self, value: T) -> bytes:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def decode(self, data: bytes) -> T:
+        raise NotImplementedError
+
+
+class JsonCodec(Codec[T]):
+    def encode(self, value: T) -> bytes:
+        return json.dumps(
+            value,
+            ensure_ascii=False,
+            separators=(",", ":"),
+            sort_keys=True,
+            allow_nan=False,
+        ).encode("utf-8")
+
+    def decode(self, data: bytes) -> T:
+        try:
+            return cast(T, json.loads(data.decode("utf-8")))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise CorruptDataError("invalid JSON value") from exc
+
+
+class BytesCodec(Codec[bytes]):
+    def encode(self, value: bytes) -> bytes:
+        return bytes(value)
+
+    def decode(self, data: bytes) -> bytes:
+        return bytes(data)
+
+
+class StringCodec(Codec[str]):
+    def encode(self, value: str) -> bytes:
+        return value.encode("utf-8")
+
+    def decode(self, data: bytes) -> str:
+        try:
+            return data.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            raise CorruptDataError("invalid UTF-8 value") from exc
+
+
+_NONE = 0x10
+_FALSE = 0x11
+_TRUE = 0x12
+_INT = 0x20
+_FLOAT = 0x21
+_BYTES = 0x30
+_STRING = 0x31
+_UUID = 0x40
+_TERMINATOR = 0x00
+_ESCAPED_ZERO = 0xFF
+
+
+def _escape(data: bytes) -> bytes:
+    return data.replace(b"\x00", b"\x00\xff") + b"\x00"
+
+
+def _unescape(data: bytes, offset: int) -> tuple[bytes, int]:
+    output = bytearray()
+    while offset < len(data):
+        value = data[offset]
+        offset += 1
+        if value != _TERMINATOR:
+            output.append(value)
+            continue
+        if offset < len(data) and data[offset] == _ESCAPED_ZERO:
+            output.append(0)
+            offset += 1
+            continue
+        return bytes(output), offset
+    raise CorruptDataError("unterminated tuple component")
+
+
+class TupleKeyCodec:
+    """Deterministic order-preserving codec for flat heterogeneous tuples.
+
+    Ordering across different Python types follows the stable type tags above.
+    Integer values are limited to signed 64-bit values.
+    """
+
+    def pack(self, parts: tuple[KeyPart, ...]) -> bytes:
+        output = bytearray()
+        for part in parts:
+            if part is None:
+                output.append(_NONE)
+            elif part is False:
+                output.append(_FALSE)
+            elif part is True:
+                output.append(_TRUE)
+            elif isinstance(part, int):
+                if not -(1 << 63) <= part < (1 << 63):
+                    raise OverflowError("tuple integers must fit signed 64 bits")
+                output.append(_INT)
+                output.extend(struct.pack(">Q", part + (1 << 63)))
+            elif isinstance(part, float):
+                if math.isnan(part):
+                    raise ValueError("NaN cannot be used in an ordered key")
+                raw = bytearray(struct.pack(">d", part))
+                if raw[0] & 0x80:
+                    raw = bytearray(value ^ 0xFF for value in raw)
+                else:
+                    raw[0] ^= 0x80
+                output.append(_FLOAT)
+                output.extend(raw)
+            elif isinstance(part, bytes):
+                output.append(_BYTES)
+                output.extend(_escape(part))
+            elif isinstance(part, str):
+                output.append(_STRING)
+                output.extend(_escape(part.encode("utf-8")))
+            elif isinstance(part, uuid.UUID):
+                output.append(_UUID)
+                output.extend(part.bytes)
+            else:
+                raise TypeError(f"unsupported tuple key component: {type(part).__name__}")
+        return bytes(output)
+
+    def unpack(self, data: bytes) -> tuple[KeyPart, ...]:
+        result: list[KeyPart] = []
+        offset = 0
+        while offset < len(data):
+            tag = data[offset]
+            offset += 1
+            if tag == _NONE:
+                result.append(None)
+            elif tag == _FALSE:
+                result.append(False)
+            elif tag == _TRUE:
+                result.append(True)
+            elif tag == _INT:
+                if offset + 8 > len(data):
+                    raise CorruptDataError("truncated integer key component")
+                result.append(struct.unpack(">Q", data[offset : offset + 8])[0] - (1 << 63))
+                offset += 8
+            elif tag == _FLOAT:
+                if offset + 8 > len(data):
+                    raise CorruptDataError("truncated float key component")
+                raw = bytearray(data[offset : offset + 8])
+                if raw[0] & 0x80:
+                    raw[0] ^= 0x80
+                else:
+                    raw = bytearray(value ^ 0xFF for value in raw)
+                result.append(struct.unpack(">d", raw)[0])
+                offset += 8
+            elif tag in (_BYTES, _STRING):
+                value, offset = _unescape(data, offset)
+                if tag == _STRING:
+                    try:
+                        result.append(value.decode("utf-8"))
+                    except UnicodeDecodeError as exc:
+                        raise CorruptDataError("invalid UTF-8 key component") from exc
+                else:
+                    result.append(value)
+            elif tag == _UUID:
+                if offset + 16 > len(data):
+                    raise CorruptDataError("truncated UUID key component")
+                result.append(uuid.UUID(bytes=data[offset : offset + 16]))
+                offset += 16
+            else:
+                raise CorruptDataError(f"unknown tuple key tag: {tag}")
+        return tuple(result)
+
+
+def prefix_end(prefix: bytes) -> bytes | None:
+    """Return the exclusive upper bound of all byte strings with ``prefix``."""
+
+    if not prefix:
+        return None
+    value = bytearray(prefix)
+    for index in range(len(value) - 1, -1, -1):
+        if value[index] != 0xFF:
+            value[index] += 1
+            return bytes(value[: index + 1])
+    return None
+
+
+class Subspace:
+    def __init__(self, prefix: tuple[KeyPart, ...], codec: TupleKeyCodec | None = None):
+        self.codec = codec or TupleKeyCodec()
+        self.prefix_parts = prefix
+        self.raw_prefix = self.codec.pack(prefix)
+
+    def key(self, *parts: KeyPart) -> bytes:
+        return self.raw_prefix + self.codec.pack(tuple(parts))
+
+    def range(self, *parts: KeyPart) -> tuple[bytes, bytes | None]:
+        start = self.key(*parts)
+        return start, prefix_end(start)
+
+    def unpack(self, key: bytes) -> tuple[KeyPart, ...]:
+        if not key.startswith(self.raw_prefix):
+            raise ValueError("key does not belong to this subspace")
+        return self.codec.unpack(key[len(self.raw_prefix) :])

tketool_storage-1.3.5/src/tketool/storage/abstractions/errors.py

@@ -0,0 +1,22 @@
+class StorageError(Exception):
+    """Base error for the portable storage contract."""
+
+
+class StorageClosedError(StorageError):
+    """The backend or transaction has already been closed."""
+
+
+class StorageConflictError(StorageError):
+    """An optimistic write condition did not match."""
+
+
+class DuplicateBatchKeyError(StorageError, ValueError):
+    """A write batch contains more than one operation for the same key."""
+
+
+class UnsupportedCapabilityError(StorageError):
+    """The selected backend cannot provide a required capability."""
+
+
+class CorruptDataError(StorageError):
+    """Stored bytes do not satisfy the declared codec or record format."""

tketool_storage-1.3.5/src/tketool/storage/abstractions/types.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Iterable, TypeAlias
+
+from .errors import DuplicateBatchKeyError
+
+
+class SupportLevel(str, Enum):
+    NATIVE = "native"
+    EMULATED = "emulated"
+    UNSUPPORTED = "unsupported"
+
+
+@dataclass(frozen=True)
+class BackendCapabilities:
+    point_read: SupportLevel = SupportLevel.NATIVE
+    ordered_scan: SupportLevel = SupportLevel.NATIVE
+    atomic_batch: SupportLevel = SupportLevel.NATIVE
+    transactions: SupportLevel = SupportLevel.NATIVE
+    conditional_write: SupportLevel = SupportLevel.NATIVE
+    durability: SupportLevel = SupportLevel.UNSUPPORTED
+
+
+@dataclass(frozen=True)
+class KeyValue:
+    key: bytes
+    value: bytes
+    version: int
+
+
+@dataclass(frozen=True)
+class Put:
+    key: bytes
+    value: bytes
+    expected_version: int | None = None
+    if_absent: bool = False
+
+    def __post_init__(self) -> None:
+        if self.expected_version is not None and self.if_absent:
+            raise ValueError("expected_version and if_absent are mutually exclusive")
+        if self.expected_version is not None and self.expected_version < 1:
+            raise ValueError("expected_version must be positive")
+
+
+@dataclass(frozen=True)
+class Delete:
+    key: bytes
+    expected_version: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.expected_version is not None and self.expected_version < 1:
+            raise ValueError("expected_version must be positive")
+
+
+WriteOperation: TypeAlias = Put | Delete
+
+
+@dataclass(frozen=True)
+class WriteBatch:
+    operations: tuple[WriteOperation, ...] = field(default_factory=tuple)
+
+    def __init__(self, operations: Iterable[WriteOperation] = ()) -> None:
+        items = tuple(operations)
+        keys = [item.key for item in items]
+        if len(keys) != len(set(keys)):
+            raise DuplicateBatchKeyError("each key may appear only once in a write batch")
+        object.__setattr__(self, "operations", items)
+
+    def __bool__(self) -> bool:
+        return bool(self.operations)
+
+    @classmethod
+    def of(cls, *operations: WriteOperation) -> "WriteBatch":
+        return cls(operations)

tketool_storage-1.3.5/src/tketool/storage/adapters/__init__.py

@@ -0,0 +1,6 @@
+from .memory import MemoryBackend
+from .redis import RedisBackend
+from .sqlalchemy import SQLAlchemyBackend
+from .sqlite import SQLiteBackend
+
+__all__ = ["MemoryBackend", "RedisBackend", "SQLAlchemyBackend", "SQLiteBackend"]