objbase 0.3.0__py3-none-any.whl

objbase/__init__.py

@@ -0,0 +1,59 @@
+"""Damn simple object store for Python dicts and Pydantic models across multiple backends."""
+
+import importlib
+from importlib.metadata import PackageNotFoundError, version
+from typing import TYPE_CHECKING, Any
+
+from objbase.asyncio.async_file_storage import AsyncDirectoryBasedInventoryStorage, AsyncFileBasedInventoryStorage
+from objbase.asyncio.async_inventory import AsyncInventory
+from objbase.asyncio.async_mongodb_storage import AsyncMongoDBInventoryStorage
+from objbase.asyncio.async_redis_storage import AsyncRedisInventoryStorage
+from objbase.asyncio.async_sqlite_storage import AsyncSQLiteInventoryStorage
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.errors import InventoryError, ItemNotFoundError
+from objbase.interface import InventoryStorage, Item
+from objbase.inventory import Inventory
+from objbase.storage.file_storage import DirectoryBasedInventoryStorage, FileBasedInventoryStorage
+from objbase.storage.inmemory_storage import InMemoryInventoryStorage
+from objbase.storage.mongodb_storage import MongoDBInventoryStorage
+from objbase.storage.redis_storage import RedisInventoryStorage
+from objbase.storage.sqlite_storage import SQLiteInventoryStorage
+
+if TYPE_CHECKING:
+    # Lets type checkers see the real classes; at runtime they are loaded lazily by __getattr__.
+    from objbase.pydantic import AsyncPydanticInventory, PydanticInventory
+
+try:
+    __version__ = version("objbase")
+except PackageNotFoundError:  # running from a source tree without installation
+    __version__ = "0.0.0"
+
+__all__ = [
+    "AsyncDirectoryBasedInventoryStorage",
+    "AsyncFileBasedInventoryStorage",
+    "AsyncInventory",
+    "AsyncInventoryStorage",
+    "AsyncMongoDBInventoryStorage",
+    "AsyncPydanticInventory",
+    "AsyncRedisInventoryStorage",
+    "AsyncSQLiteInventoryStorage",
+    "DirectoryBasedInventoryStorage",
+    "FileBasedInventoryStorage",
+    "InMemoryInventoryStorage",
+    "Inventory",
+    "InventoryError",
+    "InventoryStorage",
+    "Item",
+    "ItemNotFoundError",
+    "MongoDBInventoryStorage",
+    "PydanticInventory",
+    "RedisInventoryStorage",
+    "SQLiteInventoryStorage",
+]
+
+
+def __getattr__(name: str) -> Any:
+    # Imported lazily so `import objbase` works without pydantic installed.
+    if name in ("PydanticInventory", "AsyncPydanticInventory"):
+        return getattr(importlib.import_module("objbase.pydantic"), name)
+    raise AttributeError(f"module 'objbase' has no attribute {name!r}")

objbase/asyncio/async_file_storage.py

@@ -0,0 +1,40 @@
+import asyncio
+
+from objbase.asyncio.threaded_storage import ThreadedAsyncInventoryStorage
+from objbase.storage.file_storage import DirectoryBasedInventoryStorage, FileBasedInventoryStorage
+
+
+class AsyncFileBasedInventoryStorage(ThreadedAsyncInventoryStorage[FileBasedInventoryStorage]):
+    """Async counterpart of ``FileBasedInventoryStorage``, using the same files and locks.
+
+    Each call runs in a worker thread (``asyncio.to_thread``), so the event loop is never
+    blocked by file I/O or while waiting for a file lock. Sync and async adapters on the
+    same directory can be used side by side, in one or several processes.
+    """
+
+    def __init__(self, base_dir: str):
+        super().__init__(FileBasedInventoryStorage(base_dir))
+
+    @property
+    def inventory_dir(self) -> str:
+        return self.sync_storage.inventory_dir
+
+
+class AsyncDirectoryBasedInventoryStorage(ThreadedAsyncInventoryStorage[DirectoryBasedInventoryStorage]):
+    """Async counterpart of ``DirectoryBasedInventoryStorage``, using the same files, index and locks.
+
+    Each call runs in a worker thread (``asyncio.to_thread``), so the event loop is never
+    blocked by file I/O or while waiting for a file lock. Sync and async adapters on the
+    same directory can be used side by side, in one or several processes.
+    """
+
+    def __init__(self, base_dir: str):
+        super().__init__(DirectoryBasedInventoryStorage(base_dir))
+
+    @property
+    def inventory_dir(self) -> str:
+        return self.sync_storage.inventory_dir
+
+    async def arebuild_index(self, item_type: str) -> None:
+        """Async counterpart of ``DirectoryBasedInventoryStorage.rebuild_index``."""
+        await asyncio.to_thread(self.sync_storage.rebuild_index, item_type)

objbase/asyncio/async_inventory.py

@@ -0,0 +1,47 @@
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.errors import InventoryError, ItemNotFoundError
+from objbase.interface import Item
+from objbase.inventory import check_patch_data, require_item_id, require_read_back
+
+
+class AsyncInventory:
+    """Async counterpart of ``Inventory``, backed by an ``AsyncInventoryStorage``.
+
+    Same methods and behaviour as ``Inventory``, but every method is a coroutine.
+    """
+
+    def __init__(self, item_type: str, storage: AsyncInventoryStorage):
+        if not isinstance(storage, AsyncInventoryStorage):
+            raise TypeError(
+                f"{type(storage).__name__} is not an AsyncInventoryStorage; use Inventory for sync storage adapters."
+            )
+        self.storage = storage
+        self.item_type = item_type
+
+    async def keys(self) -> list[str]:
+        return await self.storage.akeys(self.item_type)
+
+    async def filter(self) -> list[Item]:
+        return await self.storage.aitems(self.item_type)
+
+    async def get(self, id: str) -> Item | None:
+        return await self.storage.aread(self.item_type, id)
+
+    async def save(self, item: Item) -> Item:
+        _id = require_item_id(item)
+        if not await self.storage.awrite(self.item_type, item):
+            raise InventoryError(f"Failed to save item '{_id}'.")
+        return require_read_back(await self.storage.aread(self.item_type, _id), self.item_type, _id)
+
+    async def patch(self, id: str, data: Item) -> Item:
+        check_patch_data(id, data)
+        item = await self.storage.aread(self.item_type, id)
+        if item is None:
+            raise ItemNotFoundError(self.item_type, id)
+        item.update(data)
+        if not await self.storage.awrite(self.item_type, item):
+            raise InventoryError(f"Failed to patch item '{id}'.")
+        return require_read_back(await self.storage.aread(self.item_type, id), self.item_type, id)
+
+    async def delete(self, id: str) -> bool:
+        return await self.storage.adelete(self.item_type, id)

objbase/asyncio/async_mongodb_storage.py

@@ -0,0 +1,46 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
+
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.interface import Item
+
+if TYPE_CHECKING:
+    from pymongo import AsyncMongoClient
+    from pymongo.asynchronous.collection import AsyncCollection
+
+
+class AsyncMongoDBInventoryStorage(AsyncInventoryStorage):
+    """Async counterpart of ``MongoDBInventoryStorage``, using the same data layout.
+
+    Takes an async client such as ``pymongo.AsyncMongoClient``.
+    """
+
+    def __init__(self, mongo_client: "AsyncMongoClient[Item]"):
+        self.mongo_client = mongo_client
+
+    def get_mongo_collection(self, item_type: str) -> "AsyncCollection[Item]":
+        db = self.mongo_client["inventory"]
+        return db[item_type]
+
+    async def akeys(self, item_type: str) -> list[str]:
+        collection = self.get_mongo_collection(item_type)
+        return [doc["id"] async for doc in collection.find({}, {"id": True, "_id": False})]
+
+    async def aitems(self, item_type: str, query: Mapping[str, Any] | None = None) -> list[Item]:
+        """Return all items of a type. ``query`` is a MongoDB-only extension to filter results."""
+        collection = self.get_mongo_collection(item_type)
+        return [doc async for doc in collection.find(query or {}, {"_id": False})]
+
+    async def awrite(self, item_type: str, item: Item) -> bool:
+        collection = self.get_mongo_collection(item_type)
+        await collection.replace_one({"id": item["id"]}, item, upsert=True)
+        return True
+
+    async def aread(self, item_type: str, id: str) -> Item | None:
+        collection = self.get_mongo_collection(item_type)
+        return await collection.find_one({"id": id}, {"_id": False})
+
+    async def adelete(self, item_type: str, id: str) -> bool:
+        collection = self.get_mongo_collection(item_type)
+        result = await collection.delete_one({"id": id})
+        return result.deleted_count > 0

objbase/asyncio/async_redis_storage.py

@@ -0,0 +1,36 @@
+import json
+
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.interface import Item
+from objbase.storage.redis_storage import DEFAULT_KEY_PREFIX, RedisHashClient, decode_key, redis_type_key
+
+
+class AsyncRedisInventoryStorage(AsyncInventoryStorage):
+    """Async counterpart of ``RedisInventoryStorage``, using the same data layout.
+
+    Takes an async client such as ``redis.asyncio.Redis``.
+    """
+
+    def __init__(self, redis_client: RedisHashClient, key_prefix: str = DEFAULT_KEY_PREFIX):
+        self.redis_client = redis_client
+        self.key_prefix = key_prefix
+
+    def _key(self, item_type: str) -> str:
+        return redis_type_key(self.key_prefix, item_type)
+
+    async def akeys(self, item_type: str) -> list[str]:
+        return [decode_key(key) for key in await self.redis_client.hkeys(self._key(item_type))]
+
+    async def aitems(self, item_type: str) -> list[Item]:
+        return [json.loads(value) for value in await self.redis_client.hvals(self._key(item_type))]
+
+    async def awrite(self, item_type: str, item: Item) -> bool:
+        await self.redis_client.hset(self._key(item_type), item["id"], json.dumps(item))
+        return True
+
+    async def aread(self, item_type: str, id: str) -> Item | None:
+        value = await self.redis_client.hget(self._key(item_type), id)
+        return json.loads(value) if value is not None else None
+
+    async def adelete(self, item_type: str, id: str) -> bool:
+        return bool(await self.redis_client.hdel(self._key(item_type), id))

objbase/asyncio/async_sqlite_storage.py

@@ -0,0 +1,19 @@
+from objbase.asyncio.threaded_storage import ThreadedAsyncInventoryStorage
+from objbase.storage.sqlite_storage import SQLiteInventoryStorage
+
+
+class AsyncSQLiteInventoryStorage(ThreadedAsyncInventoryStorage[SQLiteInventoryStorage]):
+    """Async counterpart of ``SQLiteInventoryStorage``, using the same data layout.
+
+    Uses the standard library ``sqlite3`` module, so it needs no extra dependencies.
+    Each call runs in a worker thread (``asyncio.to_thread``) with its own connection,
+    so the event loop is never blocked by database I/O.
+    """
+
+    def __init__(self, db_path: str):
+        # Creates the table if needed; this one-off setup runs synchronously.
+        super().__init__(SQLiteInventoryStorage(db_path))
+
+    @property
+    def db_path(self) -> str:
+        return self.sync_storage.db_path

objbase/asyncio/async_storage.py

@@ -0,0 +1,18 @@
+from typing import Protocol, runtime_checkable
+
+from objbase.interface import Item
+
+
+@runtime_checkable
+class AsyncInventoryStorage(Protocol):
+    """Async counterpart of ``InventoryStorage``, with the same contract."""
+
+    async def akeys(self, item_type: str) -> list[str]: ...
+
+    async def aitems(self, item_type: str) -> list[Item]: ...
+
+    async def aread(self, item_type: str, id: str) -> Item | None: ...
+
+    async def awrite(self, item_type: str, item: Item) -> bool: ...
+
+    async def adelete(self, item_type: str, id: str) -> bool: ...

objbase/asyncio/threaded_storage.py

@@ -0,0 +1,30 @@
+import asyncio
+
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.interface import InventoryStorage, Item
+
+
+class ThreadedAsyncInventoryStorage[S: InventoryStorage](AsyncInventoryStorage):
+    """Base for async adapters that run a blocking sync adapter in worker threads.
+
+    Each call is passed to ``asyncio.to_thread``, so the event loop is never blocked by
+    I/O. The sync adapter must be safe to call from several threads at once.
+    """
+
+    def __init__(self, sync_storage: S):
+        self.sync_storage = sync_storage
+
+    async def akeys(self, item_type: str) -> list[str]:
+        return await asyncio.to_thread(self.sync_storage.keys, item_type)
+
+    async def aitems(self, item_type: str) -> list[Item]:
+        return await asyncio.to_thread(self.sync_storage.items, item_type)
+
+    async def aread(self, item_type: str, id: str) -> Item | None:
+        return await asyncio.to_thread(self.sync_storage.read, item_type, id)
+
+    async def awrite(self, item_type: str, item: Item) -> bool:
+        return await asyncio.to_thread(self.sync_storage.write, item_type, item)
+
+    async def adelete(self, item_type: str, id: str) -> bool:
+        return await asyncio.to_thread(self.sync_storage.delete, item_type, id)

objbase/errors.py

@@ -0,0 +1,11 @@
+class InventoryError(Exception):
+    """Base class for all objbase errors."""
+
+
+class ItemNotFoundError(InventoryError, LookupError):
+    """Raised when an operation requires an item that does not exist."""
+
+    def __init__(self, item_type: str, id: str):
+        self.item_type = item_type
+        self.id = id
+        super().__init__(f"Item '{id}' of type '{item_type}' not found.")

objbase/interface.py

@@ -0,0 +1,27 @@
+from typing import Any, Protocol, runtime_checkable
+
+Item = dict[str, Any]
+"""A stored item: a JSON-serializable dict with an ``"id"`` key."""
+
+
+@runtime_checkable
+class InventoryStorage(Protocol):
+    """Storage contract shared by all adapters.
+
+    - ``keys`` returns all item ids of a type, or ``[]`` if there are none.
+    - ``items`` returns all items of a type, or ``[]`` if there are none.
+    - ``read`` returns the item, or ``None`` if it does not exist.
+    - ``write`` inserts the item or replaces an existing item with the same id entirely.
+    - ``delete`` returns ``True`` if an item was removed, ``False`` if it did not exist.
+    - Returned items are independent copies; mutating them does not change stored data.
+    """
+
+    def keys(self, item_type: str) -> list[str]: ...
+
+    def items(self, item_type: str) -> list[Item]: ...
+
+    def read(self, item_type: str, id: str) -> Item | None: ...
+
+    def write(self, item_type: str, item: Item) -> bool: ...
+
+    def delete(self, item_type: str, id: str) -> bool: ...

objbase/inventory.py

@@ -0,0 +1,59 @@
+from typing import Any
+
+from objbase.errors import InventoryError, ItemNotFoundError
+from objbase.interface import InventoryStorage, Item
+
+
+def require_item_id(item: Item) -> Any:
+    """Return the item's id, raising ``ValueError`` if it is missing or empty."""
+    _id = item.get("id")
+    if not _id:
+        raise ValueError("Item id is required.")
+    return _id
+
+
+def check_patch_data(id: str, data: Item) -> None:
+    """Raise ``ValueError`` if patch data would change the item id."""
+    if "id" in data and data["id"] != id:
+        raise ValueError("Patch data must not change the item id.")
+
+
+def require_read_back(item: Item | None, item_type: str, id: str) -> Item:
+    """Return an item read back after a successful write, raising if it vanished."""
+    if item is None:
+        raise InventoryError(f"Item '{id}' of type '{item_type}' could not be read back after writing.")
+    return item
+
+
+class Inventory:
+    def __init__(self, item_type: str, storage: InventoryStorage):
+        self.storage = storage
+        self.item_type = item_type
+
+    def keys(self) -> list[str]:
+        return self.storage.keys(self.item_type)
+
+    def filter(self) -> list[Item]:
+        return self.storage.items(self.item_type)
+
+    def get(self, id: str) -> Item | None:
+        return self.storage.read(self.item_type, id)
+
+    def save(self, item: Item) -> Item:
+        _id = require_item_id(item)
+        if not self.storage.write(self.item_type, item):
+            raise InventoryError(f"Failed to save item '{_id}'.")
+        return require_read_back(self.storage.read(self.item_type, _id), self.item_type, _id)
+
+    def patch(self, id: str, data: Item) -> Item:
+        check_patch_data(id, data)
+        item = self.storage.read(self.item_type, id)
+        if item is None:
+            raise ItemNotFoundError(self.item_type, id)
+        item.update(data)
+        if not self.storage.write(self.item_type, item):
+            raise InventoryError(f"Failed to patch item '{id}'.")
+        return require_read_back(self.storage.read(self.item_type, id), self.item_type, id)
+
+    def delete(self, id: str) -> bool:
+        return self.storage.delete(self.item_type, id)

objbase/pydantic.py

@@ -0,0 +1,123 @@
+import pydantic
+
+from objbase.asyncio.async_inventory import AsyncInventory
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.errors import ItemNotFoundError
+from objbase.interface import InventoryStorage, Item
+from objbase.inventory import Inventory, check_patch_data
+
+
+def _dump(model: pydantic.BaseModel) -> Item:
+    """Convert a model to the JSON-compatible dict that is stored."""
+    return model.model_dump(mode="json")
+
+
+def _validated[M: pydantic.BaseModel](model_class: type[M], data: Item | M) -> Item:
+    """Validate ``data`` against ``model_class`` and return the dict to store.
+
+    Runs before every write, so invalid data raises ``ValidationError`` without
+    being stored. This also catches models made invalid after construction
+    (Pydantic does not validate attribute assignment by default).
+    """
+    # dict(model) takes the raw field values, so a model with invalid values is
+    # re-validated (and rejected) rather than serialized as-is.
+    raw = dict(data) if isinstance(data, pydantic.BaseModel) else data
+    return _dump(model_class.model_validate(raw))
+
+
+def _patched[M: pydantic.BaseModel](
+    model_class: type[M], item_type: str, id: str, current: Item | None, data: Item | M
+) -> Item:
+    """Merge patch ``data`` into the ``current`` stored item and validate the result."""
+    patch = dict(data) if isinstance(data, pydantic.BaseModel) else data
+    check_patch_data(id, patch)
+    if current is None:
+        raise ItemNotFoundError(item_type, id)
+    return _validated(model_class, {**current, **patch})
+
+
+class PydanticInventory[M: pydantic.BaseModel]:
+    """Inventory that validates items against a Pydantic model.
+
+    Wraps an ``Inventory``: items are stored as plain dicts and returned as
+    ``model_class`` instances. The model type is inferred from ``model_class``,
+    so ``PydanticInventory("todo", storage, Todo).get("1")`` is typed ``Todo | None``.
+
+    ``save`` and ``patch`` validate the complete item before writing it, so data
+    that fails validation raises ``pydantic.ValidationError`` and is never stored.
+    """
+
+    def __init__(self, item_type: str, storage: InventoryStorage, model_class: type[M]):
+        self.model_class = model_class
+        self.inventory = Inventory(item_type, storage)
+
+    @property
+    def item_type(self) -> str:
+        return self.inventory.item_type
+
+    @property
+    def storage(self) -> InventoryStorage:
+        return self.inventory.storage
+
+    def keys(self) -> list[str]:
+        return self.inventory.keys()
+
+    def filter(self) -> list[M]:
+        return [self.model_class.model_validate(item) for item in self.inventory.filter()]
+
+    def save(self, model: M) -> M:
+        return self.model_class.model_validate(self.inventory.save(_validated(self.model_class, model)))
+
+    def get(self, id: str) -> M | None:
+        item = self.inventory.get(id)
+        if item is None:
+            return None
+        return self.model_class.model_validate(item)
+
+    def patch(self, id: str, data: Item | M) -> M:
+        item = _patched(self.model_class, self.item_type, id, self.inventory.get(id), data)
+        return self.model_class.model_validate(self.inventory.save(item))
+
+    def delete(self, id: str) -> bool:
+        return self.inventory.delete(id)
+
+
+class AsyncPydanticInventory[M: pydantic.BaseModel]:
+    """Async counterpart of ``PydanticInventory``, backed by an ``AsyncInventoryStorage``.
+
+    Same methods and behaviour as ``PydanticInventory``, but every method is a coroutine.
+    """
+
+    def __init__(self, item_type: str, storage: AsyncInventoryStorage, model_class: type[M]):
+        self.model_class = model_class
+        self.inventory = AsyncInventory(item_type, storage)
+
+    @property
+    def item_type(self) -> str:
+        return self.inventory.item_type
+
+    @property
+    def storage(self) -> AsyncInventoryStorage:
+        return self.inventory.storage
+
+    async def keys(self) -> list[str]:
+        return await self.inventory.keys()
+
+    async def filter(self) -> list[M]:
+        return [self.model_class.model_validate(item) for item in await self.inventory.filter()]
+
+    async def save(self, model: M) -> M:
+        return self.model_class.model_validate(await self.inventory.save(_validated(self.model_class, model)))
+
+    async def get(self, id: str) -> M | None:
+        item = await self.inventory.get(id)
+        if item is None:
+            return None
+        return self.model_class.model_validate(item)
+
+    async def patch(self, id: str, data: Item | M) -> M:
+        item = _patched(self.model_class, self.item_type, id, await self.inventory.get(id), data)
+        return self.model_class.model_validate(await self.inventory.save(item))
+
+    async def delete(self, id: str) -> bool:
+        return await self.inventory.delete(id)