objbase 0.3.0__py3-none-any.whl

objbase/storage/file_storage.py

@@ -0,0 +1,268 @@
+import json
+import os
+import sys
+
+from objbase.interface import InventoryStorage, Item
+from objbase.util.file_util import atomic_write_json, atomic_write_text, locked
+
+# Newlines are rejected because the directory storage index stores one id per line.
+_UNSAFE_CHARS = "/\\\x00\n\r"
+if sys.platform == "win32":
+    # Characters Windows doesn't allow in file names; ":" would also allow drive-relative
+    # paths ("C:x", which os.path.join doesn't anchor to the base dir) and alternate data streams.
+    _UNSAFE_CHARS += '<>:"|?*'
+
+
+def _safe_name(name: str, kind: str) -> str:
+    """Validate that an item type or id can be used as a single path component."""
+    if not isinstance(name, str) or name in ("", ".", "..") or any(c in name for c in _UNSAFE_CHARS):
+        raise ValueError(f"Invalid {kind} for file storage: {name!r}")
+    if sys.platform == "win32" and name[-1] in ". ":
+        # Windows drops trailing dots and spaces, so "a." would alias "a" and ".. " would mean "..".
+        raise ValueError(f"Invalid {kind} for file storage: {name!r}")
+    return name
+
+
+def _strip_extended_prefix(path: str) -> str:
+    """Remove a Windows extended-length prefix (``\\\\?\\`` or ``\\\\?\\UNC\\``) from ``path``.
+
+    ``os.path.realpath`` can leave the prefix on a path that doesn't exist yet when
+    its parent directory appears while it's resolving (another thread creating a
+    type directory), so the result wouldn't compare equal to the unprefixed base.
+    """
+    if sys.platform == "win32":
+        if path.startswith("\\\\?\\UNC\\"):
+            return "\\\\" + path[len("\\\\?\\UNC\\") :]
+        if path.startswith("\\\\?\\"):
+            return path[len("\\\\?\\") :]
+    return path
+
+
+def _contained_path(real_base: str, *parts: str) -> str:
+    """Join ``parts`` onto ``real_base``, checking that the real path of the result lies inside ``real_base``.
+
+    ``real_base`` must already be a real path (``os.path.realpath``). Resolving
+    symlinks catches links inside the base directory that point outside it, which
+    the name checks in ``_safe_name`` can't see. Returns the joined (unresolved)
+    path, so replacing or deleting a symlinked file acts on the link itself.
+
+    The check can't stop an attacker who can write to the base directory and swaps
+    in a symlink between the check and the file operation.
+    """
+    path = os.path.join(real_base, *parts)
+    real_path = os.path.normcase(_strip_extended_prefix(os.path.realpath(path)))
+    base = os.path.normcase(_strip_extended_prefix(real_base))
+    try:
+        inside = real_path != base and os.path.commonpath([base, real_path]) == base
+    except ValueError:  # on different drives (Windows)
+        inside = False
+    if not inside:
+        raise ValueError(f"Path for file storage resolves outside the base directory: {path!r}")
+    return path
+
+
+class FileBasedInventoryStorage(InventoryStorage):
+    """Simple file-based storage that saves all items of a given inventory type in a single JSON file.
+
+    Safe for concurrent use by multiple threads and processes on the same machine:
+    writes hold an exclusive lock on ``.{item_type}.json.lock`` for the whole
+    read-modify-write cycle, and files are replaced atomically. Locks are advisory
+    and may not work on network file systems.
+    """
+
+    def __init__(self, base_dir: str):
+        self.inventory_dir = base_dir
+        if not os.path.exists(self.inventory_dir):
+            raise ValueError(f"Base directory {self.inventory_dir} does not exist.")
+        self._real_base = os.path.realpath(base_dir)
+
+    def keys(self, item_type: str) -> list[str]:
+        return [item["id"] for item in self.items(item_type)]
+
+    def items(self, item_type: str) -> list[Item]:
+        file_path = self._file_path(item_type)
+        if not os.path.exists(file_path):
+            return []
+        with locked(self._lock_path(item_type), shared=True):
+            return self._load(file_path)
+
+    def write(self, item_type: str, item: Item) -> bool:
+        file_path = self._file_path(item_type)
+        with locked(self._lock_path(item_type)):
+            items = self._load(file_path)
+            for i, existing_item in enumerate(items):
+                if existing_item["id"] == item["id"]:
+                    items[i] = item
+                    break
+            else:
+                items.append(item)
+            atomic_write_json(file_path, items)
+        return True
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        items = self.items(item_type)
+        for item in items:
+            if item["id"] == id:
+                return item
+        return None
+
+    def delete(self, item_type: str, id: str) -> bool:
+        file_path = self._file_path(item_type)
+        if not os.path.exists(file_path):
+            return False
+        with locked(self._lock_path(item_type)):
+            items = self._load(file_path)
+            remaining = [item for item in items if item["id"] != id]
+            if len(remaining) == len(items):
+                return False
+            atomic_write_json(file_path, remaining)
+        return True
+
+    def _file_path(self, item_type: str) -> str:
+        return _contained_path(self._real_base, f"{_safe_name(item_type, 'item type')}.json")
+
+    def _lock_path(self, item_type: str) -> str:
+        return _contained_path(self._real_base, f".{_safe_name(item_type, 'item type')}.json.lock")
+
+    @staticmethod
+    def _load(file_path: str) -> list[Item]:
+        """Load a type file; the caller must hold its lock. A missing file means no items."""
+        try:
+            with open(file_path) as f:
+                items: list[Item] = json.load(f)
+        except FileNotFoundError:
+            return []
+        return items
+
+
+class DirectoryBasedInventoryStorage(InventoryStorage):
+    """Alternative file-based storage that uses a directory per inventory type and individual files per item.
+
+    Each type directory also holds an index file (``.index``) listing the ids of
+    all items of that type, one per line, so ``keys()`` doesn't have to scan the
+    directory. Writes and deletes hold an exclusive lock on ``.index.lock`` while
+    they change the item file and the index, keeping both in step across threads
+    and processes. Locks are advisory and may not work on network file systems.
+    """
+
+    INDEX_FILE = ".index"
+
+    def __init__(self, base_dir: str):
+        self.inventory_dir = base_dir
+        if not os.path.exists(self.inventory_dir):
+            raise ValueError(f"Base directory {self.inventory_dir} does not exist.")
+        self._real_base = os.path.realpath(base_dir)
+
+    def _type_dir(self, item_type: str) -> str:
+        return _contained_path(self._real_base, _safe_name(item_type, "item type"))
+
+    def _item_path(self, item_type: str, id: str) -> str:
+        return self._path_in_type_dir(item_type, f"{_safe_name(id, 'item id')}.json")
+
+    def _path_in_type_dir(self, item_type: str, filename: str) -> str:
+        return _contained_path(self._real_base, _safe_name(item_type, "item type"), filename)
+
+    def _index_path(self, item_type: str) -> str:
+        return self._path_in_type_dir(item_type, self.INDEX_FILE)
+
+    def _lock_path(self, item_type: str) -> str:
+        return self._path_in_type_dir(item_type, f"{self.INDEX_FILE}.lock")
+
+    def keys(self, item_type: str) -> list[str]:
+        type_dir = self._type_dir(item_type)
+        if not os.path.exists(type_dir):
+            return []
+        with locked(self._lock_path(item_type), shared=True):
+            ids = self._read_index(item_type)
+        # A type directory without an index predates indexing; the next write or
+        # delete creates the index, until then the directory is scanned.
+        return ids if ids is not None else self._scan(type_dir)
+
+    def items(self, item_type: str) -> list[Item]:
+        type_dir = self._type_dir(item_type)
+        if not os.path.exists(type_dir):
+            return []
+        items = []
+        for filename in os.listdir(type_dir):
+            if filename.endswith(".json"):
+                try:
+                    with open(self._path_in_type_dir(item_type, filename)) as f:
+                        items.append(json.load(f))
+                except FileNotFoundError:
+                    continue  # deleted by another process since listdir()
+        return items
+
+    def write(self, item_type: str, item: Item) -> bool:
+        item_id = item.get("id")
+        if not item_id:
+            raise ValueError("Item must have an 'id' field.")
+        item_path = self._item_path(item_type, item_id)
+        os.makedirs(self._type_dir(item_type), exist_ok=True)
+        with locked(self._lock_path(item_type)):
+            atomic_write_json(item_path, item)
+            if item_id not in self._load_index(item_type):
+                with open(self._index_path(item_type), "a") as f:
+                    f.write(f"{item_id}\n")
+                    f.flush()
+                    os.fsync(f.fileno())
+        return True
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        item_path = self._item_path(item_type, id)
+        try:
+            with open(item_path) as f:
+                item: Item = json.load(f)
+        except FileNotFoundError:
+            return None
+        return item
+
+    def delete(self, item_type: str, id: str) -> bool:
+        item_path = self._item_path(item_type, id)
+        if not os.path.exists(self._type_dir(item_type)):
+            return False
+        with locked(self._lock_path(item_type)):
+            try:
+                os.remove(item_path)
+            except FileNotFoundError:
+                return False
+            ids = self._load_index(item_type)
+            if id in ids:
+                self._write_index(item_type, [i for i in ids if i != id])
+        return True
+
+    def rebuild_index(self, item_type: str) -> None:
+        """Recreate the index of ``item_type`` from the item files in its directory.
+
+        Only needed if the index got out of step with the item files, e.g. after a
+        crash between writing an item and updating the index, or after item files
+        were added or removed by hand.
+        """
+        type_dir = self._type_dir(item_type)
+        if not os.path.exists(type_dir):
+            return
+        with locked(self._lock_path(item_type)):
+            self._write_index(item_type, self._scan(type_dir))
+
+    @staticmethod
+    def _scan(type_dir: str) -> list[str]:
+        """Ids of all item files in ``type_dir``, from the directory listing (item files are named ``{id}.json``)."""
+        return [filename.removesuffix(".json") for filename in os.listdir(type_dir) if filename.endswith(".json")]
+
+    def _read_index(self, item_type: str) -> list[str] | None:
+        """Ids in the index of ``item_type``, or ``None`` if there is no index. The caller must hold the lock."""
+        try:
+            with open(self._index_path(item_type)) as f:
+                return [line for line in f.read().splitlines() if line]
+        except FileNotFoundError:
+            return None
+
+    def _load_index(self, item_type: str) -> list[str]:
+        """Like ``_read_index``, but creates a missing index first. The caller must hold the exclusive lock."""
+        ids = self._read_index(item_type)
+        if ids is None:
+            ids = self._scan(self._type_dir(item_type))
+            self._write_index(item_type, ids)
+        return ids
+
+    def _write_index(self, item_type: str, ids: list[str]) -> None:
+        atomic_write_text(self._index_path(item_type), "".join(f"{id}\n" for id in ids))

objbase/storage/inmemory_storage.py

@@ -0,0 +1,51 @@
+import copy
+
+from objbase.asyncio.async_storage import AsyncInventoryStorage
+from objbase.interface import InventoryStorage, Item
+
+
+class InMemoryInventoryStorage(InventoryStorage, AsyncInventoryStorage):
+    """In-memory storage implementation for inventory items.
+
+    Items are deep-copied on the way in and out, so callers never share
+    mutable state with the store.
+    """
+
+    def __init__(self) -> None:
+        self.data: dict[str, dict[str, Item]] = {}
+
+    def keys(self, item_type: str) -> list[str]:
+        return list(self.data.get(item_type, {}))
+
+    def items(self, item_type: str) -> list[Item]:
+        return copy.deepcopy(list(self.data.get(item_type, {}).values()))
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        return copy.deepcopy(self.data.get(item_type, {}).get(id))
+
+    def write(self, item_type: str, item: Item) -> bool:
+        if item_type not in self.data:
+            self.data[item_type] = {}
+        self.data[item_type][item["id"]] = copy.deepcopy(item)
+        return True
+
+    def delete(self, item_type: str, id: str) -> bool:
+        if item_type in self.data and id in self.data[item_type]:
+            del self.data[item_type][id]
+            return True
+        return False
+
+    async def akeys(self, item_type: str) -> list[str]:
+        return self.keys(item_type)
+
+    async def aitems(self, item_type: str) -> list[Item]:
+        return self.items(item_type)
+
+    async def aread(self, item_type: str, id: str) -> Item | None:
+        return self.read(item_type, id)
+
+    async def awrite(self, item_type: str, item: Item) -> bool:
+        return self.write(item_type, item)
+
+    async def adelete(self, item_type: str, id: str) -> bool:
+        return self.delete(item_type, id)

objbase/storage/mongodb_storage.py

@@ -0,0 +1,42 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
+
+from objbase.interface import InventoryStorage, Item
+
+if TYPE_CHECKING:
+    from pymongo import MongoClient
+    from pymongo.collection import Collection
+
+
+class MongoDBInventoryStorage(InventoryStorage):
+    """MongoDB-based storage implementation for inventory items."""
+
+    def __init__(self, mongo_client: "MongoClient[Item]"):
+        self.mongo_client = mongo_client
+
+    def get_mongo_collection(self, item_type: str) -> "Collection[Item]":
+        db = self.mongo_client["inventory"]
+        return db[item_type]
+
+    def keys(self, item_type: str) -> list[str]:
+        collection = self.get_mongo_collection(item_type)
+        return [doc["id"] for doc in collection.find({}, {"id": True, "_id": False})]
+
+    def items(self, item_type: str, query: Mapping[str, Any] | None = None) -> list[Item]:
+        """Return all items of a type. ``query`` is a MongoDB-only extension to filter results."""
+        collection = self.get_mongo_collection(item_type)
+        return list(collection.find(query or {}, {"_id": False}))
+
+    def write(self, item_type: str, item: Item) -> bool:
+        collection = self.get_mongo_collection(item_type)
+        collection.replace_one({"id": item["id"]}, item, upsert=True)
+        return True
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        collection = self.get_mongo_collection(item_type)
+        return collection.find_one({"id": id}, {"_id": False})
+
+    def delete(self, item_type: str, id: str) -> bool:
+        collection = self.get_mongo_collection(item_type)
+        result = collection.delete_one({"id": id})
+        return result.deleted_count > 0

objbase/storage/redis_storage.py

@@ -0,0 +1,68 @@
+import json
+from typing import Any, Protocol
+
+from objbase.interface import InventoryStorage, Item
+
+DEFAULT_KEY_PREFIX = "inventory:"
+
+
+class RedisHashClient(Protocol):
+    """The Redis hash commands the Redis adapters use.
+
+    Satisfied by ``redis.Redis`` and ``redis.asyncio.Redis`` (and compatible
+    clients). Return types are ``Any`` because redis-py annotates every command
+    as returning ``Awaitable[T] | T``; the async adapter awaits the results.
+    """
+
+    def hkeys(self, name: str, /) -> Any: ...
+
+    def hvals(self, name: str, /) -> Any: ...
+
+    def hget(self, name: str, key: str, /) -> Any: ...
+
+    def hset(self, name: str, key: str, value: str, /) -> Any: ...
+
+    def hdel(self, name: str, /, *keys: str) -> Any: ...
+
+
+def decode_key(key: bytes | str) -> str:
+    """Hash field names are bytes unless the client was created with ``decode_responses=True``."""
+    return key.decode() if isinstance(key, bytes) else key
+
+
+def redis_type_key(key_prefix: str, item_type: str) -> str:
+    """Name of the Redis hash that holds all items of ``item_type``, keyed by id."""
+    return f"{key_prefix}{item_type}"
+
+
+class RedisInventoryStorage(InventoryStorage):
+    """Redis-backed storage.
+
+    Each item type is one Redis hash (``{key_prefix}{item_type}``) mapping item
+    ids to JSON-encoded items. Works with clients created with or without
+    ``decode_responses=True``.
+    """
+
+    def __init__(self, redis_client: RedisHashClient, key_prefix: str = DEFAULT_KEY_PREFIX):
+        self.redis_client = redis_client
+        self.key_prefix = key_prefix
+
+    def _key(self, item_type: str) -> str:
+        return redis_type_key(self.key_prefix, item_type)
+
+    def keys(self, item_type: str) -> list[str]:
+        return [decode_key(key) for key in self.redis_client.hkeys(self._key(item_type))]
+
+    def items(self, item_type: str) -> list[Item]:
+        return [json.loads(value) for value in self.redis_client.hvals(self._key(item_type))]
+
+    def write(self, item_type: str, item: Item) -> bool:
+        self.redis_client.hset(self._key(item_type), item["id"], json.dumps(item))
+        return True
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        value = self.redis_client.hget(self._key(item_type), id)
+        return json.loads(value) if value is not None else None
+
+    def delete(self, item_type: str, id: str) -> bool:
+        return bool(self.redis_client.hdel(self._key(item_type), id))

objbase/storage/sqlite_storage.py

@@ -0,0 +1,79 @@
+import json
+import sqlite3
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from objbase.interface import InventoryStorage, Item
+
+CREATE_TABLE_SQL = """
+    CREATE TABLE IF NOT EXISTS items (
+        item_type  TEXT NOT NULL,
+        id              TEXT NOT NULL,
+        data            TEXT NOT NULL,
+        PRIMARY KEY (item_type, id)
+    )
+"""
+
+
+class SQLiteInventoryStorage(InventoryStorage):
+    """SQLite-backed storage. Each item is stored as a JSON blob in a single table."""
+
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        with self._connect() as conn:
+            conn.execute(CREATE_TABLE_SQL)
+
+    @contextmanager
+    def _connect(self) -> Iterator[sqlite3.Connection]:
+        """Open a connection, commit on success (rollback on error), and always close it."""
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        try:
+            with conn:
+                yield conn
+        finally:
+            conn.close()
+
+    def keys(self, item_type: str) -> list[str]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT id FROM items WHERE item_type = ?",
+                (item_type,),
+            ).fetchall()
+        return [row["id"] for row in rows]
+
+    def items(self, item_type: str) -> list[Item]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT data FROM items WHERE item_type = ?",
+                (item_type,),
+            ).fetchall()
+        return [json.loads(row["data"]) for row in rows]
+
+    def read(self, item_type: str, id: str) -> Item | None:
+        with self._connect() as conn:
+            row = conn.execute(
+                "SELECT data FROM items WHERE item_type = ? AND id = ?",
+                (item_type, id),
+            ).fetchone()
+        return json.loads(row["data"]) if row else None
+
+    def write(self, item_type: str, item: Item) -> bool:
+        with self._connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO items (item_type, id, data)
+                VALUES (?, ?, ?)
+                ON CONFLICT (item_type, id) DO UPDATE SET data = excluded.data
+                """,
+                (item_type, item["id"], json.dumps(item)),
+            )
+        return True
+
+    def delete(self, item_type: str, id: str) -> bool:
+        with self._connect() as conn:
+            cursor = conn.execute(
+                "DELETE FROM items WHERE item_type = ? AND id = ?",
+                (item_type, id),
+            )
+        return cursor.rowcount > 0

objbase/util/file_util.py

@@ -0,0 +1,102 @@
+"""Cross-platform file locking and atomic JSON writes, using only the standard library."""
+
+import contextlib
+import json
+import os
+import shutil
+import sys
+import time
+import uuid
+from collections.abc import Iterator
+from typing import IO, Any
+
+if sys.platform == "win32":
+    import msvcrt
+
+    def _lock(f: IO[bytes], shared: bool) -> None:
+        # msvcrt has no shared locks, so readers lock exclusively too. LK_LOCK
+        # gives up after ~10s, so retry until the lock is acquired.
+        f.seek(0)
+        while True:
+            try:
+                msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1)
+                return
+            except OSError:
+                time.sleep(0.05)
+
+    def _unlock(f: IO[bytes]) -> None:
+        f.seek(0)
+        msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
+
+else:
+    import fcntl
+
+    def _lock(f: IO[bytes], shared: bool) -> None:
+        # flock locks belong to the open file, so they also exclude other threads
+        # in the same process, not just other processes.
+        fcntl.flock(f.fileno(), fcntl.LOCK_SH if shared else fcntl.LOCK_EX)
+
+    def _unlock(f: IO[bytes]) -> None:
+        fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+
+
+@contextlib.contextmanager
+def locked(lock_path: str, shared: bool = False) -> Iterator[None]:
+    """Hold an advisory lock on ``lock_path`` (created if missing) for the duration of the block.
+
+    Blocks until the lock is available. ``shared=True`` allows concurrent readers
+    (on Windows all locks are exclusive). The lock is released if the process dies.
+    Lock files are left in place; deleting them while others may use them is unsafe.
+    """
+    with open(lock_path, "a+b") as f:
+        _lock(f, shared)
+        try:
+            yield
+        finally:
+            _unlock(f)
+
+
+def _replace(src: str, dst: str) -> None:
+    if sys.platform != "win32":
+        os.replace(src, dst)
+        return
+    # Windows refuses to replace a file another process has open (e.g. an unlocked
+    # reader), so retry briefly before giving up.
+    for _ in range(40):
+        try:
+            os.replace(src, dst)
+            return
+        except PermissionError:
+            time.sleep(0.05)
+    os.replace(src, dst)
+
+
+def atomic_write_json(path: str, data: Any) -> None:
+    """Write ``data`` as JSON to ``path`` so readers see either the old or the new file, never a partial one.
+
+    See ``atomic_write_text``.
+    """
+    atomic_write_text(path, json.dumps(data, indent=4))
+
+
+def atomic_write_text(path: str, text: str) -> None:
+    """Write ``text`` to ``path`` so readers see either the old or the new file, never a partial one.
+
+    Writes to a temporary file in the same directory and renames it over ``path``.
+    Keeps the permissions of an existing ``path``; a new file gets the default
+    permissions for the current umask.
+    """
+    directory, name = os.path.split(path)
+    tmp_path = os.path.join(directory, f".{name}.{uuid.uuid4().hex}.tmp")
+    try:
+        with open(tmp_path, "x") as f:
+            f.write(text)
+            f.flush()
+            os.fsync(f.fileno())
+        if os.path.exists(path):
+            shutil.copymode(path, tmp_path)
+        _replace(tmp_path, path)
+    except BaseException:
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(tmp_path)
+        raise

objbase/util/mongodb_util.py

@@ -0,0 +1,45 @@
+import os
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pymongo import MongoClient
+
+
+def get_mongo_client(uri: str | None = None, ping: bool = False) -> "MongoClient[dict[str, Any]]":
+    mongodb_uri = uri or os.getenv("MONGODB_URI")
+    if not mongodb_uri:
+        raise ValueError("MONGODB_URI is not set in environment variables.")
+
+    try:
+        import pymongo
+    except ImportError:
+        raise ImportError("pymongo is not installed. Please install it with 'pip install pymongo'.") from None
+    client: MongoClient[dict[str, Any]] = pymongo.MongoClient(mongodb_uri)
+
+    if ping:
+        try:
+            # The ping command is cheap and does not require auth.
+            client.admin.command("ping")
+        except Exception as e:
+            raise ConnectionError(f"Could not connect to MongoDB: {e}") from e
+    return client
+
+
+def mongodb_results_to_json(results: list[dict[str, Any]], strip_id: bool = True) -> list[dict[str, Any]]:
+    json_results = []
+    for doc in results:
+        if strip_id and "_id" in doc:
+            doc.pop("_id")
+        elif "_id" in doc:
+            doc["_id"] = str(doc["_id"])  # Convert ObjectId to string
+        json_results.append(doc)
+    return json_results
+
+
+def mongodb_result_to_json(result: dict[str, Any], strip_id: bool = True) -> dict[str, Any]:
+    if result and "_id" in result:
+        if strip_id and "_id" in result:
+            result.pop("_id")
+        elif "_id" in result:
+            result["_id"] = str(result["_id"])  # Convert ObjectId to string
+    return result