tinymongo 1.0.0__py3-none-any.whl

tinymongo/__init__.py

@@ -0,0 +1,4 @@
+try:
+    from tinymongo.tinymongo import *  # noqa
+except ImportError:
+    from tinymongo import *  # noqa

tinymongo/errors.py

@@ -0,0 +1,61 @@
+"""Exceptions raised by TinyMongo."""
+
+
+class TinyMongoError(Exception):
+    """Base class for all TinyMongo exceptions."""
+
+
+class ConnectionFailure(TinyMongoError):
+    """Raised when a connection to the database file cannot be made or is lost.
+    """
+
+
+class ConfigurationError(TinyMongoError):
+    """Raised when something is incorrectly configured.
+    """
+
+
+class OperationFailure(TinyMongoError):
+    """Raised when a database operation fails.
+    """
+
+    def __init__(self, error, code=None, details=None):
+        self.__code = code
+        self.__details = details
+        TinyMongoError.__init__(self, error)
+
+    @property
+    def code(self):
+        """The error code returned by the server, if any.
+        """
+        return self.__code
+
+    @property
+    def details(self):
+        """The complete error document returned by the server.
+
+        Depending on the error that occurred, the error document
+        may include useful information beyond just the error
+        message. When connected to a mongos the error document
+        may contain one or more subdocuments if errors occurred
+        on multiple shards.
+        """
+        return self.__details
+
+
+class CursorNotFound(OperationFailure):
+    """Raised while iterating query results if the cursor is
+    invalidated on the server.
+    """
+
+
+class WriteError(OperationFailure):
+    """Base exception type for errors raised during write operations."""
+
+
+class DuplicateKeyError(WriteError):
+    """Raised when an insert or update fails due to a duplicate key error."""
+
+
+class InvalidOperation(TinyMongoError):
+    """Raised when a client attempts to perform an invalid operation."""

tinymongo/parquet_storage.py

@@ -0,0 +1,256 @@
+import json
+import os
+import tempfile
+from tinydb.storages import Storage
+import threading
+
+try:
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+except Exception:  # pragma: no cover - graceful fallback
+    pa = None
+    pq = None
+
+try:
+    import portalocker
+except Exception:  # pragma: no cover
+    portalocker = None
+
+
+# In-process reentrant locks per lock path to avoid nested portalocker
+# acquisitions causing AlreadyLocked errors when the same process/thread
+# re-enters storage write paths.
+_local_rlocks = {}
+
+
+def _fsync_dir(path):
+    try:
+        fd = os.open(path, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+    except Exception:
+        pass
+
+
+def _acquire_rlock(rlock):
+    try:
+        if rlock._is_owned():
+            rlock.acquire()
+            return False
+    except Exception:
+        pass
+
+    first_acquire = rlock.acquire(blocking=False)
+    if not first_acquire:
+        rlock.acquire()
+        return False
+    return True
+
+
+class ParquetStorage(Storage):
+    """TinyDB Storage that persists the entire DB as a single Parquet row
+
+    The DB dict is serialized to JSON and stored in a single column named
+    'data'. To coordinate concurrent writers we use a folder-scoped lock
+    file named '.tinymongo.lock'."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):
+        if not os.path.exists(self.path):
+            return {}
+
+        dname = os.path.dirname(self.path) or "."
+        lock_path = os.path.join(dname, ".tinymongo.lock")
+
+        rlock = _local_rlocks.setdefault(lock_path, threading.RLock())
+        first_acquire = _acquire_rlock(rlock)
+
+        portalocker_lock = None
+        try:
+            if first_acquire and portalocker is not None:
+                portalocker_lock = portalocker.Lock(lock_path, timeout=30)
+                portalocker_lock.acquire()
+
+            if pq is None:
+                try:
+                    with open(self.path, "r", encoding="utf8") as f:
+                        return json.load(f)
+                except Exception:
+                    return {}
+
+            try:
+                table = pq.read_table(self.path)
+                if "data" not in table.column_names:
+                    return {}
+                data_arr = table.column("data").to_pylist()
+                if not data_arr:
+                    return {}
+                return json.loads(data_arr[0])
+            except Exception:
+                return {}
+        finally:
+            if portalocker_lock is not None:
+                try:
+                    portalocker_lock.release()
+                except Exception:
+                    pass
+            try:
+                rlock.release()
+            except Exception:
+                pass
+
+    def write(self, data):
+        # serialize the entire TinyDB dict to a JSON string
+        json_str = json.dumps(data or {}, ensure_ascii=False)
+
+        # ensure parent dir exists
+        dname = os.path.dirname(self.path) or "."
+        os.makedirs(dname, exist_ok=True)
+
+        lock_path = os.path.join(dname, ".tinymongo.lock")
+
+        # Acquire an in-process reentrant lock to allow nested calls inside
+        # the same process to proceed without re-acquiring the OS-level
+        # portalocker lock. Only the first acquirer in the process will
+        # perform the portalocker acquisition for inter-process safety.
+        rlock = _local_rlocks.setdefault(lock_path, threading.RLock())
+        first_acquire = _acquire_rlock(rlock)
+
+        portalocker_lock = None
+        try:
+            if pq is None:
+                # No pyarrow — merge into existing JSON storage
+                existing = {}
+                if os.path.exists(self.path):
+                    try:
+                        with open(self.path, "r", encoding="utf8") as f:
+                            existing = json.load(f) or {}
+                    except Exception:
+                        existing = {}
+
+                # Merge incoming into existing, matching on logical `_id`.
+                merged = {}
+                incoming = data or {}
+                for tname, table_data in existing.items():
+                    merged[str(tname)] = {str(k): v for k, v in (table_data or {}).items()}
+
+                for tname, table_data in incoming.items():
+                    t = str(tname)
+                    incoming_table = {str(k): v for k, v in (table_data or {}).items()}
+                    existing_table = merged.get(t, {})
+
+                    # map existing _id -> eid
+                    id_to_eid = {v.get('_id'): k for k, v in existing_table.items() if isinstance(v, dict) and '_id' in v}
+                    try:
+                        next_eid = max(int(k) for k in existing_table.keys()) + 1
+                    except Exception:
+                        next_eid = 1
+
+                    for k, v in incoming_table.items():
+                        doc_id = v.get('_id') if isinstance(v, dict) else None
+                        if doc_id is not None and doc_id in id_to_eid:
+                            existing_table[id_to_eid[doc_id]] = v
+                        else:
+                            existing_table[str(next_eid)] = v
+                            next_eid += 1
+
+                    merged[t] = existing_table
+
+                json_str = json.dumps(merged, ensure_ascii=False)
+                fd, tmp = tempfile.mkstemp(prefix="tmp", dir=dname)
+                try:
+                    with os.fdopen(fd, "w", encoding="utf8") as f:
+                        f.write(json_str)
+                        f.flush()
+                        os.fsync(f.fileno())
+                    os.replace(tmp, self.path)
+                    _fsync_dir(dname)
+                finally:
+                    if os.path.exists(tmp):
+                        try:
+                            os.remove(tmp)
+                        except Exception:
+                            pass
+                return
+
+            # read existing parquet into dict
+            existing = {}
+            if os.path.exists(self.path):
+                try:
+                    table = pq.read_table(self.path)
+                    if "data" in table.column_names:
+                        data_arr = table.column("data").to_pylist()
+                        if data_arr:
+                            existing = json.loads(data_arr[0])
+                except Exception:
+                    existing = {}
+
+            # Merge incoming into existing, matching on logical `_id`.
+            merged = {}
+            incoming = data or {}
+            for tname, table_data in existing.items():
+                merged[str(tname)] = {str(k): v for k, v in (table_data or {}).items()}
+
+            for tname, table_data in incoming.items():
+                t = str(tname)
+                incoming_table = {str(k): v for k, v in (table_data or {}).items()}
+                existing_table = merged.get(t, {})
+
+                id_to_eid = {v.get('_id'): k for k, v in existing_table.items() if isinstance(v, dict) and '_id' in v}
+                try:
+                    next_eid = max(int(k) for k in existing_table.keys()) + 1
+                except Exception:
+                    next_eid = 1
+
+                for k, v in incoming_table.items():
+                    doc_id = v.get('_id') if isinstance(v, dict) else None
+                    if doc_id is not None and doc_id in id_to_eid:
+                        existing_table[id_to_eid[doc_id]] = v
+                    else:
+                        existing_table[str(next_eid)] = v
+                        next_eid += 1
+
+                merged[t] = existing_table
+
+            json_str = json.dumps(merged, ensure_ascii=False)
+
+            # write Parquet with single column 'data'
+            arr = pa.array([json_str], type=pa.string())
+            table = pa.table({"data": arr})
+
+            fd, tmp = tempfile.mkstemp(prefix="tmp", dir=dname)
+            os.close(fd)
+            try:
+                # write with Parquet v2 layout
+                pq.write_table(table, tmp, version="2.6")
+                # fsync file
+                try:
+                    with open(tmp, "rb") as f:
+                        f.flush()
+                        os.fsync(f.fileno())
+                except Exception:
+                    pass
+                os.replace(tmp, self.path)
+                _fsync_dir(dname)
+            finally:
+                if os.path.exists(tmp):
+                    try:
+                        os.remove(tmp)
+                    except Exception:
+                        pass
+        finally:
+            # Release portalocker only if we acquired it here.
+            if 'portalocker_lock' in locals() and portalocker_lock is not None:
+                try:
+                    portalocker_lock.release()
+                except Exception:
+                    pass
+            # Release in-process reentrant lock once.
+            try:
+                rlock.release()
+            except Exception:
+                pass

tinymongo/results.py

@@ -0,0 +1,115 @@
+"""Result class definitions."""
+
+
+class _WriteResult(object):
+    """Base class for write result classes."""
+
+    def __init__(self, acknowledged=True):
+        self.acknowledged = acknowledged  # here only to PyMongo compat
+
+
+class InsertOneResult(_WriteResult):
+    """The return type for :meth:`~tinymongo.TinyMongoCollection.insert_one`.
+    """
+
+    __slots__ = ("__inserted_id", "__acknowledged", "__eid")
+
+    def __init__(self, eid, inserted_id, acknowledged=True):
+        self.__eid = eid
+        self.__inserted_id = inserted_id
+        super(InsertOneResult, self).__init__(acknowledged)
+
+    @property
+    def inserted_id(self):
+        """The inserted document's _id."""
+        return self.__inserted_id
+
+    @property
+    def eid(self):
+        """The inserted document's tinyDB eid."""
+        return self.__eid
+
+
+class InsertManyResult(_WriteResult):
+    """The return type for :meth:`~tinymongo.TinyMongoCollection.insert_many`.
+    """
+
+    __slots__ = ("__inserted_ids", "__acknowledged", "__eids")
+
+    def __init__(self, eids, inserted_ids, acknowledged=True):
+        self.__eids = eids
+        self.__inserted_ids = inserted_ids
+        super(InsertManyResult, self).__init__(acknowledged)
+
+    @property
+    def inserted_ids(self):
+        """A list of _ids of the inserted documents, in the order provided."""
+        return self.__inserted_ids
+
+    @property
+    def eids(self):
+        """A list of _ids of the inserted documents, in the order provided."""
+        return self.__eids
+
+
+class UpdateResult(_WriteResult):
+    """The return type for :meth:`~tinymongo.TinyMongoCollection.update_one`,
+    :meth:`~tinymongo.TinyMongoCollection.update_many`, and
+    :meth:`~tinymongo.TinyMongoCollection.replace_one`.
+    """
+
+    __slots__ = ("__raw_result", "__acknowledged")
+
+    def __init__(self, raw_result, acknowledged=True):
+        self.__raw_result = raw_result
+        super(UpdateResult, self).__init__(acknowledged)
+
+    @property
+    def raw_result(self):
+        """The raw result document returned by the server."""
+        return self.__raw_result
+
+    @property
+    def matched_count(self):
+        """The number of documents matched for this update."""
+        if isinstance(self.raw_result, list):
+            return len(self.raw_result)
+        return 0
+
+    @property
+    def modified_count(self):
+        """The number of documents modified."""
+        if isinstance(self.raw_result, list):
+            return len(self.raw_result)
+        return 0
+
+    @property
+    def upserted_id(self):
+        """The _id of the inserted document if an upsert took place. Otherwise
+        ``None``.
+        """
+        return None
+
+
+class DeleteResult(_WriteResult):
+    """The return type for :meth:`~tinymongo.TinyMongoCollection.delete_one`
+    and :meth:`~tinymongo.TinyMongoCollection.delete_many`"""
+
+    __slots__ = ("__raw_result", "__acknowledged")
+
+    def __init__(self, raw_result, acknowledged=True):
+        self.__raw_result = raw_result
+        super(DeleteResult, self).__init__(acknowledged)
+
+    @property
+    def raw_result(self):
+        """The raw result document returned by the server."""
+        return self.__raw_result
+
+    @property
+    def deleted_count(self):
+        """The number of documents deleted."""
+        if isinstance(self.raw_result, list):
+            return len(self.raw_result)
+        else:
+            return self.raw_result

tinymongo/serializers.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+try:
+    from tinydb_serialization import Serializer
+except ImportError:
+    raise RuntimeError(
+        u'Cannot import tinydb_serialization due to {0} '
+        u'you need to run `pip install tinydb_serialization`'
+    )
+
+
+class DateTimeSerializer(Serializer):
+    OBJ_CLASS = datetime
+
+    def __init__(self, dateformat='%Y-%m-%dT%H:%M:%S', *args, **kwargs):
+        # super(DateTimeSerializer, self).__init__(*args, **kwargs)
+        self._format = dateformat
+
+    def encode(self, obj):
+        return obj.strftime(self._format)
+
+    def decode(self, s):
+        return self.OBJ_CLASS.strptime(s, self._format)

tinymongo/storage_backends.py

@@ -0,0 +1,174 @@
+import json
+import os
+import sqlite3
+import tempfile
+from tinydb.storages import Storage
+
+try:
+    import duckdb
+except Exception:
+    duckdb = None
+
+
+def _fsync_dir(path):
+    try:
+        fd = os.open(path, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+    except Exception:
+        pass
+
+
+class SQLiteStorage(Storage):
+    """TinyDB storage backend using SQLite as a single-row JSON store."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):
+        if not os.path.exists(self.path):
+            return {}
+
+        try:
+            conn = sqlite3.connect(self.path)
+            cursor = conn.cursor()
+            cursor.execute(
+                "CREATE TABLE IF NOT EXISTS tinydb(id INTEGER PRIMARY KEY, data TEXT)"
+            )
+            cursor.execute("SELECT data FROM tinydb WHERE id = 1")
+            row = cursor.fetchone()
+            conn.close()
+            if row is None or row[0] is None:
+                return {}
+            return json.loads(row[0])
+        except Exception:
+            return {}
+
+    def write(self, data):
+        json_str = json.dumps(data or {}, ensure_ascii=False)
+        dname = os.path.dirname(self.path) or "."
+        os.makedirs(dname, exist_ok=True)
+
+        fd, tmp = tempfile.mkstemp(prefix="tmp", dir=dname)
+        os.close(fd)
+        try:
+            conn = sqlite3.connect(tmp)
+            cursor = conn.cursor()
+            cursor.execute(
+                "PRAGMA journal_mode = OFF"
+            )
+            cursor.execute(
+                "CREATE TABLE IF NOT EXISTS tinydb(id INTEGER PRIMARY KEY, data TEXT)"
+            )
+            cursor.execute(
+                "INSERT OR REPLACE INTO tinydb(id, data) VALUES(1, ?)"
+                , (json_str,)
+            )
+            conn.commit()
+            conn.close()
+            os.replace(tmp, self.path)
+            _fsync_dir(dname)
+        finally:
+            if os.path.exists(tmp):
+                try:
+                    os.remove(tmp)
+                except Exception:
+                    pass
+
+
+class DuckDBStorage(Storage):
+    """TinyDB storage backend using DuckDB as a single-row JSON store."""
+
+    def __init__(self, path):
+        if duckdb is None:
+            raise ImportError(
+                "duckdb backend requires the duckdb package to be installed"
+            )
+        self.path = path
+
+    def read(self):
+        if not os.path.exists(self.path):
+            return {}
+
+        try:
+            conn = duckdb.connect(self.path)
+            result = conn.execute("SELECT data FROM tinydb WHERE id = 1").fetchone()
+            conn.close()
+            if result is None or result[0] is None:
+                return {}
+            return json.loads(result[0])
+        except Exception:
+            return {}
+
+    def write(self, data):
+        json_str = json.dumps(data or {}, ensure_ascii=False)
+        dname = os.path.dirname(self.path) or "."
+        os.makedirs(dname, exist_ok=True)
+
+        fd, tmp = tempfile.mkstemp(prefix="tmp", dir=dname)
+        os.close(fd)
+        try:
+            conn = duckdb.connect(tmp)
+            conn.execute("CREATE TABLE IF NOT EXISTS tinydb(id INTEGER PRIMARY KEY, data TEXT)")
+            conn.execute(
+                "INSERT OR REPLACE INTO tinydb(id, data) VALUES(1, ?)",
+                (json_str,)
+            )
+            conn.close()
+            os.replace(tmp, self.path)
+            _fsync_dir(dname)
+        finally:
+            if os.path.exists(tmp):
+                try:
+                    os.remove(tmp)
+                except Exception:
+                    pass
+
+
+def get_storage_class(name):
+    if name is None:
+        name = "tinydb"
+
+    if isinstance(name, type):
+        return name
+
+    backend = str(name).lower()
+
+    if backend in ("tinydb", "json"):
+        from tinydb import TinyDB
+
+        # TinyDB 3.x exposes DEFAULT_STORAGE instead of default_storage_class.
+        return getattr(TinyDB, "default_storage_class", TinyDB.DEFAULT_STORAGE)
+    if backend in ("parquet", "parquetv2"):
+        from .parquet_storage import ParquetStorage
+
+        return ParquetStorage
+    if backend == "sqlite":
+        return SQLiteStorage
+    if backend == "duckdb":
+        return DuckDBStorage
+
+    raise ValueError(
+        "Unsupported backend '{0}'. Supported backends: tinydb, parquet, parquetv2, sqlite, duckdb.".format(
+            name
+        )
+    )
+
+
+def storage_extension(name):
+    backend = str(name).lower()
+    if backend in ("tinydb", "json"):
+        return ".json"
+    if backend in ("parquet", "parquetv2"):
+        return ".parquet"
+    if backend == "sqlite":
+        return ".sqlite"
+    if backend == "duckdb":
+        return ".duckdb"
+    raise ValueError(
+        "Unsupported backend '{0}'. Supported backends: tinydb, parquet, parquetv2, sqlite, duckdb.".format(
+            name
+        )
+    )