nedb-engine 3.2.1__tar.gz → 3.2.2__tar.gz

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nedb-engine
-Version: 3.2.1
+Version: 3.2.2
 Summary: NEDB — a versioned, self-compressing, time-traveling embedded database (replay-protected, idempotent, relational, searchable) with durable AOF persistence and a server daemon (nedbd).
 Project-URL: Homepage, https://github.com/aiassistsecure/nedb
 Project-URL: Repository, https://github.com/aiassistsecure/nedb

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nedb-engine"
-version = "3.2.1"
+version = "3.2.2"
 description = "NEDB — a versioned, self-compressing, time-traveling embedded database (replay-protected, idempotent, relational, searchable) with durable AOF persistence and a server daemon (nedbd)."
 readme = "README.md"
 requires-python = ">=3.8"

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/python/nedb/__init__.py

@@ -102,4 +102,4 @@ __all__ = [
     "verify_proof", "fold_head",
     "_native", "__has_native__",
 ]
-__version__ = "3.2.1"
+__version__ = "3.2.2"

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/python/nedb/backends/dag.py

@@ -32,10 +32,17 @@ from typing import Any, Dict, List, Optional
 
 
 def has_dag_native() -> bool:
-    """True when the embedded Rust v2/v3 DAG core is importable."""
+    """True when the embedded Rust v2/v3 DAG core is actually LOADED.
+
+    This used to be `from .. import __has_native__; return True` — which tests
+    whether the NAME imports, never what it holds. `__has_native__` is always
+    defined (it is False on the universal pure-Python wheel), so this returned
+    True unconditionally and `backend="auto"` chose the DAG on machines that
+    had no compiled core at all.
+    """
     try:
-        from .. import __has_native__  # noqa: F401
-        return True
+        from .. import __has_native__
+        return bool(__has_native__)
     except Exception:
         return False
 

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/python/nedb/wrap_core.py

@@ -64,8 +64,16 @@ def open_engine(
         return NedBdProxy(nedbd_url, db_name, token=nedbd_token), None
 
     if backend == "dag" or (backend == "auto" and has_dag_native()):
-        core = DagBackend(path=dag_path, tmk=dag_tmk)
-        return core, None
+        try:
+            return _open_dag(dag_path, dag_tmk), None
+        except ImportError:
+            if backend == "dag":
+                raise   # an explicit request must never silently downgrade
+            # auto: no compiled core after all — fall through to v1 AOF, the
+            # universal fallback. wrap_redis has always done this; open_engine
+            # did not, so wrap_sqlite / wrap_mysql / wrap_mongo /
+            # wrap_postgresql raised ImportError out of the constructor on
+            # every install without a platform wheel.
 
     if backend in ("aof", "auto"):
         return _NEDB(), None
@@ -74,6 +82,24 @@ def open_engine(
                      "(expected 'auto' | 'aof' | 'dag' | 'nedbd')")
 
 
+def _open_dag(dag_path, dag_tmk):
+    """Construct the embedded DAG backend, warning when it is not durable."""
+    if dag_path is None:
+        # An embedded DAG with no path is IN-MEMORY: the chain dies with the
+        # process. These adapters have no durable in-host fallback the way
+        # wrap_redis has Redis Streams, so the choice stands -- but silently
+        # handing someone a provenance log that evaporates is exactly the
+        # failure this engine exists to prevent. Say so, once.
+        import warnings
+        warnings.warn(
+            "NEDB: embedded DAG opened with no dag_path= — this chain is "
+            "IN-MEMORY and will not survive the process. Pass "
+            'dag_path="./audit" for a durable store, or nedbd_url= to use '
+            "a server.",
+            RuntimeWarning, stacklevel=4)
+    return DagBackend(path=dag_path, tmk=dag_tmk)
+
+
 # ── The shared surface ───────────────────────────────────────────────────────
 
 class WrapSurface:
@@ -91,6 +117,36 @@ class WrapSurface:
         self._backfilled: bool = False
         self._db = engine
         self._persist = persist or _MemoryPersist()
+        # ── shadow observability ────────────────────────────────────────────
+        # A shadow failure must never break the host database call, but
+        # "don't raise into the host's stack" and "don't tell anyone" are
+        # different requirements. For a PROVENANCE layer, silently dropping a
+        # record is the worst available outcome: the chain gets a hole and
+        # verify() still returns True, because it verifies what got in, not
+        # that everything that should have got in did.
+        #
+        # Every swallowed failure is now counted and, optionally, reported.
+        self.shadow_errors: int = 0
+        self.last_shadow_error: Optional[str] = None
+        self.on_shadow_error: Optional[Callable[[BaseException], None]] = None
+        # True when the chain outlives the process — set by the adapter.
+        self.durable: bool = True
+        # strict=True re-raises instead of swallowing — for tests and for
+        # deployments that would rather fail loudly than lose provenance.
+        self.strict_shadow: bool = False
+
+    def note_shadow_error(self, exc: BaseException) -> None:
+        """Record a swallowed shadow failure. Never raises unless strict."""
+        self.shadow_errors += 1
+        self.last_shadow_error = f"{type(exc).__name__}: {exc}"
+        cb = self.on_shadow_error
+        if cb is not None:
+            try:
+                cb(exc)
+            except Exception:
+                pass  # a broken callback must not compound the original failure
+        if self.strict_shadow:
+            raise exc
 
     # ── host hooks (override in the host adapter) ────────────────────────────
 
@@ -139,7 +195,11 @@ class WrapSurface:
                 try:
                     self._db.put(m.collection, doc_id, doc, client="__backfill__")
                     total += 1
-                except Exception:
+                except Exception as e:
+                    # Counted, not silent: backfill() returns the number of rows
+                    # imported, and a caller comparing that to the host's row
+                    # count deserves to know the difference was errors.
+                    self.note_shadow_error(e)
                     continue
         self._backfilled = True
         return total
@@ -165,8 +225,8 @@ class WrapSurface:
             self._db.put(m.collection, m.extract_id(key), doc,
                          client="__shadow__")
             self._persist_after()
-        except Exception:
-            pass
+        except Exception as e:
+            self.note_shadow_error(e)
 
     def _persist_after(self) -> None:
         # DAG + nedbd persist themselves; v1 AOF needs the last op appended

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/python/nedb/wrap_redis.py

@@ -302,6 +302,13 @@ class NEDBSurface:
         self._mappings: List[CollectionMapping] = []
         self.shadow_writes: bool = False
         self._backfilled: bool = False
+        self.shadow_errors: int = 0
+        self.last_shadow_error: Optional[str] = None
+        self.on_shadow_error = None
+        self.strict_shadow: bool = False
+        # True when the chain outlives the process. Introspectable rather than
+        # inferred: durability should never be something you have to guess.
+        self.durable: bool = True
         # Set BEFORE any branching. _reload() reads this, and the
         # DagBackend-ImportError fallback below reaches _reload() without
         # passing through a branch that assigns it — which made wrap_redis()
@@ -315,13 +322,26 @@ class NEDBSurface:
             self._nedbd_mode = True
             self._db      = NedBdProxy(nedbd_url, db_name, token=nedbd_token)  # type: ignore[assignment]
             self._backend = None
-        elif backend in ("dag", "auto"):
+        elif backend == "dag" or (backend == "auto" and dag_path is not None):
+            # NOTE the `dag_path is not None`. Previously `auto` alone chose the
+            # embedded DAG the moment a platform wheel existed -- and an embedded
+            # DAG with no dag_path is IN-MEMORY. So wrap_redis(conn), which
+            # persisted through Redis Streams on the v1 AOF engine, silently
+            # stopped persisting as soon as pip installed a native wheel: same
+            # call site, different durability, no warning, discovered during an
+            # audit rather than a deploy.
+            #
+            # `auto` must never pick an engine LESS durable than the call site
+            # already had. Ask for the DAG explicitly (backend="dag") or give it
+            # somewhere to live (dag_path=) and you get it; stay on defaults and
+            # you keep Redis-Stream persistence.
             # Embedded v2/v3 DAG (Rust native core). Falls back to v1 AOF
             # when the platform wheel is absent (universal wheel).
             try:
                 from .backends.dag import DagBackend
                 self._db = DagBackend(path=dag_path, tmk=dag_tmk)  # type: ignore[assignment]
                 self._nedbd_mode = False
+                self.durable = dag_path is not None
                 self._backend = None      # DAG persists itself; no Redis streams
             except ImportError:
                 if backend == "dag":
@@ -586,8 +606,9 @@ class NEDBSurface:
                              evidence="redis_write",
                              confidence=1.0)
                 self._persist_last_op()
-            except Exception:
-                pass  # shadow failures must never break the Redis surface call
+            except Exception as e:
+                # Never breaks the Redis call, never invisible either.
+                self.note_shadow_error(e)
         else:
             # No mapping — raw tamper-evidence chain entry only
             # (not NQL-queryable, but proves the write happened)
@@ -602,8 +623,21 @@ class NEDBSurface:
                              client="__shadow__",
                              evidence="redis_write")
                 self._persist_last_op()
+            except Exception as e:
+                self.note_shadow_error(e)
+
+    def note_shadow_error(self, exc: BaseException) -> None:
+        """Record a swallowed shadow failure. Mirrors WrapSurface."""
+        self.shadow_errors = getattr(self, "shadow_errors", 0) + 1
+        self.last_shadow_error = f"{type(exc).__name__}: {exc}"
+        cb = getattr(self, "on_shadow_error", None)
+        if cb is not None:
+            try:
+                cb(exc)
             except Exception:
                 pass
+        if getattr(self, "strict_shadow", False):
+            raise exc
 
     # ── Full NEDB API ─────────────────────────────────────────────────────────
 
@@ -653,6 +687,11 @@ class NEDBSurface:
     def verify(self) -> bool:
         return self._db.verify()
 
+    # NOTE: @property. Without it `wrap_redis(...).nedb.head` handed back a
+    # bound method instead of the Merkle head string -- while `.seq` three
+    # lines below, and `.head` on every other wrapper, returned the value.
+    # Silent, because a bound method is truthy: `if surface.head:` passed.
+    @property
     def head(self) -> str:
         return self._db.head
 

{nedb_engine-3.2.1 → nedb_engine-3.2.2}/python/nedb/wrap_sqlite.py

@@ -158,56 +158,157 @@ class WrappedSqlite:
         return _passthrough
 
     def _execute(self, sql: str, params: tuple = ()):
-        """execute() with post-commit shadowing of registered-table writes."""
+        """execute() with post-write shadowing of registered-table writes.
+
+        UPDATE and DELETE need the affected rowids captured BEFORE the
+        statement runs -- see _affected_rowids for why cursor.lastrowid
+        cannot be used for them.
+        """
         conn = object.__getattribute__(self, "_conn")
         nedb = object.__getattribute__(self, "nedb")
+
+        head = sql.lstrip().upper()
+        shadowing = head.startswith(self._WRITE_PREFIXES) and nedb.shadow_writes
+
+        pre_rowids = None
+        if shadowing and not head.startswith("INSERT"):
+            try:
+                pre_rowids = self._affected_rowids(nedb, sql, params)
+            except Exception as e:
+                nedb.note_shadow_error(e)
+                pre_rowids = None
+
         cur = conn.execute(sql, params)
 
         try:
-            head = sql.lstrip().upper()
-            if head.startswith(self._WRITE_PREFIXES) and nedb.shadow_writes:
-                self._shadow_sql(nedb, sql, cur)
-        except Exception:
-            pass  # shadow failures must never break the host call
+            if shadowing:
+                self._shadow_sql(nedb, sql, cur, pre_rowids)
+        except Exception as e:
+            # Must never break the host call — but must never be invisible
+            # either. Counted on nedb.shadow_errors, reported through
+            # nedb.on_shadow_error, and re-raised when strict_shadow is set.
+            nedb.note_shadow_error(e)
         return cur
 
-    def _shadow_sql(self, nedb: SqliteSurface, sql: str, cur: sqlite3.Cursor) -> None:
+    def _affected_rowids(self, nedb, sql: str, params: tuple):
+        """Rowids an UPDATE/DELETE is about to touch, resolved before it runs.
+
+        `cursor.lastrowid` is ONLY meaningful after an INSERT. After an UPDATE
+        or DELETE it still holds the id of the last row *inserted* on that
+        connection, so shadowing off it recorded a completely unrelated row --
+        a FALSE provenance record, which is worse than none at all.
+
+        The affected set is therefore resolved up front with
+        `SELECT rowid FROM <table> WHERE <same predicate>`.
+
+        Placeholders: for UPDATE, some `?` may live in the SET clause, so the
+        leading params belonging to SET are skipped and only the remainder is
+        bound to the WHERE. Anything that cannot be parsed with confidence
+        raises, and the caller records a COUNTED shadow error rather than
+        inventing a row.
+        """
+        import re
+        mapping = self._mapping_for_sql(nedb, sql)
+        if mapping is None:
+            return None
         conn = object.__getattribute__(self, "_conn")
-        table = None
+        table = mapping.pattern
+
+        m = re.search(r"\bWHERE\b(.*)$", sql, re.IGNORECASE | re.DOTALL)
+        if not m:
+            # No predicate: an unqualified UPDATE/DELETE touches every row.
+            return [r[0] for r in conn.execute(f'SELECT rowid FROM "{table}"')]
+
+        where = m.group(1)
+        where_params = params or ()
+        if sql.lstrip().upper().startswith("UPDATE") and where_params:
+            set_part = sql[:m.start()]
+            n_set = set_part.count("?")
+            if n_set > len(where_params):
+                raise ValueError(
+                    "cannot attribute placeholders between SET and WHERE; "
+                    "refusing to shadow a row that may be the wrong one")
+            where_params = tuple(where_params)[n_set:]
+
+        rows = conn.execute(
+            f'SELECT rowid FROM "{table}" WHERE {where}', where_params)
+        return [r[0] for r in rows]
+
+    @staticmethod
+    def _mapping_for_sql(nedb, sql: str):
         for m in nedb._mappings:
-            t = m.pattern
-            if t.lower() in sql.lower():
-                table = t
-                break
-        if table is None:
+            if m.pattern.lower() in sql.lower():
+                return m
+        return None
+
+    def _shadow_sql(self, nedb: SqliteSurface, sql: str, cur: sqlite3.Cursor,
+                    pre_rowids=None) -> None:
+        """Mirror a write into the SAME collection backfill() writes to.
+
+        Two defects lived here, and the second hid the first.
+
+        1. ``cols = [d[0] for d in cur.description] or [...]`` -- sqlite3 sets
+           ``cursor.description`` to **None** after an INSERT (it is only
+           populated for SELECT). The list comprehension therefore raised
+           ``TypeError`` before ``or`` could ever evaluate its fallback, and
+           ``_execute`` swallowed it. Every automatic INSERT shadow silently
+           recorded nothing while ``verify()`` kept returning True.
+
+        2. Rows were written to ``__sql_shadow__`` while ``backfill()`` writes
+           to the registered collection. Even with (1) fixed, a user who ran
+           ``register("rides", collection="ride")`` and queried ``FROM ride``
+           got the historical rows and none of the live ones -- a partial
+           answer, silently.
+
+        Both are fixed: the description is guarded, and every op lands in
+        ``mapping.collection`` under the same id ``_host_scan`` yields
+        (``str(rowid)``), so an UPDATE supersedes the backfilled document
+        instead of accumulating beside it.
+        """
+        conn = object.__getattribute__(self, "_conn")
+        mapping = self._mapping_for_sql(nedb, sql)
+        if mapping is None:
             return
+        table = mapping.pattern
 
-        op = ("DELETE" if sql.upper().startswith("DELETE")
-              else "UPDATE" if sql.upper().startswith("UPDATE")
+        up = sql.lstrip().upper()
+        op = ("DELETE" if up.startswith("DELETE")
+              else "UPDATE" if up.startswith("UPDATE")
               else "INSERT")
-        rowid = cur.lastrowid
+
+        if op == "INSERT":
+            rowids = [cur.lastrowid]
+        else:
+            if pre_rowids is None:
+                raise ValueError(
+                    f"{op} on {table!r}: affected rows could not be resolved, "
+                    "nothing shadowed (see nedb.last_shadow_error)")
+            rowids = pre_rowids
 
         if op == "DELETE":
-            # Tombstone: put a deletion marker (NEDB delete would remove history;
-            # a marker keeps the causal chain and is NQL-queryable)
-            nedb.put("__sql_shadow__", f"{table}:del:{rowid}",
-                     {"table": table, "rowid": str(rowid), "_op": "DELETE"},
-                     client="__shadow__")
+            # Tombstone rather than a NEDB delete: deleting would drop history,
+            # and surviving history is the entire point. Written into the mapped
+            # collection so `FROM ride` shows the row as deleted rather than
+            # appearing to still exist.
+            for rid in rowids:
+                nedb.put(mapping.collection, str(rid),
+                         {"_table": table, "_op": "DELETE", "_deleted": True,
+                          "_rowid": str(rid)},
+                         client="__shadow__")
             return
 
-        # Read the row back post-write (works inside the open transaction)
-        try:
+        # Read each affected row back post-write (inside the open transaction).
+        desc = conn.execute(f'SELECT * FROM "{table}" LIMIT 0').description
+        cols = [d[0] for d in desc]
+        for rid in rowids:
             row = conn.execute(
-                f'SELECT * FROM "{table}" WHERE rowid = ?', (rowid,)).fetchone()
+                f'SELECT * FROM "{table}" WHERE rowid = ?', (rid,)).fetchone()
             if row is None:
-                return
-            cols = [d[0] for d in cur.description] or \
-                [d[0] for d in conn.execute(f'SELECT * FROM "{table}" LIMIT 0').description]
+                continue
             doc = dict(zip(cols, row))
-        except sqlite3.Error:
-            return
-        nedb.put("__sql_shadow__", f"{table}:{rowid}",
-                 {**doc, "_table": table, "_op": op}, client="__shadow__")
+            nedb.put(mapping.collection, str(rid),
+                     {**doc, "_table": table, "_op": op, "_source": "shadow"},
+                     client="__shadow__")
 
     def __repr__(self):
         conn = object.__getattribute__(self, "_conn")