nedb-engine 2.6.1__tar.gz → 2.6.3__tar.gz

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nedb-engine
-Version: 2.6.1
+Version: 2.6.3
 Summary: NEDB — a versioned, self-compressing, time-traveling embedded database (replay-protected, idempotent, relational, searchable) with durable AOF persistence and a server daemon (nedbd).
 Project-URL: Homepage, https://github.com/aiassistsecure/nedb
 Project-URL: Repository, https://github.com/aiassistsecure/nedb

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nedb-engine"
-version = "2.6.1"
+version = "2.6.3"
 description = "NEDB — a versioned, self-compressing, time-traveling embedded database (replay-protected, idempotent, relational, searchable) with durable AOF persistence and a server daemon (nedbd)."
 readme = "README.md"
 requires-python = ">=3.8"

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/python/nedb/__init__.py

@@ -16,7 +16,7 @@ as ``nedb._native`` (``nedb.__has_native__`` reports whether it loaded).
 """
 from __future__ import annotations
 
-from .engine import NEDB
+from .engine import NEDB, PreconditionFailed
 from .log import Op, OpLog, ReplayError
 from .query import Query, parse_nql
 from .snapshot import save_snapshot, load_snapshot
@@ -79,7 +79,7 @@ except ImportError:  # pure-Python install (sdist / unsupported platform)
     del _types, _sys, _NativeStub, _native_stub
 
 __all__ = [
-    "NEDB", "OpLog", "Op", "ReplayError", "Query", "parse_nql",
+    "NEDB", "PreconditionFailed", "OpLog", "Op", "ReplayError", "Query", "parse_nql",
     "save_snapshot", "load_snapshot",
     "sql_exec", "sql_to_nql", "SQLError", "SQLUnsupportedError",
     "RedisCompat", "RedisError", "RedisUnsupportedError",
@@ -89,4 +89,4 @@ __all__ = [
     "verify_proof", "fold_head",
     "_native", "__has_native__",
 ]
-__version__ = "2.6.1"
+__version__ = "2.6.3"

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/python/nedb/concurrent.py

@@ -90,6 +90,13 @@ class Sequencer:
     def put(self, coll: str, id: str, doc: dict, **kw: Any) -> Any:
         return self._submit("put", coll, id, doc, **kw)
 
+    def tx(self, ops: list, **kw: Any) -> Any:
+        """Atomic multi-op transaction (see NEDB.tx). Submitted as ONE
+        intent, so the committer validates preconditions and applies all ops
+        with nothing interleaving — the single-writer thread IS the
+        serialization point, exactly as Redis's single thread was for Lua."""
+        return self._submit("tx", ops, **kw)
+
     def delete(self, coll: str, id: str, **kw: Any) -> Any:
         return self._submit("delete", coll, id, **kw)
 
@@ -108,6 +115,11 @@ class Sequencer:
     def checkpoint(self) -> Any:
         return self._submit("checkpoint")
 
+    def sweep(self) -> Any:
+        """TTL sweep as ONE committer intent — sweep appends delete ops, so it
+        must ride the single writer, never race it from another thread."""
+        return self._submit("sweep")
+
     # ── read API: concurrent, snapshot-isolated at committed_seq ───────────────
     def query(self, nql: str) -> List[dict]:
         plan = parse_nql(nql)
@@ -207,6 +219,10 @@ class Sequencer:
             return db.put_file(*intent.args, **intent.kwargs)
         if k == "checkpoint":
             return db.checkpoint()
+        if k == "tx":
+            return db.tx(*intent.args, **intent.kwargs)
+        if k == "sweep":
+            return db.sweep()
         raise ValueError(f"unknown write kind: {k}")
 
     def close(self) -> None:

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/python/nedb/engine.py

@@ -24,6 +24,23 @@ from .relations import Relations
 from .store import MVCCStore
 
 
+class PreconditionFailed(Exception):
+    """An `if_seq` compare-and-set precondition did not hold; the transaction
+    was rejected as a unit and NOTHING was applied. `.failures` lists every
+    failed check as {index, coll, id, expected, actual}."""
+
+    def __init__(self, failures: List[dict]):
+        self.failures = failures
+        super().__init__(
+            f"{len(failures)} precondition(s) failed: "
+            + "; ".join(
+                f"[{f['index']}] {f['coll']}:{f['id']} expected seq "
+                f"{f['expected']}, actual {f['actual']}"
+                for f in failures
+            )
+        )
+
+
 def apply_op(store: MVCCStore, relations: Relations, indexes: Indexes, op: Op,
              cause_map: Optional[Dict[int, List[int]]] = None) -> None:
     """Deterministically fold one op into materialized state."""
@@ -484,6 +501,91 @@ class NEDB:
             return self._check_ttl(coll, id, doc)
         return doc  # time-travel reads never trigger lazy expiry
 
+    def last_seq(self, coll: str, id: str) -> Optional[int]:
+        """Version anchor for compare-and-set: the seq of the most recent op
+        (put OR delete) that touched this document, or None if never written."""
+        return self.store.last_seq(f"{coll}:{id}")
+
+    def tx(self, ops: List[dict], default_client: str = "local") -> dict:
+        """Atomic multi-op transaction with optional per-op CAS preconditions.
+
+        Each op is a dict:
+          {"op": "put",  "coll", "id", "doc", "if_seq"?, "idem"?, "client"?,
+           "caused_by"?, "ttl_s"?}
+          {"op": "del",  "coll", "id", "if_seq"?, "idem"?, "client"?}
+          {"op": "link", "frm", "rel", "to", "client"?}
+
+        `if_seq` semantics (the Lua-atomics replacement contract):
+          if_seq = N   -> the doc's last_seq must be exactly N (the caller
+                          read version N and nothing has touched it since)
+          if_seq = -1  -> the doc must NOT currently exist (create-once)
+          omitted/None -> unconditional
+
+        Two phases: ALL preconditions are validated against current state
+        first; if any fails, PreconditionFailed is raised and NOTHING is
+        applied. Only then are the ops applied in order (each becoming a
+        normal chained, hash-sealed op-log entry).
+
+        Concurrency contract: this method performs no locking of its own.
+        It is atomic when executed by a single writer — which is exactly how
+        the daemon runs it (one intent on the Sequencer's committer thread,
+        the same single-writer property that made Redis Lua scripts atomic).
+        Embedded multi-threaded callers must route through Sequencer.tx().
+
+        Durability: the Sequencer group-commits, so a tx submitted as one
+        intent shares a single fsync boundary — a crash before it loses the
+        whole tx together, never a prefix of it.
+
+        Precondition reads use the raw store (no TTL side effects): an
+        expired-but-unswept doc still counts as existing for if_seq == -1;
+        call sweep() first where create-once interacts with TTLs.
+        """
+        failures: List[dict] = []
+        for i, op in enumerate(ops):
+            want = op.get("if_seq")
+            if want is None:
+                continue
+            kind = str(op.get("op", "put")).lower()
+            if kind not in ("put", "del", "delete"):
+                continue  # links carry no per-doc version
+            coll, rid = str(op["coll"]), str(op["id"])
+            cur = self.store.last_seq(f"{coll}:{rid}")
+            want = int(want)
+            if want == -1:
+                if self.store.get(f"{coll}:{rid}") is not None:
+                    failures.append({"index": i, "coll": coll, "id": rid,
+                                     "expected": -1, "actual": cur})
+            elif cur != want:
+                failures.append({"index": i, "coll": coll, "id": rid,
+                                 "expected": want, "actual": cur})
+        if failures:
+            raise PreconditionFailed(failures)
+
+        results: List[dict] = []
+        for op in ops:
+            kind = str(op.get("op", "put")).lower()
+            client = str(op.get("client") or default_client)
+            if kind == "put":
+                stored = self.put(str(op["coll"]), str(op["id"]),
+                                  dict(op.get("doc") or {}),
+                                  client=client,
+                                  idem=op.get("idem"),
+                                  ttl_s=op.get("ttl_s"),
+                                  caused_by=op.get("caused_by"))
+                results.append({"op": "put", "id": op["id"], "seq": self.seq,
+                                "doc": stored})
+            elif kind in ("del", "delete"):
+                self.delete(str(op["coll"]), str(op["id"]),
+                            client=client, idem=op.get("idem"))
+                results.append({"op": "del", "id": op["id"], "seq": self.seq})
+            elif kind == "link":
+                self.link(str(op["frm"]), str(op["rel"]), str(op["to"]),
+                          client=client)
+                results.append({"op": "link", "seq": self.seq})
+            else:
+                results.append({"op": kind, "error": "unknown op"})
+        return {"results": results, "count": len(results), "seq": self.seq}
+
     def expire(self, coll: str, id: str, ttl_s: float) -> bool:
         """Set or update the TTL on an existing document. Returns False if not found."""
         doc = self.store.get(f"{coll}:{id}")

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/python/nedb/server.py

@@ -12,6 +12,7 @@ Config (env):
   NEDBD_PORT    bind port            (default 7070)
   NEDBD_DATA    data root directory  (default ./nedb-data)
   NEDBD_TOKEN   bearer token         (optional; if set, every /v1 route requires it)
+  NEDBD_SWEEP_S TTL sweep interval s (default 30; 0 disables the sweeper)
 
 Run:
   nedbd                 # console script (pip install nedb-engine)
@@ -24,7 +25,7 @@ HTTP API (all JSON):
   GET    /v1/databases/<name>
   DELETE /v1/databases/<name>
   POST   /v1/databases/<name>/query            {nql}
-  POST   /v1/databases/<name>/put              {coll, id, doc, client?, nonce?, idem?}
+  POST   /v1/databases/<name>/put              {coll, id, doc, client?, nonce?, idem?, ttl_s?}
   POST   /v1/databases/<name>/index            {coll, field, kind}
   POST   /v1/databases/<name>/link             {frm, rel, to}
   DELETE /v1/databases/<name>/rows/<coll>/<id>
@@ -35,7 +36,13 @@ HTTP API (all JSON):
   GET    /v1/databases/<name>/files/<filename>/root?version=N&tier=warm  — Merkle root (anchorable)
   POST   /v1/databases/<name>/proof             {hash}  — Merkle inclusion proof (verifiable offline)
   POST   /v1/databases/<name>/checkpoint        — on-demand checkpoint
-  GET    /v1/databases/<name>/batch             — batch writes (array of {op,coll,id,doc})
+  POST   /v1/databases/<name>/sweep             — delete TTL-expired docs now (also runs
+                                                  automatically every NEDBD_SWEEP_S, default 30)
+  POST   /v1/databases/<name>/batch             — ATOMIC multi-op tx: {ops: [{op, coll, id,
+                                                  doc?, if_seq?, idem?, client?}], client?}
+                                                  all-or-nothing; if_seq CAS preconditions
+                                                  (N = exact version, -1 = must-not-exist);
+                                                  409 {error: "precondition_failed", failures}
 """
 from __future__ import annotations
 
@@ -52,6 +59,7 @@ from . import __version__
 from .engine import NEDB
 from .concurrent import Sequencer
 from .log import ReplayError
+from .engine import PreconditionFailed
 
 NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}")
 
@@ -322,7 +330,7 @@ def make_handler(manager: Manager, token: Optional[str]):
                         if not coll or rid is None or not isinstance(doc, dict):
                             raise HttpError(400, "coll, id, and doc are required")
                         _scalar = ("client", "nonce", "idem", "evidence", "confidence",
-                                   "valid_from", "valid_to")
+                                   "valid_from", "valid_to", "ttl_s")
                         kw = {k: b[k] for k in _scalar if b.get(k) is not None}
                         # caused_by may live at the top level of the request body
                         # OR inside doc (natural for clients embedding it in the document).
@@ -364,6 +372,10 @@ def make_handler(manager: Manager, token: Optional[str]):
                         head = db.checkpoint()
                         self._send(200, {"ok": True, "head": head, "seq": db.seq})
                         return
+                    if method == "POST" and action == "sweep":
+                        swept = db.sweep()
+                        self._send(200, {"ok": True, "swept": swept, "seq": db.seq})
+                        return
                     if method == "GET" and action == "log":
                         limit = int(query.get("limit", ["50"])[0])
                         ops = [o.to_dict() for o in db.log.ops[-limit:]][::-1]
@@ -525,30 +537,43 @@ def make_handler(manager: Manager, token: Optional[str]):
                                      "size": len(data), "tier": tier})
                     return
 
-                # POST /v1/databases/<name>/batch  — multiple ops in one request
-                # Body: {ops: [{op:"put"|"del"|"link", coll, id, doc?, frm?, rel?, to?}, ...]}
+                # POST /v1/databases/<name>/batch  — ATOMIC multi-op transaction.
+                # Body: {ops: [{op:"put"|"del"|"link", coll, id, doc?, frm?, rel?,
+                #               to?, if_seq?, idem?, client?, caused_by?, ttl_s?}, ...],
+                #        client?: default client for ops that don't set one}
+                #
+                # All-or-nothing: every `if_seq` precondition is validated against
+                # current state FIRST (under the Sequencer's single committer
+                # thread — nothing can interleave); if any fails, HTTP 409 with
+                # the structured failure list and NOTHING is applied.
+                #
+                # `if_seq` contract (the Lua-atomics replacement):
+                #   if_seq = N   doc's last_seq must be exactly N (CAS)
+                #   if_seq = -1  doc must not exist (create-once)
+                #   omitted      unconditional
+                #
+                # Per-op `idem` and `client` are forwarded to the op log (idem
+                # keys dedupe engine-side; client attributes provenance).
                 if method == "POST" and len(parts) == 4 and parts[:2] == ["v1", "databases"] and parts[3] == "batch":
                     db = manager.require(parts[2])
                     b = self._body()
                     ops_list = b.get("ops") or []
                     if not isinstance(ops_list, list) or not ops_list:
                         raise HttpError(400, "ops array is required")
-                    results = []
-                    for op in ops_list:
-                        kind = str(op.get("op", "put")).lower()
-                        if kind == "put":
-                            doc = db.put(str(op["coll"]), str(op["id"]), dict(op.get("doc") or {}))
-                            results.append({"op": "put", "id": op["id"], "seq": db.seq})
-                        elif kind == "del":
-                            db.delete(str(op["coll"]), str(op["id"]))
-                            results.append({"op": "del", "id": op["id"], "seq": db.seq})
-                        elif kind == "link":
-                            db.link(str(op["frm"]), str(op["rel"]), str(op["to"]))
-                            results.append({"op": "link", "seq": db.seq})
-                        else:
-                            results.append({"op": kind, "error": "unknown op"})
-                    self._send(200, {"results": results, "count": len(results),
-                                     "seq": db.seq, "head": db.head})
+                    default_client = str(b.get("client") or "http")
+                    try:
+                        out = db.tx(ops_list, default_client=default_client)
+                    except PreconditionFailed as e:
+                        self._send(409, {"error": "precondition_failed",
+                                         "failures": e.failures,
+                                         "seq": db.seq})
+                        return
+                    except ReplayError as e:
+                        raise HttpError(409, str(e))
+                    except (KeyError, TypeError, ValueError) as e:
+                        raise HttpError(400, f"bad op: {e}")
+                    self._send(200, {"results": out["results"], "count": out["count"],
+                                     "atomic": True, "seq": db.seq, "head": db.head})
                     return
 
                 raise HttpError(404, "no such route")
@@ -895,6 +920,26 @@ def main() -> None:
     token = args.token
     resp2_port = args.resp2_port
     manager = Manager(data)
+
+    # TTL sweeper: engine expiry is lazy (as_of=None gets) but daemon reads are
+    # snapshot-pinned, so expired docs would linger forever without an active
+    # sweep. Runs every NEDBD_SWEEP_S seconds (default 30; 0 disables) through
+    # each Sequencer's committer intent — single-writer discipline preserved.
+    sweep_s = float(os.environ.get("NEDBD_SWEEP_S", "30"))
+    if sweep_s > 0:
+        def _sweeper():
+            import time as _t
+            while True:
+                _t.sleep(sweep_s)
+                for _name in list(manager._open.keys()):
+                    try:
+                        n = manager._open[_name].sweep()
+                        if n:
+                            _log(f"  [nedbd] ttl sweep [{_name}]: {n} expired", level=1)
+                    except Exception as _e:  # noqa: BLE001
+                        _log(f"  [nedbd] ttl sweep [{_name}] error: {_e}", level=1)
+        threading.Thread(target=_sweeper, name="nedbd-ttl-sweeper", daemon=True).start()
+
     httpd = ThreadingHTTPServer((host, port), make_handler(manager, token))
     auth = "on" if token else "off"
     BANNER = f"""\

{nedb_engine-2.6.1 → nedb_engine-2.6.3}/python/nedb/store.py

@@ -40,6 +40,16 @@ class MVCCStore:
             _, val = chain[i]
         return None if val is TOMB else val
 
+    def last_seq(self, key: str) -> Optional[int]:
+        """Seq of the most recent op that touched this key (including a
+        delete's tombstone), or None if the key has never been written.
+
+        This is the version anchor for compare-and-set (`if_seq`): a CAS
+        taken against version N correctly fails after ANY later write or
+        delete, because that op advanced the key's last_seq past N."""
+        seqs = self._seqs.get(key)
+        return seqs[-1] if seqs else None
+
     def keys(self, prefix: str = "", as_of: Optional[int] = None) -> List[str]:
         # Lock-free read: take an atomic snapshot of the key set, then resolve
         # versions by `as_of`. Under the concurrent Sequencer a single committer