nitrodb 2.4.3__py3-none-any.whl

nedb/index.py

@@ -0,0 +1,98 @@
+"""
+nedb.index — secondary indexes: equality (hash), ordered (bisect), full-text (inverted).
+
+Indexes are maintained incrementally on write and reflect HEAD. They turn filter,
+sort and search from O(n) scans into index lookups. Indexes are keyed by
+"collection.field" so each collection has its own index namespace.
+
+(Time-travel queries fall back to a version scan in the engine; temporally-indexed
+reads are a documented later optimization.)
+"""
+from __future__ import annotations
+
+import bisect
+import re
+from typing import Any, Dict, List, Set
+
+_TOKEN = re.compile(r"[a-z0-9]+")
+
+
+def tokenize(text: str) -> Set[str]:
+    return set(_TOKEN.findall(text.lower()))
+
+
+class Indexes:
+    def __init__(self) -> None:
+        self.eq: Dict[str, Dict[Any, Set[str]]] = {}        # key -> value -> {ids}
+        self.ordered: Dict[str, List[tuple]] = {}           # key -> sorted [(value,id)]
+        self.inv: Dict[str, Dict[str, Set[str]]] = {}       # key -> token -> {ids}
+        self.config: List[tuple] = []                       # [(coll, field, kind)]
+
+    def ensure(self, coll: str, field: str, kind: str = "eq") -> None:
+        k = f"{coll}.{field}"
+        if (coll, field, kind) not in self.config:
+            self.config.append((coll, field, kind))
+        if kind == "eq":
+            self.eq.setdefault(k, {})
+        elif kind == "ordered":
+            self.ordered.setdefault(k, [])
+        elif kind == "search":
+            self.inv.setdefault(k, {})
+        else:
+            raise ValueError(f"unknown index kind: {kind}")
+
+    def add(self, coll: str, key: str, doc: dict) -> None:
+        for field, vmap in self.eq.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and f in doc:
+                vmap.setdefault(doc[f], set()).add(key)
+        for field, lst in self.ordered.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and f in doc and isinstance(doc[f], (int, float, str)):
+                bisect.insort(lst, (doc[f], key))
+        for field, inv in self.inv.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and isinstance(doc.get(f), str):
+                for tok in tokenize(doc[f]):
+                    inv.setdefault(tok, set()).add(key)
+
+    def remove(self, coll: str, key: str, doc: dict) -> None:
+        for field, vmap in self.eq.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and f in doc and doc[f] in vmap:
+                vmap[doc[f]].discard(key)
+        for field, lst in self.ordered.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and f in doc:
+                try:
+                    lst.remove((doc[f], key))
+                except ValueError:
+                    pass
+        for field, inv in self.inv.items():
+            f = field.split(".", 1)[1]
+            if field.startswith(coll + ".") and isinstance(doc.get(f), str):
+                for tok in tokenize(doc[f]):
+                    if tok in inv:
+                        inv[tok].discard(key)
+
+    def eq_lookup(self, coll: str, field: str, value: Any):
+        return set(self.eq.get(f"{coll}.{field}", {}).get(value, set()))
+
+    def search_lookup(self, coll: str, field: str, term: str):
+        return set(self.inv.get(f"{coll}.{field}", {}).get(term, set()))
+
+    def has_eq(self, coll: str, field: str) -> bool:
+        return f"{coll}.{field}" in self.eq
+
+    def search_fields(self, coll: str) -> List[str]:
+        # Lock-free: snapshot keys with retry (a concurrent create_index may add
+        # one mid-iteration under the Sequencer's single-writer committer).
+        for _ in range(128):
+            try:
+                inv_keys = list(self.inv.keys())
+                break
+            except RuntimeError:
+                continue
+        else:
+            inv_keys = list(self.inv.keys())
+        return [k.split(".", 1)[1] for k in inv_keys if k.startswith(coll + ".")]

nedb/log.py

@@ -0,0 +1,216 @@
+"""
+nedb.log — the append-only, hash-chained, nonce-enforced, idempotent operation log.
+
+This is the single source of truth for NEDB. Every mutation in the database is an
+Op appended here. Three guarantees live in this one structure:
+
+  * Replay protection  — each client has a strictly-monotonic nonce; an op whose
+                         nonce is <= the client's last seen nonce is rejected.
+  * Idempotency        — an op carrying an idempotency key that was already applied
+                         returns the original result and is NOT appended again.
+  * Tamper evidence    — ops are chained by hash (h_n = H(h_{n-1} || op_n)), so the
+                         whole history is a verifiable chain and the head hash is a
+                         commitment to the entire log (anchorable on a blockchain).
+
+The same log is the substrate for MVCC snapshot isolation, crash recovery, and
+time-travel reads: every Op has a monotonic `seq`, and state "AS OF seq N" is just
+the log truncated at N.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+GENESIS = "0" * 64
+
+
+def canon(obj: Any) -> bytes:
+    """Deterministic canonical encoding for hashing."""
+    return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str).encode()
+
+
+def blake(data: bytes) -> str:
+    # Reference uses BLAKE2b (stdlib). The production Rust core uses BLAKE3
+    # (faster, natively tree-structured for the Merkle history).
+    return hashlib.blake2b(data, digest_size=32).hexdigest()
+
+
+class ReplayError(Exception):
+    """Raised when an op is replayed with a stale/duplicate nonce."""
+
+
+@dataclass
+class Op:
+    seq: int
+    client: str
+    nonce: int
+    op: str  # put | delete | link | unlink | put_file
+    payload: dict
+    ts: float
+    idem: Optional[str]
+    prev_hash: str
+    hash: str
+    # ── Causal provenance (v0.9.0+) ─────────────────────────────────────────
+    # Optional fields that, when present, are sealed inside the hash chain so
+    # they are tamper-evident and time-stamped at write time.
+    #   caused_by  — seqs of the ops that led to this write (backward trace).
+    #   evidence   — source type: "user_message" | "inference" | "tool_result"
+    #                             | "correction" | "external"
+    #   confidence — agent's certainty in this write (0.0 – 1.0).
+    caused_by:  Optional[List[int]] = None
+    evidence:   Optional[str]       = None
+    confidence: Optional[float]     = None
+
+    # ── Bi-temporal valid time (v1.0.0+) ─────────────────────────────────────
+    # When was this fact TRUE IN THE WORLD (independent of when it was written)?
+    #   valid_from — ISO 8601 date/datetime string; None = "from the beginning"
+    #   valid_to   — ISO 8601 date/datetime string; None = "still valid / open-ended"
+    #
+    # ISO 8601 strings sort lexicographically correctly, so comparisons are
+    # safe as plain string ops:  "2024-01-01" < "2024-06-15" ✓
+    #
+    # Backward-compatible: ops without valid-time fields are treated as always
+    # valid (they pass every VALID AS OF filter). Existing chains verify unchanged.
+    valid_from: Optional[str] = None
+    valid_to:   Optional[str] = None
+
+    def to_dict(self) -> dict:
+        """Serialize for the append-only log file (AOF)."""
+        d: dict = {
+            "seq": self.seq, "client": self.client, "nonce": self.nonce,
+            "op": self.op, "payload": self.payload, "ts": self.ts,
+            "idem": self.idem, "prev_hash": self.prev_hash, "hash": self.hash,
+        }
+        if self.caused_by  is not None: d["caused_by"]  = self.caused_by
+        if self.evidence   is not None: d["evidence"]   = self.evidence
+        if self.confidence is not None: d["confidence"] = self.confidence
+        if self.valid_from is not None: d["valid_from"] = self.valid_from
+        if self.valid_to   is not None: d["valid_to"]   = self.valid_to
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Op":
+        return cls(
+            d["seq"], d["client"], d["nonce"], d["op"], d["payload"],
+            d["ts"], d.get("idem"), d["prev_hash"], d["hash"],
+            caused_by  = d.get("caused_by"),
+            evidence   = d.get("evidence"),
+            confidence = d.get("confidence"),
+            valid_from = d.get("valid_from"),
+            valid_to   = d.get("valid_to"),
+        )
+
+
+class OpLog:
+    def __init__(self) -> None:
+        self.ops: List[Op] = []
+        self._last_nonce: Dict[str, int] = {}
+        self._idem: Dict[str, int] = {}  # idem key -> seq of original op
+        self._head = GENESIS
+
+    def append(
+        self,
+        client: str,
+        nonce: int,
+        op: str,
+        payload: dict,
+        idem: Optional[str] = None,
+        ts: Optional[float] = None,
+        caused_by:  Optional[List[int]] = None,
+        evidence:   Optional[str]       = None,
+        confidence: Optional[float]     = None,
+        valid_from: Optional[str]       = None,
+        valid_to:   Optional[str]       = None,
+    ) -> Tuple[Op, bool]:
+        """Append an op. Returns (op, created). `created` is False when the op was
+        deduplicated by its idempotency key (a no-op replay-safe return)."""
+        # Idempotency: a known key returns the original op without re-appending.
+        if idem is not None and idem in self._idem:
+            return self.ops[self._idem[idem]], False
+
+        # Replay protection: nonce must strictly exceed the client's last nonce.
+        last = self._last_nonce.get(client, 0)
+        if nonce <= last:
+            raise ReplayError(
+                f"replay/stale nonce for client '{client}': {nonce} <= {last}"
+            )
+
+        seq = len(self.ops)
+        ts = time.time() if ts is None else ts
+        body: dict = {
+            "seq": seq, "client": client, "nonce": nonce,
+            "op": op, "payload": payload, "ts": ts, "idem": idem,
+        }
+        # Provenance fields are sealed INTO the hash when present so they are
+        # tamper-evident — omitting them when absent keeps old ops verifiable.
+        if caused_by  is not None: body["caused_by"]  = caused_by
+        if evidence   is not None: body["evidence"]   = evidence
+        if confidence is not None: body["confidence"] = confidence
+        if valid_from is not None: body["valid_from"] = valid_from
+        if valid_to   is not None: body["valid_to"]   = valid_to
+        h = blake(self._head.encode() + canon(body))
+        rec = Op(seq, client, nonce, op, payload, ts, idem, self._head, h,
+                 caused_by=caused_by, evidence=evidence, confidence=confidence,
+                 valid_from=valid_from, valid_to=valid_to)
+
+        self.ops.append(rec)
+        self._last_nonce[client] = nonce
+        if idem is not None:
+            self._idem[idem] = seq
+        self._head = h
+        return rec, True
+
+    def load(self, ops: List[Op]) -> None:
+        """Rehydrate the log from persisted ops WITHOUT recomputing hashes, so the
+        original chain (and thus verify() and the head commitment) is preserved
+        exactly across a restart. Nonce, idempotency, and head state are restored
+        from the ops themselves — replay protection survives a reload."""
+        self.ops = list(ops)
+        self._last_nonce = {}
+        self._idem = {}
+        for o in self.ops:
+            if o.nonce > self._last_nonce.get(o.client, 0):
+                self._last_nonce[o.client] = o.nonce
+            if o.idem is not None and o.idem not in self._idem:
+                self._idem[o.idem] = o.seq
+        self._head = self.ops[-1].hash if self.ops else GENESIS
+
+    @staticmethod
+    def _op_body(o: "Op") -> dict:
+        """The canonical hash body for an op — must match exactly what append() hashes."""
+        body: dict = {
+            "seq": o.seq, "client": o.client, "nonce": o.nonce,
+            "op": o.op, "payload": o.payload, "ts": o.ts, "idem": o.idem,
+        }
+        # Optional fields included only when present (backward-compat with old ops).
+        if o.caused_by  is not None: body["caused_by"]  = o.caused_by
+        if o.evidence   is not None: body["evidence"]   = o.evidence
+        if o.confidence is not None: body["confidence"] = o.confidence
+        if o.valid_from is not None: body["valid_from"] = o.valid_from
+        if o.valid_to   is not None: body["valid_to"]   = o.valid_to
+        return body
+
+    def verify(self) -> bool:
+        """Re-walk the chain and confirm no op has been tampered with."""
+        prev = GENESIS
+        for o in self.ops:
+            body = self._op_body(o)
+            if o.prev_hash != prev:
+                return False
+            if o.hash != blake(prev.encode() + canon(body)):
+                return False
+            prev = o.hash
+        return True
+
+    @property
+    def head(self) -> str:
+        return self._head
+
+    def slice_until(self, as_of: int) -> List[Op]:
+        return [o for o in self.ops if o.seq <= as_of]
+
+    def __len__(self) -> int:
+        return len(self.ops)

nedb/merkle.py

@@ -0,0 +1,62 @@
+"""
+nedb.merkle — Merkle tree over content-addressed chunk hashes.
+
+Because every file version is a list of BLAKE-addressed chunks, a file version has
+a Merkle root that commits to its exact bytes. Any chunk's membership is provable in
+O(log n), and the root can be anchored on-chain (e.g. ITC) for tamper-evident,
+notarized version history.
+"""
+from __future__ import annotations
+
+import hashlib
+from typing import List, Tuple
+
+
+def _h(b: bytes) -> bytes:
+    return hashlib.blake2b(b, digest_size=32).digest()
+
+
+def _to_bytes(x) -> bytes:
+    return bytes.fromhex(x) if isinstance(x, str) else bytes(x)
+
+
+def merkle_root(leaves: List[str]) -> str:
+    if not leaves:
+        return "0" * 64
+    level = [_to_bytes(x) for x in leaves]
+    while len(level) > 1:
+        nxt = []
+        for i in range(0, len(level), 2):
+            a = level[i]
+            b = level[i + 1] if i + 1 < len(level) else level[i]
+            nxt.append(_h(a + b))
+        level = nxt
+    return level[0].hex()
+
+
+def merkle_proof(leaves: List[str], idx: int) -> List[Tuple[str, str]]:
+    """Return inclusion proof for leaf at idx: list of (sibling_hex, side)."""
+    level = [_to_bytes(x) for x in leaves]
+    path: List[Tuple[str, str]] = []
+    while len(level) > 1:
+        if idx % 2 == 0:
+            sib = level[idx + 1] if idx + 1 < len(level) else level[idx]
+            path.append((sib.hex(), "R"))
+        else:
+            path.append((level[idx - 1].hex(), "L"))
+        nxt = []
+        for i in range(0, len(level), 2):
+            a = level[i]
+            b = level[i + 1] if i + 1 < len(level) else level[i]
+            nxt.append(_h(a + b))
+        level = nxt
+        idx //= 2
+    return path
+
+
+def merkle_verify(leaf: str, path: List[Tuple[str, str]], root: str) -> bool:
+    h = _to_bytes(leaf)
+    for sib_hex, side in path:
+        sib = _to_bytes(sib_hex)
+        h = _h(h + sib) if side == "R" else _h(sib + h)
+    return h.hex() == root