nitrodb 2.4.3__py3-none-any.whl

nedb/__init__.py

@@ -0,0 +1,92 @@
+"""
+NEDB — a versioned, self-compressing, time-traveling embedded database.
+
+  * Replay-protected & idempotent: every write carries a monotonic nonce and an
+    optional idempotency key, enforced by a hash-chained append-only log.
+  * Time-travel: read the database AS OF any past sequence number.
+  * Relational: first-class, time-travel-aware relations with O(1) traversal.
+  * Filterable / sortable / searchable: equality, ordered, and full-text indexes.
+  * Queryable: NQL text queries and a fluent builder that share one plan.
+  * git-style files with Cascade compression: content-defined chunking + dedup +
+    temperature tiers, with a Merkle root per version anchorable on-chain.
+
+The pure-Python package is the reference implementation and the always-works
+fallback. When installed from a platform wheel, the compiled Rust core is available
+as ``nedb._native`` (``nedb.__has_native__`` reports whether it loaded).
+"""
+from __future__ import annotations
+
+from .engine import NEDB
+from .log import Op, OpLog, ReplayError
+from .query import Query, parse_nql
+from .snapshot import save_snapshot, load_snapshot
+from .crypto import resolve_tmk, rewrap_dek
+from .sql import sql_exec, sql_to_nql, SQLError, SQLUnsupportedError
+from .redis_compat import RedisCompat, RedisError, RedisUnsupportedError
+from .mongo import (
+    MongoCompat, MongoClient, MongoError, MongoUnsupportedError, ObjectId,
+)
+from .autoindex import AutoIndexDB
+from .concurrent import Sequencer
+from .wrap_redis import wrap_redis, WrappedRedis
+from .proof import verify_proof, fold_head
+
+try:  # compiled Rust core, present in platform wheels (PyO3 via maturin)
+    from . import _native  # type: ignore
+    __has_native__ = True
+except ImportError:  # pure-Python install (sdist / unsupported platform)
+    # Provide a stub module so `from nedb._native import NedbCore` raises an
+    # informative error instead of a bare ImportError with no guidance.
+    import types as _types, sys as _sys
+
+    import sys as _sys_tmp, os as _os_tmp
+    _is_msys2 = bool(_os_tmp.environ.get("MSYSTEM")) or "mingw" in _sys_tmp.executable.lower()
+    del _sys_tmp, _os_tmp
+
+    class _NativeStub(_types.ModuleType):
+        # Primary fix: install the Rust crate → get the nedbd server → use HTTP mode.
+        # Secondary fix (CPython only): pip reinstall to get the platform wheel with _native embedded.
+        _MSG_MSYS2 = (
+            "\n\n"
+            "  nedb._native (embedded v2 DAG core) is not available on MSYS2/MinGW Python.\n\n"
+            "  To use NEDB v2 features, install the server binary and use HTTP mode:\n\n"
+            "    cargo install nedb-engine          # install nedbd v2 server\n"
+            "    nedbd --dag ./data                 # start DAG server\n"
+            "    NEDB_URL=http://localhost:7070 python3 your_script.py\n\n"
+            "  Run 'nedbd --doctor' for a full diagnosis.\n"
+        )
+        _MSG_OTHER = (
+            "\n\n"
+            "  nedb._native (embedded v2 DAG core) is not available.\n"
+            "  You have the universal wheel — reinstall to get the platform wheel:\n\n"
+            "    pip install --force-reinstall --no-cache-dir nedb-engine\n\n"
+            "  Or install the server binary and use HTTP mode (works everywhere):\n\n"
+            "    cargo install nedb-engine          # install nedbd v2 server\n"
+            "    nedbd --dag ./data                 # start DAG server\n"
+            "    NEDB_URL=http://localhost:7070 python3 your_script.py\n\n"
+            "  Run 'nedbd --doctor' for a full diagnosis.\n"
+        )
+        _MSG = _MSG_MSYS2 if _is_msys2 else _MSG_OTHER
+
+        def __getattr__(self, name: str):
+            raise ImportError(f"nedb._native.{name} is not available.{self._MSG}")
+
+    _native_stub = _NativeStub("nedb._native")
+    _native_stub.__package__ = "nedb"
+    _sys.modules["nedb._native"] = _native_stub  # type: ignore
+    _native = _native_stub  # type: ignore
+    __has_native__ = False
+    del _types, _sys, _NativeStub, _native_stub
+
+__all__ = [
+    "NEDB", "OpLog", "Op", "ReplayError", "Query", "parse_nql",
+    "save_snapshot", "load_snapshot",
+    "sql_exec", "sql_to_nql", "SQLError", "SQLUnsupportedError",
+    "RedisCompat", "RedisError", "RedisUnsupportedError",
+    "MongoCompat", "MongoClient", "MongoError", "MongoUnsupportedError", "ObjectId",
+    "AutoIndexDB", "Sequencer",
+    "wrap_redis", "WrappedRedis",
+    "verify_proof", "fold_head",
+    "_native", "__has_native__",
+]
+__version__ = "2.4.3"

nedb/autoindex.py

@@ -0,0 +1,142 @@
+"""
+nedb.autoindex — automatic index management.
+
+Wraps a NEDB instance and intercepts query() calls. It tracks which fields are
+used in WHERE and ORDER BY clauses per collection. Once a field reaches the
+usage threshold it auto-creates the appropriate index:
+
+  - Equality conditions (= / !=)   → "eq"    index
+  - Ordered comparisons (< > ≤ ≥) → "ordered" index
+  - ORDER BY field                  → "ordered" index
+  - SEARCH clause on a field        → deferred (no per-field signal in NQL)
+
+Usage::
+
+    from nedb import NEDB
+    from nedb.autoindex import AutoIndexDB
+
+    db = AutoIndexDB(NEDB("./data"), threshold=3)
+    db.query('FROM users WHERE status = "active"')   # tallied
+    db.query('FROM users WHERE status = "active"')
+    db.query('FROM users WHERE status = "active"')   # threshold reached → index created
+"""
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
+
+
+_WHERE_RE = re.compile(r"\bWHERE\b([\s\S]*?)(?:\bSEARCH\b|\bORDER\b|\bTRAVERSE\b|\bLIMIT\b|$)", re.IGNORECASE)
+_ORDER_RE = re.compile(r"\bORDER\s+BY\s+(\w+)", re.IGNORECASE)
+_FROM_RE  = re.compile(r"\bFROM\s+(\w+)", re.IGNORECASE)
+_COND_RE  = re.compile(r"(\w+)\s*(=|!=|<>|<=|>=|<|>)", re.IGNORECASE)
+
+
+def _parse_signals(nql: str) -> List[Tuple[str, str, str]]:
+    """Return [(collection, field, 'eq'|'ordered')] from a NQL query string."""
+    signals = []
+    fm = _FROM_RE.search(nql)
+    if not fm:
+        return signals
+    coll = fm.group(1)
+
+    wm = _WHERE_RE.search(nql)
+    if wm:
+        for m in _COND_RE.finditer(wm.group(1)):
+            field, op = m.group(1), m.group(2)
+            kind = "eq" if op in ("=", "!=", "<>") else "ordered"
+            signals.append((coll, field, kind))
+
+    om = _ORDER_RE.search(nql)
+    if om:
+        signals.append((coll, om.group(1), "ordered"))
+
+    return signals
+
+
+class AutoIndexDB:
+    """
+    NEDB wrapper that creates indexes automatically based on query usage.
+
+    Parameters
+    ----------
+    db : NEDB
+        A NEDB database instance (embedded or opened with a path).
+    threshold : int
+        Number of times a (collection, field, kind) combination must be
+        observed before the index is created. Default: 5.
+    verbose : bool
+        Print a message when an index is auto-created. Default: False.
+    """
+
+    def __init__(self, db: Any, threshold: int = 5, verbose: bool = False):
+        self._db = db
+        self.threshold = threshold
+        self.verbose = verbose
+        # counts[(coll, field, kind)] = n
+        self._counts: Dict[Tuple[str, str, str], int] = defaultdict(int)
+        # indexes already created so we don't re-create
+        self._created: set = set()
+        # Seed from existing index config if available
+        if hasattr(db, "indexes") and hasattr(db.indexes, "config"):
+            for coll, field, kind in db.indexes.config:
+                self._created.add((coll, field, kind))
+
+    # ── Proxy every NEDB attribute ────────────────────────────────────────────
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._db, name)
+
+    # ── Instrumented query ────────────────────────────────────────────────────
+
+    def query(self, nql: str) -> List[dict]:
+        """Execute a NQL query, tally field usage, and auto-create indexes."""
+        signals = _parse_signals(nql)
+        for coll, field, kind in signals:
+            key = (coll, field, kind)
+            if key in self._created:
+                continue
+            # "ordered" supersedes "eq" — if we already have eq, upgrade to ordered
+            eq_key = (coll, field, "eq")
+            if kind == "ordered" and eq_key not in self._created:
+                self._counts[key] += 1
+            elif kind == "eq" and (coll, field, "ordered") not in self._created:
+                self._counts[key] += 1
+            else:
+                self._counts[key] += 1
+
+            if self._counts[key] >= self.threshold:
+                self._auto_create(coll, field, kind)
+
+        return self._db.query(nql)
+
+    def _auto_create(self, coll: str, field: str, kind: str) -> None:
+        key = (coll, field, kind)
+        if key in self._created:
+            return
+        # Don't index internal NEDB fields
+        if field.startswith("_") and field not in ("_id",):
+            return
+        self._db.create_index(coll, field, kind)
+        self._created.add(key)
+        if self.verbose:
+            print(f"[autoindex] created {kind} index on {coll}.{field} (threshold={self.threshold})")
+
+    # ── Manual analysis ───────────────────────────────────────────────────────
+
+    def analyze(self) -> Dict[str, Any]:
+        """Return current tallies and the indexes already created."""
+        return {
+            "tallies": {f"{c}.{f} ({k})": n for (c, f, k), n in self._counts.items()},
+            "indexes_created": [f"{c}.{f} ({k})" for (c, f, k) in sorted(self._created)],
+            "threshold": self.threshold,
+        }
+
+    def suggest(self) -> List[str]:
+        """Return suggestions for indexes that are close to the threshold."""
+        out = []
+        for (coll, field, kind), count in sorted(self._counts.items(), key=lambda x: -x[1]):
+            if (coll, field, kind) not in self._created:
+                out.append(f"{coll}.{field} ({kind}) — {count}/{self.threshold} queries")
+        return out

nedb/backends/redis_backend.py

@@ -0,0 +1,115 @@
+"""
+nedb.backends.redis_backend — Redis Streams as the NEDB append-only log.
+
+Alice's existing Redis keys are NEVER touched. NEDB operates in a strictly
+isolated namespace:
+
+    nedb:{db_name}:oplog       Redis Stream  — hash-chained op log
+    nedb:{db_name}:snapshot    Redis Hash    — checkpoint for fast restart
+    nedb:{db_name}:events      Pub/Sub chan  — live subscriptions (future)
+    nedb:{db_name}:meta        Redis Hash    — version, index config
+
+On startup NEDB replays the stream to rebuild its in-memory MVCC store.
+On every write a new entry is XADD'd. One Redis connection, zero impact on
+the user's existing keys.
+
+© INTERCHAINED LLC × Claude Sonnet 4.6
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional
+
+
+class RedisBackend:
+    """
+    Redis-Streams-backed persistence for NEDB.
+
+    Pass an instance to NEDB as the `backend` parameter::
+
+        import redis
+        from nedb.backends.redis_backend import RedisBackend
+        from nedb import NEDB
+
+        r = redis.Redis("localhost", 6379)
+        db = NEDB(backend=RedisBackend(r, "rideshare"))
+    """
+
+    def __init__(self, r: Any, db_name: str):
+        self._r        = r
+        self.db_name   = db_name
+        self.stream    = f"nedb:{db_name}:oplog"
+        self.snap_key  = f"nedb:{db_name}:snapshot"
+        self.meta_key  = f"nedb:{db_name}:meta"
+        self.events_ch = f"nedb:{db_name}:events"
+
+    # ── Op log ──────────────────────────────────────────────────────────────────
+
+    def append(self, op_json: str) -> None:
+        """Append one JSON-serialised op to the stream."""
+        self._r.xadd(self.stream, {"op": op_json})
+
+    def append_batch(self, ops: List[str]) -> None:
+        """Append multiple ops in a single pipeline (one round-trip)."""
+        pipe = self._r.pipeline(transaction=False)
+        for op_json in ops:
+            pipe.xadd(self.stream, {"op": op_json})
+        pipe.execute()
+
+    def read_all(self) -> List[str]:
+        """Return all ops from the stream in insertion order."""
+        entries = self._r.xrange(self.stream, "-", "+")
+        return [e[1][b"op"].decode() for e in entries]
+
+    def read_after(self, last_id: str = "0") -> List[str]:
+        """Return ops appended after `last_id` (for incremental replay)."""
+        entries = self._r.xrange(self.stream, f"({last_id}", "+")
+        return [e[1][b"op"].decode() for e in entries]
+
+    # ── Snapshot / checkpoint ────────────────────────────────────────────────────
+
+    def save_snapshot(self, data: Dict[str, Any]) -> None:
+        """Persist a checkpoint so restart replay only needs the delta."""
+        self._r.hset(self.snap_key, mapping={
+            k: json.dumps(v, separators=(",", ":"), default=str)
+            for k, v in data.items()
+        })
+
+    def load_snapshot(self) -> Optional[Dict[str, Any]]:
+        """Load the last checkpoint, or None if none exists."""
+        raw = self._r.hgetall(self.snap_key)
+        if not raw:
+            return None
+        return {k.decode(): json.loads(v) for k, v in raw.items()}
+
+    # ── Pub/sub live events ──────────────────────────────────────────────────────
+
+    def publish_ops(self, ops: List[str]) -> None:
+        """Publish committed ops to the events channel for live subscribers."""
+        if ops:
+            payload = json.dumps(ops, separators=(",", ":"))
+            self._r.publish(self.events_ch, payload)
+
+    # ── Meta ─────────────────────────────────────────────────────────────────────
+
+    def save_meta(self, meta: Dict[str, Any]) -> None:
+        self._r.hset(self.meta_key, mapping={
+            k: json.dumps(v, separators=(",", ":"), default=str)
+            for k, v in meta.items()
+        })
+
+    def load_meta(self) -> Dict[str, Any]:
+        raw = self._r.hgetall(self.meta_key)
+        if not raw:
+            return {}
+        return {k.decode(): json.loads(v) for k, v in raw.items()}
+
+    # ── Utility ──────────────────────────────────────────────────────────────────
+
+    def stream_len(self) -> int:
+        return self._r.xlen(self.stream)
+
+    def flush(self) -> None:
+        """Delete all NEDB shadow keys for this database (non-destructive to user keys)."""
+        for key in [self.stream, self.snap_key, self.meta_key]:
+            self._r.delete(key)

nedb/cascade.py

@@ -0,0 +1,130 @@
+"""
+nedb.cascade — the Cascade compression pipeline + content-addressed blob store.
+
+This is what makes NEDB double as a git-style file manager with maximum compression
+WITHOUT inventing a new entropy coder. The novelty is the pipeline composition:
+
+  1. Content-defined chunking (Gear rolling hash) — boundaries follow content, so a
+     one-byte insert only changes the chunk(s) around it, not everything after it.
+  2. Content-addressed dedup (BLAKE) — identical chunks across all files and all
+     versions are stored exactly once.
+  3. Temperature tiers — warm data uses a fast codec (zstd in prod; zlib in this
+     reference), cold/archival history uses a maximum-ratio codec (LZMA).
+
+The production pipeline adds similarity-picked binary deltas (zstd --patch-from) and
+schema-aware columnar transforms before the entropy stage; both are documented in
+docs/SPEC.md and stubbed for the reference engine.
+"""
+from __future__ import annotations
+
+import hashlib
+import lzma
+import random
+import zlib
+from typing import Dict, List
+
+from .merkle import merkle_root
+
+# --- Gear-hash content-defined chunking -------------------------------------
+_MASK = (1 << 13) - 1            # ~8 KiB average chunk
+_MIN = 2 * 1024
+_MAX = 64 * 1024
+_M64 = 0xFFFFFFFFFFFFFFFF
+_GEAR = [random.Random(0x12345678 + i).getrandbits(64) for i in range(256)]
+
+
+def chunk(data: bytes) -> List[bytes]:
+    chunks: List[bytes] = []
+    n = len(data)
+    i = 0
+    while i < n:
+        limit = min(i + _MAX, n)
+        h = 0
+        pos = i
+        cut = limit
+        while pos < limit:
+            h = ((h << 1) + _GEAR[data[pos]]) & _M64
+            pos += 1
+            if (pos - i) >= _MIN and (h & _MASK) == 0:
+                cut = pos
+                break
+        chunks.append(data[i:cut])
+        i = cut
+    return chunks
+
+
+def _blake(b: bytes) -> str:
+    return hashlib.blake2b(b, digest_size=32).hexdigest()
+
+
+# --- temperature tiers ------------------------------------------------------
+def warm_compress(b: bytes) -> bytes:    # zstd stand-in in the reference
+    return zlib.compress(b, 6)
+
+
+def warm_decompress(b: bytes) -> bytes:
+    return zlib.decompress(b)
+
+
+def cold_compress(b: bytes) -> bytes:    # real LZMA — the maximum-ratio archival tier
+    return lzma.compress(b, preset=9 | lzma.PRESET_EXTREME)
+
+
+def cold_decompress(b: bytes) -> bytes:
+    return lzma.decompress(b)
+
+
+class BlobStore:
+    """Content-addressed, deduplicated, tiered blob store with versioned files."""
+
+    def __init__(self, tier: str = "warm") -> None:
+        self.tier = tier
+        self.chunks: Dict[str, bytes] = {}                      # hash -> compressed bytes
+        self.files: Dict[str, Dict[str, list]] = {}            # name -> {versions, roots}
+        self.logical_bytes = 0
+        self.dedup_hits = 0
+
+    def _compress(self, b: bytes) -> bytes:
+        return cold_compress(b) if self.tier == "cold" else warm_compress(b)
+
+    def _decompress(self, b: bytes) -> bytes:
+        return cold_decompress(b) if self.tier == "cold" else warm_decompress(b)
+
+    def put_file(self, name: str, data: bytes) -> int:
+        recipe: List[str] = []
+        for c in chunk(data):
+            hh = _blake(c)
+            recipe.append(hh)
+            if hh in self.chunks:
+                self.dedup_hits += 1
+            else:
+                self.chunks[hh] = self._compress(c)
+        self.logical_bytes += len(data)
+        f = self.files.setdefault(name, {"versions": [], "roots": []})
+        f["versions"].append(recipe)
+        f["roots"].append(merkle_root(recipe))
+        return len(f["versions"]) - 1
+
+    def get_file(self, name: str, version: int = -1) -> bytes:
+        recipe = self.files[name]["versions"][version]
+        out = bytearray()
+        for hh in recipe:
+            out += self._decompress(self.chunks[hh])
+        return bytes(out)
+
+    def root(self, name: str, version: int = -1) -> str:
+        return self.files[name]["roots"][version]
+
+    def stored_bytes(self) -> int:
+        return sum(len(v) for v in self.chunks.values())
+
+    def stats(self) -> dict:
+        stored = self.stored_bytes()
+        return {
+            "tier": self.tier,
+            "unique_chunks": len(self.chunks),
+            "dedup_hits": self.dedup_hits,
+            "logical_bytes": self.logical_bytes,
+            "stored_bytes": stored,
+            "ratio": round(self.logical_bytes / stored, 2) if stored else 0.0,
+        }