hybriddb 0.3.0__py3-none-any.whl

hybriddb/__init__.py

@@ -0,0 +1,31 @@
+"""HybridDB: SQLite + FTS5 + ChromaDB + Graph + DuckDB with self-healing journal.
+
+TEXT columns get keyword search. LONGTEXT columns get keyword + semantic search.
+Graph capabilities: SQLite-backed nodes/edges, recursive CTE traversal, NetworkX algorithms.
+Analytics: DuckDB columnar store synced via unified journal for fast OLAP queries.
+"""
+
+__version__ = "0.3.0"
+
+from hybriddb.embedding import default_embedding_fn
+from hybriddb.db import HybridDB
+from hybriddb.types import (
+    BOOLEAN,
+    HYBRID,
+    INTEGER,
+    JSON,
+    KEYWORD,
+    LONGTEXT,
+    REAL,
+    SEMANTIC,
+    TEXT,
+    Column,
+    EmbeddingModelError,
+    SearchMode,
+)
+
+__all__ = [
+    "HybridDB", "SearchMode", "EmbeddingModelError", "Column", "default_embedding_fn",
+    "TEXT", "LONGTEXT", "INTEGER", "REAL", "BOOLEAN", "JSON",
+    "KEYWORD", "SEMANTIC", "HYBRID",
+]

hybriddb/analytics.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import sqlite3
+import struct
+import tempfile
+import uuid
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from hybriddb.embedding import EMBEDDING_DIM
+from hybriddb.types import Column, SearchMode
+from hybriddb.utils import (
+    CHROMA_BATCH,
+    JOURNAL_CAP,
+    RRF_K,
+    _CHROMA_INDEX_MAX_ELEMENTS,
+    _CHROMA_INDEX_MAX_M0,
+    _CHROMA_INDEX_WARN_FACTOR,
+    _CHROMA_REBUILD_BATCH,
+    _SKIP_SEARCH_COLUMNS,
+    _SYSTEM_TABLES,
+    _coerce_search_mode,
+    _column_spec,
+    _is_safe_identifier,
+    _now_iso,
+    _sanitize_fts_query,
+    _validate_identifier,
+    _validate_order_by,
+)
+
+logger = logging.getLogger("hybriddb")
+
+class AnalyticsMixin:
+    def _init_duckdb(self) -> None:
+        self._duckdb_path = ""
+        self._duckdb_synced_tables: dict[str, dict] = {}
+        self._duckdb_conn = None
+
+        try:
+            import duckdb
+
+            self._duckdb_path = str((self.path / "analytics.duckdb").resolve())
+            self._duckdb_conn = duckdb.connect(self._duckdb_path)
+            self._duckdb_conn.execute("SET threads = 4")
+            self._duckdb_conn.execute("""
+                CREATE TABLE IF NOT EXISTS _duckdb_sync (
+                    table_name TEXT PRIMARY KEY,
+                    synced_count INTEGER DEFAULT 0
+                )
+            """)
+            existing = self._duckdb_conn.execute(
+                "SELECT table_name FROM _duckdb_sync"
+            ).fetchall()
+            for (tname,) in existing:
+                quoted_tname = self._duckdb_quote_identifier(tname)
+                count = self._duckdb_conn.execute(
+                    f"SELECT count(*) FROM {quoted_tname}"
+                ).fetchone()[0]
+                cols_info = self._duckdb_conn.execute(
+                    "SELECT column_name, data_type FROM information_schema.columns "
+                    "WHERE table_name = ?",
+                    (tname,),
+                ).fetchall()
+                columns = {c: t for c, t in cols_info}
+                self._duckdb_synced_tables[tname] = {"columns": columns, "count": count}
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning("duckdb.init_failed error=%s", e)
+            self._duckdb_path = ""
+            self._duckdb_conn = None
+
+    def _auto_register_duckdb_tables(self) -> None:
+        if not self._duckdb_path:
+            return
+        app_tables = self.list_tables()
+        new_tables = [t for t in app_tables if t not in self._duckdb_synced_tables]
+        if not new_tables:
+            return
+        for table in new_tables:
+            self.register_duckdb_table(table)
+        logger.info("duckdb.auto_registered tables=%s count=%d", new_tables, len(new_tables))
+
+    def register_duckdb_table(self, table: str) -> bool:
+        _validate_identifier(table, "table")
+        if not self._duckdb_path or self._duckdb_conn is None:
+            return False
+
+        meta = self._table_meta(table)
+        if not meta:
+            logger.warning("duckdb.register_missing_table table=%s", table)
+            return False
+
+        cols = []
+        for col_name, col_type in meta["columns"].items():
+            base = col_type.replace("_PK", "")
+            quoted_col = self._duckdb_quote_identifier(col_name)
+            if base == "BOOLEAN":
+                cols.append(f"{quoted_col} INTEGER")
+            elif base == "JSON":
+                cols.append(f"{quoted_col} TEXT")
+            elif base in ("TEXT", "LONGTEXT"):
+                cols.append(f"{quoted_col} TEXT")
+            elif base == "INTEGER" or base == "INTEGER_PK":
+                cols.append(f"{quoted_col} BIGINT")
+            else:
+                cols.append(f"{quoted_col} {base}")
+        if "id" not in meta["columns"]:
+            cols.insert(0, "id BIGINT")
+
+        quoted_table = self._duckdb_quote_identifier(table)
+        with self._db_lock:
+            dk = self._duckdb_conn
+            dk.execute(f"DROP TABLE IF EXISTS {quoted_table}")
+            dk.execute(f"CREATE TABLE {quoted_table} ({', '.join(cols)})")
+            dk.execute(
+                "INSERT OR REPLACE INTO _duckdb_sync (table_name, synced_count) VALUES (?, 0)",
+                (table,),
+            )
+
+        self._duckdb_synced_tables[table] = {"columns": dict(meta["columns"]), "count": 0}
+        self._full_sync_duckdb_table(table)
+        return True
+
+    @staticmethod
+    def _duckdb_quote_identifier(identifier: str) -> str:
+        return '"' + identifier.replace('"', '""') + '"'
+
+    def _refresh_duckdb_table_if_registered(self, table: str) -> None:
+        if table in self._duckdb_synced_tables:
+            self.register_duckdb_table(table)
+
+    def _full_sync_duckdb_table(self, table: str) -> None:
+        if not self._duckdb_path or table not in self._duckdb_synced_tables:
+            return
+
+        with self._db_lock:
+            dk = self._duckdb_conn
+            quoted_table = self._duckdb_quote_identifier(table)
+            dk.execute(f"DELETE FROM {quoted_table}")
+            try:
+                dk.execute("DETACH src")
+            except Exception:
+                pass
+            try:
+                dk.execute(f"ATTACH '{self._db_path}' AS src (TYPE sqlite)")
+                dk.execute(f"INSERT INTO {quoted_table} SELECT * FROM src.{quoted_table}")
+            finally:
+                dk.execute("DETACH src")
+            count = dk.execute(f"SELECT count(*) FROM {quoted_table}").fetchone()[0]
+            dk.execute(
+                "UPDATE _duckdb_sync SET synced_count = ? WHERE table_name = ?",
+                (count, table),
+            )
+            self._duckdb_synced_tables[table]["count"] = count
+
+    def unregister_duckdb_table(self, table: str) -> bool:
+        _validate_identifier(table, "table")
+        if table not in self._duckdb_synced_tables:
+            return False
+
+        with self._db_lock:
+            dk = self._duckdb_conn
+            dk.execute(f"DROP TABLE IF EXISTS {self._duckdb_quote_identifier(table)}")
+            dk.execute("DELETE FROM _duckdb_sync WHERE table_name = ?", (table,))
+        self._duckdb_synced_tables.pop(table, None)
+        return True
+
+    def analytics(self, sql: str) -> list[dict]:
+        if not self._duckdb_path:
+            raise RuntimeError(
+                "DuckDB analytics not available — DuckDB initialization failed "
+                "or module not installed"
+            )
+        with self._db_lock:
+            dk = self._duckdb_conn
+            result = dk.execute(sql)
+            columns = [desc[0] for desc in result.description]
+            rows = [dict(zip(columns, row)) for row in result.fetchall()]
+        return rows
+
+    def _sync_duckdb_from_journal(self, entries: list[dict]) -> None:
+        if not self._duckdb_path or not self._duckdb_synced_tables:
+            return
+
+        by_table: dict[str, dict[str, list[int]]] = {}
+        seen_ids: set[int] = set()
+
+        for e in entries:
+            if e["id"] in seen_ids:
+                continue
+            seen_ids.add(e["id"])
+            tbl = e["app_table"]
+            if tbl not in self._duckdb_synced_tables:
+                continue
+            if tbl not in by_table:
+                by_table[tbl] = {"add": [], "delete": []}
+            row_id = e["row_id"]
+            if row_id is not None:
+                by_table[tbl]["delete"].append(row_id)
+                if e["op"] != "row_delete":
+                    by_table[tbl]["add"].append(row_id)
+
+        if not by_table:
+            return
+
+        with self._db_lock:
+            dk = self._duckdb_conn
+            dk.execute(f"ATTACH '{self._db_path}' AS src (TYPE sqlite)")
+            for tbl, ops in by_table.items():
+                quoted_tbl = self._duckdb_quote_identifier(tbl)
+                if ops["delete"]:
+                    ids = ",".join(str(i) for i in ops["delete"])
+                    dk.execute(f"DELETE FROM {quoted_tbl} WHERE id IN ({ids})")
+                if ops["add"]:
+                    ids = ",".join(str(i) for i in ops["add"])
+                    dk.execute(
+                        f"INSERT INTO {quoted_tbl} SELECT * FROM src.{quoted_tbl} WHERE id IN ({ids})"
+                    )
+                count = dk.execute(f"SELECT count(*) FROM {quoted_tbl}").fetchone()[0]
+                dk.execute(
+                    "UPDATE _duckdb_sync SET synced_count = ? WHERE table_name = ?",
+                    (count, tbl),
+                )
+                self._duckdb_synced_tables[tbl]["count"] = count
+            dk.execute("DETACH src")
+
+    def sync_duckdb_table(self, table: str) -> None:
+        _validate_identifier(table, "table")
+        self._full_sync_duckdb_table(table)
+

hybriddb/async_api.py

@@ -0,0 +1,68 @@
+"""Async wrappers for HybridDB."""
+
+import asyncio
+from typing import Any
+
+from hybriddb.types import Column
+
+
+class AsyncMixin:
+    async def acreate_table(self, table: str, columns: dict[str, str | Column]) -> None:
+        await asyncio.to_thread(self.create_table, table, columns)
+
+    async def aadd_column(self, table: str, column: str, col_type: str) -> None:
+        await asyncio.to_thread(self.add_column, table, column, col_type)
+
+    async def adrop_column(self, table: str, column: str) -> None:
+        await asyncio.to_thread(self.drop_column, table, column)
+
+    async def arename_column(self, table: str, old_name: str, new_name: str) -> None:
+        await asyncio.to_thread(self.rename_column, table, old_name, new_name)
+
+    async def ainsert(self, table: str, data: dict, sync: bool = True) -> int | str:
+        return await asyncio.to_thread(self.insert, table, data, sync)
+
+    async def ainsert_batch(self, table: str, rows: list[dict], sync: bool = True) -> list[int | str]:
+        return await asyncio.to_thread(self.insert_batch, table, rows, sync)
+
+    async def aupdate(self, table: str, row_id: int | str, data: dict, sync: bool = True) -> bool:
+        return await asyncio.to_thread(self.update, table, row_id, data, sync)
+
+    async def adelete(self, table: str, row_id: int | str, sync: bool = True) -> bool:
+        return await asyncio.to_thread(self.delete, table, row_id, sync)
+
+    async def aget(self, table: str, row_id: int | str) -> dict | None:
+        return await asyncio.to_thread(self.get, table, row_id)
+
+    async def aquery(
+        self, table: str, where: str = "", params: tuple = (),
+        order_by: str = "", limit: int = 100,
+    ) -> list[dict]:
+        return await asyncio.to_thread(self.query, table, where, params, order_by, limit)
+
+    async def araw_query(self, sql: str, params: tuple = ()) -> list[dict]:
+        return await asyncio.to_thread(self.raw_query, sql, params)
+
+    async def aread_query(self, sql: str, params: tuple = ()) -> list[dict]:
+        return await asyncio.to_thread(self.read_query, sql, params)
+
+    async def acount(self, table: str, where: str = "", params: tuple = ()) -> int:
+        return await asyncio.to_thread(self.count, table, where, params)
+
+    async def asearch(self, table: str, column: str, query: str | None = None, **kwargs: Any) -> list[dict]:
+        return await asyncio.to_thread(self.search, table, column, query, **kwargs)
+
+    async def asearch_all(self, table: str, query: str, **kwargs: Any) -> list[dict]:
+        return await asyncio.to_thread(self.search_all, table, query, **kwargs)
+
+    async def ahealth(self, table: str) -> dict:
+        return await asyncio.to_thread(self.health, table)
+
+    async def areconcile(self, table: str) -> dict:
+        return await asyncio.to_thread(self.reconcile, table)
+
+    async def aprocess_journal(self, limit: int = 5000) -> int:
+        return await asyncio.to_thread(self.process_journal, limit)
+
+    async def aclose(self) -> None:
+        await asyncio.to_thread(self.close)

hybriddb/crud.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import sqlite3
+import struct
+import tempfile
+import uuid
+from collections import defaultdict
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from hybriddb.embedding import EMBEDDING_DIM
+from hybriddb.types import Column, SearchMode
+from hybriddb.utils import (
+    CHROMA_BATCH,
+    JOURNAL_CAP,
+    RRF_K,
+    _CHROMA_INDEX_MAX_ELEMENTS,
+    _CHROMA_INDEX_MAX_M0,
+    _CHROMA_INDEX_WARN_FACTOR,
+    _CHROMA_REBUILD_BATCH,
+    _SKIP_SEARCH_COLUMNS,
+    _SYSTEM_TABLES,
+    _coerce_search_mode,
+    _column_spec,
+    _is_safe_identifier,
+    _now_iso,
+    _sanitize_fts_query,
+    _validate_identifier,
+    _validate_order_by,
+)
+
+logger = logging.getLogger("hybriddb")
+
+class CrudMixin:
+    def _row_to_metadata(self, table: str, row: dict[str, Any]) -> dict[str, Any]:
+        meta = self._table_meta(table)
+        if not meta:
+            return {}
+        result = {}
+        for col, ctype in meta["columns"].items():
+            if ctype in ("LONGTEXT", "JSON"):
+                continue
+            val = row.get(col)
+            if val is None:
+                continue
+            base = ctype.replace("_PK", "")
+            if base == "BOOLEAN":
+                result[col] = bool(val)
+            else:
+                result[col] = val
+        return result
+
+    def insert(
+        self, table: str, data: dict, sync: bool = True,
+        skip_journal_columns: set[str] | None = None,
+    ) -> int | str:
+        _validate_identifier(table, "table")
+        meta = self._table_meta(table)
+        if not meta:
+            raise ValueError(f"Table '{table}' not found")
+        filtered = {k: v for k, v in data.items() if k in meta["columns"]}
+        columns = list(filtered.keys())
+        placeholders = ", ".join("?" * len(columns))
+        values = list(filtered.values())
+
+        with self._connect() as cur:
+            cur.execute(f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})", values)
+            internal_rowid = cur.lastrowid
+            has_auto_id = self._has_autoincrement_id(table)
+            if has_auto_id:
+                user_pk = internal_rowid
+                row = dict(cur.execute(f"SELECT * FROM {table} WHERE id = ?", (user_pk,)).fetchone())
+            elif "id" in filtered:
+                user_pk = filtered["id"]
+                row = dict(cur.execute(f"SELECT * FROM {table} WHERE id = ?", (user_pk,)).fetchone())
+            else:
+                user_pk = internal_rowid
+                row = dict(cur.execute(f"SELECT * FROM {table} WHERE rowid = ?", (internal_rowid,)).fetchone())
+
+            metadata = self._row_to_metadata(table, row)
+            now = _now_iso()
+            for col in self._get_longtext_columns(table):
+                if skip_journal_columns and col in skip_journal_columns:
+                    continue
+                cur.execute(
+                    "INSERT INTO _journal (app_table, row_id, column_name, op, data, metadata, created_at) "
+                    "VALUES (?, ?, ?, 'add', ?, ?, ?)",
+                    (table, internal_rowid, col, row.get(col, ""), json.dumps(metadata), now),
+                )
+            cur.execute(
+                "INSERT INTO _journal (app_table, row_id, op, data, created_at) "
+                "VALUES (?, ?, 'row_add', ?, ?)",
+                (table, internal_rowid, json.dumps(dict(row), default=str), now),
+            )
+        if sync:
+            self._process_journal()
+        return user_pk or 0
+
+    def row_to_metadata(self, table: str, row: dict[str, Any]) -> dict[str, Any]:
+        _validate_identifier(table, "table")
+        return self._row_to_metadata(table, row)
+
+    def vector_upsert(
+        self, collection_name: str, row_id: int | str, document: str,
+        embedding: list[float], metadata: dict[str, Any] | None = None,
+    ) -> bool:
+        _validate_identifier(collection_name, "collection")
+        collection = self._get_collection(collection_name)
+        if collection is None:
+            return False
+        collection.upsert(
+            ids=[str(row_id)], embeddings=[embedding], documents=[document],
+            metadatas=[metadata or {}],
+        )
+        return True
+
+    def insert_batch(self, table: str, rows: list[dict], sync: bool = True) -> list[int | str]:
+        _validate_identifier(table, "table")
+        if len(rows) > JOURNAL_CAP:
+            logger.warning("insert_batch.large_batch table=%s rows=%d limit=%d", table, len(rows), JOURNAL_CAP)
+        meta = self._table_meta(table)
+        if not meta:
+            raise ValueError(f"Table '{table}' not found")
+        ids: list[int | str] = []
+        with self._connect() as cur:
+            now = _now_iso()
+            for data in rows:
+                filtered = {k: v for k, v in data.items() if k in meta["columns"]}
+                columns = list(filtered.keys())
+                placeholders = ", ".join("?" * len(columns))
+                values = list(filtered.values())
+                cur.execute(f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})", values)
+                internal_rowid = cur.lastrowid
+                has_auto_id = self._has_autoincrement_id(table)
+                if has_auto_id:
+                    user_pk = internal_rowid
+                    row = dict(cur.execute(f"SELECT * FROM {table} WHERE id = ?", (user_pk,)).fetchone())
+                elif "id" in filtered:
+                    user_pk = filtered["id"]
+                    row = dict(cur.execute(f"SELECT * FROM {table} WHERE id = ?", (user_pk,)).fetchone())
+                else:
+                    assert internal_rowid is not None
+                    user_pk = internal_rowid
+                    row = dict(cur.execute(f"SELECT * FROM {table} WHERE rowid = ?", (internal_rowid,)).fetchone())
+                ids.append(user_pk)
+                metadata = self._row_to_metadata(table, row)
+                for col in self._get_longtext_columns(table):
+                    cur.execute(
+                        "INSERT INTO _journal (app_table, row_id, column_name, op, data, metadata, created_at) "
+                        "VALUES (?, ?, ?, 'add', ?, ?, ?)",
+                        (table, internal_rowid, col, row.get(col, ""), json.dumps(metadata), now),
+                    )
+                cur.execute(
+                    "INSERT INTO _journal (app_table, row_id, op, data, created_at) "
+                    "VALUES (?, ?, 'row_add', ?, ?)",
+                    (table, internal_rowid, json.dumps(dict(row), default=str), now),
+                )
+        if sync:
+            self._process_journal()
+        return ids
+
+    def update(self, table: str, row_id: int | str, data: dict, sync: bool = True) -> bool:
+        _validate_identifier(table, "table")
+        meta = self._table_meta(table)
+        if not meta:
+            raise ValueError(f"Table '{table}' not found")
+        filtered = {k: v for k, v in data.items() if k in meta["columns"]}
+        if not filtered:
+            return False
+
+        with self._connect() as cur:
+            internal_rowid = self._resolve_internal_rowid(cur, table, row_id)
+            if internal_rowid is None:
+                return False
+            set_clause = ", ".join(f"{k} = ?" for k in filtered.keys())
+            cur.execute(f"UPDATE {table} SET {set_clause} WHERE id = ?", list(filtered.values()) + [row_id])
+            if cur.rowcount == 0:
+                return False
+            row = dict(cur.execute(f"SELECT * FROM {table} WHERE id = ?", (row_id,)).fetchone())
+            metadata = self._row_to_metadata(table, row)
+            for col in self._get_longtext_columns(table):
+                now = _now_iso()
+                cur.execute(
+                    "INSERT INTO _journal (app_table, row_id, column_name, op, data, metadata, created_at) "
+                    "VALUES (?, ?, ?, 'update', ?, ?, ?)",
+                    (table, internal_rowid, col, row.get(col, ""), json.dumps(metadata), now),
+                )
+            now = _now_iso()
+            cur.execute(
+                "INSERT INTO _journal (app_table, row_id, op, data, created_at) "
+                "VALUES (?, ?, 'row_update', ?, ?)",
+                (table, internal_rowid, json.dumps(dict(row), default=str), now),
+            )
+        if sync:
+            self._process_journal()
+        return True
+
+    def delete(self, table: str, row_id: int | str, sync: bool = True) -> bool:
+        _validate_identifier(table, "table")
+        with self._connect() as cur:
+            internal_rowid = self._resolve_internal_rowid(cur, table, row_id)
+            if internal_rowid is None:
+                return False
+            cur.execute(f"DELETE FROM {table} WHERE id = ?", (row_id,))
+            if cur.rowcount == 0:
+                return False
+            for col in self._get_longtext_columns(table):
+                now = _now_iso()
+                cur.execute(
+                    "INSERT INTO _journal (app_table, row_id, column_name, op, created_at) "
+                    "VALUES (?, ?, ?, 'delete', ?)",
+                    (table, internal_rowid, col, now),
+                )
+            now = _now_iso()
+            cur.execute(
+                "INSERT INTO _journal (app_table, row_id, op, created_at) "
+                "VALUES (?, ?, 'row_delete', ?)",
+                (table, internal_rowid, now),
+            )
+        if sync:
+            self._process_journal()
+        return True
+
+    def get(self, table: str, row_id: int | str) -> dict | None:
+        _validate_identifier(table, "table")
+        with self._connect() as cur:
+            cur.execute(f"SELECT * FROM {table} WHERE id = ?", (row_id,))
+            row = cur.fetchone()
+        if not row:
+            return None
+        return dict(row)
+
+    def query(
+        self, table: str, where: str = "", params: tuple = (),
+        order_by: str = "", limit: int = 100,
+    ) -> list[dict]:
+        _validate_identifier(table, "table")
+        _validate_order_by(order_by)
+        sql = f"SELECT * FROM {table}"
+        if where:
+            sql += f" WHERE {where}"
+        if order_by:
+            sql += f" ORDER BY {order_by}"
+        sql += f" LIMIT {limit}"
+        with self._connect() as cur:
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+        return [dict(r) for r in rows]
+
+    def raw_query(self, sql: str, params: tuple = ()) -> list[dict]:
+        with self._connect() as cur:
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+        return [dict(r) for r in rows]
+
+    def read_query(self, sql: str, params: tuple = ()) -> list[dict]:
+        """Run a read-only SQL query against SQLite."""
+        first = sql.lstrip().split(None, 1)[0].upper() if sql.strip() else ""
+        if first not in {"SELECT", "WITH", "PRAGMA", "EXPLAIN"}:
+            raise ValueError("read_query only accepts read-only SQL")
+        return self.raw_query(sql, params)
+
+    def count(self, table: str, where: str = "", params: tuple = ()) -> int:
+        _validate_identifier(table, "table")
+        sql = f"SELECT COUNT(*) FROM {table}"
+        if where:
+            sql += f" WHERE {where}"
+        with self._connect() as cur:
+            result = cur.execute(sql, params).fetchone()
+        return result[0] if result else 0
+