heavenbase 0.1.0.1.dev0__py3-none-any.whl

heavenbase/__init__.py

@@ -0,0 +1,154 @@
+"""HeavenBase public API.
+
+The root package exposes user-facing workspace, entity, query, prompt, LLM,
+Capsule, Toolkit, interop, grouped backend, and utility entry points. Backend
+registry, handler, storage, and low-level extension contracts live under
+``heavenbase.ext``.
+"""
+
+__all__ = [
+    "__version__",
+    "Array",
+    "Artifact",
+    "Catalog",
+    "capabilities",
+    "backends",
+    "BackendType",
+    "Boolean",
+    "Categorical",
+    "Date",
+    "Entity",
+    "ExternalRef",
+    "Float",
+    "GraphEdge",
+    "HeavenBase",
+    "HyperG",
+    "Identifier",
+    "InlineColumn",
+    "Integer",
+    "InvertedIndex",
+    "Json",
+    "JsonField",
+    "LogicalType",
+    "LongText",
+    "MediumText",
+    "MetaSchema",
+    "Prompt",
+    "QueryBuilder",
+    "QuerySpec",
+    "ResultFrame",
+    "ShortText",
+    "SideTable",
+    "StorageStrategy",
+    "Timestamp",
+    "Vector",
+    "VectorIndex",
+    "field",
+    "fast_prompt_section",
+    "CM_HVNB",
+    "ConfigEngine",
+    "ConfigSpec",
+    "resolve_interpolations",
+    "Database",
+    "DBEngine",
+    "DBSpec",
+    "DB_ENGINE",
+    "DatabaseEngineRegistry",
+    "SQLResponse",
+    "StableRNG",
+    "dumps_json",
+    "loads_json",
+    "md_section",
+    "LLM",
+    "LLMCache",
+    "LLMEmbeddingEntry",
+    "LLMEngine",
+    "LLMImage",
+    "LLMImageEntry",
+    "LLMResponse",
+    "LLMSpec",
+    "LLMTextEntry",
+    "LLMToolCallRepair",
+    "Capsule",
+    "capsule_registry",
+    "Tool",
+    "Toolkit",
+    "InteropIssue",
+    "InteropReport",
+    "InteropSet",
+    "from_daft",
+    "from_numpy",
+    "from_pandas",
+    "from_pyarrow",
+    "from_pydantic",
+    "from_sql_result",
+    "register_interop_type_inferer",
+    "ext",
+]
+
+from heavenbase.utils.config import CM_HVNB
+
+from .version import __version__
+from ._bootstrap import bootstrap as _bootstrap
+
+_bootstrap()
+
+from . import backends as backends
+from .backends import BackendType
+from .discovery import capabilities
+from .frame import ResultFrame
+from .entity import Catalog, Entity, MetaSchema, field
+from .interop import (
+    InteropIssue,
+    InteropReport,
+    InteropSet,
+    from_daft,
+    from_numpy,
+    from_pandas,
+    from_pydantic,
+    from_pyarrow,
+    from_sql_result,
+    register_interop_type_inferer,
+)
+from .utils.llm import LLM, LLMCache, LLMEmbeddingEntry, LLMEngine, LLMImage, LLMImageEntry, LLMResponse, LLMSpec, LLMTextEntry, LLMToolCallRepair
+from .utils.db import Database, DatabaseEngineRegistry, DBEngine, DBSpec, DB_ENGINE, SQLResponse
+from .utils.rng import StableRNG
+from .utils.serialize import dumps_json, loads_json
+from .utils.spec import ConfigEngine, ConfigSpec, resolve_interpolations
+from .utils.strings import md_section
+from .query import QueryBuilder, QuerySpec
+from .strategies import (
+    ExternalRef,
+    GraphEdge,
+    InlineColumn,
+    InvertedIndex,
+    JsonField,
+    SideTable,
+    StorageStrategy,
+    VectorIndex,
+)
+from .types import (
+    Array,
+    Artifact,
+    Boolean,
+    Categorical,
+    Date,
+    Float,
+    HyperG,
+    Identifier,
+    Integer,
+    Json,
+    LogicalType,
+    LongText,
+    MediumText,
+    ShortText,
+    Timestamp,
+    Vector,
+)
+from .workspace import HeavenBase
+from .capsule import Capsule, capsule_registry
+from .toolkit import Tool, Toolkit
+from .prompt import Prompt, fast_prompt_section
+
+
+from . import ext as ext

heavenbase/_bootstrap.py

@@ -0,0 +1,90 @@
+"""Explicit bootstrap — centralised registration entry point.
+
+Called once during ``import heavenbase``. Keeps module-level side effects
+out of individual ``__init__.py`` files. Importing any ``heavenbase.*``
+submodule loads the root package first and therefore runs bootstrap.
+Tests can reset state by calling ``_bootstrap._reset()`` followed by
+``_bootstrap.bootstrap()``.
+"""
+
+from __future__ import annotations
+
+_BOOTSTRAPPED = False
+
+
+def bootstrap() -> None:
+    """Register all built-in logical types, strategies, and handlers."""
+    global _BOOTSTRAPPED
+    if _BOOTSTRAPPED:
+        return
+    _register_types()
+    _register_strategies()
+    _BOOTSTRAPPED = True
+
+
+def _reset() -> None:
+    """Reset bootstrap state (test helper)."""
+    global _BOOTSTRAPPED
+    _BOOTSTRAPPED = False
+
+
+def _register_types() -> None:
+    from .types import (
+        TYPE_FAMILY_FILTER,
+        TYPE_FAMILY_FILTER_ARRAY,
+        TYPE_FAMILY_FILTER_BOOL,
+        TYPE_FAMILY_FILTER_ORDER,
+        TYPE_FAMILY_FILTER_SIDE,
+        TYPE_FAMILY_FILTER_STRUCT,
+        TYPE_FAMILY_FILTER_TEXT,
+        TYPE_FAMILY_FILTER_VECTOR,
+        TYPE_FAMILY_STORAGE_JSON,
+        TYPE_FAMILY_STORAGE_LONG_TEXT,
+        TYPE_FAMILY_STORAGE_SIDE,
+        TYPE_FAMILY_STORAGE_VECTOR,
+        register_logical_type,
+    )
+    from .types.artifact import Artifact
+    from .types.array import Array
+    from .types.boolean import Boolean
+    from .types.categorical import Categorical
+    from .types.date import Date
+    from .types.float import Float
+    from .types.hyperg import HyperG
+    from .types.identifier import Identifier
+    from .types.integer import Integer
+    from .types.json import Json
+    from .types.long_text import LongText
+    from .types.medium_text import MediumText
+    from .types.short_text import ShortText
+    from .types.timestamp import Timestamp
+    from .types.vector import Vector
+
+    register_logical_type(Artifact)
+    register_logical_type(Array, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_ARRAY, TYPE_FAMILY_FILTER_SIDE, TYPE_FAMILY_STORAGE_SIDE))
+    register_logical_type(Boolean, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_BOOL))
+    register_logical_type(Categorical, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_TEXT, TYPE_FAMILY_FILTER_ORDER))
+    register_logical_type(Date, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_ORDER, TYPE_FAMILY_FILTER_TEXT))
+    register_logical_type(Float, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_ORDER))
+    register_logical_type(HyperG, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_STRUCT, TYPE_FAMILY_FILTER_SIDE, TYPE_FAMILY_STORAGE_SIDE))
+    register_logical_type(Identifier, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_TEXT))
+    register_logical_type(Integer, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_ORDER))
+    register_logical_type(Json, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_STRUCT, TYPE_FAMILY_STORAGE_JSON))
+    register_logical_type(LongText, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_TEXT, TYPE_FAMILY_STORAGE_LONG_TEXT))
+    register_logical_type(MediumText, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_TEXT))
+    register_logical_type(ShortText, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_TEXT))
+    register_logical_type(Timestamp, families=(TYPE_FAMILY_FILTER, TYPE_FAMILY_FILTER_ORDER))
+    register_logical_type(Vector, families=(TYPE_FAMILY_FILTER_VECTOR, TYPE_FAMILY_STORAGE_VECTOR))
+
+
+def _register_strategies() -> None:
+    from .strategies import STRATEGY_FAMILY_FILTER, STRATEGY_FAMILY_SIDE_FILTER, STRATEGY_FAMILY_VECTOR, register_strategy
+    from .strategies.base import ExternalRef, GraphEdge, InlineColumn, InvertedIndex, JsonField, SideTable, VectorIndex
+
+    register_strategy(ExternalRef)
+    register_strategy(GraphEdge)
+    register_strategy(InlineColumn, families=(STRATEGY_FAMILY_FILTER,))
+    register_strategy(InvertedIndex, families=(STRATEGY_FAMILY_FILTER,))
+    register_strategy(JsonField, families=(STRATEGY_FAMILY_FILTER,))
+    register_strategy(SideTable, families=(STRATEGY_FAMILY_SIDE_FILTER,))
+    register_strategy(VectorIndex, families=(STRATEGY_FAMILY_VECTOR,))

heavenbase/backends/__init__.py

@@ -0,0 +1,80 @@
+"""Built-in backend implementations."""
+
+__all__ = [
+    "Backend",
+    "BackendRegistry",
+    "BackendType",
+    "backend_families",
+    "backend_identifier_in_family",
+    "backend_identifiers_for",
+    "BACKEND_FAMILY_ALL",
+    "BACKEND_FAMILY_HANDLER",
+    "BACKEND_FAMILY_STORAGE_ROW",
+    "BACKEND_FAMILY_STORAGE_TEXT",
+    "BACKEND_FAMILY_STORAGE_VECTOR",
+    "BACKEND_FAMILY_SEARCH",
+    "BACKEND_FAMILY_SQL",
+    "build_backend",
+    "ChromaBackend",
+    "QueryFragment",
+    "DuckDBBackend",
+    "ElasticsearchBackend",
+    "FileBackend",
+    "InMemBackend",
+    "JsonFileBackend",
+    "LanceBackend",
+    "MilvusBackend",
+    "MsSQLBackend",
+    "MySQLBackend",
+    "OceanBaseBackend",
+    "OracleBackend",
+    "PgVectorBackend",
+    "PickleFileBackend",
+    "PineconeBackend",
+    "HandlerContext",
+    "PostgresBackend",
+    "RowOp",
+    "SQLBackend",
+    "ScanningBackend",
+    "SQLiteBackend",
+    "StarRocksBackend",
+    "TrinoBackend",
+    "get_backend_builder",
+    "load_configured_backends",
+    "register_backend_builder",
+    "register_backend_class",
+    "registered_backend_identifiers",
+]
+
+from .base import Backend, QueryFragment, HandlerContext, RowOp, ScanningBackend
+from .elasticsearch import ElasticsearchBackend
+from .file import FileBackend, JsonFileBackend, PickleFileBackend
+from .inmem import InMemBackend
+from .families import (
+    backend_families,
+    backend_identifier_in_family,
+    backend_identifiers_for,
+    BACKEND_FAMILY_ALL,
+    BACKEND_FAMILY_HANDLER,
+    BACKEND_FAMILY_STORAGE_ROW,
+    BACKEND_FAMILY_STORAGE_TEXT,
+    BACKEND_FAMILY_STORAGE_VECTOR,
+    BACKEND_FAMILY_SEARCH,
+    BACKEND_FAMILY_SQL,
+    register_backend_class,
+)
+from .type_registry import BackendType
+from .registry import BackendRegistry, build_backend, get_backend_builder, load_configured_backends, register_backend_builder, registered_backend_identifiers
+from .sql import (
+    DuckDBBackend,
+    MsSQLBackend,
+    MySQLBackend,
+    OceanBaseBackend,
+    OracleBackend,
+    PostgresBackend,
+    SQLBackend,
+    SQLiteBackend,
+    StarRocksBackend,
+    TrinoBackend,
+)
+from .vector import ChromaBackend, LanceBackend, MilvusBackend, PgVectorBackend, PineconeBackend

heavenbase/backends/base.py

@@ -0,0 +1,280 @@
+"""Base backend protocol and query fragment primitives."""
+
+from __future__ import annotations
+
+__all__ = [
+    "Backend",
+    "QueryFragment",
+    "LOGICAL_OPS",
+    "HandlerContext",
+    "RowOp",
+    "ScanningBackend",
+    "safe_error",
+]
+
+import threading
+from functools import wraps
+
+from heavenbase.utils import Any, Callable, Dict, Iterable, List, Optional, Sequence
+
+from heavenbase.utils import dataclass, field, safe_error
+from heavenbase.utils.config import CM_HVNB
+
+from ..frame import ResultFrame
+from ..schema import EntitySchema
+from ..utils.ids import check_id, check_workspace_id
+
+LOGICAL_OPS = ("and", "or", "not")
+
+
+def backend_locked(fn: Callable[..., Any]) -> Callable[..., Any]:
+    """Run a backend method under the instance re-entrant lock."""
+
+    @wraps(fn)
+    def _wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
+        with self._lock:
+            return fn(self, *args, **kwargs)
+
+    return _wrapped
+
+
+@dataclass
+class RowOp:
+    """One object_id row mutation routed to a backend."""
+
+    entity_id: str
+    object_id: Any
+    values: Dict[str, Any] = field(default_factory=dict)
+    op: str = "upsert"
+
+
+@dataclass
+class QueryFragment:
+    """Backend-ready operation fragment produced by a handler."""
+
+    backend: str
+    op: str
+    field: str
+    payload: Any
+    returns: List[str] = field(default_factory=list)
+    notes: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class HandlerContext:
+    """Handler compile context keyed by backend instance, backend type, and strategy."""
+
+    backend: str
+    type: str
+    strategy: str
+    top_k: int = field(default_factory=lambda: int(CM_HVNB.get("heavenbase.query.near.default_top_k", default=50)))
+    metric: str = field(default_factory=lambda: str(CM_HVNB.get("heavenbase.query.near.default_metric", default="cosine")))
+
+
+class Backend:
+    """Thin physical adapter. It executes fragments; it does not parse QuerySpec."""
+
+    identifier = "backend"
+    binary_storage = "bytes"
+    near_filter_modes_supported: Sequence[str] = ()
+
+    def __init__(self, name: str, ws_id: str | None = None):
+        self.name = check_id(name, "backend instance name")
+        self.type = type(self).identifier
+        self.ws_id = check_workspace_id(ws_id) if ws_id else None
+        self._lock = threading.RLock()
+
+    def bind_workspace(self, ws_id: str) -> "Backend":
+        """Bind this backend instance to a validated workspace namespace."""
+        with self._lock:
+            next_id = check_workspace_id(ws_id)
+            if self.ws_id != next_id:
+                self.ws_id = next_id
+                self._refresh_workspace()
+        return self
+
+    def _refresh_workspace(self) -> None:
+        """Refresh state derived from the workspace namespace."""
+        return None
+
+    def drop(self) -> None:
+        """Drop this backend's current workspace namespace, if it has one."""
+        return None
+
+    def ensure(self, schema: EntitySchema, bindings: Sequence[Any]) -> None:
+        """Prepare storage for an entity schema and its bindings."""
+        return None
+
+    def upsert(self, rows: Sequence[RowOp]) -> None:
+        """Insert, replace, or delete backend rows."""
+        raise NotImplementedError
+
+    def delete(self, entity_id: str, object_ids: Iterable[Any]) -> None:
+        """Delete object_ids for one entity."""
+        self.upsert([RowOp(entity_id, object_id, op="delete") for object_id in object_ids])
+
+    def get_many(self, entity_id: str, object_ids: Iterable[Any]) -> List[Optional[Dict[str, Any]]]:
+        """Return rows for object_ids in input order, using scan as the portable fallback."""
+        ordered_ids = list(object_ids)
+        wanted = set(ordered_ids)
+        rows = {row.get("object_id"): row for row in self.scan(entity_id) if row.get("object_id") in wanted}
+        return [rows.get(row_id) for row_id in ordered_ids]
+
+    def exists_many(self, entity_id: str, object_ids: Iterable[Any]) -> List[bool]:
+        """Return existence flags for object_ids in input order."""
+        return [row is not None for row in self.get_many(entity_id, object_ids)]
+
+    def count(self, entity_id: str) -> int:
+        """Return row count for one entity."""
+        return len(self.scan(entity_id))
+
+    def execute(self, fragment: QueryFragment) -> ResultFrame:
+        """Execute a compiled query fragment and return a ResultFrame."""
+        raise NotImplementedError
+
+    def scan(self, entity_id: str) -> List[Dict[str, Any]]:
+        """Return all rows for one entity as dictionaries."""
+        raise NotImplementedError
+
+    def to_dict(self) -> dict:
+        """Return backend metadata suitable for system catalog rows."""
+        return {"name": self.name, "type": self.type}
+
+    def near_filter_modes(self) -> List[str]:
+        """Return near/filter planning modes this backend type can participate in."""
+        return list(self.near_filter_modes_supported)
+
+    def candidate_id_fragment(self, object_ids: Sequence[Any]) -> QueryFragment:
+        """Return a backend-local object_id filter fragment for candidate prefiltering."""
+        id_set = set(object_ids)
+
+        def _payload(row: dict) -> bool:
+            return row.get("object_id") in id_set
+
+        return QueryFragment(self.name, "in", "object_id", _payload, notes={"mode": "py_scan", "row_payload": _payload, "candidate_ids": list(object_ids)})
+
+    def capabilities(self) -> Dict[str, Any]:
+        """Return backend capabilities exposed through workspace metadata."""
+        modes = self.near_filter_modes()
+        return {
+            "near_filter_modes": modes,
+            "candidate_id_prefilter": "candidate_id_prefilter" in modes,
+            "row_lookup": self.row_lookup_capabilities(),
+            "binary_storage": self.binary_storage,
+        }
+
+    def row_lookup_capabilities(self) -> Dict[str, str]:
+        """Return object_id lookup implementation modes for this backend."""
+        return {"get_many": "scan", "exists_many": "scan", "count": "scan"}
+
+    def to_mcp(
+        self,
+        *,
+        name: Optional[str] = None,
+        workspace_name: Optional[str] = None,
+        description: Optional[str] = None,
+        profile: Optional[str] = None,
+        tools: Optional[Iterable[str]] = None,
+    ) -> Any:
+        """Return a native Toolkit bound to this backend instance."""
+        from ..mcp import _backend_toolkit
+
+        return _backend_toolkit(self, name=name, workspace_name=workspace_name, description=description, profile=profile, tools=tools)
+
+    def to_mcp_json(
+        self,
+        *,
+        name: Optional[str] = None,
+        workspace_name: Optional[str] = None,
+        description: Optional[str] = None,
+        profile: Optional[str] = None,
+        tools: Optional[Iterable[str]] = None,
+        transport: Optional[str] = None,
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+    ) -> str:
+        """Return MCP client config JSON for this backend instance."""
+        from ..mcp import _mcp_transport
+
+        return self.to_mcp(name=name, workspace_name=workspace_name, description=description, profile=profile, tools=tools).to_mcp_json(
+            transport=_mcp_transport(transport),
+            host=host,
+            port=port,
+        )
+
+    def serve(
+        self,
+        *,
+        name: Optional[str] = None,
+        workspace_name: Optional[str] = None,
+        description: Optional[str] = None,
+        profile: Optional[str] = None,
+        tools: Optional[Iterable[str]] = None,
+        transport: Optional[str] = None,
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+        wait: Optional[bool] = None,
+        **server_kwargs: Any,
+    ) -> Any:
+        """Serve this backend instance as an MCP server."""
+        from ..mcp import _mcp_host, _mcp_port, _mcp_transport, _mcp_wait
+
+        return self.to_mcp(name=name, workspace_name=workspace_name, description=description, profile=profile, tools=tools).serve(
+            transport=_mcp_transport(transport),
+            host=_mcp_host(host),
+            port=_mcp_port(port),
+            wait=_mcp_wait(wait),
+            **server_kwargs,
+        )
+
+
+class ScanningBackend(Backend):
+    """Backend base for providers that can scan rows and execute Python fragments."""
+
+    def _row_payload(self, fragment: QueryFragment) -> Callable[[Dict[str, Any]], Any]:
+        payload = fragment.notes.get("row_payload", fragment.payload)
+        if not callable(payload):
+            raise TypeError(f"{type(self).__name__} expects a callable row payload, got {type(payload).__name__}")
+        return payload
+
+    def _row_ok(self, fragment: QueryFragment, row: Dict[str, Any]) -> bool:
+        if fragment.op == "and":
+            return all(self._row_ok(child, row) for child in fragment.payload)
+        if fragment.op == "or":
+            return any(self._row_ok(child, row) for child in fragment.payload)
+        if fragment.op == "not":
+            return not self._row_ok(fragment.payload[0], row)
+        return bool(self._row_payload(fragment)(row))
+
+    def _filter_fragment(self, fragment: QueryFragment) -> Any:
+        return (
+            fragment.payload.get("filter")
+            if isinstance(fragment.payload, dict) and fragment.payload.get("filter") is not None
+            else fragment.notes.get("filter")
+        )
+
+    def _top_k(self, fragment: QueryFragment) -> int:
+        default_top_k = int(CM_HVNB.get("heavenbase.query.near.default_top_k", default=50))
+        return fragment.notes.get("top_k", fragment.payload.get("top_k", default_top_k) if isinstance(fragment.payload, dict) else default_top_k)
+
+    def execute(self, fragment: QueryFragment) -> ResultFrame:
+        """Execute a fragment by scanning rows with Python predicates."""
+        entity_id = fragment.notes.get("entity_id")
+        rows = self.scan(entity_id) if entity_id else []
+        if fragment.op in LOGICAL_OPS:
+            return ResultFrame.from_rows([row for row in rows if self._row_ok(fragment, row)])
+        payload = self._row_payload(fragment)
+        if fragment.op == "near":
+            filt = self._filter_fragment(fragment)
+            scored = []
+            for row in rows:
+                if filt is not None and not self._row_ok(filt, row):
+                    continue
+                score = payload(row)
+                if score is not None:
+                    out = dict(row)
+                    out["score"] = score
+                    scored.append(out)
+            scored.sort(key=lambda row: row.get("score", 0.0), reverse=True)
+            return ResultFrame.from_rows(scored[: self._top_k(fragment)])
+        return ResultFrame.from_rows([row for row in rows if payload(row)])

heavenbase/backends/elasticsearch/__init__.py

@@ -0,0 +1,7 @@
+"""Elasticsearch backend package."""
+
+__all__ = [
+    "ElasticsearchBackend",
+]
+
+from .backend import ElasticsearchBackend