dbt-benostreamdb 1.8.0__tar.gz

dbt_benostreamdb-1.8.0/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: dbt-benostreamdb
+Version: 1.8.0
+Summary: The BenoStreamDB adapter plugin for dbt
+Author-email: Richard Albright <rla3rd@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/benolabsai/benostreamdb
+Project-URL: Repository, https://github.com/benolabsai/benostreamdb
+Project-URL: Documentation, https://github.com/benolabsai/benostreamdb/blob/main/dbt-benostreamdb/README.md
+Keywords: dbt,benostreamdb,iceberg,vector-search,analytics
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: dbt-core<1.9.0,>=1.8.0
+Requires-Dist: benostreamdb>=0.12.0
+Requires-Dist: pyarrow>=15.0.0
+Requires-Dist: pandas
+
+# dbt Adapter for BenoStreamDB (`dbt-benostreamdb`)
+
+`dbt-benostreamdb` is the official dbt adapter enabling analytics engineering, transformation models, and data lineage directly against BenoStreamDB.
+
+## Features
+
+- **Table & View Materializations**: Build standard and optimized Iceberg-format tables.
+- **Incremental Materializations**: Efficient append and merge updates leveraging BenoStreamDB's primary key upserts and delete vector acceleration.
+- **Vector & Graph Extensions**: Built-in macros for vector distance scoring, hybrid search, and graph relationship traversals (`benostreamdb.vector_search(...)`, `benostreamdb.graph_neighbors(...)`).
+
+## Installation
+
+```bash
+pip install dbt-benostreamdb
+```
+
+## Configuration (`profiles.yml`)
+
+```yaml
+benostreamdb_profile:
+  target: dev
+  outputs:
+    dev:
+      type: benostreamdb
+      database: default
+      schema: public
+      threads: 4
+      uri: "file:///path/to/benostreamdb/warehouse"
+```

dbt_benostreamdb-1.8.0/README.md

@@ -0,0 +1,29 @@
+# dbt Adapter for BenoStreamDB (`dbt-benostreamdb`)
+
+`dbt-benostreamdb` is the official dbt adapter enabling analytics engineering, transformation models, and data lineage directly against BenoStreamDB.
+
+## Features
+
+- **Table & View Materializations**: Build standard and optimized Iceberg-format tables.
+- **Incremental Materializations**: Efficient append and merge updates leveraging BenoStreamDB's primary key upserts and delete vector acceleration.
+- **Vector & Graph Extensions**: Built-in macros for vector distance scoring, hybrid search, and graph relationship traversals (`benostreamdb.vector_search(...)`, `benostreamdb.graph_neighbors(...)`).
+
+## Installation
+
+```bash
+pip install dbt-benostreamdb
+```
+
+## Configuration (`profiles.yml`)
+
+```yaml
+benostreamdb_profile:
+  target: dev
+  outputs:
+    dev:
+      type: benostreamdb
+      database: default
+      schema: public
+      threads: 4
+      uri: "file:///path/to/benostreamdb/warehouse"
+```

dbt_benostreamdb-1.8.0/dbt/adapters/benostreamdb/__init__.py

@@ -0,0 +1,12 @@
+from dbt.adapters.base import AdapterPlugin
+from dbt.include import benostreamdb
+
+from .connections import BenoStreamDBConnectionManager
+from .connections import BenoStreamDBCredentials
+from .impl import BenoStreamDBAdapter
+
+Plugin = AdapterPlugin(
+    adapter=BenoStreamDBAdapter,
+    credentials=BenoStreamDBCredentials,
+    include_path=benostreamdb.PACKAGE_PATH,
+)

dbt_benostreamdb-1.8.0/dbt/adapters/benostreamdb/__version__.py

@@ -0,0 +1,4 @@
+# dbt adapters version as `<dbt-core-major>.<minor>.<patch>` so pip can resolve
+# the adapter for the installed dbt-core (e.g. 1.8.x supports dbt-core 1.8.x).
+# This is independent of the BenoStreamDB engine version.
+version = "1.8.0"

dbt_benostreamdb-1.8.0/dbt/adapters/benostreamdb/connections.py

@@ -0,0 +1,206 @@
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Optional, Tuple, Any
+
+from dbt.adapters.contracts.connection import Credentials, Connection, AdapterResponse
+from dbt.adapters.sql import SQLConnectionManager
+from dbt.exceptions import DbtRuntimeError
+from dbt.adapters.events.logging import AdapterLogger
+
+import benostreamdb
+
+logger = AdapterLogger("BenoStreamDB")
+
+
+@dataclass
+class BenoStreamDBCredentials(Credentials):
+    path: str = ":memory:"
+    database: Optional[str] = "benostreamdb"
+    # Unified lakehouse default schema (matches the engine's DEFAULT_SCHEMA,
+    # Spark's default namespace, and Trino's default schema).
+    schema: str = "default"
+    # Optional GPU device for the embedded engine:
+    # auto | cpu | cuda[:N] | mps | intel | rocm.
+    gpu_device: Optional[str] = None
+
+    @property
+    def type(self):
+        return "benostreamdb"
+
+    @property
+    def unique_field(self):
+        return self.path
+
+    def _connection_keys(self):
+        return ("path", "database", "schema", "gpu_device")
+
+
+class BenoStreamDBCursor:
+    def __init__(self, session):
+        self.session = session
+        self.description = None
+        self._results = []
+
+    @staticmethod
+    def _prepare_sql(sql: str) -> str:
+        """Normalize a dbt-issued SQL script for the engine's single-statement
+        `session.sql` guard (which rejects ';' and '--' anywhere).
+
+        dbt materializations legitimately emit line comments and a trailing
+        semicolon. We strip `--` line comments (quote-aware, so `--` inside a
+        string/identifier survives), drop a single trailing semicolon, and
+        still reject any *embedded* semicolon so the multi-statement injection
+        protection is preserved end to end.
+        """
+        out = []
+        i, n = 0, len(sql)
+        in_str = None
+        while i < n:
+            c = sql[i]
+            if in_str is not None:
+                out.append(c)
+                if c == in_str:
+                    if i + 1 < n and sql[i + 1] == in_str:  # doubled-quote escape
+                        out.append(sql[i + 1])
+                        i += 2
+                        continue
+                    in_str = None
+                i += 1
+                continue
+            if c in ("'", '"'):
+                in_str = c
+                out.append(c)
+                i += 1
+                continue
+            if c == "-" and i + 1 < n and sql[i + 1] == "-":
+                nl = sql.find("\n", i)
+                i = n if nl == -1 else nl  # skip comment, keep the newline
+                continue
+            out.append(c)
+            i += 1
+        prepared = "".join(out).strip()
+        if prepared.endswith(";"):
+            prepared = prepared[:-1].rstrip()
+        if ";" in prepared:
+            raise DbtRuntimeError(
+                "BenoStreamDB executes one statement at a time; "
+                "embedded ';' is not supported."
+            )
+        return prepared
+
+    def execute(self, sql: str, bindings=None):
+        if bindings:
+            # Note: dbt-core handles most string formatting. For complex bindings, DBAPI compliance would be needed.
+            pass
+
+        sql = self._prepare_sql(sql)
+        try:
+            arrow_table = self.session.sql(sql)
+            if arrow_table is not None and hasattr(arrow_table, "schema"):
+                self.description = [(f.name, f.type) for f in arrow_table.schema]
+                df = arrow_table.to_pandas()
+                self._results = [tuple(x) for x in df.to_numpy()]
+            else:
+                self.description = None
+                self._results = []
+        except Exception as e:
+            raise Exception(f"Error executing {sql}: {e}")
+            
+    def fetchall(self):
+        return self._results
+
+    def fetchone(self):
+        if self._results:
+            return self._results.pop(0)
+        return None
+
+    def close(self):
+        pass
+
+
+class BenoStreamDBConnectionWrapper:
+    def __init__(self, session):
+        self.session = session
+        
+    def cursor(self):
+        return BenoStreamDBCursor(self.session)
+        
+    def commit(self):
+        pass
+        
+    def rollback(self):
+        pass
+        
+    def close(self):
+        pass
+
+
+class BenoStreamDBConnectionManager(SQLConnectionManager):
+    TYPE = "benostreamdb"
+    _session = None
+
+    @classmethod
+    def get_session(cls, credentials):
+        if cls._session is None:
+            # Instantiate a single embedded engine session for all connections.
+            # The profile `path` is the warehouse base location so CREATE
+            # TABLE / CREATE TABLE AS SELECT can derive table URIs; the
+            # default ":memory:" maps onto the engine's in-memory store.
+            path = getattr(credentials, "path", None) or ":memory:"
+            warehouse = "memory:/dbt" if path in (":memory:", "") else path
+            cls._session = benostreamdb.Session(warehouse=warehouse)
+            # Pin the GPU device for the embedded engine (profile `gpu_device`).
+            gpu_device = getattr(credentials, "gpu_device", None)
+            if gpu_device:
+                try:
+                    resolved = benostreamdb.set_gpu_device(gpu_device)
+                    logger.debug(f"GPU device set to {resolved}")
+                except Exception as e:
+                    logger.warning(f"Could not set GPU device '{gpu_device}': {e}")
+        return cls._session
+
+    @classmethod
+    def open(cls, connection: Connection) -> Connection:
+        if connection.state == "open":
+            logger.debug("Connection is already open, skipping open.")
+            return connection
+
+        credentials = connection.credentials
+        
+        try:
+            session = cls.get_session(credentials)
+            connection.handle = BenoStreamDBConnectionWrapper(session)
+            connection.state = "open"
+        except Exception as e:
+            logger.error(f"Error initializing BenoStreamDB embedded engine: {e}")
+            connection.handle = None
+            connection.state = "fail"
+            raise DbtRuntimeError(f"Failed to connect to BenoStreamDB: {e}")
+
+        return connection
+
+    @classmethod
+    def get_response(cls, cursor: Any) -> AdapterResponse:
+        return AdapterResponse(_message="OK")
+
+    def cancel(self, connection: Connection):
+        pass
+
+    def begin(self):
+        pass
+
+    def commit(self):
+        pass
+
+    def clear_transaction(self):
+        pass
+
+    @contextmanager
+    def exception_handler(self, sql: str):
+        try:
+            yield
+        except Exception as e:
+            logger.error(f"Error running SQL: {sql}")
+            logger.error(f"Exception: {e}")
+            raise DbtRuntimeError(str(e))
+

dbt_benostreamdb-1.8.0/dbt/adapters/benostreamdb/impl.py

@@ -0,0 +1,97 @@
+# Portions of this code were inspired by fal-ai/dbt-datafusion
+# Copyright (c) 2022 fal-ai
+# Licensed under the MIT License (or corresponding open source license)
+
+from typing import Optional, List, Any
+
+from dbt.adapters.base.relation import BaseRelation, RelationType
+from dbt.adapters.sql import SQLAdapter
+from dbt.adapters.events.logging import AdapterLogger
+import dbt.exceptions
+import pandas as pd
+
+from .connections import BenoStreamDBConnectionManager
+
+logger = AdapterLogger("BenoStreamDB")
+
+class BenoStreamDBAdapter(SQLAdapter):
+    ConnectionManager = BenoStreamDBConnectionManager
+
+    @classmethod
+    def date_function(cls):
+        return "datenow()"
+
+    @classmethod
+    def is_cancelable(cls) -> bool:
+        return False
+
+    def convert_boolean_type(self, column=None):
+        return "boolean"
+
+    def convert_date_type(self, column=None):
+        return "date"
+
+    def convert_datetime_type(self, column=None):
+        raise dbt.exceptions.NotImplementedException(
+            "`datetime` is not implemented for this adapter!"
+        )
+
+    def convert_number_type(self, column=None):
+        return "double"
+
+    def convert_text_type(self, column=None):
+        return "string"
+
+    def convert_time_type(self, column=None):
+        return "time"
+
+    def debug_query(self) -> None:
+        self.execute("SELECT 1 as id")
+
+    # Datafusion / Flight SQL currently prefers standard unquoted or double-quoted identifiers
+    def quote(self, identifier: str) -> str:
+        return f'"{identifier}"'
+
+    def list_schemas(self, database: str) -> List[str]:
+        # Using standard information schema to fetch schemas
+        # DataFusion natively supports information_schema.schemata
+        try:
+            _, results = self.execute(
+                f"SELECT schema_name FROM information_schema.schemata",
+                fetch=True
+            )
+            return [row[0] for row in results]
+        except Exception as e:
+            logger.error(f"Failed to list schemas: {e}")
+            return ["public"]
+
+    def list_relations_without_caching(
+        self, schema_relation: BaseRelation
+    ) -> List[BaseRelation]:
+        schema = schema_relation.schema
+        if not schema:
+            schema = "public"
+            
+        sql = f"""
+            SELECT table_name, table_type
+            FROM information_schema.tables
+            WHERE table_schema = '{schema}'
+        """
+        try:
+            _, results = self.execute(sql, fetch=True)
+            relations = []
+            for row in results:
+                table_name = row[0]
+                table_type = row[1]
+                
+                rel_type = RelationType.Table if "TABLE" in table_type else RelationType.View
+                relations.append(self.Relation.create(
+                    database=schema_relation.database,
+                    schema=schema,
+                    identifier=table_name,
+                    type=rel_type
+                ))
+            return relations
+        except Exception as e:
+            logger.error(f"Failed to list relations: {e}")
+            return []

dbt_benostreamdb-1.8.0/dbt/include/benostreamdb/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

dbt_benostreamdb-1.8.0/dbt/include/benostreamdb/dbt_project.yml

@@ -0,0 +1,5 @@
+name: dbt_benostreamdb
+version: 0.1.0
+config-version: 2
+
+macro-paths: ["macros"]

dbt_benostreamdb-1.8.0/dbt/include/benostreamdb/macros/adapters.sql

@@ -0,0 +1,90 @@
+{% macro benostreamdb__create_table_as(temporary, relation, sql) -%}
+  {%- set sql_header = config.get('sql_header', none) -%}
+  {%- set partition_by = config.get('partition_by') -%}
+  {%- if partition_by is none -%}
+    {%- set partition_by = config.get('partitioned_by') -%}
+  {%- endif -%}
+
+  {{ sql_header if sql_header is not none }}
+
+  create table {{ relation }}
+  {% if partition_by %}
+    {% if partition_by is string %}
+      PARTITIONED BY ({{ partition_by }})
+    {% else %}
+      PARTITIONED BY ({{ partition_by | join(', ') }})
+    {% endif %}
+  {% endif %}
+  as (
+    {{ sql }}
+  );
+{%- endmacro %}
+
+{% macro benostreamdb__create_view_as(relation, sql) -%}
+  {%- set sql_header = config.get('sql_header', none) -%}
+
+  {{ sql_header if sql_header is not none }}
+
+  create view {{ relation }} as (
+    {{ sql }}
+  );
+{%- endmacro %}
+
+{% macro benostreamdb__drop_relation(relation) -%}
+  {% call statement('drop_relation', auto_begin=False) -%}
+    drop {{ relation.type or 'table' }} if exists {{ relation }}
+  {%- endcall %}
+{% endmacro %}
+
+{% macro benostreamdb__rename_relation(from_relation, to_relation) -%}
+  {% call statement('rename_relation') -%}
+    alter {{ from_relation.type }} {{ from_relation }} rename to {{ to_relation }}
+  {%- endcall %}
+{% endmacro %}
+
+{% macro benostreamdb__create_schema(relation) -%}
+  {%- call statement('create_schema') -%}
+    create schema if not exists {{ relation.without_identifier().include(database=False) }}
+  {%- endcall -%}
+{% endmacro %}
+
+{% macro benostreamdb__drop_schema(relation) -%}
+  {%- call statement('drop_schema') -%}
+    drop schema if exists {{ relation.without_identifier().include(database=False) }}
+  {%- endcall -%}
+{% endmacro %}
+
+{% macro benostreamdb__check_schema_exists(information_schema, schema) -%}
+  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}
+    select count(*)
+    from information_schema.schemata
+    where schema_name = '{{ schema }}'
+  {% endcall %}
+  {{ return(load_result('check_schema_exists').table) }}
+{% endmacro %}
+
+{% macro benostreamdb__get_columns_in_relation(relation) -%}
+  {% call statement('get_columns_in_relation', fetch_result=True) %}
+      select
+          column_name,
+          data_type,
+          character_maximum_length,
+          numeric_precision,
+          numeric_scale
+      from information_schema.columns
+      where table_name = '{{ relation.identifier }}'
+        {% if relation.schema %}
+        and table_schema = '{{ relation.schema }}'
+        {% endif %}
+      order by ordinal_position
+  {% endcall %}
+  {% set table = load_result('get_columns_in_relation').table %}
+  {{ return(sql_convert_columns_in_relation(table)) }}
+{% endmacro %}
+
+{% macro benostreamdb__get_create_index_sql(relation, index_dict) -%}
+  {%- set index_columns = index_dict.get('columns', []) -%}
+  {%- set index_name = index_dict.get('name', relation.identifier ~ '_' ~ index_columns | join('_') ~ '_idx') -%}
+  {{ log("BenoStreamDB manages indices natively; skipping CREATE INDEX for " ~ index_name, info=True) }}
+  {% do return(None) %}
+{%- endmacro %}

dbt_benostreamdb-1.8.0/dbt/include/benostreamdb/macros/ddl.sql

@@ -0,0 +1,63 @@
+-- =============================================================================
+-- BenoStreamDB dbt DDL Macros
+-- Index lifecycle (CREATE INDEX / DROP INDEX) over Apache Iceberg tables.
+--
+-- The engine intercepts these statements before DataFusion planning and
+-- dispatches them to the core `Table` API, so they work from any dbt model or
+-- hook. `type` is one of the engine's index algorithms (hnsw, hnsw_pq,
+-- hnsw_tq4, hnsw_tq8, bm25, bloom, bitmap, composite_bitmap, csr_graph,
+-- json_path).
+-- =============================================================================
+
+{% macro create_index(relation, column, type='bitmap', name=none) -%}
+  {{ return(adapter.dispatch('create_index', 'dbt')(relation, column, type, name)) }}
+{%- endmacro %}
+
+{% macro default__create_index(relation, column, type='bitmap', name=none) -%}
+  {{ exceptions.raise_compiler_error("create_index is not supported on this adapter") }}
+{%- endmacro %}
+
+{% macro benostreamdb__create_index(relation, column, type='bitmap', name=none) -%}
+  {%- set idx_name = name if name is not none else relation.identifier ~ '_' ~ column ~ '_idx' -%}
+  create index {{ idx_name }} on {{ relation }} ({{ column }}) using {{ type }}
+{%- endmacro %}
+
+
+{% macro drop_index(relation, name) -%}
+  {{ return(adapter.dispatch('drop_index', 'dbt')(relation, name)) }}
+{%- endmacro %}
+
+{% macro default__drop_index(relation, name) -%}
+  {{ exceptions.raise_compiler_error("drop_index is not supported on this adapter") }}
+{%- endmacro %}
+
+{% macro benostreamdb__drop_index(relation, name) -%}
+  alter table {{ relation }} drop index {{ name }}
+{%- endmacro %}
+
+
+{% macro add_primary_key(relation, columns) -%}
+  {{ return(adapter.dispatch('add_primary_key', 'dbt')(relation, columns)) }}
+{%- endmacro %}
+
+{% macro default__add_primary_key(relation, columns) -%}
+  {{ exceptions.raise_compiler_error("add_primary_key is not supported on this adapter") }}
+{%- endmacro %}
+
+{% macro benostreamdb__add_primary_key(relation, columns) -%}
+  {%- set cols = columns if columns is string else columns | join(', ') -%}
+  alter table {{ relation }} add primary key ({{ cols }})
+{%- endmacro %}
+
+
+{% macro drop_primary_key(relation) -%}
+  {{ return(adapter.dispatch('drop_primary_key', 'dbt')(relation)) }}
+{%- endmacro %}
+
+{% macro default__drop_primary_key(relation) -%}
+  {{ exceptions.raise_compiler_error("drop_primary_key is not supported on this adapter") }}
+{%- endmacro %}
+
+{% macro benostreamdb__drop_primary_key(relation) -%}
+  alter table {{ relation }} drop primary key
+{%- endmacro %}