dbt-redis-adbc 0.1.0__tar.gz

dbt_redis_adbc-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+__pycache__/
+*.egg-info/
+target/
+dbt_packages/
+logs/
+.user.yml
+data/
+driver/
+build/
+dist/

dbt_redis_adbc-0.1.0/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.5
+Name: dbt-redis-adbc
+Version: 0.1.0
+Summary: dbt adapter for Redis 8.x using the Redis ADBC driver
+Project-URL: Homepage, https://github.com/alberttwong/redis-dbt-adapter-with-project
+Project-URL: Documentation, https://github.com/alberttwong/redis-dbt-adapter-with-project#adapter-reference
+Project-URL: Issues, https://github.com/alberttwong/redis-dbt-adapter-with-project/issues
+Project-URL: Source, https://github.com/alberttwong/redis-dbt-adapter-with-project/tree/main/adapter
+Author: Albert T. Wong
+Keywords: adbc,analytics,dbt,redis
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Database
+Requires-Python: >=3.10
+Requires-Dist: adbc-driver-manager>=1.6
+Requires-Dist: dbt-adapters<2,>=1.16
+Requires-Dist: dbt-core<2,>=1.10
+Requires-Dist: pyarrow>=18
+Description-Content-Type: text/markdown
+
+# dbt-redis-adbc
+
+A community dbt adapter for Redis 8.x using the
+[Redis Arrow Database Connectivity (ADBC) driver](https://github.com/alberttwong/redis-adbc-driver).
+Supports Python-based dbt 1.x; dbt 2.x and the dbt platform are not supported.
+
+## Install
+
+Requires Python 3.10 or later and Redis 8.x with the Query Engine (Search)
+enabled. The adapter is tested against Redis 8.6.2.
+
+```shell
+python -m pip install dbt-redis-adbc
+```
+
+The Python package does not include the native Redis ADBC driver. Download and
+extract the shared library for your platform from the
+[driver releases](https://github.com/alberttwong/redis-adbc-driver/releases).
+The example project uses driver v0.0.14. Prebuilt libraries are available for
+macOS Apple Silicon, Linux x86-64, and Linux arm64.
+
+Set `REDIS_ADBC_DRIVER` to the library's absolute path. See the driver's
+[server requirements and ACL permissions](https://github.com/alberttwong/redis-adbc-driver#server-requirements)
+before connecting.
+
+## Configure
+
+Set your dbt project's `profile` to `redis_project` and add this profile to
+`~/.dbt/profiles.yml`:
+
+```yaml
+redis_project:
+  target: dev
+  outputs:
+    dev:
+      type: redis_adbc
+      driver: "{{ env_var('REDIS_ADBC_DRIVER') }}"
+      uri: "{{ env_var('REDIS_URI', 'redis://localhost:6379/0') }}"
+      username: "{{ env_var('REDIS_USERNAME', '') }}"
+      password: "{{ env_var('REDIS_PASSWORD', '') }}"
+      database: redis
+      schema: analytics
+      threads: 4
+```
+
+Use a `rediss://` URI for TLS. Keep credentials in the separate username and
+password fields. From your project, verify the connection with:
+
+```shell
+dbt debug --profile redis_project
+```
+
+## Features and limitations
+
+- Tables, views, ephemeral models, Arrow bulk-loaded seeds, and snapshots
+- SQL incremental models with `append`, `delete+insert`, `merge`, and `microbatch`
+- Model contracts, persisted relation and column descriptions, and Python table models
+- No SQL transactions or rollback: statements autocommit, and interrupted writes
+  may need cleanup or a rerun
+- No materialized views or SQL grants; configure access with Redis ACLs
+- Python models run in the dbt process and load their inputs into its memory;
+  incremental Python models are not supported
+
+See the [adapter reference](https://github.com/alberttwong/redis-dbt-adapter-with-project#adapter-reference)
+for all profile options, supported features, and limitations. The
+[example project and functional tests](https://github.com/alberttwong/redis-dbt-adapter-with-project)
+demonstrate the adapter with NYC taxi data.

dbt_redis_adbc-0.1.0/README.md

@@ -0,0 +1,67 @@
+# dbt-redis-adbc
+
+A community dbt adapter for Redis 8.x using the
+[Redis Arrow Database Connectivity (ADBC) driver](https://github.com/alberttwong/redis-adbc-driver).
+Supports Python-based dbt 1.x; dbt 2.x and the dbt platform are not supported.
+
+## Install
+
+Requires Python 3.10 or later and Redis 8.x with the Query Engine (Search)
+enabled. The adapter is tested against Redis 8.6.2.
+
+```shell
+python -m pip install dbt-redis-adbc
+```
+
+The Python package does not include the native Redis ADBC driver. Download and
+extract the shared library for your platform from the
+[driver releases](https://github.com/alberttwong/redis-adbc-driver/releases).
+The example project uses driver v0.0.14. Prebuilt libraries are available for
+macOS Apple Silicon, Linux x86-64, and Linux arm64.
+
+Set `REDIS_ADBC_DRIVER` to the library's absolute path. See the driver's
+[server requirements and ACL permissions](https://github.com/alberttwong/redis-adbc-driver#server-requirements)
+before connecting.
+
+## Configure
+
+Set your dbt project's `profile` to `redis_project` and add this profile to
+`~/.dbt/profiles.yml`:
+
+```yaml
+redis_project:
+  target: dev
+  outputs:
+    dev:
+      type: redis_adbc
+      driver: "{{ env_var('REDIS_ADBC_DRIVER') }}"
+      uri: "{{ env_var('REDIS_URI', 'redis://localhost:6379/0') }}"
+      username: "{{ env_var('REDIS_USERNAME', '') }}"
+      password: "{{ env_var('REDIS_PASSWORD', '') }}"
+      database: redis
+      schema: analytics
+      threads: 4
+```
+
+Use a `rediss://` URI for TLS. Keep credentials in the separate username and
+password fields. From your project, verify the connection with:
+
+```shell
+dbt debug --profile redis_project
+```
+
+## Features and limitations
+
+- Tables, views, ephemeral models, Arrow bulk-loaded seeds, and snapshots
+- SQL incremental models with `append`, `delete+insert`, `merge`, and `microbatch`
+- Model contracts, persisted relation and column descriptions, and Python table models
+- No SQL transactions or rollback: statements autocommit, and interrupted writes
+  may need cleanup or a rerun
+- No materialized views or SQL grants; configure access with Redis ACLs
+- Python models run in the dbt process and load their inputs into its memory;
+  incremental Python models are not supported
+
+See the [adapter reference](https://github.com/alberttwong/redis-dbt-adapter-with-project#adapter-reference)
+for all profile options, supported features, and limitations. The
+[example project and functional tests](https://github.com/alberttwong/redis-dbt-adapter-with-project)
+demonstrate the adapter with NYC taxi data.

dbt_redis_adbc-0.1.0/dbt/adapters/redis_adbc/__init__.py

@@ -0,0 +1,13 @@
+from dbt.adapters.base import AdapterPlugin
+
+from dbt.adapters.redis_adbc.connections import RedisAdbcConnectionManager, RedisAdbcCredentials
+from dbt.adapters.redis_adbc.impl import RedisAdbcAdapter
+from dbt.include import redis_adbc
+
+Plugin = AdapterPlugin(
+    adapter=RedisAdbcAdapter,
+    credentials=RedisAdbcCredentials,
+    include_path=redis_adbc.PACKAGE_PATH,
+)
+
+__all__ = ["Plugin", "RedisAdbcAdapter", "RedisAdbcConnectionManager", "RedisAdbcCredentials"]

dbt_redis_adbc-0.1.0/dbt/adapters/redis_adbc/__version__.py

@@ -0,0 +1 @@
+version = "0.1.0"

dbt_redis_adbc-0.1.0/dbt/adapters/redis_adbc/connections.py

@@ -0,0 +1,286 @@
+"""Connection management: dbt <-> Redis ADBC driver via adbc_driver_manager's DB-API."""
+
+import os
+import sys
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union
+from urllib.parse import urlsplit, urlunsplit
+
+import adbc_driver_manager
+import adbc_driver_manager.dbapi as dbapi
+from adbc_driver_manager import AdbcStatusCode
+from dbt_common.exceptions import DbtDatabaseError, DbtRuntimeError
+
+from dbt.adapters.contracts.connection import AdapterResponse, Connection, ConnectionState, Credentials
+from dbt.adapters.events.logging import AdapterLogger
+from dbt.adapters.sql import SQLConnectionManager
+
+logger = AdapterLogger("RedisAdbc")
+
+# Driver options that profile fields set (option -> field), so
+# driver_options can't.
+_PROFILE_OPTIONS = {
+    "uri": "uri",
+    "username": "username",
+    "password": "password",
+    "adbc.redis.default_schema": "schema",
+    "adbc.redis.aggregate_pushdown": "aggregate_pushdown",
+    "adbc.redis.rename_rekey": "rename_rekey",
+    "adbc.redis.time_zone": "time_zone",
+    "adbc.redis.read_timeout": "read_timeout",
+    "adbc.redis.write_timeout": "write_timeout",
+}
+
+# Unknown profile keys already warned about (profiles can load more than once).
+_warned_keys: Set[str] = set()
+
+
+def _option_value(value: Any) -> str:
+    # ADBC options are strings; booleans as the driver spells them.
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+@dataclass
+class RedisAdbcCredentials(Credentials):
+    # Path to libadbc_driver_redis.{so,dylib,dll}; falls back to $REDIS_ADBC_DRIVER.
+    driver: Optional[str] = None
+    uri: str = "redis://localhost:6379/0"
+    username: Optional[str] = None
+    password: Optional[str] = None
+    # exact | all | none (see the driver README).
+    aggregate_pushdown: Optional[str] = None
+    # Move a renamed table's rows to its new name's keys (driver option
+    # adbc.redis.rename_rekey). dbt renames every table it builds, so this
+    # copies every row once more per build, in exchange for key names that
+    # match the table's.
+    rename_rekey: Optional[bool] = None
+    # The session time zone (adbc.redis.time_zone): what current_timestamp's
+    # text, localtimestamp and TIMESTAMP <-> TIMESTAMP WITH TIME ZONE
+    # conversions use. UTC by default.
+    time_zone: Optional[str] = None
+    # How long the client waits for each reply, and to send each command:
+    # "30s", "10m", a number of seconds, or 0 for no timeout.
+    read_timeout: Optional[Union[str, int]] = None
+    write_timeout: Optional[Union[str, int]] = None
+    # Any other driver option, passed as is (see the driver README's Options).
+    driver_options: Optional[Dict[str, Any]] = None
+
+    @classmethod
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
+        # The driver exposes a single catalog named "redis".
+        data.setdefault("database", "redis")
+        # dbt accepts profile keys the adapter doesn't have, and drops them.
+        for key in sorted(data.keys() - cls.__dataclass_fields__.keys() - _warned_keys):
+            _warned_keys.add(key)
+            logger.warning(f"`{key}` isn't a redis_adbc profile option; it's ignored")
+        for option, value in (data.get("driver_options") or {}).items():
+            if option in _PROFILE_OPTIONS:
+                raise DbtRuntimeError(
+                    f"driver_options can't set {option}; use the profile's `{_PROFILE_OPTIONS[option]}`"
+                )
+            if not isinstance(value, (str, int, float, bool)):
+                raise DbtRuntimeError(f"driver_options: {option} must be a string, number or boolean")
+        return data
+
+    @classmethod
+    def validate(cls, data):
+        # dbt validates the profile before __pre_deserialize__ sets the default.
+        super().validate({"database": "redis", **data})
+
+    @property
+    def type(self) -> str:
+        return "redis_adbc"
+
+    @property
+    def unique_field(self) -> str:
+        # Hashed for dbt's anonymous usage stats: the server, not credentials.
+        return _redact_uri(self.uri, keep_user=False)
+
+    def _connection_keys(self) -> Tuple[str, ...]:
+        return (
+            "driver",
+            "uri",
+            "database",
+            "schema",
+            "username",
+            "aggregate_pushdown",
+            "rename_rekey",
+            "time_zone",
+            "read_timeout",
+            "write_timeout",
+            "driver_options",
+        )
+
+    def connection_info(self, *, with_aliases: bool = False) -> Iterable[Tuple[str, Any]]:
+        # dbt debug prints these and logs them: never the URI's password.
+        for key, value in super().connection_info(with_aliases=with_aliases):
+            yield key, _redact_uri(value) if key == "uri" and value else value
+
+    def driver_path(self) -> str:
+        path = self.driver or os.environ.get("REDIS_ADBC_DRIVER")
+        if not path:
+            raise DbtRuntimeError(
+                "No Redis ADBC driver configured: set `driver` in profiles.yml or $REDIS_ADBC_DRIVER"
+            )
+        path = os.path.expanduser(path)
+        # `driver/libadbc_driver_redis` means the platform's shared library.
+        if not os.path.splitext(os.path.basename(path))[1]:
+            path += {"darwin": ".dylib", "win32": ".dll"}.get(sys.platform, ".so")
+        return path
+
+    def db_kwargs(self) -> Dict[str, str]:
+        """The driver's database options for this profile."""
+        kwargs = {option: _option_value(v) for option, v in (self.driver_options or {}).items()}
+        for option, name in _PROFILE_OPTIONS.items():
+            value = getattr(self, name)
+            # None, or "" from an unset env_var, leaves the driver's default.
+            if value is not None and value != "":
+                kwargs[option] = _option_value(value)
+        return kwargs
+
+
+def _redact_uri(uri: str, keep_user: bool = True) -> str:
+    """The URI with its password replaced by ****, or (keep_user=False)
+    without any credentials."""
+    try:
+        parts = urlsplit(uri)
+        host = parts.hostname or ""
+        if parts.port:
+            host = f"{host}:{parts.port}"
+    except ValueError:
+        return "<unparseable uri>"
+    if keep_user and (parts.username or parts.password):
+        user = parts.username or ""
+        host = f"{user}:****@{host}" if parts.password else f"{user}@{host}"
+    return urlunsplit((parts.scheme, host, parts.path, parts.query, parts.fragment))
+
+
+class RedisAdbcHandle:
+    """Wraps an ADBC DB-API connection so cursors dbt never closes get closed.
+
+    ADBC refuses to close a connection while it still has open statements.
+    """
+
+    def __init__(self, conn: dbapi.Connection):
+        self.conn = conn
+        self._cursors: list = []
+
+    def cursor(self):
+        # Close finished cursors eagerly so statements don't pile up.
+        self._cursors = [c for c in self._cursors if not c._closed]
+        cur = self.conn.cursor()
+        self._cursors.append(cur)
+        return cur
+
+    def commit(self):
+        pass  # autocommit only
+
+    def rollback(self):
+        pass  # no transactions
+
+    def cancel(self):
+        """Cancel the statement running on this connection, or the streamed
+        result it is reading (from another thread). Either then fails with
+        CANCELLED (see exception_handler).
+
+        A cursor with neither refuses with INVALID_STATE.
+        """
+        for cur in list(self._cursors):
+            if cur._closed:
+                continue
+            try:
+                cur.adbc_cancel()
+            except adbc_driver_manager.Error as e:
+                if e.status_code != AdbcStatusCode.INVALID_STATE:
+                    logger.warning(f"Redis ADBC: cancelling a statement failed: {e}")
+
+    def close(self):
+        for cur in self._cursors:
+            try:
+                cur.close()
+            except Exception:
+                pass
+        self._cursors = []
+        self.conn.close()
+
+
+class RedisAdbcSession:
+    """What a Python model gets as `session`: the thread's ADBC DB-API
+    connection, except that its cursors go through the handle (so dbt
+    closes them, and Ctrl-C cancels them) and it can't be closed."""
+
+    def __init__(self, handle: RedisAdbcHandle):
+        self._handle = handle
+
+    def cursor(self):
+        return self._handle.cursor()
+
+    def close(self):
+        raise DbtRuntimeError("A Python model's session is dbt's connection; it can't be closed")
+
+    def __getattr__(self, name):
+        return getattr(self._handle.conn, name)
+
+
+class RedisAdbcConnectionManager(SQLConnectionManager):
+    TYPE = "redis_adbc"
+
+    @classmethod
+    def open(cls, connection: Connection) -> Connection:
+        if connection.state == ConnectionState.OPEN:
+            return connection
+        creds: RedisAdbcCredentials = connection.credentials
+
+        def connect():
+            conn = dbapi.connect(driver=creds.driver_path(), db_kwargs=creds.db_kwargs(), autocommit=True)
+            return RedisAdbcHandle(conn)
+
+        return cls.retry_connection(
+            connection,
+            connect=connect,
+            logger=logger,
+            retryable_exceptions=[adbc_driver_manager.OperationalError],
+        )
+
+    @contextmanager
+    def exception_handler(self, sql: str):
+        try:
+            yield
+        except adbc_driver_manager.Error as e:
+            if e.status_code == AdbcStatusCode.CANCELLED:
+                # Cancelled (Ctrl-C, --fail-fast): not the database's error.
+                # A write keeps the rows it wrote before the cancel.
+                logger.debug(f"Redis ADBC: cancelled: {e}")
+                raise DbtRuntimeError(str(e).strip()) from e
+            logger.debug(f"Redis ADBC error: {e}")
+            raise DbtDatabaseError(str(e).strip()) from e
+        except Exception as e:
+            if isinstance(e, (DbtRuntimeError, DbtDatabaseError)):
+                raise
+            raise DbtRuntimeError(str(e)) from e
+
+    def execute(self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None):
+        # The driver streams a long result (over 65,536 rows), so an error
+        # past its first batch comes from fetching the rows, which dbt does
+        # outside add_query's exception_handler.
+        with self.exception_handler(sql):
+            return super().execute(sql, auto_begin=auto_begin, fetch=fetch, limit=limit)
+
+    def cancel(self, connection: Connection):
+        # dbt calls this from the main thread on Ctrl-C and --fail-fast.
+        connection.handle.cancel()
+
+    @classmethod
+    def get_response(cls, cursor: Any) -> AdapterResponse:
+        rows = cursor.rowcount if cursor is not None else -1
+        return AdapterResponse(_message="OK", rows_affected=rows if rows is not None else -1)
+
+    @classmethod
+    def data_type_code_to_name(cls, type_code) -> str:
+        from dbt.adapters.redis_adbc.impl import arrow_type_to_sql
+
+        return arrow_type_to_sql(type_code)