interloper-sql 0.110.2__tar.gz → 0.111.0__tar.gz

{interloper_sql-0.110.2 → interloper_sql-0.111.0}/PKG-INFO

@@ -1,24 +1,31 @@
 Metadata-Version: 2.3
 Name: interloper-sql
-Version: 0.110.2
-Summary: Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases
+Version: 0.111.0
+Summary: Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: ibis-framework>=12,<13
 Requires-Dist: interloper-core
-Requires-Dist: psycopg[binary]>=3.1
-Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: psycopg[binary]>=3.1 ; extra == 'postgres'
+Requires-Dist: sqlalchemy>=2.0 ; extra == 'postgres'
 Requires-Python: >=3.10
+Provides-Extra: postgres
 Description-Content-Type: text/markdown
 
 # interloper-sql
 
-A SQL database destination for interloper: `SQLDestination`, a
-`DatabaseDestination` over SQLAlchemy Core, and the `SQLConnection` that holds
-the database URL.
+SQL for interloper: `ibis_view`, portable views authored as Ibis expressions,
+and behind the `postgres` extra a SQL database destination, `SQLDestination`,
+a `DatabaseDestination` over SQLAlchemy Core with the `SQLConnection` that
+holds the database URL.
+
+The base package depends on Ibis only, so a package that defines `ibis_view`s
+pulls in no database driver. `interloper-sql[postgres]` adds SQLAlchemy and
+`psycopg[binary]`; the destination and its connection exist only with it.
 
 One backend serves every SQLAlchemy dialect whose driver is installed.
-PostgreSQL works out of the box (the package depends on `psycopg[binary]`);
-MySQL, SQLite, Redshift, SQL Server and the rest need their own driver.
+PostgreSQL works with the extra; MySQL, SQLite, Redshift, SQL Server and the
+rest need their own driver on top.
 
 ## Setup
 
@@ -26,7 +33,7 @@ The connection is a single SQLAlchemy URL:
 
 | Database | URL | Driver to install |
 |----------|-----|-------------------|
-| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | none, ships with the package |
+| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | `interloper-sql[postgres]` |
 | MySQL / MariaDB | `mysql+pymysql://user:password@host:3306/db` | `pymysql` |
 | SQL Server | `mssql+pyodbc://user:password@host/db?driver=ODBC+Driver+18+for+SQL+Server` | `pyodbc` (and the ODBC driver) |
 | Redshift | `redshift+redshift_connector://user:password@host:5439/db` | `sqlalchemy-redshift`, `redshift_connector` |

{interloper_sql-0.110.2 → interloper_sql-0.111.0}/README.md

@@ -1,12 +1,17 @@
 # interloper-sql
 
-A SQL database destination for interloper: `SQLDestination`, a
-`DatabaseDestination` over SQLAlchemy Core, and the `SQLConnection` that holds
-the database URL.
+SQL for interloper: `ibis_view`, portable views authored as Ibis expressions,
+and behind the `postgres` extra a SQL database destination, `SQLDestination`,
+a `DatabaseDestination` over SQLAlchemy Core with the `SQLConnection` that
+holds the database URL.
+
+The base package depends on Ibis only, so a package that defines `ibis_view`s
+pulls in no database driver. `interloper-sql[postgres]` adds SQLAlchemy and
+`psycopg[binary]`; the destination and its connection exist only with it.
 
 One backend serves every SQLAlchemy dialect whose driver is installed.
-PostgreSQL works out of the box (the package depends on `psycopg[binary]`);
-MySQL, SQLite, Redshift, SQL Server and the rest need their own driver.
+PostgreSQL works with the extra; MySQL, SQLite, Redshift, SQL Server and the
+rest need their own driver on top.
 
 ## Setup
 
@@ -14,7 +19,7 @@ The connection is a single SQLAlchemy URL:
 
 | Database | URL | Driver to install |
 |----------|-----|-------------------|
-| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | none, ships with the package |
+| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | `interloper-sql[postgres]` |
 | MySQL / MariaDB | `mysql+pymysql://user:password@host:3306/db` | `pymysql` |
 | SQL Server | `mssql+pyodbc://user:password@host/db?driver=ODBC+Driver+18+for+SQL+Server` | `pyodbc` (and the ODBC driver) |
 | Redshift | `redshift+redshift_connector://user:password@host:5439/db` | `sqlalchemy-redshift`, `redshift_connector` |

{interloper_sql-0.110.2 → interloper_sql-0.111.0}/pyproject.toml

@@ -1,19 +1,24 @@
 [project]
 name = "interloper-sql"
-version = "0.110.2"
-description = "Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
+version = "0.111.0"
+description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "ibis-framework>=12,<13",
     "interloper-core",
-    "psycopg[binary]>=3.1",
-    "sqlalchemy>=2.0",
 ]
 
 [[project.authors]]
 name = "Guillaume Onfroy"
 email = "guillaume@digitlcloud.com"
 
+[project.optional-dependencies]
+postgres = [
+    "psycopg[binary]>=3.1",
+    "sqlalchemy>=2.0",
+]
+
 [project.entry-points."interloper.components"]
 sql = "interloper_sql"
 

{interloper_sql-0.110.2 → interloper_sql-0.111.0}/pyproject.toml.orig

@@ -3,17 +3,19 @@
 # ###############
 [project]
 name = "interloper-sql"
-version = "0.110.2"
-description = "Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
+version = "0.111.0"
+description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
 requires-python = ">=3.10"
 dependencies = [
+    "ibis-framework>=12,<13",
     "interloper-core",
-    "psycopg[binary]>=3.1",
-    "sqlalchemy>=2.0",
 ]
 
+[project.optional-dependencies]
+postgres = ["psycopg[binary]>=3.1", "sqlalchemy>=2.0"]
+
 [project.entry-points."interloper.components"]
 sql = "interloper_sql"
 

interloper_sql-0.111.0/src/interloper_sql/__init__.py

@@ -0,0 +1,24 @@
+"""Interloper SQL integration: portable views, and a SQLAlchemy-backed destination behind the postgres extra."""
+
+from interloper_sql.view import IbisView, ibis_view, union
+
+try:
+    from interloper_sql.connection import SQLConnection
+    from interloper_sql.destination import SQLDestination
+except ModuleNotFoundError as error:
+    # the destination and its connection need the postgres extra; the views do not
+    if error.name != "sqlalchemy":
+        raise
+    __all__ = [
+        "IbisView",
+        "ibis_view",
+        "union",
+    ]
+else:
+    __all__ = [
+        "IbisView",
+        "SQLConnection",
+        "SQLDestination",
+        "ibis_view",
+        "union",
+    ]

{interloper_sql-0.110.2 → interloper_sql-0.111.0}/src/interloper_sql/connection.py

@@ -22,7 +22,7 @@ class SQLConnection(Connection):
     """Connection resource holding the URL of a SQL database.
 
     Any SQLAlchemy dialect whose driver is installed works; PostgreSQL through
-    psycopg ships with the package.
+    psycopg ships with the ``postgres`` extra.
     """
 
     model_config = SettingsConfigDict(env_prefix="sql_")
@@ -31,7 +31,7 @@ class SQLConnection(Connection):
         label="Database URL",
         description="SQLAlchemy URL, e.g. postgresql+psycopg://user:password@host:5432/db",
         info=(
-            "The URL carries the password, so it is stored as a secret. PostgreSQL works out of the box "
+            "The URL carries the password, so it is stored as a secret. PostgreSQL works with the postgres extra "
             "(postgresql+psycopg://); other databases need their SQLAlchemy driver installed, e.g. "
             "mysql+pymysql:// or mssql+pyodbc://."
         ),

interloper_sql-0.111.0/src/interloper_sql/view/__init__.py

@@ -0,0 +1,7 @@
+"""Portable views: Ibis expressions over typed upstream tables, compiled per warehouse dialect."""
+
+from interloper_sql.view.base import IbisDefinition, IbisView, union
+from interloper_sql.view.decorator import ibis_view
+from interloper_sql.view.types import IBIS_DIALECTS, ibis_schema
+
+__all__ = ["IBIS_DIALECTS", "IbisDefinition", "IbisView", "ibis_schema", "ibis_view", "union"]

interloper_sql-0.111.0/src/interloper_sql/view/base.py

@@ -0,0 +1,166 @@
+"""IbisView: a view authored as an Ibis expression over its upstreams' declared schemas."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import TYPE_CHECKING, Any
+
+import ibis
+from interloper.asset import Upstream, View
+from interloper.destination.database import TableReference
+from interloper.errors import ConfigError
+
+from interloper_sql.view.types import IBIS_DIALECTS, ibis_schema
+
+if TYPE_CHECKING:
+    from interloper.asset import Asset
+    from interloper.dag.base import DAG
+    from interloper.partitioning.base import Partition, PartitionWindow
+
+
+class IbisDefinition:
+    """A view definition that compiles an Ibis expression for a dialect."""
+
+    def __init__(self, expression: ibis.Table) -> None:
+        """Wrap an expression.
+
+        Args:
+            expression: The view's query as an Ibis table expression.
+        """
+        self.expression = expression
+
+    def compile(self, dialect: str) -> str:
+        """Render the expression as SQL for a warehouse dialect.
+
+        Args:
+            dialect: The destination's dialect name.
+
+        Returns:
+            The SQL text.
+
+        Raises:
+            ConfigError: If the dialect has no Ibis counterpart.
+        """
+        if dialect not in IBIS_DIALECTS:
+            raise ConfigError(f"No Ibis dialect for '{dialect}'")
+        return str(ibis.to_sql(self.expression, dialect=IBIS_DIALECTS[dialect]))
+
+
+class IbisView(View):
+    """A view whose ``data()`` receives Ibis tables and returns an Ibis expression.
+
+    Each upstream's ``data`` is an ``ibis.Table`` named after the table the
+    destination references and typed from the upstream's declared schema,
+    so the expression is checked as it is built. The result must carry the
+    view's declared columns with their declared types; the destination then
+    compiles it for its own dialect.
+    """
+
+    async def _read_upstreams(
+        self,
+        name: str,
+        targets: list[Asset],
+        dag: DAG | None,
+        partition_or_window: Partition | PartitionWindow | None,
+        metadata: dict[str, Any],
+    ) -> list[Upstream]:
+        """Turn every table reference into a typed Ibis table.
+
+        Args:
+            name: The ``data()`` parameter the upstreams are built for.
+            targets: The bound upstream assets.
+            dag: Forwarded to `View._read_upstreams`.
+            partition_or_window: Forwarded.
+            metadata: Forwarded.
+
+        Returns:
+            One `Upstream` per target, its ``data`` an ``ibis.Table``.
+
+        Raises:
+            ConfigError: If an upstream declares no schema, or its reference is not a table.
+        """
+        upstreams = await super()._read_upstreams(name, targets, dag, partition_or_window, metadata)
+        tables: list[Upstream] = []
+        for upstream in upstreams:
+            reference = upstream.data
+            if not isinstance(reference, TableReference):
+                raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' is not a table: {reference!r}")
+            if reference.schema is None:
+                raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' declares no schema to type its table")
+            *outer, table = reference.parts
+            location = {}
+            if outer:
+                location["database"] = outer[-1]
+            if len(outer) > 1:
+                location["catalog"] = outer[-2]
+            tables.append(
+                Upstream(asset=upstream.asset, data=ibis.table(ibis_schema(reference.schema), name=table, **location))
+            )
+        return tables
+
+    def _definition(self, result: Any) -> IbisDefinition:
+        """Check the expression against the declared schema and wrap it.
+
+        Args:
+            result: The value ``data()`` returned.
+
+        Returns:
+            The compilable definition.
+
+        Raises:
+            ConfigError: If the result is not an Ibis table, the view
+                declares no schema, or the expression's columns differ from
+                the declared ones.
+        """
+        if not isinstance(result, ibis.Table):
+            raise ConfigError(f"View '{self.qualified_key}' returned {type(result).__name__}, not an Ibis table")
+        if self.schema is None:
+            raise ConfigError(f"View '{self.qualified_key}' declares no schema")
+        declared = {name: str(dtype) for name, dtype in ibis_schema(self.schema).items()}
+        produced = {name: str(dtype) for name, dtype in result.schema().items()}
+        if declared != produced:
+            missing = sorted(set(declared) - set(produced))
+            extra = sorted(set(produced) - set(declared))
+            retyped = sorted(name for name in declared.keys() & produced.keys() if declared[name] != produced[name])
+            raise ConfigError(
+                f"View '{self.qualified_key}' declares columns its expression does not produce: "
+                f"missing {missing}, extra {extra}, retyped {retyped}"
+            )
+        return IbisDefinition(result)
+
+
+def union(upstreams: Sequence[Upstream], *columns: str, **stamps: Callable[[Upstream], Any]) -> ibis.Table:
+    """The UNION ALL of every upstream's table, optionally stamping a column per upstream.
+
+    Each table is projected to *columns*, or to every declared column, before
+    the union, so the branches line up by name: a bare ``SELECT *`` unions by
+    position, and two accounts' tables created at different times need not
+    share a column order.
+
+    Args:
+        upstreams: The bound upstreams of a many-valued relation, each carrying an ``ibis.Table``.
+        *columns: The columns to keep from every table, in this order; every declared column by default.
+        **stamps: Column name to a function of the upstream giving that
+            upstream's constant value, such as the account it came from.
+
+    Returns:
+        One table over every upstream.
+
+    Raises:
+        ConfigError: If no upstream is bound; a view over nothing has no columns.
+    """
+    if not upstreams:
+        raise ConfigError("no upstream bound: a view needs at least one")
+    names = list(columns or upstreams[0].data.columns)
+    tables = []
+    for upstream in upstreams:
+        # a projection keeping every column in order compiles to SELECT *, which unions by
+        # position; selecting in reverse forces the column list, and the outer select restores it
+        table = upstream.data.select(*reversed(names))
+        values = {}
+        for name, stamp in stamps.items():
+            stamped = stamp(upstream)
+            is_integer = isinstance(stamped, int) and not isinstance(stamped, bool)
+            values[name] = ibis.literal(stamped, type="int64") if is_integer else ibis.literal(stamped)
+        tables.append(table.mutate(**values))
+    return ibis.union(*tables, distinct=False).select(*names, *stamps)

interloper_sql-0.111.0/src/interloper_sql/view/decorator.py

@@ -0,0 +1,84 @@
+"""Decorator for creating IbisView subclasses from functions."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, cast, overload
+
+from interloper.asset.decorator import build_asset_class
+from interloper.component.decorator import decorate
+from interloper.component.relation import Relation
+
+from interloper_sql.view.base import IbisView
+
+
+@overload
+def ibis_view(fn: Callable[..., Any], /) -> type[IbisView]: ...
+@overload
+def ibis_view(
+    *,
+    relations: dict[str, Any] = ...,
+    **overrides: Any,
+) -> Callable[[Callable[..., Any]], type[IbisView]]: ...
+def ibis_view(
+    fn: Callable[..., Any] | None = None,
+    /,
+    *,
+    relations: dict[str, Any] | None = None,
+    **overrides: Any,
+) -> type[IbisView] | Callable[..., type[IbisView]]:
+    """Create an IbisView subclass from a decorated function.
+
+    Like ``il.view``, but portable: each upstream parameter receives a typed
+    ``ibis.Table`` in ``data``, the function returns an Ibis expression, and
+    the destination compiles it for its own dialect:
+
+    ```py
+    @ibis_view(schema=Totals, relations={"orders": il.Relation("asset", "shop.orders", many=True)})
+    def totals(orders: list[il.Upstream]) -> ibis.Table:
+        rows = union(orders)
+        return rows.group_by(rows.date).aggregate(total=rows.total.sum())
+    ```
+
+    Args:
+        fn: The function to turn into a view, passed positionally when the
+            decorator is used bare. ``None`` in the parenthesised form.
+        relations: As for ``il.asset``.
+        **overrides: As for ``il.asset``; a schema is required.
+
+    Returns:
+        An IbisView subclass with the function as its ``data()`` method.
+    """
+    if fn is not None:
+        return decorate(IbisView, fn, build=_build_ibis_view_class, relations=relations, **overrides)
+
+    def wrapper(fn: Callable[..., Any]) -> type[IbisView]:
+        return decorate(IbisView, fn, build=_build_ibis_view_class, relations=relations, **overrides)
+
+    return wrapper
+
+
+# -- Internals -----------------------------------------------------------------
+
+
+def _build_ibis_view_class(
+    fn: Any,
+    *,
+    classvars: dict[str, Any],
+    fields: dict[str, Any],
+    relations: dict[str, Relation],
+) -> type[IbisView]:
+    """Build an IbisView subclass from a function or method.
+
+    Args:
+        fn: The function (or method) backing the view's ``data()``.
+        classvars: Class-level attributes to set on the generated subclass.
+        fields: Pydantic field values to set on the generated subclass.
+        relations: Relations the decorator declares, name to relation.
+
+    Returns:
+        A dynamically created IbisView subclass.
+    """
+    return cast(
+        "type[IbisView]", build_asset_class(fn, base=IbisView, classvars=classvars, fields=fields, relations=relations)
+    )

interloper_sql-0.111.0/src/interloper_sql/view/types.py

@@ -0,0 +1,46 @@
+"""Ibis's view of interloper's types: schema fields and warehouse dialects, in one place."""
+
+from __future__ import annotations
+
+import datetime as dt
+from decimal import Decimal
+from typing import Any
+
+import ibis
+from interloper.schema import Schema
+
+IBIS_TYPES: dict[Any, str] = {
+    str: "string",
+    int: "int64",
+    float: "float64",
+    bool: "boolean",
+    dt.date: "date",
+    dt.datetime: "timestamp",
+    Decimal: "decimal",
+    bytes: "binary",
+}
+
+IBIS_DIALECTS: dict[str, str] = {
+    "bigquery": "bigquery",
+    "duckdb": "duckdb",
+    "snowflake": "snowflake",
+    "clickhouse": "clickhouse",
+    "databricks": "databricks",
+    "tsql": "mssql",
+    "postgres": "postgres",
+}
+
+
+def ibis_schema(schema: type[Schema]) -> ibis.Schema:
+    """The Ibis schema of a declared schema, every column nullable.
+
+    A field whose type has no Ibis counterpart (a nested model, ``Any``) is
+    typed ``string``: a view selects such a column through, never computes on it.
+
+    Args:
+        schema: The declared schema.
+
+    Returns:
+        The Ibis schema, in declaration order.
+    """
+    return ibis.schema({spec.name: IBIS_TYPES.get(spec.type, "string") for spec in schema.field_specs()})

interloper_sql-0.110.2/src/interloper_sql/__init__.py

@@ -1,9 +0,0 @@
-"""Interloper SQL integration: a SQLAlchemy-backed destination and its connection."""
-
-from interloper_sql.connection import SQLConnection
-from interloper_sql.destination import SQLDestination
-
-__all__ = [
-    "SQLConnection",
-    "SQLDestination",
-]