interloper-sql 0.110.2__tar.gz → 0.113.0__tar.gz

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/PKG-INFO

@@ -1,24 +1,31 @@
 Metadata-Version: 2.3
 Name: interloper-sql
-Version: 0.110.2
-Summary: Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases
+Version: 0.113.0
+Summary: Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: ibis-framework>=12,<13
 Requires-Dist: interloper-core
-Requires-Dist: psycopg[binary]>=3.1
-Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: psycopg[binary]>=3.1 ; extra == 'postgres'
+Requires-Dist: sqlalchemy>=2.0 ; extra == 'postgres'
 Requires-Python: >=3.10
+Provides-Extra: postgres
 Description-Content-Type: text/markdown
 
 # interloper-sql
 
-A SQL database destination for interloper: `SQLDestination`, a
-`DatabaseDestination` over SQLAlchemy Core, and the `SQLConnection` that holds
-the database URL.
+SQL for interloper: `ibis_view`, portable views authored as Ibis expressions,
+and behind the `postgres` extra a SQL database destination, `SQLDestination`,
+a `DatabaseDestination` over SQLAlchemy Core with the `SQLConnection` that
+holds the database URL.
+
+The base package depends on Ibis only, so a package that defines `ibis_view`s
+pulls in no database driver. `interloper-sql[postgres]` adds SQLAlchemy and
+`psycopg[binary]`; the destination and its connection exist only with it.
 
 One backend serves every SQLAlchemy dialect whose driver is installed.
-PostgreSQL works out of the box (the package depends on `psycopg[binary]`);
-MySQL, SQLite, Redshift, SQL Server and the rest need their own driver.
+PostgreSQL works with the extra; MySQL, SQLite, Redshift, SQL Server and the
+rest need their own driver on top.
 
 ## Setup
 
@@ -26,7 +33,7 @@ The connection is a single SQLAlchemy URL:
 
 | Database | URL | Driver to install |
 |----------|-----|-------------------|
-| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | none, ships with the package |
+| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | `interloper-sql[postgres]` |
 | MySQL / MariaDB | `mysql+pymysql://user:password@host:3306/db` | `pymysql` |
 | SQL Server | `mssql+pyodbc://user:password@host/db?driver=ODBC+Driver+18+for+SQL+Server` | `pyodbc` (and the ODBC driver) |
 | Redshift | `redshift+redshift_connector://user:password@host:5439/db` | `sqlalchemy-redshift`, `redshift_connector` |

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/README.md

@@ -1,12 +1,17 @@
 # interloper-sql
 
-A SQL database destination for interloper: `SQLDestination`, a
-`DatabaseDestination` over SQLAlchemy Core, and the `SQLConnection` that holds
-the database URL.
+SQL for interloper: `ibis_view`, portable views authored as Ibis expressions,
+and behind the `postgres` extra a SQL database destination, `SQLDestination`,
+a `DatabaseDestination` over SQLAlchemy Core with the `SQLConnection` that
+holds the database URL.
+
+The base package depends on Ibis only, so a package that defines `ibis_view`s
+pulls in no database driver. `interloper-sql[postgres]` adds SQLAlchemy and
+`psycopg[binary]`; the destination and its connection exist only with it.
 
 One backend serves every SQLAlchemy dialect whose driver is installed.
-PostgreSQL works out of the box (the package depends on `psycopg[binary]`);
-MySQL, SQLite, Redshift, SQL Server and the rest need their own driver.
+PostgreSQL works with the extra; MySQL, SQLite, Redshift, SQL Server and the
+rest need their own driver on top.
 
 ## Setup
 
@@ -14,7 +19,7 @@ The connection is a single SQLAlchemy URL:
 
 | Database | URL | Driver to install |
 |----------|-----|-------------------|
-| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | none, ships with the package |
+| PostgreSQL | `postgresql+psycopg://user:password@host:5432/db` | `interloper-sql[postgres]` |
 | MySQL / MariaDB | `mysql+pymysql://user:password@host:3306/db` | `pymysql` |
 | SQL Server | `mssql+pyodbc://user:password@host/db?driver=ODBC+Driver+18+for+SQL+Server` | `pyodbc` (and the ODBC driver) |
 | Redshift | `redshift+redshift_connector://user:password@host:5439/db` | `sqlalchemy-redshift`, `redshift_connector` |

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/pyproject.toml

@@ -1,19 +1,24 @@
 [project]
 name = "interloper-sql"
-version = "0.110.2"
-description = "Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
+version = "0.113.0"
+description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "ibis-framework>=12,<13",
     "interloper-core",
-    "psycopg[binary]>=3.1",
-    "sqlalchemy>=2.0",
 ]
 
 [[project.authors]]
 name = "Guillaume Onfroy"
 email = "guillaume@digitlcloud.com"
 
+[project.optional-dependencies]
+postgres = [
+    "psycopg[binary]>=3.1",
+    "sqlalchemy>=2.0",
+]
+
 [project.entry-points."interloper.components"]
 sql = "interloper_sql"
 

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/pyproject.toml.orig

@@ -3,17 +3,19 @@
 # ###############
 [project]
 name = "interloper-sql"
-version = "0.110.2"
-description = "Interloper SQL integration: a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
+version = "0.113.0"
+description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
 requires-python = ">=3.10"
 dependencies = [
+    "ibis-framework>=12,<13",
     "interloper-core",
-    "psycopg[binary]>=3.1",
-    "sqlalchemy>=2.0",
 ]
 
+[project.optional-dependencies]
+postgres = ["psycopg[binary]>=3.1", "sqlalchemy>=2.0"]
+
 [project.entry-points."interloper.components"]
 sql = "interloper_sql"
 

interloper_sql-0.113.0/src/interloper_sql/__init__.py

@@ -0,0 +1,24 @@
+"""Interloper SQL integration: portable views, and a SQLAlchemy-backed destination behind the postgres extra."""
+
+from interloper_sql.view import IbisView, ibis_view, union
+
+try:
+    from interloper_sql.connection import SQLConnection
+    from interloper_sql.destination import SQLDestination
+except ModuleNotFoundError as error:
+    # the destination and its connection need the postgres extra; the views do not
+    if error.name != "sqlalchemy":
+        raise
+    __all__ = [
+        "IbisView",
+        "ibis_view",
+        "union",
+    ]
+else:
+    __all__ = [
+        "IbisView",
+        "SQLConnection",
+        "SQLDestination",
+        "ibis_view",
+        "union",
+    ]

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/src/interloper_sql/connection.py

@@ -6,7 +6,7 @@ from functools import cached_property
 
 import sqlalchemy
 from interloper.connection import Connection, connection
-from interloper.resource.fields import SecretField, fetch_field_provider
+from interloper.resource import SecretField, fetch_field_provider
 from pydantic_settings import SettingsConfigDict
 from sqlalchemy.engine import Engine
 
@@ -22,7 +22,7 @@ class SQLConnection(Connection):
     """Connection resource holding the URL of a SQL database.
 
     Any SQLAlchemy dialect whose driver is installed works; PostgreSQL through
-    psycopg ships with the package.
+    psycopg ships with the ``postgres`` extra.
     """
 
     model_config = SettingsConfigDict(env_prefix="sql_")
@@ -31,7 +31,7 @@ class SQLConnection(Connection):
         label="Database URL",
         description="SQLAlchemy URL, e.g. postgresql+psycopg://user:password@host:5432/db",
         info=(
-            "The URL carries the password, so it is stored as a secret. PostgreSQL works out of the box "
+            "The URL carries the password, so it is stored as a secret. PostgreSQL works with the postgres extra "
             "(postgresql+psycopg://); other databases need their SQLAlchemy driver installed, e.g. "
             "mysql+pymysql:// or mssql+pyodbc://."
         ),

{interloper_sql-0.110.2 → interloper_sql-0.113.0}/src/interloper_sql/destination.py

@@ -10,11 +10,11 @@ from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
 from typing import Any
 
-from interloper.destination import IOContext, destination
-from interloper.destination.database import DatabaseDestination, PartitionFilter
+from interloper.destination import DatabaseDestination, IOContext, destination
 from interloper.errors import DataNotFoundError
-from interloper.representation import Representation
-from interloper.resource.fields import InputField
+from interloper.partitioning import PartitionFilter
+from interloper.representation import TableData
+from interloper.resource import InputField
 from interloper.schema import FieldSpec
 from pydantic import PrivateAttr
 from sqlalchemy import Column, MetaData, String, Table, and_, cast, func, inspect, select
@@ -186,13 +186,13 @@ class SQLDestination(DatabaseDestination):
             context: IO context carrying the effective schema.
         """
         dataset = self._resolve_dataset(dataset)
-        view = Representation.of(data)
+        table_data = TableData.from_data(data)
         with self._begin() as conn:
             sql_table = self._table(table, dataset, conn)
             if sql_table is None:
-                schema = context.schema if context.schema is not None else view.infer()
+                schema = context.schema if context.schema is not None else table_data.infer()
                 sql_table = self._create_table(table, dataset, schema.field_specs(), conn)
-            rows = _align(view.records, [c.name for c in sql_table.columns], self._ref(table, dataset))
+            rows = _align(table_data.rows, [c.name for c in sql_table.columns], self._ref(table, dataset))
             for start in range(0, len(rows), _BATCH_SIZE):
                 conn.execute(sql_table.insert(), rows[start : start + _BATCH_SIZE])
 
@@ -312,22 +312,23 @@ def _predicate(table: Table, where: PartitionFilter) -> ColumnElement[bool]:
 
     Args:
         table: The reflected table.
-        where: The filter: equality on ``value``, or half-open ``bounds``.
+        where: The filter: membership of ``values``, or half-open ``bounds``.
 
     Returns:
         The ``WHERE`` clause.
     """
     column = table.c[where.column]
-    if where.bounds is not None:
-        start, end = where.bounds
-        return and_(column >= _coerce(column, start), column < _coerce(column, end))
-    return column == _coerce(column, where.value)
+    if where.values is not None:
+        return column.in_([_coerce(column, value) for value in where.values])
+    assert where.bounds is not None
+    start, end = where.bounds
+    return and_(column >= _coerce(column, start), column < _coerce(column, end))
 
 
 def _coerce(column: Column[Any], value: Any) -> Any:
     """Coerce a filter value to the Python type a date or timestamp column binds.
 
-    A partition id arrives as a string (``"2024-01-01"``) and a day's bounds
+    A partition key arrives as a string (``"2024-01-01"``) and a day's bounds
     as dates; bound as-is against a ``DATE`` or ``TIMESTAMP`` column, they
     would be sent as ``VARCHAR`` or rejected by the dialect's type.
 

interloper_sql-0.113.0/src/interloper_sql/view/__init__.py

@@ -0,0 +1,7 @@
+"""Portable views: Ibis expressions over typed upstream tables, compiled per warehouse dialect."""
+
+from interloper_sql.view.base import IbisDefinition, IbisView, union
+from interloper_sql.view.decorator import ibis_view
+from interloper_sql.view.types import IBIS_DIALECTS, ibis_schema
+
+__all__ = ["IBIS_DIALECTS", "IbisDefinition", "IbisView", "ibis_schema", "ibis_view", "union"]

interloper_sql-0.113.0/src/interloper_sql/view/base.py

@@ -0,0 +1,154 @@
+"""IbisView: a view authored as an Ibis expression over its upstreams' declared schemas."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import TYPE_CHECKING, Any
+
+import ibis
+from interloper.asset import Output, Upstream, View
+from interloper.destination import TableReference
+from interloper.errors import ConfigError
+
+from interloper_sql.view.types import IBIS_DIALECTS, ibis_schema
+
+if TYPE_CHECKING:
+    from interloper.asset import Asset
+    from interloper.partitioning import PartitionWindow
+
+
+class IbisDefinition:
+    """A view definition that compiles an Ibis expression for a dialect."""
+
+    def __init__(self, expression: ibis.Table) -> None:
+        """Wrap an expression.
+
+        Args:
+            expression: The view's query as an Ibis table expression.
+        """
+        self.expression = expression
+
+    def compile(self, dialect: str) -> str:
+        """Render the expression as SQL for a warehouse dialect.
+
+        Args:
+            dialect: The destination's dialect name.
+
+        Returns:
+            The SQL text.
+
+        Raises:
+            ConfigError: If the dialect has no Ibis counterpart.
+        """
+        if dialect not in IBIS_DIALECTS:
+            raise ConfigError(f"No Ibis dialect for '{dialect}'")
+        return str(ibis.to_sql(self.expression, dialect=IBIS_DIALECTS[dialect]))
+
+
+class IbisView(View):
+    """A view whose ``data()`` receives Ibis tables and returns an Ibis expression.
+
+    Each upstream's ``data`` is an ``ibis.Table`` named after the table the
+    destination references and typed from the upstream's declared schema,
+    so the expression is checked as it is built. The result must carry the
+    view's declared columns with their declared types; the destination then
+    compiles it for its own dialect.
+    """
+
+    async def _upstream(
+        self, name: str, target: Asset, window: PartitionWindow | None, metadata: dict[str, Any]
+    ) -> Upstream:
+        """Turn the upstream's table reference into a typed Ibis table.
+
+        Args:
+            name: The ``data()`` parameter the upstream is bound to.
+            target: The upstream asset.
+            window: Forwarded; a reference has no scope.
+            metadata: Forwarded.
+
+        Returns:
+            The upstream, its ``data`` an ``ibis.Table``.
+
+        Raises:
+            ConfigError: If the upstream declares no schema, or its reference is not a table.
+        """
+        upstream = await super()._upstream(name, target, window, metadata)
+        reference = upstream.data
+        if not isinstance(reference, TableReference):
+            raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' is not a table: {reference!r}")
+        if reference.schema is None:
+            raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' declares no schema to type its table")
+        *outer, table = reference.parts
+        location = {}
+        if outer:
+            location["database"] = outer[-1]
+        if len(outer) > 1:
+            location["catalog"] = outer[-2]
+        return Upstream(asset=upstream.asset, data=ibis.table(ibis_schema(reference.schema), name=table, **location))
+
+    def _output(self, result: Any) -> Output:
+        """Check the expression against the declared schema and wrap it.
+
+        Args:
+            result: The value ``data()`` returned.
+
+        Returns:
+            The output, its data the compilable definition.
+
+        Raises:
+            ConfigError: If the result is not an Ibis table, the view
+                declares no schema, or the expression's columns differ from
+                the declared ones.
+        """
+        if not isinstance(result, ibis.Table):
+            raise ConfigError(f"View '{self.qualified_key}' returned {type(result).__name__}, not an Ibis table")
+        if self.schema is None:
+            raise ConfigError(f"View '{self.qualified_key}' declares no schema")
+        declared = {name: str(dtype) for name, dtype in ibis_schema(self.schema).items()}
+        produced = {name: str(dtype) for name, dtype in result.schema().items()}
+        if declared != produced:
+            missing = sorted(set(declared) - set(produced))
+            extra = sorted(set(produced) - set(declared))
+            retyped = sorted(name for name in declared.keys() & produced.keys() if declared[name] != produced[name])
+            raise ConfigError(
+                f"View '{self.qualified_key}' declares columns its expression does not produce: "
+                f"missing {missing}, extra {extra}, retyped {retyped}"
+            )
+        return Output(IbisDefinition(result))
+
+
+def union(upstreams: Sequence[Upstream], *columns: str, **stamps: Callable[[Upstream], Any]) -> ibis.Table:
+    """The UNION ALL of every upstream's table, optionally stamping a column per upstream.
+
+    Each table is projected to *columns*, or to every declared column, before
+    the union, so the branches line up by name: a bare ``SELECT *`` unions by
+    position, and two accounts' tables created at different times need not
+    share a column order.
+
+    Args:
+        upstreams: The bound upstreams of a many-valued relation, each carrying an ``ibis.Table``.
+        *columns: The columns to keep from every table, in this order; every declared column by default.
+        **stamps: Column name to a function of the upstream giving that
+            upstream's constant value, such as the account it came from.
+
+    Returns:
+        One table over every upstream.
+
+    Raises:
+        ConfigError: If no upstream is bound; a view over nothing has no columns.
+    """
+    if not upstreams:
+        raise ConfigError("no upstream bound: a view needs at least one")
+    names = list(columns or upstreams[0].data.columns)
+    tables = []
+    for upstream in upstreams:
+        # a projection keeping every column in order compiles to SELECT *, which unions by
+        # position; selecting in reverse forces the column list, and the outer select restores it
+        table = upstream.data.select(*reversed(names))
+        values = {}
+        for name, stamp in stamps.items():
+            stamped = stamp(upstream)
+            is_integer = isinstance(stamped, int) and not isinstance(stamped, bool)
+            values[name] = ibis.literal(stamped, type="int64") if is_integer else ibis.literal(stamped)
+        tables.append(table.mutate(**values))
+    return ibis.union(*tables, distinct=False).select(*names, *stamps)

interloper_sql-0.113.0/src/interloper_sql/view/decorator.py

@@ -0,0 +1,29 @@
+"""The ibis_view decorator: an IbisView subclass from a decorated function."""
+
+from __future__ import annotations
+
+from interloper.component import Decorator
+
+from interloper_sql.view.base import IbisView
+
+
+class IbisViewDecorator(Decorator[IbisView]):
+    """Create an IbisView subclass from a decorated function.
+
+    Like ``il.view``, but portable: each upstream parameter receives a typed
+    ``ibis.Table`` in ``data``, the function returns an Ibis expression, and the
+    destination compiles it for its own dialect:
+
+    ```py
+    @ibis_view(schema=Totals, relations={"orders": il.Relation("asset", "shop.orders", many=True)})
+    def totals(orders: list[il.Upstream]) -> ibis.Table:
+        rows = union(orders)
+        return rows.group_by(rows.date).aggregate(total=rows.total.sum())
+    ```
+
+    It takes the settings ``il.asset`` takes (a schema is required) and returns an
+    IbisView subclass with the function as its ``data()``.
+    """
+
+
+ibis_view = IbisViewDecorator(IbisView)

interloper_sql-0.113.0/src/interloper_sql/view/types.py

@@ -0,0 +1,46 @@
+"""Ibis's view of interloper's types: schema fields and warehouse dialects, in one place."""
+
+from __future__ import annotations
+
+import datetime as dt
+from decimal import Decimal
+from typing import Any
+
+import ibis
+from interloper.schema import Schema
+
+IBIS_TYPES: dict[Any, str] = {
+    str: "string",
+    int: "int64",
+    float: "float64",
+    bool: "boolean",
+    dt.date: "date",
+    dt.datetime: "timestamp",
+    Decimal: "decimal",
+    bytes: "binary",
+}
+
+IBIS_DIALECTS: dict[str, str] = {
+    "bigquery": "bigquery",
+    "duckdb": "duckdb",
+    "snowflake": "snowflake",
+    "clickhouse": "clickhouse",
+    "databricks": "databricks",
+    "tsql": "mssql",
+    "postgres": "postgres",
+}
+
+
+def ibis_schema(schema: type[Schema]) -> ibis.Schema:
+    """The Ibis schema of a declared schema, every column nullable.
+
+    A field whose type has no Ibis counterpart (a nested model, ``Any``) is
+    typed ``string``: a view selects such a column through, never computes on it.
+
+    Args:
+        schema: The declared schema.
+
+    Returns:
+        The Ibis schema, in declaration order.
+    """
+    return ibis.schema({spec.name: IBIS_TYPES.get(spec.type, "string") for spec in schema.field_specs()})

interloper_sql-0.110.2/src/interloper_sql/__init__.py

@@ -1,9 +0,0 @@
-"""Interloper SQL integration: a SQLAlchemy-backed destination and its connection."""
-
-from interloper_sql.connection import SQLConnection
-from interloper_sql.destination import SQLDestination
-
-__all__ = [
-    "SQLConnection",
-    "SQLDestination",
-]