interloper-sql 0.111.0__tar.gz → 0.113.0__tar.gz

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: interloper-sql
-Version: 0.111.0
+Version: 0.113.0
 Summary: Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "interloper-sql"
-version = "0.111.0"
+version = "0.113.0"
 description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 requires-python = ">=3.10"

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/pyproject.toml.orig

@@ -3,7 +3,7 @@
 # ###############
 [project]
 name = "interloper-sql"
-version = "0.111.0"
+version = "0.113.0"
 description = "Interloper SQL integration: portable views over SQL warehouses, and a SQLAlchemy-backed destination for PostgreSQL and other SQL databases"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/src/interloper_sql/connection.py

@@ -6,7 +6,7 @@ from functools import cached_property
 
 import sqlalchemy
 from interloper.connection import Connection, connection
-from interloper.resource.fields import SecretField, fetch_field_provider
+from interloper.resource import SecretField, fetch_field_provider
 from pydantic_settings import SettingsConfigDict
 from sqlalchemy.engine import Engine
 

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/src/interloper_sql/destination.py

@@ -10,11 +10,11 @@ from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
 from typing import Any
 
-from interloper.destination import IOContext, destination
-from interloper.destination.database import DatabaseDestination, PartitionFilter
+from interloper.destination import DatabaseDestination, IOContext, destination
 from interloper.errors import DataNotFoundError
-from interloper.representation import Representation
-from interloper.resource.fields import InputField
+from interloper.partitioning import PartitionFilter
+from interloper.representation import TableData
+from interloper.resource import InputField
 from interloper.schema import FieldSpec
 from pydantic import PrivateAttr
 from sqlalchemy import Column, MetaData, String, Table, and_, cast, func, inspect, select
@@ -186,13 +186,13 @@ class SQLDestination(DatabaseDestination):
             context: IO context carrying the effective schema.
         """
         dataset = self._resolve_dataset(dataset)
-        view = Representation.of(data)
+        table_data = TableData.from_data(data)
         with self._begin() as conn:
             sql_table = self._table(table, dataset, conn)
             if sql_table is None:
-                schema = context.schema if context.schema is not None else view.infer()
+                schema = context.schema if context.schema is not None else table_data.infer()
                 sql_table = self._create_table(table, dataset, schema.field_specs(), conn)
-            rows = _align(view.records, [c.name for c in sql_table.columns], self._ref(table, dataset))
+            rows = _align(table_data.rows, [c.name for c in sql_table.columns], self._ref(table, dataset))
             for start in range(0, len(rows), _BATCH_SIZE):
                 conn.execute(sql_table.insert(), rows[start : start + _BATCH_SIZE])
 
@@ -312,22 +312,23 @@ def _predicate(table: Table, where: PartitionFilter) -> ColumnElement[bool]:
 
     Args:
         table: The reflected table.
-        where: The filter: equality on ``value``, or half-open ``bounds``.
+        where: The filter: membership of ``values``, or half-open ``bounds``.
 
     Returns:
         The ``WHERE`` clause.
     """
     column = table.c[where.column]
-    if where.bounds is not None:
-        start, end = where.bounds
-        return and_(column >= _coerce(column, start), column < _coerce(column, end))
-    return column == _coerce(column, where.value)
+    if where.values is not None:
+        return column.in_([_coerce(column, value) for value in where.values])
+    assert where.bounds is not None
+    start, end = where.bounds
+    return and_(column >= _coerce(column, start), column < _coerce(column, end))
 
 
 def _coerce(column: Column[Any], value: Any) -> Any:
     """Coerce a filter value to the Python type a date or timestamp column binds.
 
-    A partition id arrives as a string (``"2024-01-01"``) and a day's bounds
+    A partition key arrives as a string (``"2024-01-01"``) and a day's bounds
     as dates; bound as-is against a ``DATE`` or ``TIMESTAMP`` column, they
     would be sent as ``VARCHAR`` or rejected by the dialect's type.
 

{interloper_sql-0.111.0 → interloper_sql-0.113.0}/src/interloper_sql/view/base.py

@@ -6,16 +6,15 @@ from collections.abc import Callable, Sequence
 from typing import TYPE_CHECKING, Any
 
 import ibis
-from interloper.asset import Upstream, View
-from interloper.destination.database import TableReference
+from interloper.asset import Output, Upstream, View
+from interloper.destination import TableReference
 from interloper.errors import ConfigError
 
 from interloper_sql.view.types import IBIS_DIALECTS, ibis_schema
 
 if TYPE_CHECKING:
     from interloper.asset import Asset
-    from interloper.dag.base import DAG
-    from interloper.partitioning.base import Partition, PartitionWindow
+    from interloper.partitioning import PartitionWindow
 
 
 class IbisDefinition:
@@ -56,56 +55,45 @@ class IbisView(View):
     compiles it for its own dialect.
     """
 
-    async def _read_upstreams(
-        self,
-        name: str,
-        targets: list[Asset],
-        dag: DAG | None,
-        partition_or_window: Partition | PartitionWindow | None,
-        metadata: dict[str, Any],
-    ) -> list[Upstream]:
-        """Turn every table reference into a typed Ibis table.
+    async def _upstream(
+        self, name: str, target: Asset, window: PartitionWindow | None, metadata: dict[str, Any]
+    ) -> Upstream:
+        """Turn the upstream's table reference into a typed Ibis table.
 
         Args:
-            name: The ``data()`` parameter the upstreams are built for.
-            targets: The bound upstream assets.
-            dag: Forwarded to `View._read_upstreams`.
-            partition_or_window: Forwarded.
+            name: The ``data()`` parameter the upstream is bound to.
+            target: The upstream asset.
+            window: Forwarded; a reference has no scope.
             metadata: Forwarded.
 
         Returns:
-            One `Upstream` per target, its ``data`` an ``ibis.Table``.
+            The upstream, its ``data`` an ``ibis.Table``.
 
         Raises:
-            ConfigError: If an upstream declares no schema, or its reference is not a table.
+            ConfigError: If the upstream declares no schema, or its reference is not a table.
         """
-        upstreams = await super()._read_upstreams(name, targets, dag, partition_or_window, metadata)
-        tables: list[Upstream] = []
-        for upstream in upstreams:
-            reference = upstream.data
-            if not isinstance(reference, TableReference):
-                raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' is not a table: {reference!r}")
-            if reference.schema is None:
-                raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' declares no schema to type its table")
-            *outer, table = reference.parts
-            location = {}
-            if outer:
-                location["database"] = outer[-1]
-            if len(outer) > 1:
-                location["catalog"] = outer[-2]
-            tables.append(
-                Upstream(asset=upstream.asset, data=ibis.table(ibis_schema(reference.schema), name=table, **location))
-            )
-        return tables
-
-    def _definition(self, result: Any) -> IbisDefinition:
+        upstream = await super()._upstream(name, target, window, metadata)
+        reference = upstream.data
+        if not isinstance(reference, TableReference):
+            raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' is not a table: {reference!r}")
+        if reference.schema is None:
+            raise ConfigError(f"Upstream '{upstream.asset.qualified_key}' declares no schema to type its table")
+        *outer, table = reference.parts
+        location = {}
+        if outer:
+            location["database"] = outer[-1]
+        if len(outer) > 1:
+            location["catalog"] = outer[-2]
+        return Upstream(asset=upstream.asset, data=ibis.table(ibis_schema(reference.schema), name=table, **location))
+
+    def _output(self, result: Any) -> Output:
         """Check the expression against the declared schema and wrap it.
 
         Args:
             result: The value ``data()`` returned.
 
         Returns:
-            The compilable definition.
+            The output, its data the compilable definition.
 
         Raises:
             ConfigError: If the result is not an Ibis table, the view
@@ -126,7 +114,7 @@ class IbisView(View):
                 f"View '{self.qualified_key}' declares columns its expression does not produce: "
                 f"missing {missing}, extra {extra}, retyped {retyped}"
             )
-        return IbisDefinition(result)
+        return Output(IbisDefinition(result))
 
 
 def union(upstreams: Sequence[Upstream], *columns: str, **stamps: Callable[[Upstream], Any]) -> ibis.Table:

interloper_sql-0.113.0/src/interloper_sql/view/decorator.py

@@ -0,0 +1,29 @@
+"""The ibis_view decorator: an IbisView subclass from a decorated function."""
+
+from __future__ import annotations
+
+from interloper.component import Decorator
+
+from interloper_sql.view.base import IbisView
+
+
+class IbisViewDecorator(Decorator[IbisView]):
+    """Create an IbisView subclass from a decorated function.
+
+    Like ``il.view``, but portable: each upstream parameter receives a typed
+    ``ibis.Table`` in ``data``, the function returns an Ibis expression, and the
+    destination compiles it for its own dialect:
+
+    ```py
+    @ibis_view(schema=Totals, relations={"orders": il.Relation("asset", "shop.orders", many=True)})
+    def totals(orders: list[il.Upstream]) -> ibis.Table:
+        rows = union(orders)
+        return rows.group_by(rows.date).aggregate(total=rows.total.sum())
+    ```
+
+    It takes the settings ``il.asset`` takes (a schema is required) and returns an
+    IbisView subclass with the function as its ``data()``.
+    """
+
+
+ibis_view = IbisViewDecorator(IbisView)

interloper_sql-0.111.0/src/interloper_sql/view/decorator.py

@@ -1,84 +0,0 @@
-"""Decorator for creating IbisView subclasses from functions."""
-
-from __future__ import annotations
-
-from collections.abc import Callable
-from typing import Any, cast, overload
-
-from interloper.asset.decorator import build_asset_class
-from interloper.component.decorator import decorate
-from interloper.component.relation import Relation
-
-from interloper_sql.view.base import IbisView
-
-
-@overload
-def ibis_view(fn: Callable[..., Any], /) -> type[IbisView]: ...
-@overload
-def ibis_view(
-    *,
-    relations: dict[str, Any] = ...,
-    **overrides: Any,
-) -> Callable[[Callable[..., Any]], type[IbisView]]: ...
-def ibis_view(
-    fn: Callable[..., Any] | None = None,
-    /,
-    *,
-    relations: dict[str, Any] | None = None,
-    **overrides: Any,
-) -> type[IbisView] | Callable[..., type[IbisView]]:
-    """Create an IbisView subclass from a decorated function.
-
-    Like ``il.view``, but portable: each upstream parameter receives a typed
-    ``ibis.Table`` in ``data``, the function returns an Ibis expression, and
-    the destination compiles it for its own dialect:
-
-    ```py
-    @ibis_view(schema=Totals, relations={"orders": il.Relation("asset", "shop.orders", many=True)})
-    def totals(orders: list[il.Upstream]) -> ibis.Table:
-        rows = union(orders)
-        return rows.group_by(rows.date).aggregate(total=rows.total.sum())
-    ```
-
-    Args:
-        fn: The function to turn into a view, passed positionally when the
-            decorator is used bare. ``None`` in the parenthesised form.
-        relations: As for ``il.asset``.
-        **overrides: As for ``il.asset``; a schema is required.
-
-    Returns:
-        An IbisView subclass with the function as its ``data()`` method.
-    """
-    if fn is not None:
-        return decorate(IbisView, fn, build=_build_ibis_view_class, relations=relations, **overrides)
-
-    def wrapper(fn: Callable[..., Any]) -> type[IbisView]:
-        return decorate(IbisView, fn, build=_build_ibis_view_class, relations=relations, **overrides)
-
-    return wrapper
-
-
-# -- Internals -----------------------------------------------------------------
-
-
-def _build_ibis_view_class(
-    fn: Any,
-    *,
-    classvars: dict[str, Any],
-    fields: dict[str, Any],
-    relations: dict[str, Relation],
-) -> type[IbisView]:
-    """Build an IbisView subclass from a function or method.
-
-    Args:
-        fn: The function (or method) backing the view's ``data()``.
-        classvars: Class-level attributes to set on the generated subclass.
-        fields: Pydantic field values to set on the generated subclass.
-        relations: Relations the decorator declares, name to relation.
-
-    Returns:
-        A dynamically created IbisView subclass.
-    """
-    return cast(
-        "type[IbisView]", build_asset_class(fn, base=IbisView, classvars=classvars, fields=fields, relations=relations)
-    )