interloper-sql 0.2.0rc1__py3-none-any.whl

interloper_sql/__init__.py

@@ -0,0 +1,10 @@
+"""Interloper SQL integration for relational database IO via SQLAlchemy."""
+
+from interloper_sql.io import MySQLIO, PostgresIO, SqlIO, SqliteIO
+
+__all__ = [
+    "MySQLIO",
+    "PostgresIO",
+    "SqlIO",
+    "SqliteIO",
+]

interloper_sql/io/__init__.py

@@ -0,0 +1,13 @@
+"""SQL IO managers for reading and writing to databases via SQLAlchemy."""
+
+from interloper_sql.io.base import SqlIO
+from interloper_sql.io.mysql import MySQLIO
+from interloper_sql.io.postgres import PostgresIO
+from interloper_sql.io.sqlite import SqliteIO
+
+__all__ = [
+    "MySQLIO",
+    "PostgresIO",
+    "SqlIO",
+    "SqliteIO",
+]

interloper_sql/io/base.py

@@ -0,0 +1,281 @@
+"""SQLAlchemy IO implementation."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+from interloper.errors import TableNotFoundError
+from interloper.io.database import DatabaseIO, WriteDisposition
+from sqlalchemy import Column, MetaData, Table, create_engine
+from sqlalchemy import inspect as sa_inspect
+
+if TYPE_CHECKING:
+    from interloper.io.adapter import DataAdapter
+    from sqlalchemy.engine import URL, Connection, Engine
+
+
+def _infer_sa_type(value: Any) -> Any:
+    """Infer a SQLAlchemy column type from a Python value.
+
+    Args:
+        value: A sample Python value used to determine the column type.
+
+    Returns:
+        A SQLAlchemy type instance.
+    """
+    import datetime
+    from decimal import Decimal
+
+    from sqlalchemy import BigInteger, Boolean, Date, DateTime, Float, LargeBinary, Numeric, Text
+
+    if isinstance(value, bool):
+        return Boolean()
+    if isinstance(value, int):
+        return BigInteger()
+    if isinstance(value, float):
+        return Float()
+    if isinstance(value, Decimal):
+        return Numeric()
+    if isinstance(value, datetime.datetime):
+        return DateTime()
+    if isinstance(value, datetime.date):
+        return Date()
+    if isinstance(value, bytes):
+        return LargeBinary()
+    return Text()
+
+
+class SqlIO(DatabaseIO):
+    """Base IO implementation for SQL databases via SQLAlchemy.
+
+    Provides connection management, transactional writes, table reflection, and
+    automatic table creation.  Not intended for direct instantiation — use a
+    dialect subclass (:class:`PostgresIO`, :class:`MySQLIO`, :class:`SqliteIO`)
+    which accepts explicit connection parameters and implements ``to_spec``.
+
+    The IO is fully stateless with respect to table identity — the table name
+    and schema are passed through from the asset context on every call, so a
+    single instance can safely serve multiple assets.
+
+    Args:
+        url: SQLAlchemy connection URL or :class:`~sqlalchemy.engine.URL` object
+            (constructed by dialect subclasses).
+        write_disposition: Controls whether existing rows are deleted before
+            writing.  Defaults to :attr:`WriteDisposition.REPLACE`.
+        chunk_size: Number of rows per insert batch
+        adapter: Optional data adapter for type conversion
+    """
+
+    def __init__(
+        self,
+        url: str | URL,
+        write_disposition: WriteDisposition = WriteDisposition.REPLACE,
+        chunk_size: int = 1000,
+        adapter: DataAdapter | str | None = None,
+    ) -> None:
+        super().__init__(write_disposition, chunk_size, adapter)
+        self._engine: Engine = create_engine(url)
+        self._table_cache: dict[tuple[str, str | None], Table] = {}
+        self._conn: Connection | None = None
+
+    # ------------------------------------------------------------------
+    # Table helpers
+    # ------------------------------------------------------------------
+
+    def _resolve_table(self, table: str, schema: str | None) -> Table | None:
+        """Reflect and cache the SQLAlchemy Table, or return None if it doesn't exist."""
+        key = (table, schema)
+        if key not in self._table_cache:
+            if sa_inspect(self._engine).has_table(table, schema=schema):
+                metadata = MetaData(schema=schema)
+                self._table_cache[key] = Table(table, metadata, autoload_with=self._engine)
+        return self._table_cache.get(key)
+
+    def _require_table(self, table: str, schema: str | None) -> Table:
+        """Resolve the table, raising if it does not exist."""
+        sa_table = self._resolve_table(table, schema)
+        if sa_table is None:
+            qualified = f"{schema}.{table}" if schema else table
+            raise TableNotFoundError(f"Table '{qualified}' does not exist. Has the asset been materialized?")
+        return sa_table
+
+    def _create_table(self, table: str, schema: str | None, rows: list[dict[str, Any]]) -> Table:
+        """Create a new table from the structure of the first row.
+
+        Column types are inferred from the Python values in the sample row
+        using :func:`_infer_sa_type`.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+            rows: Row data (at least one row required for schema inference).
+
+        Returns:
+            The newly created :class:`~sqlalchemy.schema.Table`.
+        """
+        assert self._conn is not None
+        sample = rows[0]
+        columns = [Column(name, _infer_sa_type(value)) for name, value in sample.items()]
+        sa_metadata = MetaData(schema=schema)
+        sa_table = Table(table, sa_metadata, *columns)
+        sa_metadata.create_all(self._conn)
+        self._table_cache[(table, schema)] = sa_table
+        return sa_table
+
+    # ------------------------------------------------------------------
+    # Transaction management
+    # ------------------------------------------------------------------
+
+    @contextmanager
+    def _transaction(self) -> Iterator[None]:
+        """Open a SQLAlchemy transactional connection for write operations.
+
+        Sets ``self._conn`` for the duration of the block.  The connection is
+        committed on success and rolled back on exception (``engine.begin()``
+        semantics).
+
+        Yields:
+            None
+        """
+        with self._engine.begin() as conn:
+            self._conn = conn
+            try:
+                yield
+            finally:
+                self._conn = None
+
+    # ------------------------------------------------------------------
+    # DatabaseIO hooks
+    # ------------------------------------------------------------------
+
+    def _insert(self, table: str, schema: str | None, rows: list[dict[str, Any]]) -> None:
+        """Insert rows in chunks using the active transaction connection.
+
+        If the table does not exist yet, it is created from the row data
+        before inserting.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+            rows: Row data as list of dicts
+        """
+        assert self._conn is not None
+        sa_table = self._resolve_table(table, schema)
+        if sa_table is None:
+            sa_table = self._create_table(table, schema, rows)
+        for i in range(0, len(rows), self.chunk_size):
+            self._conn.execute(sa_table.insert(), rows[i : i + self.chunk_size])
+
+    def _delete_all(self, table: str, schema: str | None) -> None:
+        """Delete all rows from the table using the active transaction connection.
+
+        No-op when the table does not exist yet.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+        """
+        assert self._conn is not None
+        sa_table = self._resolve_table(table, schema)
+        if sa_table is None:
+            return
+        self._conn.execute(sa_table.delete())
+
+    def _delete_partition(self, table: str, schema: str | None, column: str, value: Any) -> None:
+        """Delete rows matching a partition value using the active transaction connection.
+
+        No-op when the table does not exist yet.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+            column: Partition column name
+            value: Partition value to match
+        """
+        assert self._conn is not None
+        sa_table = self._resolve_table(table, schema)
+        if sa_table is None:
+            return
+        self._conn.execute(sa_table.delete().where(sa_table.c[column] == value))
+
+    def _select_all(self, table: str, schema: str | None) -> list[dict[str, Any]]:
+        """Select all rows from the table.
+
+        Opens a dedicated read connection (not part of the write transaction).
+
+        Args:
+            table: Target table name
+            schema: Database schema
+
+        Returns:
+            All rows as list of dicts
+
+        Raises:
+            ValueError: If the table does not exist
+        """
+        sa_table = self._require_table(table, schema)
+        with self._engine.connect() as conn:
+            result = conn.execute(sa_table.select())
+            return [dict(row._mapping) for row in result]
+
+    def _select_partition(self, table: str, schema: str | None, column: str, value: Any) -> list[dict[str, Any]]:
+        """Select rows matching a partition value.
+
+        Opens a dedicated read connection (not part of the write transaction).
+
+        Args:
+            table: Target table name
+            schema: Database schema
+            column: Partition column name
+            value: Partition value to match
+
+        Returns:
+            Matching rows as list of dicts
+
+        Raises:
+            ValueError: If the table does not exist
+        """
+        sa_table = self._require_table(table, schema)
+        with self._engine.connect() as conn:
+            result = conn.execute(sa_table.select().where(sa_table.c[column] == value))
+            return [dict(row._mapping) for row in result]
+
+    # ------------------------------------------------------------------
+    # Introspection
+    # ------------------------------------------------------------------
+
+    def _count_by_partition(
+        self, table: str, schema: str | None, column: str,
+    ) -> dict[str, int]:
+        """Return row counts grouped by partition column via SQL ``GROUP BY``.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+            column: Column to group by
+
+        Returns:
+            Mapping from partition value (as string) to row count.
+
+        Raises:
+            TableNotFoundError: If the table does not exist.
+        """
+        from sqlalchemy import func
+
+        sa_table = self._require_table(table, schema)
+        col = sa_table.c[column]
+        stmt = sa_table.select().with_only_columns(col, func.count()).group_by(col)
+        with self._engine.connect() as conn:
+            result = conn.execute(stmt)
+            return {str(row[0]): row[1] for row in result}
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def dispose(self) -> None:
+        """Dispose the SQLAlchemy engine and clear the table cache."""
+        self._engine.dispose()
+        self._table_cache.clear()

interloper_sql/io/mysql.py

@@ -0,0 +1,70 @@
+"""MySQL IO implementation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from interloper.io.database import WriteDisposition
+from interloper.serialization.io import IOSpec
+from sqlalchemy.engine import URL
+
+from interloper_sql.io.base import SqlIO
+
+if TYPE_CHECKING:
+    from interloper.io.adapter import DataAdapter
+
+
+class MySQLIO(SqlIO):
+    """MySQL-specific IO manager.
+
+    Extends :class:`SqlIO` for MySQL connections. Uses standard ``DELETE`` for
+    row removal because MySQL's ``TRUNCATE`` causes an implicit commit and
+    cannot participate in a transaction.
+
+    Args:
+        host: Database server hostname
+        database: Database name
+        port: Database server port
+        user: Database user
+        password: Database password
+        driver: SQLAlchemy driver (e.g. ``pymysql``, ``mysqlconnector``)
+        write_disposition: Controls whether existing rows are deleted before writing
+        chunk_size: Number of rows per insert batch
+        adapter: Optional data adapter for type conversion
+    """
+
+    def __init__(
+        self,
+        host: str,
+        database: str,
+        port: int = 3306,
+        user: str = "root",
+        password: str | None = None,
+        driver: str | None = None,
+        write_disposition: WriteDisposition = WriteDisposition.REPLACE,
+        chunk_size: int = 1000,
+        adapter: DataAdapter | str | None = None,
+    ) -> None:
+        self.host = host
+        self.port = port
+        self.database = database
+        self.user = user
+        self.password = password
+        self.driver = driver
+
+        drivername = f"mysql+{driver}" if driver else "mysql"
+        url = URL.create(drivername, user, password, host, port, database)
+        super().__init__(url, write_disposition, chunk_size, adapter)
+
+    def to_spec(self) -> IOSpec:
+        """Convert to serializable spec."""
+        init = self._base_init_kwargs()
+        init["host"] = self.host
+        init["port"] = self.port
+        init["database"] = self.database
+        init["user"] = self.user
+        if self.password is not None:
+            init["password"] = self.password
+        if self.driver is not None:
+            init["driver"] = self.driver
+        return IOSpec(path=self.path, init=init)

interloper_sql/io/postgres.py

@@ -0,0 +1,87 @@
+"""PostgreSQL IO implementation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from interloper.io.database import WriteDisposition
+from interloper.serialization.io import IOSpec
+from sqlalchemy import text
+from sqlalchemy.engine import URL
+
+from interloper_sql.io.base import SqlIO
+
+if TYPE_CHECKING:
+    from interloper.io.adapter import DataAdapter
+
+
+class PostgresIO(SqlIO):
+    """PostgreSQL-specific IO manager.
+
+    Extends :class:`SqlIO` with PostgreSQL optimisations:
+
+    * Uses ``TRUNCATE`` (transactional in Postgres) instead of ``DELETE`` for
+      full-table replacements, which is significantly faster on large tables.
+
+    Args:
+        host: Database server hostname
+        port: Database server port
+        database: Database name
+        user: Database user
+        password: Database password
+        driver: SQLAlchemy driver (e.g. ``psycopg2``, ``asyncpg``)
+        write_disposition: Controls whether existing rows are deleted before writing
+        chunk_size: Number of rows per insert batch
+        adapter: Optional data adapter for type conversion
+    """
+
+    def __init__(
+        self,
+        host: str,
+        port: int = 5432,
+        database: str = "postgres",
+        user: str = "postgres",
+        password: str | None = None,
+        driver: str | None = None,
+        write_disposition: WriteDisposition = WriteDisposition.REPLACE,
+        chunk_size: int = 1000,
+        adapter: DataAdapter | str | None = None,
+    ) -> None:
+        self.host = host
+        self.port = port
+        self.database = database
+        self.user = user
+        self.password = password
+        self.driver = driver
+
+        drivername = f"postgresql+{driver}" if driver else "postgresql"
+        url = URL.create(drivername, user, password, host, port, database)
+        super().__init__(url, write_disposition, chunk_size, adapter)
+
+    def _delete_all(self, table: str, schema: str | None) -> None:
+        """Use TRUNCATE for full-table deletes (transactional in PostgreSQL).
+
+        No-op when the table does not exist yet.
+
+        Args:
+            table: Target table name
+            schema: Database schema
+        """
+        assert self._conn is not None
+        sa_table = self._resolve_table(table, schema)
+        if sa_table is None:
+            return
+        self._conn.execute(text(f"TRUNCATE TABLE {sa_table.fullname}"))
+
+    def to_spec(self) -> IOSpec:
+        """Convert to serializable spec."""
+        init = self._base_init_kwargs()
+        init["host"] = self.host
+        init["port"] = self.port
+        init["database"] = self.database
+        init["user"] = self.user
+        if self.password is not None:
+            init["password"] = self.password
+        if self.driver is not None:
+            init["driver"] = self.driver
+        return IOSpec(path=self.path, init=init)

interloper_sql/io/sqlite.py

@@ -0,0 +1,45 @@
+"""SQLite IO implementation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from interloper.io.database import WriteDisposition
+from interloper.serialization.io import IOSpec
+
+from interloper_sql.io.base import SqlIO
+
+if TYPE_CHECKING:
+    from interloper.io.adapter import DataAdapter
+
+
+class SqliteIO(SqlIO):
+    """SQLite-specific IO manager.
+
+    Extends :class:`SqlIO` for SQLite connections. Useful for local development
+    and testing without requiring an external database server.
+
+    Args:
+        database: Path to the SQLite database file, or ``":memory:"`` for an
+            in-memory database.
+        write_disposition: Controls whether existing rows are deleted before writing
+        chunk_size: Number of rows per insert batch
+        adapter: Optional data adapter for type conversion
+    """
+
+    def __init__(
+        self,
+        database: str = ":memory:",
+        write_disposition: WriteDisposition = WriteDisposition.REPLACE,
+        chunk_size: int = 1000,
+        adapter: DataAdapter | str | None = None,
+    ) -> None:
+        self.database = database
+        url = f"sqlite:///{database}"
+        super().__init__(url, write_disposition, chunk_size, adapter)
+
+    def to_spec(self) -> IOSpec:
+        """Convert to serializable spec."""
+        init = self._base_init_kwargs()
+        init["database"] = self.database
+        return IOSpec(path=self.path, init=init)

interloper_sql-0.2.0rc1.dist-info/METADATA

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: interloper-sql
+Version: 0.2.0rc1
+Summary: Interloper SQLAlchemy IO managers
+Author: Guillaume Onfroy
+Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: interloper-core
+Requires-Dist: pymysql>=1.1 ; extra == 'mysql'
+Requires-Dist: psycopg2-binary>=2.9 ; extra == 'postgres'
+Requires-Python: >=3.10
+Provides-Extra: mysql
+Provides-Extra: postgres
+Description-Content-Type: text/markdown
+
+# interloper-sql
+
+SQLAlchemy IO managers for the Interloper framework.

interloper_sql-0.2.0rc1.dist-info/RECORD

@@ -0,0 +1,9 @@
+interloper_sql/__init__.py,sha256=__WqY0mvjudCFv3LwyHExUO5vGxd-JnWNXiE2ujA_4E,221
+interloper_sql/io/__init__.py,sha256=gf_2w_A3d2yMujwWd5RNpyUDLkCyjL2DYFXV_81Tj7s,334
+interloper_sql/io/base.py,sha256=jRwXI5LCT5xAoBtUun1SBzKANKtVd5wU_z44HDyFqqw,10386
+interloper_sql/io/mysql.py,sha256=hMzFCYNUERUVp-LScZJ9PzZNAKQ44CBGTfN0XlRwJA0,2273
+interloper_sql/io/postgres.py,sha256=xkuw7-OvfJbR_qZm3z7mtdmshxxFoZ5cF8SpmtohC1o,2863
+interloper_sql/io/sqlite.py,sha256=w8KCzYF4SiG9ROSwB2198ljQTL8H7OFI2ec38jAFFUU,1428
+interloper_sql-0.2.0rc1.dist-info/WHEEL,sha256=iHtWm8nRfs0VRdCYVXocAWFW8ppjHL-uTJkAdZJKOBM,80
+interloper_sql-0.2.0rc1.dist-info/METADATA,sha256=aNYYSBu9vBkzg18V-veVHrJvB4RMIFGi9EfSfWAQLoE,538
+interloper_sql-0.2.0rc1.dist-info/RECORD,,

interloper_sql-0.2.0rc1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.9.30
+Root-Is-Purelib: true
+Tag: py3-none-any