onestep-sql 0.1.0__py3-none-any.whl

onestep_sql/__init__.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from onestep.resource_registry import ResourceRegistry
+
+from . import mysql, postgres
+from .mysql import register_resources as _register_mysql
+from .postgres import register_resources as _register_postgres
+
+__all__ = ["register_resources", "mysql", "postgres"]
+
+
+def register_resources(registry: ResourceRegistry) -> None:
+    """Register every onestep-sql resource type (MySQL + PostgreSQL).
+
+    This is the single entry point for the canonical ``onestep-sql``
+    distribution (declared as ``sql`` in the ``onestep.resources`` group).
+    It delegates to the per-backend registrars without changing any of the
+    14 existing YAML type names, their catalog roles, or their fields.
+    """
+    _register_mysql(registry)
+    _register_postgres(registry)

onestep_sql/_shared/__init__.py

@@ -0,0 +1,26 @@
+"""Internal shared SQL behaviour for the canonical ``onestep-sql`` package.
+
+Phase 2 of the ``onestep-mysql`` / ``onestep-postgres`` consolidation
+(issue #133, design in PR #134) keeps exactly one copy of the machinery that
+previously lived in parallel in both backend subpackages (as monitored by the
+now-retired ``scripts/check_plugin_drift.py``):
+
+* :mod:`onestep_sql._shared.state_sqlalchemy` — the SQLAlchemy state/cursor
+  stores; backends subclass them and override only the asyncio driver mapping
+  (``mysql+asyncmy`` vs ``postgresql+psycopg``) and install hint;
+* :mod:`onestep_sql._shared.table_sink_policy` — the table-sink
+  ``update_columns`` write policies (``overwrite`` / ``skip_null`` /
+  ``backfill``), the ``_normalize_update_columns`` validator and the
+  ``_update_payload`` / ``_coerce_json_values`` mixin;
+* :mod:`onestep_sql._shared.state_keys` — the default incremental state-key
+  derivation feeding the at-least-once cursor state;
+* :mod:`onestep_sql._shared.resilience` — the secret-token collection, message
+  redaction, redacted error-cause base class and the
+  ``ConnectorOperationError`` factory; the dialect-specific SQLAlchemy error
+  classification tables stay in each backend's ``resilience.py``.
+
+This package is **not** public API. Backend capabilities never move here:
+MySQL binlog CDC stays in ``onestep_sql.mysql`` and the PostgreSQL tracked
+execution modules stay in ``onestep_sql.postgres``. Dual-backend contract
+tests live in ``tests/contract/test_onestep_sql_shared.py``.
+"""

onestep_sql/_shared/resilience.py

@@ -0,0 +1,147 @@
+"""Shared secret-redaction scaffolding for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of the resilience machinery that previously existed in
+parallel in both backend ``resilience.py`` modules and differed only in the
+backend name (see the retired ``scripts/check_plugin_drift.py``):
+
+* the secret-token collection used to scrub error causes
+  (:func:`collect_sensitive_tokens`);
+* the message redaction itself (:func:`redact_message`);
+* the redacted error-cause base class (:class:`SQLErrorCause`) that backends
+  subclass with their own ``backend`` label so ``str(cause)`` keeps its
+  per-database prefix;
+* the :class:`~onestep.resilience.ConnectorOperationError` factory
+  (:func:`as_connector_operation_error`).
+
+The SQLAlchemy error-classification tables stay in each backend's
+``resilience.py`` because they encode genuinely different server error
+messages (``server has gone away`` vs ``server closed the connection``,
+``deadlock found`` vs ``deadlock detected`` and so on).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import ClassVar
+from urllib.parse import urlsplit
+
+from onestep.resilience import (
+    ConnectorErrorKind,
+    ConnectorOperation,
+    ConnectorOperationError,
+)
+
+REDACTED = "<redacted>"
+MAX_MESSAGE_LENGTH = 500
+SECRET_OPTION_KEYS = frozenset(
+    {
+        "password",
+        "passwd",
+        "pwd",
+        "secret",
+        "token",
+        "access_token",
+        "api_key",
+        "apikey",
+        "credentials",
+        "authorization",
+    }
+)
+
+
+def collect_sensitive_tokens(*config_values: object) -> list[str]:
+    """Collect secret substrings that may surface in SQL backend error messages.
+
+    SQLAlchemy masks the password in its rendered URL, but the underlying DBAPI
+    ``orig`` exception can still echo credentials (e.g. after echoing the full
+    connection string). Tokens are derived from connector config (raw DSN +
+    parsed userinfo + known secret option values) and used to scrub error
+    causes before they leave the plugin.
+    """
+    tokens: list[str] = []
+    seen: set[str] = set()
+
+    def add(value: object) -> None:
+        if value is None:
+            return
+        text = str(value)
+        if text and text not in seen:
+            seen.add(text)
+            tokens.append(text)
+
+    def collect_mapping(value: Mapping[object, object]) -> None:
+        for key, item in value.items():
+            if str(key).lower() in SECRET_OPTION_KEYS:
+                add(item)
+            elif isinstance(item, Mapping):
+                collect_mapping(item)
+
+    for value in config_values:
+        if isinstance(value, Mapping):
+            collect_mapping(value)
+            continue
+        add(value)
+        try:
+            parsed = urlsplit(str(value))
+        except ValueError:
+            continue
+        username = parsed.username
+        password = parsed.password
+        if username and password:
+            add(f"{username}:{password}")
+            add(f"{username}:{password}@")
+        add(password)
+    return tokens
+
+
+def redact_message(message: str, tokens: list[str]) -> str:
+    redacted = message
+    for token in sorted(tokens, key=len, reverse=True):
+        if token:
+            redacted = redacted.replace(token, REDACTED)
+    return redacted[:MAX_MESSAGE_LENGTH]
+
+
+@dataclass(frozen=True)
+class SQLErrorCause(Exception):
+    """Redacted cause attached to normalized SQL connector errors.
+
+    Backends subclass this with their own ``backend`` label so the rendered
+    message keeps its per-database prefix (``mysql error: ...`` /
+    ``postgres error: ...``) while the dataclass shape, freezing and identity
+    semantics are implemented once here.
+    """
+
+    message: str
+
+    backend: ClassVar[str] = "sql"
+
+    def __str__(self) -> str:
+        return f"{self.backend} error: {self.message}"
+
+
+def as_connector_operation_error(
+    *,
+    backend: str,
+    operation: ConnectorOperation,
+    exc: BaseException,
+    classify: Callable[[BaseException], ConnectorErrorKind | None],
+    redacted_cause: Callable[..., Exception],
+    source_name: str | None = None,
+    retry_delay_s: float | None = None,
+    secrets: list[str] | None = None,
+) -> ConnectorOperationError | None:
+    """Normalize ``exc`` using the backend's classifier and redacted cause."""
+    kind = classify(exc)
+    if kind is None:
+        return None
+    return ConnectorOperationError(
+        backend=backend,
+        operation=operation,
+        kind=kind,
+        source_name=source_name,
+        retry_delay_s=retry_delay_s,
+        cause=redacted_cause(exc, secrets=secrets),
+    )

onestep_sql/_shared/state_keys.py

@@ -0,0 +1,31 @@
+"""Shared default incremental state-key derivation for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of ``_default_incremental_state_key``, which previously
+existed in parallel in both backend ``connector.py`` modules (see the retired
+``scripts/check_plugin_drift.py``). The derived key feeds the at-least-once
+cursor state, so its format is stable contract: changing it would orphan
+existing persisted cursors.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+
+
+def _default_incremental_state_key(
+    *,
+    table: str,
+    cursor: Sequence[str],
+    key: str,
+    where: str | None,
+) -> str:
+    normalized_where = " ".join((where or "").split())
+    if normalized_where:
+        where_fragment = normalized_where
+        if len(where_fragment) > 64:
+            where_fragment = f"sha1:{hashlib.sha1(where_fragment.encode('utf-8')).hexdigest()}"
+    else:
+        where_fragment = "-"
+    return f"{table}:{','.join(cursor)}:key={key}:where={where_fragment}"

onestep_sql/_shared/state_sqlalchemy.py

@@ -0,0 +1,209 @@
+"""Shared SQLAlchemy state/cursor stores for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of the store implementation that previously lived in parallel
+in both backend ``state_sqlalchemy.py`` modules (see the retired
+``scripts/check_plugin_drift.py``). Dual-backend contract tests in
+``tests/contract/test_onestep_sql_shared.py`` pin the behaviour for both
+backends.
+
+Only two things are genuinely per-database and stay in the backend adapters:
+
+* the asyncio driver mapping (``mysql+asyncmy`` vs ``postgresql+psycopg``) —
+  backends override :meth:`SQLAlchemyStateStore._resolve_async_driver`;
+* the installation hint in the missing-SQLAlchemy error message — backends
+  override ``_install_hint``.
+
+Backends subclass :class:`SQLAlchemyStateStore` /
+:class:`SQLAlchemyCursorStore` so their public class identities stay inside
+``onestep_sql.mysql`` / ``onestep_sql.postgres`` while the behaviour is
+implemented here once.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+try:
+    import sqlalchemy as sa
+    from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+except ImportError:  # pragma: no cover - exercised when optional deps are missing
+    sa = None
+    AsyncEngine = None
+    create_async_engine = None
+
+
+def sqlite_async_driver(drivername: str) -> str | None:
+    """Return the asyncio sqlite driver for sqlite URLs, else ``None``."""
+    if drivername in ("sqlite", "sqlite+pysqlite"):
+        return "sqlite+aiosqlite"
+    return None
+
+
+def async_dsn(dsn: str, *, resolve_driver: Any) -> str:
+    """Return a SQLAlchemy URL backed by an asyncio-compatible dialect.
+
+    ``resolve_driver`` maps the URL's current drivername to the
+    asyncio-compatible drivername to use, or ``None`` to keep the DSN
+    unchanged. The per-database mappings live in the backend adapters.
+    """
+    if sa is None:
+        return dsn
+    url = sa.engine.make_url(dsn)
+    async_drivername = resolve_driver(url.drivername)
+    if async_drivername is None:
+        return dsn
+    return url.set(drivername=async_drivername).render_as_string(hide_password=False)
+
+
+class SQLAlchemyStateStore:
+    _install_hint = "Install onestep-sql."
+
+    @staticmethod
+    def _resolve_async_driver(drivername: str) -> str | None:
+        return sqlite_async_driver(drivername)
+
+    def __init__(
+        self,
+        *,
+        dsn: str | None = None,
+        engine: Any | None = None,
+        table: str = "onestep_state",
+        key_column: str = "state_key",
+        value_column: str = "state_value",
+        updated_at_column: str = "updated_at",
+        auto_create: bool = True,
+        **engine_options: Any,
+    ) -> None:
+        if create_async_engine is None or sa is None:
+            raise RuntimeError(f"SQLAlchemyStateStore requires SQLAlchemy. {self._install_hint}")
+        if engine is None and dsn is None:
+            raise ValueError("dsn or engine is required")
+        if engine is not None and dsn is not None:
+            raise ValueError("pass either dsn or engine, not both")
+        engine_options.setdefault("pool_pre_ping", True)
+        self.engine = engine or create_async_engine(
+            async_dsn(dsn, resolve_driver=self._resolve_async_driver), **engine_options
+        )
+        self._owns_engine = engine is None
+        self.table_name = table
+        self.key_column_name = key_column
+        self.value_column_name = value_column
+        self.updated_at_column_name = updated_at_column
+        self.auto_create = auto_create
+        self._metadata = sa.MetaData()
+        self._table = sa.Table(
+            table,
+            self._metadata,
+            sa.Column(key_column, sa.String(255), primary_key=True),
+            sa.Column(value_column, sa.Text(), nullable=False),
+            sa.Column(updated_at_column, sa.DateTime(timezone=True), nullable=False),
+        )
+        self._ready = False
+        self._ready_lock: asyncio.Lock | None = None
+
+    async def load(self, key: str) -> Any | None:
+        await self._ensure_ready()
+        key_column = self._table.c[self.key_column_name]
+        value_column = self._table.c[self.value_column_name]
+        async with self.engine.begin() as conn:
+            row = (
+                await conn.execute(sa.select(value_column).where(key_column == key))
+            ).scalar_one_or_none()
+        if row is None:
+            return None
+        return json.loads(row)
+
+    async def save(self, key: str, value: Any) -> None:
+        await self._ensure_ready()
+        key_column = self._table.c[self.key_column_name]
+        payload = {
+            self.key_column_name: key,
+            self.value_column_name: json.dumps(value, ensure_ascii=False),
+            self.updated_at_column_name: datetime.now(timezone.utc),
+        }
+        async with self.engine.begin() as conn:
+            exists = (
+                await conn.execute(sa.select(key_column).where(key_column == key))
+            ).scalar_one_or_none()
+            if exists is None:
+                await conn.execute(sa.insert(self._table).values(**payload))
+                return
+            await conn.execute(
+                sa.update(self._table)
+                .where(key_column == key)
+                .values(
+                    **{
+                        self.value_column_name: payload[self.value_column_name],
+                        self.updated_at_column_name: payload[self.updated_at_column_name],
+                    }
+                )
+            )
+
+    async def delete(self, key: str) -> None:
+        await self._ensure_ready()
+        key_column = self._table.c[self.key_column_name]
+        async with self.engine.begin() as conn:
+            await conn.execute(sa.delete(self._table).where(key_column == key))
+
+    async def close(self) -> None:
+        if self._owns_engine:
+            await self.engine.dispose()
+
+    async def _ensure_ready(self) -> None:
+        if self._ready or not self.auto_create:
+            return
+        lock = self._ready_lock
+        if lock is None:
+            lock = asyncio.Lock()
+            self._ready_lock = lock
+        async with lock:
+            if self._ready:
+                return
+            async with self.engine.begin() as conn:
+                await conn.run_sync(
+                    lambda sync_conn: self._metadata.create_all(
+                        sync_conn, tables=[self._table], checkfirst=True
+                    )
+                )
+            self._ready = True
+
+
+class SQLAlchemyCursorStore(SQLAlchemyStateStore):
+    _DATETIME_TYPE_KEY = "__onestep_cursor_type__"
+    _DATETIME_TYPE_VALUE = "datetime"
+    _DATETIME_VALUE_KEY = "value"
+
+    async def load(self, key: str) -> Any | None:
+        value = await super().load(key)
+        if not isinstance(value, list):
+            return value
+        return [self._decode_cursor_component(component) for component in value]
+
+    async def save(self, key: str, value: Any) -> None:
+        if isinstance(value, (list, tuple)):
+            value = [self._encode_cursor_component(component) for component in value]
+        await super().save(key, value)
+
+    @classmethod
+    def _encode_cursor_component(cls, value: Any) -> Any:
+        if isinstance(value, datetime):
+            return {
+                cls._DATETIME_TYPE_KEY: cls._DATETIME_TYPE_VALUE,
+                cls._DATETIME_VALUE_KEY: value.isoformat(),
+            }
+        return value
+
+    @classmethod
+    def _decode_cursor_component(cls, value: Any) -> Any:
+        if (
+            isinstance(value, dict)
+            and set(value) == {cls._DATETIME_TYPE_KEY, cls._DATETIME_VALUE_KEY}
+            and value[cls._DATETIME_TYPE_KEY] == cls._DATETIME_TYPE_VALUE
+            and isinstance(value[cls._DATETIME_VALUE_KEY], str)
+        ):
+            return datetime.fromisoformat(value[cls._DATETIME_VALUE_KEY])
+        return value

onestep_sql/_shared/table_sink_policy.py

@@ -0,0 +1,119 @@
+"""Shared table-sink update-policy machinery for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of the update-policy helpers that previously lived in
+parallel in both backend ``connector.py`` modules (see the retired
+``scripts/check_plugin_drift.py``):
+
+* ``_UPDATE_COLUMN_POLICIES`` / ``_normalize_update_columns`` — validation of
+  the ``update_columns`` write policies (``overwrite`` / ``skip_null`` /
+  ``backfill``) against keys and ``update_expr``;
+* :class:`TableSinkUpdatePolicy` — the ``_update_payload`` column-write
+  policy and ``_coerce_json_values`` JSON serialization mixin used by both
+  ``TableSink`` (mysql) and ``PostgresTableSink``.
+
+Everything that genuinely differs between the backends stays in their
+``connector.py``: constructor validation order, ``_build_statement`` SQL
+dialect branches (``ON DUPLICATE KEY UPDATE`` vs ``ON CONFLICT DO UPDATE``)
+and the ``_send`` logging.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+try:
+    import sqlalchemy as sa
+except ImportError:  # pragma: no cover - exercised when optional deps are missing
+    sa = None
+
+_UPDATE_COLUMN_POLICIES = frozenset({"overwrite", "skip_null", "backfill"})
+
+
+def _normalize_update_columns(
+    update_columns: Sequence[str | Mapping[str, str]] | None,
+    *,
+    keys: tuple[str, ...],
+    update_expr: Mapping[str, str] | None = None,
+) -> tuple[tuple[str, ...] | None, dict[str, str]]:
+    if update_columns is None:
+        return None, {}
+    names: list[str] = []
+    policies: dict[str, str] = {}
+    for entry in update_columns:
+        if isinstance(entry, str):
+            if not entry:
+                raise ValueError("update_columns entries must be non-empty")
+            name, policy = entry, "overwrite"
+        elif isinstance(entry, Mapping):
+            unknown_keys = set(entry) - {"name", "policy"}
+            if unknown_keys:
+                raise ValueError(f"unknown update_columns entry keys: {', '.join(sorted(unknown_keys))}")
+            name = entry.get("name")
+            policy = entry.get("policy", "overwrite")
+            if not isinstance(name, str) or not name:
+                raise ValueError("update_columns entry requires a non-empty 'name'")
+            if policy not in _UPDATE_COLUMN_POLICIES:
+                raise ValueError(
+                    "update_columns policy must be one of 'overwrite', 'skip_null' or 'backfill', "
+                    f"got {policy!r}"
+                )
+            if name in keys:
+                raise ValueError(f"update_columns policy cannot apply to key column {name!r}")
+        else:
+            raise TypeError("update_columns entries must be strings or mappings")
+        if name in policies:
+            raise ValueError(f"duplicate update column {name!r}")
+        names.append(name)
+        policies[name] = policy
+    update_expr_keys = set(update_expr) if update_expr else set()
+    conflicting = sorted(set(policies) & update_expr_keys)
+    if conflicting:
+        raise ValueError(f"update_columns policy conflicts with update_expr for: {', '.join(conflicting)}")
+    return tuple(names), policies
+
+
+class TableSinkUpdatePolicy:
+    """Mixin implementing the shared SQL table-sink column-write policy.
+
+    Host sinks must set these instance attributes (both existing backends
+    already do): ``keys``, ``update_columns``, ``column_policies``,
+    ``update_expr`` and ``serialize_json``.
+    """
+
+    def _update_payload(self, payload: dict[str, Any], table: sa.Table) -> tuple[dict[str, Any], bool]:
+        if self.update_columns is not None:
+            candidates = {key: value for key, value in payload.items() if key in self.update_columns}
+        else:
+            candidates = {key: value for key, value in payload.items() if key not in self.keys}
+        update_payload: dict[str, Any] = {}
+        skipped = False
+        for column, value in candidates.items():
+            policy = self.column_policies.get(column, "overwrite")
+            if policy == "skip_null" and value is None:
+                skipped = True
+                continue
+            if policy == "backfill":
+                update_payload[column] = sa.func.coalesce(table.columns[column], value)
+            else:
+                update_payload[column] = value
+        for column, expr in self.update_expr.items():
+            update_payload[column] = sa.literal_column(expr)
+        return update_payload, skipped
+
+    def _coerce_json_values(self, payload: dict[str, Any], table: sa.Table) -> dict[str, Any]:
+        if self.serialize_json == "never":
+            return payload
+        coerced = dict(payload)
+        for column_name, value in list(payload.items()):
+            if not isinstance(value, (list, dict)):
+                continue
+            column = table.columns.get(column_name)
+            if column is None:
+                continue
+            is_json_column = isinstance(column.type, sa.JSON)
+            if self.serialize_json == "always" or not is_json_column:
+                coerced[column_name] = json.dumps(value, ensure_ascii=False)
+        return coerced

onestep_sql/mysql/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version as _package_version
+
+from .connector import (
+    BinlogDelivery,
+    BinlogSource,
+    IncrementalDelivery,
+    IncrementalTableSource,
+    MySQLConnector,
+    TableQueueDelivery,
+    TableQueueSource,
+    TableSink,
+)
+from .resources import register_resources
+from .resilience import classify_sqlalchemy_error
+from .state_sqlalchemy import SQLAlchemyCursorStore, SQLAlchemyStateStore
+
+try:
+    __version__ = _package_version("onestep-sql")
+except PackageNotFoundError:  # pragma: no cover - local source tree before install
+    __version__ = "dev"
+
+register = register_resources
+
+__all__ = [
+    "BinlogDelivery",
+    "BinlogSource",
+    "IncrementalDelivery",
+    "IncrementalTableSource",
+    "MySQLConnector",
+    "SQLAlchemyCursorStore",
+    "SQLAlchemyStateStore",
+    "TableQueueDelivery",
+    "TableQueueSource",
+    "TableSink",
+    "__version__",
+    "classify_sqlalchemy_error",
+    "register",
+    "register_resources",
+]