onestep-sql 0.1.0__tar.gz

onestep_sql-0.1.0/.gitignore

@@ -0,0 +1,146 @@
+# created by virtualenv automatically
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+!deploy/env/
+!deploy/env/onestep-app.env.example
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pycharm
+.idea
+poetry.lock
+
+# Ignore dynaconf secret files
+*.secrets.*
+node_modules/
+.DS_Store
+
+.release-smoke-check/
+.release-smoke-local/
+.release-smoke-local2/
+.uv-cache/

onestep_sql-0.1.0/PKG-INFO

@@ -0,0 +1,85 @@
+Metadata-Version: 2.5
+Name: onestep-sql
+Version: 0.1.0
+Summary: Unified MySQL and PostgreSQL connector plugin for onestep.
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: onestep>=1.9.0
+Requires-Dist: sqlalchemy[asyncio]>=2.0.0
+Provides-Extra: all
+Requires-Dist: aiosqlite>=0.20.0; extra == 'all'
+Requires-Dist: asyncmy>=0.2.10; extra == 'all'
+Requires-Dist: mysql-replication>=1.0.15; extra == 'all'
+Requires-Dist: psycopg[binary]>=3.2.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Provides-Extra: mysql
+Requires-Dist: asyncmy>=0.2.10; extra == 'mysql'
+Requires-Dist: mysql-replication>=1.0.15; extra == 'mysql'
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgres'
+Provides-Extra: sqlite
+Requires-Dist: aiosqlite>=0.20.0; extra == 'sqlite'
+Provides-Extra: test
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'test'
+Requires-Dist: pytest>=8.0.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# onestep-sql
+
+Canonical, unified MySQL **and** PostgreSQL connector plugin for
+[onestep](https://github.com/mic1on/onestep).
+
+This package merges the previously separate `onestep-mysql` and
+`onestep-postgres` distributions behind a single entry point and a single
+namespace (`onestep_sql`). It is introduced incrementally per
+[the consolidation design](../../docs/superpowers/specs/2026-08-20-onestep-sql-consolidation-design.md)
+(tracking issue #133).
+
+## Status
+
+**Phase 3 — canonical distribution; legacy packages are thin forwarding shims.**
+
+* `onestep-sql[mysql]`, `[postgres]`, `[sqlite]`, and `[all]` build, discover
+  their resources, and pass their suites.
+* `onestep_sql.mysql` and `onestep_sql.postgres` carry the canonical
+  implementations; shared SQL behaviour lives once in `onestep_sql._shared`.
+* The root `onestep` extras (`mysql`, `postgres`, `sql`, `all`, `dev`,
+  `integration`) resolve through `onestep-sql`.
+* The legacy `onestep-mysql` / `onestep-postgres` distributions remain
+  available as thin forwarding shims without their own resource entry points;
+  existing `pip install onestep-mysql` and `from onestep_mysql import ...`
+  imports keep working unchanged.
+* All 14 YAML resource type names, catalog roles, fields, defaults, and
+  connector boundaries are unchanged.
+
+## Install
+
+```bash
+pip install "onestep-sql[all]"      # MySQL + PostgreSQL
+pip install "onestep-sql[mysql]"    # MySQL only
+pip install "onestep-sql[postgres]" # PostgreSQL only
+```
+
+## Usage
+
+The package registers all 14 YAML resource types through a single entry point
+(`sql` in the `onestep.resources` group). No import is required to use the
+types in a YAML pipeline — onestep discovers them automatically.
+
+For programmatic access to the connector classes:
+
+```python
+from onestep_sql.mysql import MySQLConnector, BinlogSource
+from onestep_sql.postgres import PostgresConnector, PostgresExecutionSource
+```
+
+## What is NOT changing
+
+* The 14 YAML type names (`mysql_*`, `postgres_*`) and their catalog roles,
+  fields, defaults, and connector boundaries are unchanged.
+* `mysql_binlog` stays MySQL-only; `postgres_execution_source` / tracked
+  execution stays PostgreSQL-only.
+* See the design doc for the full non-goals and the phased rollout plan.

onestep_sql-0.1.0/README.md

@@ -0,0 +1,56 @@
+# onestep-sql
+
+Canonical, unified MySQL **and** PostgreSQL connector plugin for
+[onestep](https://github.com/mic1on/onestep).
+
+This package merges the previously separate `onestep-mysql` and
+`onestep-postgres` distributions behind a single entry point and a single
+namespace (`onestep_sql`). It is introduced incrementally per
+[the consolidation design](../../docs/superpowers/specs/2026-08-20-onestep-sql-consolidation-design.md)
+(tracking issue #133).
+
+## Status
+
+**Phase 3 — canonical distribution; legacy packages are thin forwarding shims.**
+
+* `onestep-sql[mysql]`, `[postgres]`, `[sqlite]`, and `[all]` build, discover
+  their resources, and pass their suites.
+* `onestep_sql.mysql` and `onestep_sql.postgres` carry the canonical
+  implementations; shared SQL behaviour lives once in `onestep_sql._shared`.
+* The root `onestep` extras (`mysql`, `postgres`, `sql`, `all`, `dev`,
+  `integration`) resolve through `onestep-sql`.
+* The legacy `onestep-mysql` / `onestep-postgres` distributions remain
+  available as thin forwarding shims without their own resource entry points;
+  existing `pip install onestep-mysql` and `from onestep_mysql import ...`
+  imports keep working unchanged.
+* All 14 YAML resource type names, catalog roles, fields, defaults, and
+  connector boundaries are unchanged.
+
+## Install
+
+```bash
+pip install "onestep-sql[all]"      # MySQL + PostgreSQL
+pip install "onestep-sql[mysql]"    # MySQL only
+pip install "onestep-sql[postgres]" # PostgreSQL only
+```
+
+## Usage
+
+The package registers all 14 YAML resource types through a single entry point
+(`sql` in the `onestep.resources` group). No import is required to use the
+types in a YAML pipeline — onestep discovers them automatically.
+
+For programmatic access to the connector classes:
+
+```python
+from onestep_sql.mysql import MySQLConnector, BinlogSource
+from onestep_sql.postgres import PostgresConnector, PostgresExecutionSource
+```
+
+## What is NOT changing
+
+* The 14 YAML type names (`mysql_*`, `postgres_*`) and their catalog roles,
+  fields, defaults, and connector boundaries are unchanged.
+* `mysql_binlog` stays MySQL-only; `postgres_execution_source` / tracked
+  execution stays PostgreSQL-only.
+* See the design doc for the full non-goals and the phased rollout plan.

onestep_sql-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "onestep-sql"
+version = "0.1.0"
+description = "Unified MySQL and PostgreSQL connector plugin for onestep."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+dependencies = [
+    "onestep>=1.9.0",
+    "SQLAlchemy[asyncio]>=2.0.0",
+    "aiosqlite>=0.20.0",
+]
+
+[project.optional-dependencies]
+mysql = [
+    "asyncmy>=0.2.10",
+    "mysql-replication>=1.0.15",
+]
+postgres = [
+    "psycopg[binary]>=3.2.0",
+]
+sqlite = [
+    "aiosqlite>=0.20.0",
+]
+all = [
+    "asyncmy>=0.2.10",
+    "mysql-replication>=1.0.15",
+    "psycopg[binary]>=3.2.0",
+    "aiosqlite>=0.20.0",
+]
+test = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+]
+
+[project.entry-points."onestep.resources"]
+sql = "onestep_sql:register_resources"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/onestep_sql"]
+
+[tool.uv.sources]
+onestep = { workspace = true }

onestep_sql-0.1.0/src/onestep_sql/__init__.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from onestep.resource_registry import ResourceRegistry
+
+from . import mysql, postgres
+from .mysql import register_resources as _register_mysql
+from .postgres import register_resources as _register_postgres
+
+__all__ = ["register_resources", "mysql", "postgres"]
+
+
+def register_resources(registry: ResourceRegistry) -> None:
+    """Register every onestep-sql resource type (MySQL + PostgreSQL).
+
+    This is the single entry point for the canonical ``onestep-sql``
+    distribution (declared as ``sql`` in the ``onestep.resources`` group).
+    It delegates to the per-backend registrars without changing any of the
+    14 existing YAML type names, their catalog roles, or their fields.
+    """
+    _register_mysql(registry)
+    _register_postgres(registry)

onestep_sql-0.1.0/src/onestep_sql/_shared/__init__.py

@@ -0,0 +1,26 @@
+"""Internal shared SQL behaviour for the canonical ``onestep-sql`` package.
+
+Phase 2 of the ``onestep-mysql`` / ``onestep-postgres`` consolidation
+(issue #133, design in PR #134) keeps exactly one copy of the machinery that
+previously lived in parallel in both backend subpackages (as monitored by the
+now-retired ``scripts/check_plugin_drift.py``):
+
+* :mod:`onestep_sql._shared.state_sqlalchemy` — the SQLAlchemy state/cursor
+  stores; backends subclass them and override only the asyncio driver mapping
+  (``mysql+asyncmy`` vs ``postgresql+psycopg``) and install hint;
+* :mod:`onestep_sql._shared.table_sink_policy` — the table-sink
+  ``update_columns`` write policies (``overwrite`` / ``skip_null`` /
+  ``backfill``), the ``_normalize_update_columns`` validator and the
+  ``_update_payload`` / ``_coerce_json_values`` mixin;
+* :mod:`onestep_sql._shared.state_keys` — the default incremental state-key
+  derivation feeding the at-least-once cursor state;
+* :mod:`onestep_sql._shared.resilience` — the secret-token collection, message
+  redaction, redacted error-cause base class and the
+  ``ConnectorOperationError`` factory; the dialect-specific SQLAlchemy error
+  classification tables stay in each backend's ``resilience.py``.
+
+This package is **not** public API. Backend capabilities never move here:
+MySQL binlog CDC stays in ``onestep_sql.mysql`` and the PostgreSQL tracked
+execution modules stay in ``onestep_sql.postgres``. Dual-backend contract
+tests live in ``tests/contract/test_onestep_sql_shared.py``.
+"""

onestep_sql-0.1.0/src/onestep_sql/_shared/resilience.py

@@ -0,0 +1,147 @@
+"""Shared secret-redaction scaffolding for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of the resilience machinery that previously existed in
+parallel in both backend ``resilience.py`` modules and differed only in the
+backend name (see the retired ``scripts/check_plugin_drift.py``):
+
+* the secret-token collection used to scrub error causes
+  (:func:`collect_sensitive_tokens`);
+* the message redaction itself (:func:`redact_message`);
+* the redacted error-cause base class (:class:`SQLErrorCause`) that backends
+  subclass with their own ``backend`` label so ``str(cause)`` keeps its
+  per-database prefix;
+* the :class:`~onestep.resilience.ConnectorOperationError` factory
+  (:func:`as_connector_operation_error`).
+
+The SQLAlchemy error-classification tables stay in each backend's
+``resilience.py`` because they encode genuinely different server error
+messages (``server has gone away`` vs ``server closed the connection``,
+``deadlock found`` vs ``deadlock detected`` and so on).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import ClassVar
+from urllib.parse import urlsplit
+
+from onestep.resilience import (
+    ConnectorErrorKind,
+    ConnectorOperation,
+    ConnectorOperationError,
+)
+
+REDACTED = "<redacted>"
+MAX_MESSAGE_LENGTH = 500
+SECRET_OPTION_KEYS = frozenset(
+    {
+        "password",
+        "passwd",
+        "pwd",
+        "secret",
+        "token",
+        "access_token",
+        "api_key",
+        "apikey",
+        "credentials",
+        "authorization",
+    }
+)
+
+
+def collect_sensitive_tokens(*config_values: object) -> list[str]:
+    """Collect secret substrings that may surface in SQL backend error messages.
+
+    SQLAlchemy masks the password in its rendered URL, but the underlying DBAPI
+    ``orig`` exception can still echo credentials (e.g. after echoing the full
+    connection string). Tokens are derived from connector config (raw DSN +
+    parsed userinfo + known secret option values) and used to scrub error
+    causes before they leave the plugin.
+    """
+    tokens: list[str] = []
+    seen: set[str] = set()
+
+    def add(value: object) -> None:
+        if value is None:
+            return
+        text = str(value)
+        if text and text not in seen:
+            seen.add(text)
+            tokens.append(text)
+
+    def collect_mapping(value: Mapping[object, object]) -> None:
+        for key, item in value.items():
+            if str(key).lower() in SECRET_OPTION_KEYS:
+                add(item)
+            elif isinstance(item, Mapping):
+                collect_mapping(item)
+
+    for value in config_values:
+        if isinstance(value, Mapping):
+            collect_mapping(value)
+            continue
+        add(value)
+        try:
+            parsed = urlsplit(str(value))
+        except ValueError:
+            continue
+        username = parsed.username
+        password = parsed.password
+        if username and password:
+            add(f"{username}:{password}")
+            add(f"{username}:{password}@")
+        add(password)
+    return tokens
+
+
+def redact_message(message: str, tokens: list[str]) -> str:
+    redacted = message
+    for token in sorted(tokens, key=len, reverse=True):
+        if token:
+            redacted = redacted.replace(token, REDACTED)
+    return redacted[:MAX_MESSAGE_LENGTH]
+
+
+@dataclass(frozen=True)
+class SQLErrorCause(Exception):
+    """Redacted cause attached to normalized SQL connector errors.
+
+    Backends subclass this with their own ``backend`` label so the rendered
+    message keeps its per-database prefix (``mysql error: ...`` /
+    ``postgres error: ...``) while the dataclass shape, freezing and identity
+    semantics are implemented once here.
+    """
+
+    message: str
+
+    backend: ClassVar[str] = "sql"
+
+    def __str__(self) -> str:
+        return f"{self.backend} error: {self.message}"
+
+
+def as_connector_operation_error(
+    *,
+    backend: str,
+    operation: ConnectorOperation,
+    exc: BaseException,
+    classify: Callable[[BaseException], ConnectorErrorKind | None],
+    redacted_cause: Callable[..., Exception],
+    source_name: str | None = None,
+    retry_delay_s: float | None = None,
+    secrets: list[str] | None = None,
+) -> ConnectorOperationError | None:
+    """Normalize ``exc`` using the backend's classifier and redacted cause."""
+    kind = classify(exc)
+    if kind is None:
+        return None
+    return ConnectorOperationError(
+        backend=backend,
+        operation=operation,
+        kind=kind,
+        source_name=source_name,
+        retry_delay_s=retry_delay_s,
+        cause=redacted_cause(exc, secrets=secrets),
+    )

onestep_sql-0.1.0/src/onestep_sql/_shared/state_keys.py

@@ -0,0 +1,31 @@
+"""Shared default incremental state-key derivation for the onestep-sql backends.
+
+Phase 2 of the mysql/postgres consolidation (issue #133, design §3.1) keeps
+exactly one copy of ``_default_incremental_state_key``, which previously
+existed in parallel in both backend ``connector.py`` modules (see the retired
+``scripts/check_plugin_drift.py``). The derived key feeds the at-least-once
+cursor state, so its format is stable contract: changing it would orphan
+existing persisted cursors.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+
+
+def _default_incremental_state_key(
+    *,
+    table: str,
+    cursor: Sequence[str],
+    key: str,
+    where: str | None,
+) -> str:
+    normalized_where = " ".join((where or "").split())
+    if normalized_where:
+        where_fragment = normalized_where
+        if len(where_fragment) > 64:
+            where_fragment = f"sha1:{hashlib.sha1(where_fragment.encode('utf-8')).hexdigest()}"
+    else:
+        where_fragment = "-"
+    return f"{table}:{','.join(cursor)}:key={key}:where={where_fragment}"