any2heliosdb 0.9.1__py3-none-any.whl

any2heliosdb/cdc/replicat.py

@@ -0,0 +1,109 @@
+"""Replicat: apply captured change records to the target, idempotently.
+
+Records are bucketed by table and applied through the target driver's upsert
+(insert/update) and delete-by-key seams, so re-applying the same trail slice is
+a no-op on row state. Tables without a primary key can't be keyed and are
+skipped with a warning.
+
+Within a table the trail is an *ordered* stream of explicit I/U/D change
+records, so order is load-bearing: ``DELETE id=7`` then ``INSERT id=7`` must
+leave the row present. The apply therefore walks each table's records in
+arrival order, batching only maximal contiguous runs of the same op class
+(upsert vs delete) so the relative delete/upsert order is preserved while still
+amortizing the round-trips. (This is distinct from :meth:`reconcile_deletes`,
+the snapshot-source key-set diff that has no per-record order to honor.)
+"""
+from __future__ import annotations
+
+from typing import Dict, Iterator, List, Tuple
+
+from ..core.change_record import DELETE, ChangeRecord
+
+
+def _runs_by_op_class(recs: List[ChangeRecord]) -> Iterator[Tuple[bool, List[ChangeRecord]]]:
+    """Split records into maximal contiguous runs of one op class, in order.
+
+    Yields ``(is_delete, run)``: ``is_delete`` True for a run of DELETEs, False
+    for a run of INSERT/UPDATE upserts. Splitting only at the upsert<->delete
+    boundary preserves the relative ordering that distinguishes ``D`` then ``I``
+    (row ends present) from ``I`` then ``D`` (row ends absent), while letting
+    same-class neighbors batch into one driver round-trip.
+    """
+    run: List[ChangeRecord] = []
+    cur_is_delete = False
+    for r in recs:
+        is_delete = r.op == DELETE
+        if run and is_delete != cur_is_delete:
+            yield cur_is_delete, run
+            run = []
+        cur_is_delete = is_delete
+        run.append(r)
+    if run:
+        yield cur_is_delete, run
+
+
+class Replicat:
+    def __init__(self, target, schema_ir, preserve_case: bool = False) -> None:  # type: ignore[no-untyped-def]
+        self.target = target
+        self.preserve_case = preserve_case
+        self._by_name = {t.name.upper(): t for t in schema_ir.tables}
+
+    def _ident(self, name: str) -> str:
+        return name if self.preserve_case else name.lower()
+
+    def apply(self, records: List[ChangeRecord]) -> Tuple[int, List[str]]:
+        buckets: Dict[str, List[ChangeRecord]] = {}
+        for r in records:
+            buckets.setdefault(r.table, []).append(r)
+
+        applied = 0
+        warnings: List[str] = []
+        for tname, recs in buckets.items():
+            t = self._by_name.get(tname.upper())
+            if t is None or not (t.primary_key and t.primary_key.columns):
+                warnings.append("{}: no primary key; skipped {} change(s)".format(tname, len(recs)))
+                continue
+            target_table = t.target_name(self.preserve_case)
+            cols = [c.name for c in t.columns]
+            tcols = [self._ident(c) for c in cols]
+            key_cols = [self._ident(c) for c in t.primary_key.columns]
+            pk = t.primary_key.columns
+
+            # Apply in arrival order, flushing one op class at a time so that an
+            # earlier DELETE is never resurrected by a later batched upsert (and
+            # vice versa). Contiguous same-class records are coalesced into a
+            # single driver call to keep the round-trip count low.
+            for is_delete, run in _runs_by_op_class(recs):
+                if is_delete:
+                    keys = [[r.key.get(c) for c in pk] for r in run]
+                    applied += self.target.delete_keys(target_table, key_cols, keys)
+                else:
+                    rows = [[r.after.get(c) for c in cols] for r in run]
+                    applied += self.target.upsert(target_table, key_cols, tcols, rows)
+        return applied, warnings
+
+    def reconcile_deletes(self, source_adapter) -> Tuple[int, List[str]]:  # type: ignore[no-untyped-def]
+        """Delete target rows whose PK is absent from the source's current keys.
+
+        v1 SCN-watermark capture cannot observe DELETEs (the rows are already
+        gone), so the replicat reconciles them with a full key-set diff: keys on
+        the target but not in the source are removed. This is a full pass
+        (cost O(keys)); incremental delete capture is the log-based roadmap.
+        """
+        deleted = 0
+        warnings: List[str] = []
+        for t in self._by_name.values():
+            if not (t.primary_key and t.primary_key.columns):
+                continue
+            pk = t.primary_key.columns
+            try:
+                src_keys = {tuple(r) for r in source_adapter.stream_rows(t, pk)}
+                target_table = t.target_name(self.preserve_case)
+                key_idents = [self._ident(c) for c in pk]
+                tgt_keys = {tuple(r) for r in self.target.select_keys(target_table, key_idents)}
+                extra = [list(k) for k in (tgt_keys - src_keys)]
+                if extra:
+                    deleted += self.target.delete_keys(target_table, key_idents, extra)
+            except Exception as e:  # noqa: BLE001
+                warnings.append("delete reconcile {}: {}".format(t.name, e))
+        return deleted, warnings

any2heliosdb/cdc/sources/mysql_binlog.py

@@ -0,0 +1,193 @@
+"""MySQL binlog CDC capture (log-based).
+
+Reads ROW-format binlog events (`mysql-replication`) and turns them into
+`ChangeRecord`s — real inserts/updates **and deletes**, unlike the SCN-watermark
+source. The capture cursor is the binlog coordinate ``"<file>:<pos>"``; on the
+first cycle it anchors at the server's *current* position (so only changes after
+the baseline load are captured) and returns no records.
+
+Requires the source MySQL to have ``log_bin=ON`` and ``binlog_format=ROW``, and
+the connecting user to hold ``REPLICATION SLAVE``/``REPLICATION CLIENT``.
+
+Correctness depends on **full** row images and column metadata: with
+``binlog_row_image=MINIMAL`` an UPDATE only logs the changed columns (the
+replicat would then NULL the omitted ones), and with
+``binlog_row_metadata=MINIMAL`` events carry ``UNKNOWN_COL0..`` placeholders
+instead of real column names. The source therefore fails closed — it verifies
+``binlog_format=ROW``, ``binlog_row_metadata=FULL`` and ``binlog_row_image=FULL``
+before anchoring/capture, and rejects any captured event whose after-image does
+not cover the table's columns.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Tuple
+
+from ...core.change_record import DELETE, INSERT, UPDATE, ChangeRecord
+from ...errors import Any2HeliosError
+
+# Server variables that must be FULL/ROW for log-based CDC to be lossless.
+_REQUIRED_VARS = (
+    ("binlog_format", "ROW",
+     "binlog row events are required; STATEMENT/MIXED don't carry per-row before/after images"),
+    ("binlog_row_metadata", "FULL",
+     "MINIMAL omits column names, so events expose UNKNOWN_COL0.. instead of real columns"),
+    ("binlog_row_image", "FULL",
+     "MINIMAL logs only changed columns on UPDATE, so unchanged columns would be written as NULL"),
+)
+
+
+def _require_full_row_image(settings: Dict[str, str]) -> None:
+    """Raise unless ROW binlog with FULL metadata + FULL row image is in effect.
+
+    ``settings`` maps server-variable name -> value (case-insensitively), as read
+    from ``SHOW VARIABLES``. Fails closed with a clear, actionable message naming
+    the offending variable rather than letting capture silently corrupt the
+    target. Pure (no I/O) so it can be unit-tested with plain dicts.
+    """
+    norm = {str(k).lower(): ("" if v is None else str(v)) for k, v in settings.items()}
+    for var, want, why in _REQUIRED_VARS:
+        got = norm.get(var.lower())
+        if got is None:
+            raise Any2HeliosError(
+                "MySQL CDC: could not read '{}' (need {}={}). {}. Grant the connecting "
+                "user access to read server variables, or set it server-side.".format(
+                    var, var, want, why))
+        if got.strip().upper() != want:
+            raise Any2HeliosError(
+                "MySQL CDC requires {}={} but the server reports {}={!r}. {}. Fix it with "
+                "`SET GLOBAL {}={}` (needs SYSTEM_VARIABLES_ADMIN; for binlog_format also "
+                "restart replication threads) or set it in my.cnf and restart.".format(
+                    var, want, var, got, why, var, want))
+
+
+def _check_image_columns(table: str, op: str, image_keys, expected_cols) -> None:  # type: ignore[no-untyped-def]
+    """Raise if a captured row image is missing columns or carries placeholders.
+
+    A FULL row image/metadata stream names every column; a MINIMAL one drops
+    unchanged columns and/or surfaces ``UNKNOWN_COL0..`` keys. Either case would
+    have the replicat write NULLs over real data, so reject the event loudly.
+    ``image_keys`` is the dict-key set of the captured value map; ``expected_cols``
+    is the table's full column list from the source schema.
+    """
+    have = set(image_keys)
+    bad = sorted(k for k in have if str(k).upper().startswith("UNKNOWN_COL"))
+    if bad:
+        raise Any2HeliosError(
+            "MySQL CDC: {} on {} carried unnamed columns {} — binlog_row_metadata is not "
+            "FULL. Set binlog_row_metadata=FULL on the source and re-anchor.".format(
+                op, table, bad))
+    missing = [c for c in expected_cols if c not in have]
+    if missing:
+        raise Any2HeliosError(
+            "MySQL CDC: {} on {} omitted columns {} from its row image — binlog_row_image "
+            "is not FULL (partial images would be written as NULL). Set "
+            "binlog_row_image=FULL on the source and re-anchor.".format(op, table, missing))
+
+
+class MySqlBinlogSource:
+    def __init__(self, dsn, schema, tables, server_id: int = 4279):  # type: ignore[no-untyped-def]
+        self.dsn = dsn
+        self.schema = schema
+        self.server_id = server_id
+        self._pk = {t.name: (list(t.primary_key.columns) if t.primary_key else []) for t in tables}
+        self._cols = {t.name: [c.name for c in t.columns] for t in tables}
+        self._tables = [t.name for t in tables]
+
+    def _conn_settings(self) -> dict:
+        return {"host": self.dsn.host, "port": int(self.dsn.port),
+                "user": self.dsn.user, "passwd": self.dsn.password or ""}
+
+    @staticmethod
+    def _read_row_image_vars(cur) -> Dict[str, str]:  # type: ignore[no-untyped-def]
+        """Read the binlog_* variables that gate lossless capture into a dict."""
+        out: Dict[str, str] = {}
+        for var, _want, _why in _REQUIRED_VARS:
+            try:
+                cur.execute("SHOW VARIABLES LIKE %s", (var,))
+                row = cur.fetchone()
+            except Exception:  # noqa: BLE001
+                row = None
+            if row:
+                # SHOW VARIABLES returns (Variable_name, Value).
+                out[str(row[0])] = "" if row[1] is None else str(row[1])
+        return out
+
+    def current_position(self) -> str:
+        import pymysql
+
+        c = pymysql.connect(**{k: v for k, v in self._conn_settings().items() if k != "passwd"},
+                            password=self.dsn.password or "")
+        try:
+            cur = c.cursor()
+            # Binlog row events only carry column *names* when row metadata is FULL
+            # (the default MINIMAL yields UNKNOWN_COL0..). Set it best-effort so
+            # events written after this anchor map to real column names. Requires
+            # SYSTEM_VARIABLES_ADMIN; if denied, set it server-side (a documented
+            # prerequisite alongside log_bin=ON / binlog_format=ROW).
+            try:
+                cur.execute("SET GLOBAL binlog_row_metadata = FULL")
+            except Exception:  # noqa: BLE001
+                pass
+            # Fail closed: anchoring here means every later cycle resumes from this
+            # coordinate, so the row-image guarantees must already hold *now*.
+            # (binlog_row_image can't be fixed by a SET that only takes effect for
+            # new sessions, so verify rather than assume the best-effort SET stuck.)
+            _require_full_row_image(self._read_row_image_vars(cur))
+            for q in ("SHOW BINARY LOG STATUS", "SHOW MASTER STATUS"):  # 8.4 renamed it
+                try:
+                    cur.execute(q)
+                    row = cur.fetchone()
+                    if row:
+                        return "{}:{}".format(row[0], row[1])
+                except Exception:  # noqa: BLE001
+                    continue
+            return ""
+        finally:
+            c.close()
+
+    def capture(self, position: str) -> Tuple[List[ChangeRecord], str]:
+        if not position:
+            # First cycle: anchor at the current position, capture nothing yet.
+            return [], self.current_position()
+
+        from pymysqlreplication import BinLogStreamReader
+        from pymysqlreplication.row_event import (
+            DeleteRowsEvent,
+            UpdateRowsEvent,
+            WriteRowsEvent,
+        )
+
+        log_file, _, log_pos = position.rpartition(":")
+        stream = BinLogStreamReader(
+            connection_settings=self._conn_settings(), server_id=self.server_id,
+            only_schemas=[self.schema], only_tables=self._tables,
+            only_events=[WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent],
+            log_file=log_file, log_pos=int(log_pos), resume_stream=True, blocking=False)
+        records: List[ChangeRecord] = []
+        try:
+            for ev in stream:
+                tbl = ev.table
+                pk = self._pk.get(tbl, [])
+                expected = self._cols.get(tbl, [])
+                for row in ev.rows:
+                    if isinstance(ev, WriteRowsEvent):
+                        vals = row["values"]
+                        _check_image_columns(tbl, "INSERT", vals.keys(), expected)
+                        records.append(ChangeRecord(op=INSERT, schema=self.schema, table=tbl,
+                                                    key={k: vals.get(k) for k in pk}, after=dict(vals)))
+                    elif isinstance(ev, UpdateRowsEvent):
+                        vals = row["after_values"]
+                        _check_image_columns(tbl, "UPDATE", vals.keys(), expected)
+                        records.append(ChangeRecord(op=UPDATE, schema=self.schema, table=tbl,
+                                                    key={k: vals.get(k) for k in pk}, after=dict(vals)))
+                    elif isinstance(ev, DeleteRowsEvent):
+                        vals = row["values"]
+                        # Delete only needs a sound key, but UNKNOWN_COL / missing PK
+                        # still signals a non-FULL image, so verify the key columns.
+                        _check_image_columns(tbl, "DELETE", vals.keys(), pk)
+                        records.append(ChangeRecord(op=DELETE, schema=self.schema, table=tbl,
+                                                    key={k: vals.get(k) for k in pk}, after={}))
+            new_pos = "{}:{}".format(stream.log_file, stream.log_pos)
+        finally:
+            stream.close()
+        return records, new_pos

any2heliosdb/cdc/sources/oracle_scn.py

@@ -0,0 +1,43 @@
+"""v1 capture: Oracle SCN-watermark.
+
+Captures rows whose ``ORA_ROWSCN`` exceeds the extract's watermark (a full
+snapshot on the first cycle, when the watermark is 0) and emits them as upsert
+change records. ``ORA_ROWSCN`` is block-granular without ``ROWDEPENDENCIES``, so
+this may re-emit unchanged neighbours — harmless because the sink upserts on the
+key. Deletes are not visible to a watermark scan; they are a log-based (v2)
+concern. This is the guaranteed-portable Oracle "CDC" for shops without
+LogMiner/supplemental-logging access.
+"""
+from __future__ import annotations
+
+from typing import List, Tuple
+
+from ..registry import Extract  # noqa: F401  (type reference for callers)
+from ...core.change_record import UPDATE, ChangeRecord
+
+
+class OracleScnSource:
+    def __init__(self, adapter, schema, tables) -> None:  # type: ignore[no-untyped-def]
+        self.adapter = adapter
+        self.schema = schema
+        self.tables = tables
+
+    def capture(self, since_scn: int) -> Tuple[List[ChangeRecord], int, List[str]]:
+        # Anchor the new watermark *before* scanning, so concurrent commits during
+        # the scan are picked up next cycle rather than skipped.
+        start_scn = self.adapter.current_scn()
+        records: List[ChangeRecord] = []
+        skipped: List[str] = []
+        for t in self.tables:
+            if not (t.primary_key and t.primary_key.columns):
+                skipped.append(t.name)
+                continue
+            cols = [c.name for c in t.columns]
+            where = None if since_scn <= 0 else "ORA_ROWSCN > {}".format(int(since_scn))
+            for row in self.adapter.stream_rows(t, cols, where=where):
+                after = {col: row[i] for i, col in enumerate(cols)}
+                key = {pk: after[pk] for pk in t.primary_key.columns}
+                records.append(ChangeRecord(op=UPDATE, schema=self.schema, table=t.name,
+                                            key=key, after=after, scn=start_scn))
+        new_watermark = start_scn if start_scn > 0 else since_scn
+        return records, new_watermark, skipped

any2heliosdb/cdc/trail.py

@@ -0,0 +1,51 @@
+"""Durable append-only change trail (one file per extract).
+
+Records are appended as JSON lines and fsync'd before the append returns, so a
+committed record survives a crash. The reader is a simple line cursor: reading
+from cursor N returns every record after line N and the new line count, which
+the replicat persists only *after* a successful apply (at-least-once; combined
+with idempotent upserts on the key, effectively-once per row).
+"""
+from __future__ import annotations
+
+import os
+from typing import List, Tuple
+
+from ..core.change_record import ChangeRecord
+
+
+class Trail:
+    def __init__(self, trail_dir: str) -> None:
+        os.makedirs(trail_dir, exist_ok=True)
+        self.path = os.path.join(trail_dir, "trail.jsonl")
+
+    def append(self, records: List[ChangeRecord]) -> int:
+        if not records:
+            return 0
+        with open(self.path, "a", encoding="utf-8") as f:
+            for r in records:
+                f.write(r.to_json() + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+        return len(records)
+
+    def line_count(self) -> int:
+        if not os.path.exists(self.path):
+            return 0
+        with open(self.path, "r", encoding="utf-8") as f:
+            return sum(1 for _ in f)
+
+    def read(self, cursor: int) -> Tuple[List[ChangeRecord], int]:
+        """Return records after line ``cursor`` and the new cursor (total lines)."""
+        if not os.path.exists(self.path):
+            return [], cursor
+        out: List[ChangeRecord] = []
+        n = 0
+        with open(self.path, "r", encoding="utf-8") as f:
+            for n, line in enumerate(f, start=1):
+                if n <= cursor:
+                    continue
+                line = line.strip()
+                if line:
+                    out.append(ChangeRecord.from_json(line))
+        return out, max(cursor, n)

any2heliosdb/chunking/pk_range.py

@@ -0,0 +1,65 @@
+"""Split a table into key-range chunks for parallel + resumable load.
+
+A chunk is a half-open integer-PK range ``[lo, hi)``. Chunks are deterministic
+for a given source state (derived from ``MIN``/``MAX`` of the PK), so a resumed
+run regenerates the identical ``chunk_id``s and can skip the ones the manifest
+already recorded as loaded. Tables with no single integer PK fall back to one
+whole-table chunk.
+
+The same range is rendered for the source (Oracle, quoted/uppercase columns) and
+the target (lowercased unless ``preserve_case``), so the loader can stream the
+chunk from the source and idempotently DELETE the same range on the target.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+from ..core.catalog_model import Table
+from ..core.identifiers import render_ident
+
+
+@dataclass
+class Chunk:
+    table: Table
+    chunk_id: str
+    pk_col: Optional[str] = None  # None => whole-table chunk
+    lo: Optional[int] = None      # inclusive
+    hi: Optional[int] = None      # exclusive
+
+    def source_where(self) -> Optional[str]:
+        if self.pk_col is None:
+            return None
+        return '"{c}" >= {lo} AND "{c}" < {hi}'.format(c=self.pk_col, lo=self.lo, hi=self.hi)
+
+    def target_where(self, preserve_case: bool = False) -> Optional[str]:
+        if self.pk_col is None:
+            return None
+        # Render through the shared quoter so a reserved/mixed-case PK column
+        # (e.g. "order", "User") in the idempotent range DELETE matches the name
+        # the DDL/loader created, instead of a bare token that errors or folds.
+        c = render_ident(self.pk_col, preserve_case)
+        return "{c} >= {lo} AND {c} < {hi}".format(c=c, lo=self.lo, hi=self.hi)
+
+
+def compute_chunks(source, table: Table, target_chunks: int = 4) -> List[Chunk]:  # type: ignore[no-untyped-def]
+    """Return the chunk list for *table*, aiming for ~*target_chunks* pieces."""
+    pk = table.primary_key
+    if pk and len(pk.columns) == 1 and target_chunks > 1:
+        col = pk.columns[0]
+        bounds = source.numeric_pk_bounds(table, col)
+        if bounds is not None:
+            lo, hi = bounds
+            span = hi - lo + 1
+            n = max(1, min(target_chunks, span))
+            step = (span + n - 1) // n  # ceil
+            chunks: List[Chunk] = []
+            start = lo
+            ordinal = 0
+            while start <= hi:
+                end = min(start + step, hi + 1)  # exclusive; last covers hi
+                chunks.append(Chunk(table, "{}:{}".format(table.name, ordinal), col, start, end))
+                start = end
+                ordinal += 1
+            return chunks
+    return [Chunk(table, "{}:0".format(table.name))]