hindsightdb 0.1.0__py3-none-any.whl

hindsight/__init__.py

@@ -0,0 +1,251 @@
+"""Public API for Hindsight capture."""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+from collections.abc import Callable
+from typing import Any
+
+from hindsight.config import CaptureConfig
+from hindsight.contract import SDK_VERSION
+from hindsight.keys import parse_key, resolve_key, resolve_url
+from hindsight.queue import RowQueue
+from hindsight.shipper import Shipper
+from hindsight.transport import Transport, UrllibTransport
+
+__all__ = ["init", "shutdown", "flush", "stats", "Client", "CaptureConfig", "SDK_VERSION"]
+
+log = logging.getLogger("hindsight")
+_client: Client | None = None
+
+
+class Client:
+    def __init__(
+        self,
+        *,
+        slug: str,
+        transport: Transport,
+        cfg: CaptureConfig,
+        queue_size: int = 10_000,
+        batch_rows: int = 500,
+        batch_interval: float = 2.0,
+        spool_dir: str | None = None,
+        producer: str = "orm",
+    ):
+        self.slug = slug
+        self.cfg = cfg
+        self.queue = RowQueue(queue_size)
+        self.shipper = Shipper(
+            transport,
+            slug,
+            self.queue,
+            producer=producer,
+            batch_rows=batch_rows,
+            batch_interval=batch_interval,
+            spool_dir=spool_dir,
+        )
+        self.adapter: Any = None
+        self.outbox: Any = None
+        self.manifest: dict[str, Any] | None = None
+
+    def set_metadata(self, metadata: Any) -> None:
+        from hindsight.sqlalchemy.manifest import build_manifest
+
+        self.manifest = build_manifest(metadata, self.cfg)
+        self.shipper.set_manifest(self.manifest)
+
+    def attach_sqlalchemy(
+        self, engine: Any, context_provider: Callable[[], dict[str, Any] | None] | None
+    ) -> None:
+        from hindsight.sqlalchemy.adapter import SQLAlchemyAdapter
+
+        self.adapter = SQLAlchemyAdapter(
+            self.queue,
+            self.cfg,
+            engine=engine,
+            context_provider=context_provider,
+            on_manifest=self.set_metadata,
+        )
+        self.adapter.install()
+
+    def attach_outbox(
+        self,
+        dsn: str,
+        *,
+        engine: Any = None,
+        context_provider: Callable[[], dict[str, Any] | None] | None = None,
+        metadata: Any = None,
+        auto_install: bool = False,
+        batch_rows: int = 500,
+        batch_interval: float = 2.0,
+    ) -> None:
+        from hindsight.outbox.adapter import OutboxAdapter
+        from hindsight.outbox.catalog import installed_triggers, introspect, manifest_from_catalog
+        from hindsight.outbox.db import Database, connector
+        from hindsight.outbox.shipper import OutboxShipper
+        from hindsight.outbox.spec import specs_from_manifest
+        from hindsight.outbox.sql import install_sql
+
+        db = Database(connector(dsn))
+        if metadata is not None:
+            from hindsight.sqlalchemy.manifest import build_manifest
+
+            self.manifest = build_manifest(metadata, self.cfg, mode="outbox")
+        else:
+            self.manifest = manifest_from_catalog(introspect(db), self.cfg)
+        self.shipper.set_manifest(self.manifest)
+        specs = specs_from_manifest(self.manifest, self.cfg)
+        if auto_install:
+            db.run(install_sql(specs))
+        have = installed_triggers(db)
+        missing = [s.name for s in specs if s.name not in have]
+        if missing:
+            log.warning(
+                "hindsight: %d captured tables have no trigger (run `hindsight apply`): %s",
+                len(missing),
+                ", ".join(missing[:10]),
+            )
+        begin_ok = db.fetchrow("SELECT to_regprocedure('capture_begin(jsonb)') IS NOT NULL")
+        if begin_ok and begin_ok[0]:
+            self.adapter = OutboxAdapter(engine=engine, context_provider=context_provider)
+            self.adapter.install()
+        else:
+            log.warning("hindsight: capture_begin() is missing — actor attribution is off")
+        self.outbox = OutboxShipper(
+            db,
+            self.shipper,
+            [s.name for s in specs],
+            batch_rows=batch_rows,
+            interval=batch_interval,
+            max_rows=self.cfg.outbox_max_rows,
+            max_bytes=self.cfg.outbox_max_bytes,
+        )
+
+    def start(self) -> None:
+        if self.outbox is not None:
+            self.outbox.start()
+        else:
+            self.shipper.start()
+
+    def flush(self, timeout: float = 10.0) -> int:
+        import time
+
+        deadline = time.monotonic() + timeout
+        if self.outbox is not None:
+            return int(self.outbox.drain(deadline=deadline))
+        return self.shipper.flush(deadline=deadline)
+
+    def close(self, timeout: float = 5.0) -> None:
+        if self.adapter is not None:
+            self.adapter.uninstall()
+        if self.outbox is not None:
+            self.outbox.stop(timeout)
+        self.shipper.stop(timeout)
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "slug": self.slug,
+            "boot_id": self.shipper.boot_id,
+            "queue": self.queue.stats(),
+            "shipper": dict(self.shipper.stats),
+            "adapter": dict(self.adapter.stats) if self.adapter else {},
+            "outbox": dict(self.outbox.stats) if self.outbox else {},
+        }
+
+
+def init(
+    engine: Any = None,
+    *,
+    key: str | None = None,
+    url: str | None = None,
+    config: str | CaptureConfig | None = None,
+    context_provider: Callable[[], dict[str, Any] | None] | None = None,
+    metadata: Any = None,
+    mode: str | None = None,
+    dsn: str | None = None,
+    auto_install: bool = False,
+    spool_dir: str | None = None,
+    queue_size: int = 10_000,
+    batch_rows: int = 500,
+    batch_interval: float = 2.0,
+    transport: Transport | None = None,
+    start: bool = True,
+) -> Client | None:
+    """Start capture, returning ``None`` when capture is unavailable."""
+    global _client
+    try:
+        raw_key = resolve_key(key)
+        if not raw_key:
+            log.warning("hindsight: no HINDSIGHT_KEY — capture is off")
+            return None
+        slug, raw_key = parse_key(raw_key)
+        cfg = config if isinstance(config, CaptureConfig) else CaptureConfig.load(config)
+        if mode:
+            cfg.mode = mode
+        if cfg.mode not in ("orm", "outbox"):
+            log.warning("hindsight: unknown mode %r; using orm", cfg.mode)
+            cfg.mode = "orm"
+        if _client is not None:
+            _client.close(0.0)
+        client = Client(
+            slug=slug,
+            transport=transport or UrllibTransport(resolve_url(url), raw_key),
+            cfg=cfg,
+            queue_size=queue_size,
+            batch_rows=batch_rows,
+            batch_interval=batch_interval,
+            spool_dir=spool_dir,
+            producer=cfg.mode,
+        )
+        if cfg.mode == "outbox":
+            from hindsight.outbox.db import dsn_of_engine
+
+            dsn = dsn or os.environ.get("HINDSIGHT_DSN") or dsn_of_engine(engine)
+            if not dsn:
+                raise RuntimeError("outbox mode needs dsn= (or HINDSIGHT_DSN, or an engine)")
+            client.attach_outbox(
+                dsn,
+                engine=engine,
+                context_provider=context_provider,
+                metadata=metadata,
+                auto_install=auto_install,
+                batch_rows=batch_rows,
+                batch_interval=batch_interval,
+            )
+        else:
+            if metadata is not None:
+                client.set_metadata(metadata)
+            if engine is not None or metadata is not None:
+                client.attach_sqlalchemy(engine, context_provider)
+        if start:
+            client.start()
+        _client = client
+        atexit.register(_atexit)
+        log.info("hindsight: capturing for source %r (boot %s)", slug, client.shipper.boot_id)
+        return client
+    except Exception:
+        log.exception("hindsight: init failed — capture is off, the app is unaffected")
+        return None
+
+
+def _atexit() -> None:
+    shutdown(timeout=3.0)
+
+
+def shutdown(timeout: float = 5.0) -> None:
+    global _client
+    if _client is not None:
+        try:
+            _client.close(timeout)
+        finally:
+            _client = None
+
+
+def flush(timeout: float = 10.0) -> int:
+    return _client.flush(timeout) if _client else 0
+
+
+def stats() -> dict[str, Any]:
+    return _client.stats() if _client else {}

hindsight/canonical.py

@@ -0,0 +1,145 @@
+"""Type-aware normalization and hashing for captured rows."""
+
+from __future__ import annotations
+
+import datetime as dt
+import decimal
+import hashlib
+import json
+import uuid
+from typing import Any
+
+_NUMERIC = ("int", "numeric", "decimal", "real", "double", "money", "serial", "float")
+_TZ = ("timestamptz", "with time zone")
+_JSON = ("json",)
+
+
+def kind_of(pg_type: str) -> str:
+    """Map a PostgreSQL type to its normalization family."""
+    t = (pg_type or "").lower().strip()
+    if t.endswith("[]") or t.startswith("_") or t.startswith("array"):
+        return "array"
+    if any(k in t for k in _JSON):
+        return "json"
+    if t.startswith("bool"):
+        return "bool"
+    if any(k in t for k in _TZ):
+        return "timestamptz"
+    if t.startswith("timestamp") or t.startswith("date") or t.startswith("time"):
+        return "datetime"
+    if t == "uuid":
+        return "uuid"
+    if any(k in t for k in _NUMERIC):
+        return "numeric"
+    return "text"
+
+
+def element_type(pg_type: str) -> str:
+    t = (pg_type or "").lower().strip()
+    if t.endswith("[]"):
+        return t[:-2]
+    if t.startswith("_"):
+        return t[1:]
+    return "text"
+
+
+def _decimal(value: Any) -> str:
+    try:
+        d = decimal.Decimal(str(value))
+    except (decimal.InvalidOperation, ValueError):
+        return str(value)
+    if not d.is_finite():
+        return str(d)
+    d = d.normalize()
+    if d == 0:
+        return "0"
+    return format(d, "f")
+
+
+def _parse_dt(value: Any) -> Any:
+    if isinstance(value, dt.datetime):
+        return value
+    if isinstance(value, str):
+        s = value.strip().replace(" ", "T", 1) if " " in value.strip() else value.strip()
+        if s.endswith("Z"):
+            s = s[:-1] + "+00:00"
+        try:
+            return dt.datetime.fromisoformat(s)
+        except ValueError:
+            return None
+    return None
+
+
+def canonical_value(value: Any, pg_type: str) -> Any:
+    """Normalize a value using its PostgreSQL type."""
+    if value is None:
+        return None
+    kind = kind_of(pg_type)
+    if kind == "numeric":
+        if isinstance(value, bool):
+            return "1" if value else "0"
+        return _decimal(value)
+    if kind == "bool":
+        if isinstance(value, str):
+            return value.strip().lower() in ("t", "true", "1", "yes", "on")
+        return bool(value)
+    if kind == "timestamptz":
+        parsed = _parse_dt(value)
+        if parsed is None:
+            return str(value)
+        if parsed.tzinfo is None:
+            parsed = parsed.replace(tzinfo=dt.UTC)
+        return parsed.astimezone(dt.UTC).isoformat()
+    if kind == "datetime":
+        if isinstance(value, (dt.datetime, dt.date, dt.time)):
+            return value.isoformat()
+        return str(value).strip().replace(" ", "T", 1)
+    if kind == "uuid":
+        if isinstance(value, uuid.UUID):
+            return str(value)
+        return str(value).strip().lower()
+    if kind == "json":
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except ValueError:
+                return value
+        return value
+    if kind == "array":
+        if isinstance(value, str):
+            try:
+                value = json.loads(value)
+            except ValueError:
+                return value
+        if isinstance(value, (list, tuple)):
+            et = element_type(pg_type)
+            return [canonical_value(v, et) for v in value]
+        return value
+    if isinstance(value, (dict, list)):
+        return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return bytes(value).hex()
+    return str(value)
+
+
+def captured_columns(table: dict[str, Any]) -> dict[str, str]:
+    """Return captured column names and types."""
+    cols = table.get("columns") or {}
+    return {
+        name: str((spec or {}).get("pg_type") or "text")
+        for name, spec in sorted(cols.items())
+        if (spec or {}).get("captured", True) and not name.startswith("_")
+    }
+
+
+def canonical_row(values: dict[str, Any], columns: dict[str, str]) -> dict[str, Any]:
+    return {name: canonical_value(values.get(name), pg_type) for name, pg_type in columns.items()}
+
+
+def row_hash(values: dict[str, Any], columns: dict[str, str]) -> str:
+    canonical = json.dumps(
+        canonical_row(values, columns), sort_keys=True, separators=(",", ":"), default=str
+    )
+    return hashlib.sha256(canonical.encode()).hexdigest()[:32]

hindsight/cli/apply.py

@@ -0,0 +1,93 @@
+"""Install, validate, or resume outbox capture."""
+
+from __future__ import annotations
+
+import secrets
+from typing import Any
+
+from hindsight.config import CaptureConfig
+from hindsight.outbox.catalog import installed_triggers, introspect, manifest_from_catalog
+from hindsight.outbox.db import Database, connector
+from hindsight.outbox.spec import TableSpec, specs_from_manifest
+from hindsight.outbox.sql import emit_migration, install_sql
+
+
+def open_db(dsn: str) -> Database:
+    return Database(connector(dsn))
+
+
+def build_specs(
+    cfg: CaptureConfig, *, metadata: Any = None, db: Database | None = None
+) -> list[TableSpec]:
+    if metadata is not None:
+        from hindsight.sqlalchemy.manifest import build_manifest
+
+        return specs_from_manifest(build_manifest(metadata, cfg, mode="outbox"), cfg)
+    if db is None:
+        raise ValueError("apply needs --metadata or --dsn")
+    return specs_from_manifest(manifest_from_catalog(introspect(db), cfg), cfg)
+
+
+def apply(
+    cfg: CaptureConfig,
+    *,
+    metadata: Any = None,
+    dsn: str | None = None,
+    emit_migration_to: str | None = None,
+    down_revision: str | None = None,
+    dry_run: bool = False,
+) -> dict[str, Any]:
+    db = open_db(dsn) if dsn else None
+    try:
+        specs = build_specs(cfg, metadata=metadata, db=db)
+        statements = install_sql(specs)
+        if emit_migration_to is not None:
+            text = emit_migration(specs, revision=secrets.token_hex(6), down_revision=down_revision)
+            if emit_migration_to != "-":
+                with open(emit_migration_to, "w") as fh:
+                    fh.write(text)
+            return {"tables": [s.name for s in specs], "migration": text, "applied": False}
+        if dry_run or db is None:
+            return {"tables": [s.name for s in specs], "sql": statements, "applied": False}
+        db.run(statements)
+        return {
+            "tables": [s.name for s in specs],
+            "applied": True,
+            "triggers": installed_triggers(db),
+        }
+    finally:
+        if db is not None:
+            db.close()
+
+
+def resume(cfg: CaptureConfig, *, dsn: str, metadata: Any = None) -> dict[str, Any]:
+    from hindsight.outbox.shipper import read_state
+    from hindsight.outbox.shipper import resume as _resume
+
+    db = open_db(dsn)
+    try:
+        tables = [s.name for s in build_specs(cfg, metadata=metadata, db=db)]
+        previous = _resume(db, tables)
+        return {"tables": tables, "previous": previous, "state": read_state(db, "breaker")}
+    finally:
+        db.close()
+
+
+def trigger_problems(cfg: CaptureConfig, *, dsn: str, metadata: Any = None) -> list[str]:
+    """Return missing or disabled trigger problems."""
+    db = open_db(dsn)
+    try:
+        specs = build_specs(cfg, metadata=metadata, db=db)
+        have = installed_triggers(db)
+        out: list[str] = []
+        for s in specs:
+            if s.name not in have:
+                out.append(f"{s.name}: no zz_capture trigger (run `hindsight apply`)")
+            elif not have[s.name]:
+                out.append(f"{s.name}: trigger disabled (breaker tripped? `hindsight resume`)")
+        for name in have:
+            if name not in {s.name for s in specs} and not cfg.table_captured(name):
+                out.append(f"{name}: excluded in config but still has a trigger")
+        return out
+    finally:
+        db.close()

hindsight/cli/audit.py

@@ -0,0 +1,180 @@
+"""Import an existing audit table into Hindsight history."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, fields
+from typing import Any
+
+from hindsight.cli.client import Session, Source, Throttle, ident, started
+from hindsight.config import CaptureConfig
+from hindsight.contract import Row, json_value, now_iso
+
+
+@dataclass
+class AuditColumns:
+    """Audit-table column mapping."""
+
+    id: str = "id"
+    entity_type: str = "entity_type"
+    entity_id: str = "entity_id"
+    tenant: str | None = "provider_id"
+    action: str | None = "action"
+    actor_type: str | None = "actor_type"
+    actor_id: str | None = "actor_id"
+    actor_name: str | None = "actor_name"
+    changes: str = "changes"
+    created_at: str = "created_at"
+
+    @classmethod
+    def from_overrides(cls, overrides: dict[str, str]) -> AuditColumns:
+        names = {f.name for f in fields(cls)}
+        bad = sorted(set(overrides) - names)
+        if bad:
+            raise ValueError(f"unknown audit column keys: {', '.join(bad)}")
+        return cls(**{k: (v or None) for k, v in overrides.items()})  # type: ignore[arg-type]
+
+
+def delta_of(raw: Any) -> dict[str, dict[str, Any]]:
+    """Normalize audit changes to the capture contract."""
+    if isinstance(raw, (str, bytes)):
+        try:
+            raw = json.loads(raw)
+        except ValueError:
+            return {}
+    if not isinstance(raw, dict):
+        return {}
+    out: dict[str, dict[str, Any]] = {}
+    for key, v in raw.items():
+        name = str(key)
+        if name.startswith("_"):
+            continue
+        if isinstance(v, dict) and ("to" in v or "from" in v):
+            entry = {k: json_value(v[k]) for k in ("from", "to") if k in v}
+            if "to" not in entry:
+                continue
+            out[name] = entry
+        elif isinstance(v, dict) and ("new" in v or "old" in v):
+            if "new" not in v:
+                continue
+            entry = {"to": json_value(v["new"])}
+            if "old" in v:
+                entry["from"] = json_value(v["old"])
+            out[name] = entry
+        elif isinstance(v, (list, tuple)) and len(v) == 2:
+            out[name] = {"from": json_value(v[0]), "to": json_value(v[1])}
+        else:
+            out[name] = {"to": json_value(v)}
+    return out
+
+
+def import_audit(
+    cfg: CaptureConfig,
+    *,
+    dsn: str,
+    session: Session,
+    mapping: dict[str, str],
+    table: str = "audit_events",
+    columns: AuditColumns | None = None,
+    metadata: Any = None,
+    run_id: str | None = None,
+    force: bool = False,
+    rows_per_second: float | None = None,
+    batch_rows: int = 500,
+) -> dict[str, Any]:
+    if not mapping:
+        raise ValueError("import-audit needs at least one --map entity_type=table")
+    run_id = run_id or session.boot_id.split("-", 1)[-1]
+    cols = columns or AuditColumns()
+    status = session.status()
+    if status.get("genesis") and not force:
+        raise RuntimeError(
+            "a genesis run is already recorded for this source; audit rows would land on "
+            "top of the current state. Import audit first on a fresh source, or --force."
+        )
+    source = Source.open(cfg, dsn, metadata=metadata)
+    unknown = sorted(set(mapping.values()) - set(source.manifest["tables"]))
+    if unknown:
+        source.close()
+        raise ValueError(f"--map targets are not captured tables: {', '.join(unknown)}")
+    session.require_same_manifest(source.manifest)
+    session.shipper.set_manifest(source.manifest)
+    optional = [cols.tenant, cols.action, cols.actor_type, cols.actor_id, cols.actor_name]
+    select = [cols.id, cols.entity_type, cols.entity_id, cols.changes, cols.created_at] + [
+        c for c in optional if c
+    ]
+    placeholders = ", ".join(["%s"] * len(mapping))
+    sql = (
+        f"SELECT {', '.join(ident(c) for c in select)} FROM {ident(table)} "
+        f"WHERE {ident(cols.entity_type)} IN ({placeholders}) "
+        f"ORDER BY {ident(cols.created_at)}, {ident(cols.id)}"
+    )
+    throttle = Throttle(rows_per_second)
+    report: dict[str, Any] = {
+        "run_id": run_id,
+        "boot_id": session.boot_id,
+        "table": table,
+        "mapping": mapping,
+        "started_at": started(),
+        "rows": 0,
+        "shipped": 0,
+        "skipped_empty": 0,
+        "per_table": {t: 0 for t in mapping.values()},
+    }
+    seq = 0
+    batch: list[Row] = []
+
+    def ship() -> None:
+        nonlocal batch
+        if batch:
+            session.ship(batch)
+            throttle(len(batch))
+            report["shipped"] += len(batch)
+            batch = []
+
+    try:
+        for raw in source.db.iter(sql, tuple(mapping), batch=batch_rows):
+            rec = dict(zip(select, raw, strict=True))
+            report["rows"] += 1
+            changes = delta_of(rec[cols.changes])
+            if not changes:
+                report["skipped_empty"] += 1
+                continue
+            target = mapping[str(rec[cols.entity_type])]
+            created = rec[cols.created_at]
+            when = created.isoformat() if hasattr(created, "isoformat") else str(created)
+            tenant = rec.get(cols.tenant) if cols.tenant else None
+            seq += 1
+            batch.append(
+                Row(
+                    client_seq=seq,
+                    table_name=target,
+                    row_id=str(json_value(rec[cols.entity_id])),
+                    op="A",
+                    changes=changes,
+                    source_time=when,
+                    tx_id=f"audit:{json_value(rec[cols.id])}",
+                    row_tenant_id=None if tenant is None else str(json_value(tenant)),
+                    actor_type=(
+                        str(rec.get(cols.actor_type) or "user") if cols.actor_type else "user"
+                    ),
+                    actor_id=_text(rec.get(cols.actor_id)) if cols.actor_id else None,
+                    actor_name=_text(rec.get(cols.actor_name)) if cols.actor_name else None,
+                    correlation_id=_text(rec.get(cols.action)) if cols.action else None,
+                    causation_id=f"audit-import-{run_id}",
+                )
+            )
+            report["per_table"][target] += 1
+            if len(batch) >= batch_rows:
+                ship()
+        ship()
+    finally:
+        source.close()
+    report["finished_at"] = now_iso()
+    report["accepted"] = int(session.shipper.stats["accepted"])
+    report["duplicates"] = int(session.shipper.stats["duplicates"])
+    return report
+
+
+def _text(value: Any) -> str | None:
+    return None if value is None else str(json_value(value))