dataforge-07 0.1.0__py3-none-any.whl

dataforge/stores/patch_plan.py

@@ -0,0 +1,178 @@
+"""Backend-neutral patch plan models for verified tabular repairs."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import secrets
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from dataforge.transactions.txn import CellFix
+
+PATCH_PLAN_SCHEMA_VERSION: Literal["patch_plan_v1"] = "patch_plan_v1"
+
+
+class RowIdentity(BaseModel):
+    """Stable row locator required before a table-store repair can apply."""
+
+    kind: Literal["csv_position", "column_values", "unavailable"]
+    columns: tuple[str, ...] = Field(default_factory=tuple)
+    values: dict[str, str] = Field(default_factory=dict)
+    stable: bool = False
+    reason: str = Field(min_length=1)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class PatchOperation(BaseModel):
+    """One cell-level mutation in a backend-neutral repair plan."""
+
+    operation: Literal["update"] = "update"
+    relation: str = Field(min_length=1)
+    row: int = Field(ge=0)
+    row_identity: RowIdentity
+    column: str = Field(min_length=1)
+    old_value: str
+    new_value: str
+    detector_id: str = Field(min_length=1)
+    reason: str = Field(min_length=1)
+    confidence: float = Field(ge=0.0, le=1.0)
+    provenance: str = Field(min_length=1)
+    precondition_sql: str | None = None
+    forward_sql: str | None = None
+    rollback_sql: str | None = None
+    verification_sql: tuple[str, ...] = Field(default_factory=tuple)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    @classmethod
+    def from_cell_fix(
+        cls,
+        fix: CellFix,
+        *,
+        relation: str,
+        row_identity: RowIdentity,
+        reason: str,
+        confidence: float,
+        provenance: str,
+        precondition_sql: str | None = None,
+        forward_sql: str | None = None,
+        rollback_sql: str | None = None,
+        verification_sql: tuple[str, ...] = (),
+    ) -> PatchOperation:
+        """Build an operation from an existing DataForge cell fix."""
+        if fix.operation != "update":
+            raise ValueError("PatchPlan v1 supports cell updates only.")
+        return cls(
+            relation=relation,
+            row=fix.row,
+            row_identity=row_identity,
+            column=fix.column,
+            old_value=fix.old_value,
+            new_value=fix.new_value,
+            detector_id=fix.detector_id,
+            reason=reason,
+            confidence=confidence,
+            provenance=provenance,
+            precondition_sql=precondition_sql,
+            forward_sql=forward_sql,
+            rollback_sql=rollback_sql,
+            verification_sql=verification_sql,
+        )
+
+
+class CostEstimate(BaseModel):
+    """Small, backend-agnostic estimate surfaced before mutation."""
+
+    rows_scanned: int = Field(ge=0)
+    rows_written: int = Field(ge=0)
+    bytes_scanned: int | None = Field(default=None, ge=0)
+    quota_units: float = Field(default=0.0, ge=0.0)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+
+class PatchPlan(BaseModel):
+    """The only write contract accepted by non-CSV DataForge stores."""
+
+    schema_version: Literal["patch_plan_v1"] = PATCH_PLAN_SCHEMA_VERSION
+    plan_id: str = Field(pattern=r"^plan-[0-9a-f]{12}$")
+    created_at: datetime
+    backend: str = Field(min_length=1)
+    target: str = Field(min_length=1)
+    relation: str = Field(min_length=1)
+    row_identity_columns: tuple[str, ...] = Field(default_factory=tuple)
+    stable_row_identity: bool
+    operations: tuple[PatchOperation, ...] = Field(default_factory=tuple)
+    forward_sql: tuple[str, ...] = Field(default_factory=tuple)
+    rollback_sql: tuple[str, ...] = Field(default_factory=tuple)
+    preflight_probes: tuple[str, ...] = Field(default_factory=tuple)
+    verification_queries: tuple[str, ...] = Field(default_factory=tuple)
+    touched_constraints: tuple[str, ...] = Field(default_factory=tuple)
+    smt_obligations: tuple[str, ...] = Field(default_factory=tuple)
+    cost_estimate: CostEstimate
+    safety_verdict: str = Field(min_length=1)
+    reversible: bool
+    apply_supported: bool
+    apply_requires_approval: bool = True
+    audit_metadata: dict[str, str] = Field(default_factory=dict)
+    reason: str = Field(min_length=1)
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    @classmethod
+    def new(
+        cls,
+        *,
+        backend: str,
+        target: str,
+        relation: str,
+        row_identity_columns: tuple[str, ...],
+        operations: tuple[PatchOperation, ...],
+        safety_verdict: str,
+        rows_scanned: int,
+        reason: str,
+        touched_constraints: tuple[str, ...] = (),
+        smt_obligations: tuple[str, ...] = (),
+        audit_metadata: dict[str, str] | None = None,
+        apply_supported: bool | None = None,
+        reversible: bool | None = None,
+    ) -> PatchPlan:
+        """Construct a stable plan with derived SQL and support flags."""
+        stable = bool(row_identity_columns) and all(op.row_identity.stable for op in operations)
+        has_operations = bool(operations)
+        supported = stable and has_operations if apply_supported is None else apply_supported
+        is_reversible = supported if reversible is None else reversible
+        return cls(
+            plan_id=f"plan-{secrets.token_hex(6)}",
+            created_at=datetime.now(UTC),
+            backend=backend,
+            target=target,
+            relation=relation,
+            row_identity_columns=row_identity_columns,
+            stable_row_identity=stable,
+            operations=operations,
+            forward_sql=tuple(sql for op in operations if (sql := op.forward_sql)),
+            rollback_sql=tuple(sql for op in operations if (sql := op.rollback_sql)),
+            preflight_probes=tuple(sql for op in operations if (sql := op.precondition_sql)),
+            verification_queries=tuple(query for op in operations for query in op.verification_sql),
+            touched_constraints=touched_constraints,
+            smt_obligations=smt_obligations,
+            cost_estimate=CostEstimate(rows_scanned=rows_scanned, rows_written=len(operations)),
+            safety_verdict=safety_verdict,
+            reversible=is_reversible,
+            apply_supported=supported,
+            audit_metadata=audit_metadata or {},
+            reason=reason,
+        )
+
+    def canonical_json(self) -> str:
+        """Return deterministic JSON suitable for audit hashing."""
+        return json.dumps(self.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))
+
+    def sha256(self) -> str:
+        """Return a SHA-256 digest of the canonical plan."""
+        return hashlib.sha256(self.canonical_json().encode("utf-8")).hexdigest()

dataforge/stores/registry.py

@@ -0,0 +1,82 @@
+"""Table-store URI parsing and adapter selection."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import parse_qs, unquote, urlparse
+
+from dataforge.stores.base import TableStore, TableStoreError
+from dataforge.stores.cloud import CloudWarehouseStore
+from dataforge.stores.duckdb import DuckDBStore
+
+_CLOUD_BACKENDS = {"snowflake", "bigquery", "databricks", "databricks_delta"}
+
+
+@dataclass(frozen=True)
+class TableStoreSpec:
+    """Parsed table-store URI."""
+
+    backend: str
+    target: str
+    relation: str
+    database_path: Path | None
+    row_identity_columns: tuple[str, ...]
+
+
+def is_table_store_uri(raw: str) -> bool:
+    """Return whether a CLI target string names a DataForge table store."""
+    return raw.startswith("warehouse://")
+
+
+def parse_table_store_uri(raw: str, *, row_ids: tuple[str, ...] = ()) -> TableStoreSpec:
+    """Parse a ``warehouse://`` URI into an adapter spec.
+
+    Supported local form:
+    ``warehouse://duckdb?database=/tmp/dev.duckdb&relation=main.model&row_id=id``.
+    """
+    parsed = urlparse(raw)
+    if parsed.scheme != "warehouse":
+        raise TableStoreError("Table-store URIs must use the warehouse:// scheme.")
+    backend = (parsed.netloc or parsed.path.strip("/").split("/", 1)[0]).lower()
+    if not backend:
+        raise TableStoreError("Warehouse URI must include a backend name.")
+    query = {key: values[-1] for key, values in parse_qs(parsed.query).items() if values}
+    relation = unquote(query.get("relation", ""))
+    if not relation:
+        raise TableStoreError("Warehouse URI must include relation=<schema.table>.")
+    database = query.get("database") or query.get("path")
+    row_id_query = query.get("row_id") or query.get("key")
+    resolved_row_ids = row_ids
+    if row_id_query:
+        resolved_row_ids = tuple(part.strip() for part in row_id_query.split(",") if part.strip())
+    return TableStoreSpec(
+        backend=backend,
+        target=raw,
+        relation=relation,
+        database_path=Path(unquote(database)).expanduser() if database else None,
+        row_identity_columns=resolved_row_ids,
+    )
+
+
+def store_from_uri(raw: str, *, row_ids: tuple[str, ...] = ()) -> TableStore:
+    """Create a table-store adapter from a CLI URI."""
+    spec = parse_table_store_uri(raw, row_ids=row_ids)
+    if spec.backend == "duckdb":
+        if spec.database_path is None:
+            raise TableStoreError("DuckDB warehouse URI requires database=<path>.")
+        return DuckDBStore(
+            database_path=spec.database_path,
+            relation=spec.relation,
+            row_identity_columns=spec.row_identity_columns,
+            target=spec.target,
+        )
+    if spec.backend in _CLOUD_BACKENDS:
+        backend = "databricks" if spec.backend == "databricks_delta" else spec.backend
+        return CloudWarehouseStore(
+            backend=backend,
+            target=spec.target,
+            relation=spec.relation,
+            row_identity_columns=spec.row_identity_columns,
+        )
+    raise TableStoreError(f"Unsupported warehouse backend: {spec.backend}")

dataforge/stores/repair.py

@@ -0,0 +1,121 @@
+"""Repair pipeline entrypoints for table-store targets."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from dataforge.detectors import run_all_detectors
+from dataforge.detectors.base import Issue, Schema
+from dataforge.engine.repair import propose_repairs
+from dataforge.safety import SafetyFilter, SafetyVerdict
+from dataforge.stores.base import StoreApplyReceipt, TableStore, TableStoreError
+from dataforge.stores.patch_plan import PatchPlan
+from dataforge.table import copy_table
+
+
+class TableStoreRepairResult(BaseModel):
+    """Repair result for warehouse/table-store CLI calls."""
+
+    schema_version: str = "table_store_repair_result_v1"
+    mode: str
+    target: str
+    backend: str
+    issues: list[Issue]
+    fixes: list[dict[str, object]]
+    patch_plan: PatchPlan
+    apply_receipt: StoreApplyReceipt | None = None
+
+    model_config = ConfigDict(
+        strict=True, arbitrary_types_allowed=True, extra="forbid", frozen=True
+    )
+
+
+def run_table_store_repair(
+    store: TableStore,
+    *,
+    mode: str,
+    schema: Schema | None,
+    allow_llm: bool = False,
+    model: str = "gemini-2.0-flash",
+    allow_pii: bool = False,
+    confirm_pii: bool = False,
+    confirm_escalations: bool = False,
+    state_root: Path | None = None,
+    only_column: str | None = None,
+) -> TableStoreRepairResult:
+    """Detect, verify, plan, and optionally apply repairs for a table store."""
+    if mode not in {"dry_run", "apply"}:
+        raise TableStoreError("Table-store repair mode must be dry_run or apply.")
+
+    if store.backend in {"snowflake", "bigquery", "databricks"}:
+        plan = store.build_patch_plan(
+            [],
+            schema=schema,
+            safety_verdict="dry_run_only",
+            touched_constraints=(),
+            smt_obligations=(),
+        )
+        if mode == "apply":
+            raise TableStoreError(plan.reason)
+        return TableStoreRepairResult(
+            mode=mode,
+            target=store.target,
+            backend=store.backend,
+            issues=[],
+            fixes=[],
+            patch_plan=plan,
+        )
+
+    table = store.read_table()
+    issues = run_all_detectors(table, schema)
+    if only_column is not None:
+        issues = [issue for issue in issues if issue.column == only_column]
+    accepted_fixes, attempt_groups = propose_repairs(
+        issues,
+        Path.cwd() / ".dataforge" / "warehouse-target.csv",
+        copy_table(table),
+        schema,
+        allow_llm=allow_llm,
+        model=model,
+        allow_pii=allow_pii,
+        confirm_pii=confirm_pii,
+        confirm_escalations=confirm_escalations,
+        interactive=False,
+    )
+    batch_safety = SafetyFilter().evaluate_batch(accepted_fixes)
+    if batch_safety.verdict != SafetyVerdict.ALLOW:
+        accepted_fixes = []
+    plan = store.build_patch_plan(
+        accepted_fixes,
+        schema=schema,
+        safety_verdict=batch_safety.verdict.value,
+        touched_constraints=(),
+        smt_obligations=("SMTVerifier.verify",) if accepted_fixes else (),
+    )
+    apply_receipt = None
+    if mode == "apply":
+        apply_receipt = store.apply_patch_plan(plan, state_root=state_root)
+
+    return TableStoreRepairResult(
+        mode=mode,
+        target=store.target,
+        backend=store.backend,
+        issues=issues,
+        fixes=[
+            {
+                "row": fix.fix.row,
+                "column": fix.fix.column,
+                "old_value": fix.fix.old_value,
+                "new_value": fix.fix.new_value,
+                "detector_id": fix.fix.detector_id,
+                "reason": fix.reason,
+                "confidence": fix.confidence,
+                "provenance": fix.provenance,
+            }
+            for fix in accepted_fixes
+        ],
+        patch_plan=plan,
+        apply_receipt=apply_receipt,
+    )

dataforge/stores/revert.py

@@ -0,0 +1,22 @@
+"""Rollback helpers for table-store transactions."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from dataforge.stores.base import TableStoreError
+from dataforge.stores.duckdb import load_duckdb_transaction
+from dataforge.transactions.log import load_transaction
+from dataforge.transactions.txn import RepairTransaction
+
+
+def revert_table_store_transaction(log_path: Path) -> RepairTransaction:
+    """Revert a table-store transaction using its recorded backend patch plan."""
+    transaction = load_transaction(log_path)
+    if transaction.backend == "duckdb":
+        store, loaded = load_duckdb_transaction(log_path)
+        store.revert_transaction(loaded, log_path=log_path)
+        return load_transaction(log_path)
+    raise TableStoreError(
+        f"Table-store revert is not implemented for backend {transaction.backend!r}."
+    )

dataforge/stores/sql.py

@@ -0,0 +1,27 @@
+"""SQL rendering helpers for DataForge table-store adapters."""
+
+from __future__ import annotations
+
+import re
+
+_SAFE_RELATION_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*|\.\"[^\"]+\")*$")
+_SAFE_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def ensure_safe_relation(relation: str) -> str:
+    """Return a validated relation identifier or raise ``ValueError``."""
+    if not _SAFE_RELATION_RE.fullmatch(relation):
+        raise ValueError(f"Unsafe relation identifier: {relation}")
+    return relation
+
+
+def quote_identifier(identifier: str) -> str:
+    """Return a double-quoted SQL identifier after strict validation."""
+    if not _SAFE_IDENTIFIER_RE.fullmatch(identifier):
+        raise ValueError(f"Unsafe column identifier: {identifier}")
+    return f'"{identifier}"'
+
+
+def sql_literal(value: object) -> str:
+    """Render a SQL string literal for generated patch statements."""
+    return "'" + str(value).replace("'", "''") + "'"

dataforge/table.py

@@ -0,0 +1,228 @@
+"""Small string-preserving table primitives for DataForge core paths.
+
+The CLI hot path should not need pandas just to profile or repair a CSV.
+This module provides the narrow DataFrame-like surface that DataForge's
+detectors, repairers, and verifier actually need.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+from collections.abc import Iterable, Iterator, Sequence
+from pathlib import Path
+from typing import Any, Protocol, cast, overload
+
+
+class TableLike(Protocol):
+    """Protocol for the tabular surface consumed by DataForge core logic."""
+
+    @property
+    def columns(self) -> Any: ...
+
+    @property
+    def index(self) -> Any: ...
+
+    @property
+    def at(self) -> Any: ...
+
+    def __getitem__(self, key: str) -> Any: ...
+
+    def copy(self, deep: bool = True) -> Any: ...
+
+    def to_csv(
+        self,
+        buffer: io.StringIO,
+        *,
+        index: bool = False,
+        lineterminator: str = "\n",
+    ) -> None: ...
+
+
+class ColumnView(Sequence[str]):
+    """Read-only column view with the small API repairers expect."""
+
+    def __init__(self, values: Sequence[str]) -> None:
+        self._values = values
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._values)
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    @overload
+    def __getitem__(self, index: int) -> str: ...
+
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[str]: ...
+
+    def __getitem__(self, index: int | slice) -> str | Sequence[str]:
+        return self._values[index]
+
+    def tolist(self) -> list[str]:
+        """Return a list copy, matching pandas Series enough for detectors."""
+        return list(self._values)
+
+
+class _AtIndexer:
+    """``table.at[row, column]`` getter/setter compatibility shim."""
+
+    def __init__(self, table: Table) -> None:
+        self._table = table
+
+    def __getitem__(self, key: tuple[int, str]) -> str:
+        row, column = key
+        return self._table.cell(row, column)
+
+    def __setitem__(self, key: tuple[int, str], value: object) -> None:
+        row, column = key
+        self._table.set_cell(row, column, value)
+
+
+class Table:
+    """In-memory CSV table with string-preserving cells."""
+
+    def __init__(self, columns: Sequence[str], rows: Iterable[dict[str, object]]) -> None:
+        self._columns = [str(column) for column in columns]
+        self._rows: list[dict[str, str]] = [
+            {
+                column: "" if row.get(column) is None else str(row.get(column, ""))
+                for column in self._columns
+            }
+            for row in rows
+        ]
+        self.at = _AtIndexer(self)
+
+    @property
+    def columns(self) -> list[str]:
+        """Return column names in CSV order."""
+        return list(self._columns)
+
+    @property
+    def index(self) -> range:
+        """Return zero-based row positions."""
+        return range(len(self._rows))
+
+    @property
+    def empty(self) -> bool:
+        """Return whether the table has no rows."""
+        return not self._rows
+
+    @overload
+    def __getitem__(self, key: str) -> ColumnView: ...
+
+    @overload
+    def __getitem__(self, key: list[str]) -> Table: ...
+
+    @overload
+    def __getitem__(self, key: tuple[str, ...]) -> Table: ...
+
+    def __getitem__(self, key: str | list[str] | tuple[str, ...]) -> ColumnView | Table:
+        if isinstance(key, str):
+            if key not in self._columns:
+                raise KeyError(key)
+            return ColumnView([row.get(key, "") for row in self._rows])
+        columns = [str(column) for column in key]
+        for column in columns:
+            if column not in self._columns:
+                raise KeyError(column)
+        return Table(
+            columns, ({column: row.get(column, "") for column in columns} for row in self._rows)
+        )
+
+    def __len__(self) -> int:
+        return len(self._rows)
+
+    def copy(self, deep: bool = True) -> Table:
+        """Return an independent table copy."""
+        del deep
+        return Table(self._columns, (dict(row) for row in self._rows))
+
+    def cell(self, row: int, column: str) -> str:
+        """Return a cell value."""
+        if column not in self._columns:
+            raise KeyError(column)
+        return self._rows[row].get(column, "")
+
+    def set_cell(self, row: int, column: str, value: object) -> None:
+        """Set a cell value after validating the column."""
+        if column not in self._columns:
+            raise KeyError(column)
+        self._rows[row][column] = "" if value is None else str(value)
+
+    def iter_records(self, columns: Sequence[str] | None = None) -> Iterator[dict[str, str]]:
+        """Yield row dictionaries in table order."""
+        selected = self._columns if columns is None else [str(column) for column in columns]
+        for row in self._rows:
+            yield {column: row.get(column, "") for column in selected}
+
+    def to_dict(self, orient: str = "records") -> list[dict[str, str]]:
+        """Return records in the pandas-compatible orientation used by DataForge."""
+        if orient != "records":
+            raise ValueError("Only orient='records' is supported.")
+        return list(self.iter_records())
+
+    def to_csv(
+        self, buffer: io.StringIO, *, index: bool = False, lineterminator: str = "\n"
+    ) -> None:
+        """Write the table as CSV to a text buffer."""
+        if index:
+            raise ValueError("Table.to_csv does not support index=True.")
+        writer = csv.DictWriter(buffer, fieldnames=self._columns, lineterminator=lineterminator)
+        writer.writeheader()
+        for row in self._rows:
+            writer.writerow({column: row.get(column, "") for column in self._columns})
+
+
+def read_csv(path: Path) -> Table:
+    """Read a CSV as a string-preserving ``Table``."""
+    with path.open("r", encoding="utf-8-sig", newline="") as handle:
+        reader = csv.DictReader(handle)
+        columns = list(reader.fieldnames or [])
+        return Table(columns, reader)
+
+
+def table_to_csv_bytes(table: TableLike) -> bytes:
+    """Serialize a table-like object to UTF-8 CSV bytes."""
+    output = io.StringIO()
+    if isinstance(table, Table):
+        table.to_csv(output, index=False, lineterminator="\n")
+    else:
+        # pandas-compatible fallback for tests and optional integrations.
+        table.to_csv(output, index=False, lineterminator="\n")
+    return output.getvalue().encode("utf-8")
+
+
+def column_names(table: TableLike) -> list[str]:
+    """Return table column names as strings."""
+    return [str(column) for column in table.columns]
+
+
+def row_count(table: TableLike) -> int:
+    """Return the number of rows in a table-like object."""
+    return len(table.index)
+
+
+def column_values(table: TableLike, column: str) -> list[Any]:
+    """Return all values for one column."""
+    values = table[column]
+    if hasattr(values, "tolist"):
+        return list(values.tolist())
+    return list(values)
+
+
+def cell_value(table: TableLike, row: int, column: str) -> str:
+    """Return a cell value as a string."""
+    return str(table.at[row, column])
+
+
+def set_cell_value(table: TableLike, row: int, column: str, value: object) -> None:
+    """Set a cell value on a table-like object."""
+    table.at[row, column] = value
+
+
+def copy_table(table: TableLike) -> TableLike:
+    """Return a deep copy of a table-like object."""
+    copied = table.copy(deep=True)
+    return cast(TableLike, copied)

dataforge/transactions/__init__.py

@@ -0,0 +1,34 @@
+"""Transaction exports for DataForge."""
+
+from dataforge.transactions.log import (
+    LEGACY_SCHEMA_NAME,
+    SCHEMA_NAME,
+    TransactionAuditReport,
+    TransactionAuditVerdict,
+    append_applied_event,
+    append_created_transaction,
+    append_reverted_event,
+    find_transaction_log,
+    load_transaction,
+    verify_transaction_log,
+)
+from dataforge.transactions.revert import TransactionRevertError, revert_transaction
+from dataforge.transactions.txn import CellFix, RepairTransaction, generate_txn_id
+
+__all__ = [
+    "CellFix",
+    "LEGACY_SCHEMA_NAME",
+    "SCHEMA_NAME",
+    "TransactionAuditReport",
+    "TransactionAuditVerdict",
+    "RepairTransaction",
+    "TransactionRevertError",
+    "append_applied_event",
+    "append_created_transaction",
+    "append_reverted_event",
+    "find_transaction_log",
+    "generate_txn_id",
+    "load_transaction",
+    "revert_transaction",
+    "verify_transaction_log",
+]