dataforge-07 0.1.0__py3-none-any.whl

dataforge/cli/release.py

@@ -0,0 +1,166 @@
+"""CLI group for local release verification."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from dataforge.release.doctor import DEFAULT_KAGGLE_CREDENTIALS, run_doctor
+from dataforge.release.full_vision import (
+    DEFAULT_BACKEND_URL as FULL_VISION_BACKEND_URL,
+)
+from dataforge.release.full_vision import (
+    DEFAULT_FRONTEND_URL as FULL_VISION_FRONTEND_URL,
+)
+from dataforge.release.full_vision import (
+    run_full_vision_gate,
+)
+from dataforge.release.gate import run_release_gate
+
+release_app = typer.Typer(help="Release verification utilities.", no_args_is_help=True)
+
+
+@release_app.command(name="doctor")
+def doctor(
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print machine-readable JSON."),
+    ] = False,
+    core: Annotated[
+        bool,
+        typer.Option("--core", help="Run OSS core release checks."),
+    ] = False,
+    maintainer_deploy: Annotated[
+        bool,
+        typer.Option(
+            "--maintainer-deploy",
+            help="Run maintainer-specific deploy/auth checks.",
+        ),
+    ] = False,
+    kaggle_credentials: Annotated[
+        Path,
+        typer.Option(
+            "--kaggle-credentials",
+            help="Path to Kaggle OAuth credentials.json. Legacy kaggle.json is never read.",
+        ),
+    ] = DEFAULT_KAGGLE_CREDENTIALS,
+) -> None:
+    """Verify local release/deploy auth without printing secrets."""
+    run_core = core or not maintainer_deploy
+    report = run_doctor(
+        kaggle_credentials=kaggle_credentials,
+        core=run_core,
+        maintainer_deploy=maintainer_deploy,
+    )
+    if json_output:
+        typer.echo(json.dumps(report.to_dict(), indent=2, sort_keys=True))
+    else:
+        for check in report.checks:
+            status = "ok" if check.ok else "fail"
+            typer.echo(f"{status:4} {check.name}: {check.detail}")
+    raise typer.Exit(code=0 if report.ok else 2)
+
+
+@release_app.command(name="gate")
+def gate(
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print machine-readable JSON."),
+    ] = False,
+    keep_artifacts: Annotated[
+        bool,
+        typer.Option(
+            "--keep-artifacts",
+            help="Copy the temporary gate workspace to dist/release-gate-latest.",
+        ),
+    ] = False,
+) -> None:
+    """Build, audit, offline-install, and smoke-test the release wheel."""
+    report = run_release_gate(keep_artifacts=keep_artifacts)
+    if json_output:
+        typer.echo(json.dumps(report.to_dict(), indent=2, sort_keys=True))
+    else:
+        for step in report.steps:
+            status = "ok" if step.ok else "fail"
+            typer.echo(f"{status:4} {step.name}: {step.detail}")
+    raise typer.Exit(code=0 if report.ok else 1)
+
+
+@release_app.command(name="full-vision")
+def full_vision(
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print machine-readable JSON."),
+    ] = False,
+    evidence_root: Annotated[
+        Path | None,
+        typer.Option("--evidence-root", help="Directory containing external proof manifests."),
+    ] = None,
+    frontend_url: Annotated[
+        str,
+        typer.Option("--frontend-url", help="Cloudflare Workers Playground URL."),
+    ] = FULL_VISION_FRONTEND_URL,
+    backend_url: Annotated[
+        str,
+        typer.Option("--backend-url", help="Hugging Face Playground backend URL."),
+    ] = FULL_VISION_BACKEND_URL,
+    expected_git_sha: Annotated[
+        str | None,
+        typer.Option("--expected-git-sha", help="Release git SHA expected in backend health."),
+    ] = None,
+) -> None:
+    """Verify the external gates for the full original DataForge vision."""
+    report = run_full_vision_gate(
+        evidence_root=evidence_root,
+        frontend_url=frontend_url,
+        backend_url=backend_url,
+        expected_git_sha=expected_git_sha,
+    )
+    if json_output:
+        typer.echo(json.dumps(report.to_dict(), indent=2, sort_keys=True))
+    else:
+        for check in report.checks:
+            status = "ok" if check.ok else "fail"
+            typer.echo(f"{status:4} {check.name}: {check.detail}")
+    raise typer.Exit(code=0 if report.ok else 1)
+
+
+@release_app.command(name="playground-check")
+def playground_check(
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print machine-readable JSON."),
+    ] = False,
+    frontend_url: Annotated[
+        str,
+        typer.Option("--frontend-url", help="Cloudflare Playground frontend URL."),
+    ] = "https://dataforge.praneshrajan15.workers.dev/playground",
+    backend_url: Annotated[
+        str,
+        typer.Option("--backend-url", help="Hugging Face Playground backend URL."),
+    ] = "https://Praneshrajan15-dataforge-playground.hf.space",
+    latency_threshold_ms: Annotated[
+        float,
+        typer.Option("--latency-threshold-ms", help="Warm health latency threshold."),
+    ] = 5_000.0,
+) -> None:
+    """Verify the deployed Playground release checklist."""
+    from dataforge.release.playground_check import report_to_json, run_playground_check
+
+    report = run_playground_check(
+        frontend_url=frontend_url,
+        backend_url=backend_url,
+        latency_threshold_ms=latency_threshold_ms,
+        include_doctor=True,
+        include_smoke=True,
+    )
+    if json_output:
+        typer.echo(report_to_json(report))
+    else:
+        for check in report.checks:
+            status = "ok" if check.ok else "fail"
+            typer.echo(f"{status:4} {check.name}: {check.detail}")
+    raise typer.Exit(code=0 if report.ok else 1)

dataforge/cli/repair.py

@@ -0,0 +1,407 @@
+"""CLI subcommand: ``dataforge repair <path> [--dry-run | --apply]``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+
+from dataforge.cli.common import load_schema, resolve_cli_path
+from dataforge.detectors.base import Issue, Schema
+from dataforge.repairers.base import ProposedFix, RepairAttempt
+from dataforge.safety import SafetyContext, SafetyFilter, SafetyResult
+from dataforge.schema_inference import ConstraintReviewArtifact, load_constraint_review_artifact
+from dataforge.stores import (
+    TableStoreError,
+    is_table_store_uri,
+    run_table_store_repair,
+    store_from_uri,
+)
+from dataforge.transactions.txn import CellFix
+from dataforge.ui.repair_diff import render_repair_diff
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from dataforge.engine.repair import RepairPipelineResult
+
+_console = Console(stderr=True)
+
+
+def apply_fixes_to_csv(path: Path, fixes: list[CellFix]) -> str:
+    """Apply ordered cell fixes to a CSV and return the post-state SHA-256.
+
+    Args:
+        path: Source CSV path.
+        fixes: Ordered list of cell fixes to apply.
+
+    Returns:
+        SHA-256 of the written file bytes.
+
+    Raises:
+        ValueError: If a fix references a missing row/column or stale old value.
+    """
+    from dataforge.engine.repair import apply_fixes_to_csv as engine_apply_fixes_to_csv
+
+    return engine_apply_fixes_to_csv(path, fixes)
+
+
+def _resolve_schema(schema_path: Path | None) -> Schema | None:
+    """Resolve an optional schema path into a parsed Schema."""
+    if schema_path is None:
+        return None
+    resolved_schema = resolve_cli_path(schema_path)
+    if not resolved_schema.exists():
+        raise typer.BadParameter(f"Schema file '{schema_path}' does not exist.")
+    return load_schema(resolved_schema)
+
+
+def _resolve_constraints(
+    constraints_path: Path | None,
+) -> tuple[ConstraintReviewArtifact | None, str | None]:
+    """Resolve an optional reviewed constraints artifact."""
+    if constraints_path is None:
+        return None, None
+    resolved_constraints = resolve_cli_path(constraints_path)
+    if not resolved_constraints.exists():
+        raise typer.BadParameter(f"Constraints file '{constraints_path}' does not exist.")
+    return load_constraint_review_artifact(resolved_constraints)
+
+
+def _print_error(message: str, *, hint: str | None = None) -> None:
+    """Render a rich-formatted CLI error."""
+    body = f"[bold red]{message}[/bold red]"
+    if hint:
+        body = f"{body}\n\n[dim]{hint}[/dim]"
+    _console.print(Panel(body, title="Repair Error", style="red"))
+
+
+def _propose_repairs(
+    issues: list[Issue],
+    path: Path,
+    working_df: pd.DataFrame,
+    schema: Schema | None,
+    *,
+    allow_llm: bool,
+    model: str,
+    allow_pii: bool,
+    confirm_pii: bool,
+    confirm_escalations: bool,
+    interactive: bool,
+) -> tuple[list[ProposedFix], list[list[RepairAttempt]]]:
+    """Compatibility wrapper around the shared repair engine proposal stage."""
+    from dataforge.engine.repair import propose_repairs as engine_propose_repairs
+
+    return engine_propose_repairs(
+        issues,
+        path,
+        working_df,
+        schema,
+        allow_llm=allow_llm,
+        model=model,
+        allow_pii=allow_pii,
+        confirm_pii=confirm_pii,
+        confirm_escalations=confirm_escalations,
+        interactive=interactive,
+        escalation_resolver=_resolve_escalation,
+    )
+
+
+def _resolve_escalation(
+    candidate: ProposedFix,
+    schema: Schema | None,
+    context: SafetyContext,
+    safety_filter: SafetyFilter,
+    safety_result: SafetyResult,
+) -> tuple[SafetyContext, SafetyResult]:
+    """Prompt for safety escalations and re-evaluate if the user confirms."""
+    if "NO_PII_OVERWRITE" in safety_result.rule_ids:
+        confirmed = typer.confirm(
+            f"Candidate fix for row {candidate.fix.row}, column '{candidate.fix.column}' "
+            "touches PII. Confirm this edit?",
+            default=False,
+        )
+        if confirmed:
+            updated = context.model_copy(update={"confirm_pii": True})
+            return updated, safety_filter.evaluate(candidate, schema, updated)
+        return context, safety_result
+
+    confirmed = typer.confirm(
+        f"Candidate fix for row {candidate.fix.row}, column '{candidate.fix.column}' "
+        f"requires confirmation ({', '.join(safety_result.rule_ids)}). Confirm this edit?",
+        default=False,
+    )
+    if confirmed:
+        updated = context.model_copy(update={"confirm_escalations": True})
+        return updated, safety_filter.evaluate(candidate, schema, updated)
+    return context, safety_result
+
+
+def _render_attempt_summary(
+    attempt_groups: list[list[RepairAttempt]],
+    console: Console,
+) -> int:
+    """Render a summary for issues that were not accepted."""
+    failed_groups = [
+        attempts for attempts in attempt_groups if attempts and attempts[-1].status != "accepted"
+    ]
+    if not failed_groups:
+        return 0
+
+    lines: list[str] = []
+    for attempts in failed_groups:
+        final_attempt = attempts[-1]
+        issue = final_attempt.issue
+        prefix = ""
+        if any(label.startswith("fd::") for label in final_attempt.unsat_core):
+            prefix = "functional dependency rejection - "
+        elif any(label.startswith("domain::") for label in final_attempt.unsat_core):
+            prefix = "domain bound rejection - "
+        lines.append(
+            f"{issue.issue_type} at {issue.row}:{issue.column} "
+            f"after {len(attempts)} attempt(s): {prefix}{final_attempt.reason}"
+        )
+
+    console.print("[bold yellow]Attempted But Not Fixed[/bold yellow]")
+    for line in lines:
+        console.print(line, overflow="fold")
+    return len(failed_groups)
+
+
+def _render_failure_summary(result: RepairPipelineResult, console: Console) -> int:
+    """Render a summary for issues that the shared engine could not repair."""
+    if not result.failures:
+        return 0
+
+    console.print("[bold yellow]Attempted But Not Fixed[/bold yellow]")
+    for failure in result.failures:
+        prefix = ""
+        if any(label.startswith("fd::") for label in failure.unsat_core):
+            prefix = "functional dependency rejection - "
+        elif any(label.startswith("domain::") for label in failure.unsat_core):
+            prefix = "domain bound rejection - "
+        console.print(
+            f"{failure.issue_type} at {failure.row}:{failure.column} "
+            f"after {failure.attempt_count} attempt(s): {prefix}{failure.reason}",
+            overflow="fold",
+        )
+    return len(result.failures)
+
+
+def _json_result(result: RepairPipelineResult) -> str:
+    """Serialize a repair result for CLI/MCP/CI consumers."""
+    return json.dumps(result.model_dump(mode="json"), indent=2, sort_keys=True)
+
+
+def _apply_transaction(
+    path: Path,
+    fixes: list[ProposedFix],
+    source_bytes: bytes,
+) -> str:
+    """Compatibility wrapper around the shared repair engine transaction path."""
+    from dataforge.engine.repair import apply_transaction as engine_apply_transaction
+
+    return engine_apply_transaction(path, fixes, source_bytes)
+
+
+def repair(
+    path: Annotated[
+        str,
+        typer.Argument(
+            help="Path to the CSV file, or warehouse:// backend URI, to repair.",
+        ),
+    ],
+    schema: Annotated[
+        Path | None,
+        typer.Option(
+            "--schema",
+            help="Path to a YAML schema file with column types and FDs.",
+        ),
+    ] = None,
+    constraints: Annotated[
+        Path | None,
+        typer.Option(
+            "--constraints",
+            help="Path to a reviewed constraints artifact from profile --constraints-out.",
+        ),
+    ] = None,
+    dry_run: Annotated[
+        bool,
+        typer.Option("--dry-run", help="Show proposed fixes without changing the file."),
+    ] = False,
+    apply: Annotated[
+        bool,
+        typer.Option("--apply", help="Apply fixes and record a reversible transaction."),
+    ] = False,
+    allow_llm: Annotated[
+        bool,
+        typer.Option(
+            "--allow-llm",
+            help="Allow fd_violation repair to call the configured LLM provider if needed.",
+        ),
+    ] = False,
+    allow_pii: Annotated[
+        bool,
+        typer.Option(
+            "--allow-pii",
+            help="Allow PII-targeting fixes to be considered by the safety layer.",
+        ),
+    ] = False,
+    confirm_pii: Annotated[
+        bool,
+        typer.Option(
+            "--confirm-pii",
+            help="Non-interactively confirm any PII-targeting fixes allowed via --allow-pii.",
+        ),
+    ] = False,
+    confirm_escalations: Annotated[
+        bool,
+        typer.Option(
+            "--confirm-escalations",
+            help="Non-interactively confirm soft safety escalations such as aggregate-sensitive edits.",
+        ),
+    ] = False,
+    llm_model: Annotated[
+        str,
+        typer.Option("--llm-model", help="Model name for fd_violation LLM fallback."),
+    ] = "gemini-2.0-flash",
+    row_id: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--row-id",
+            help="Stable row identity column for warehouse apply. Repeat for composite keys.",
+        ),
+    ] = None,
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print repair result as JSON."),
+    ] = False,
+) -> None:
+    """Detect, propose, and optionally apply reversible repairs to a CSV."""
+    if dry_run == apply:
+        _print_error(
+            "Choose exactly one of --dry-run or --apply.",
+            hint="Example: dataforge repair data.csv --dry-run",
+        )
+        raise typer.Exit(code=2)
+
+    try:
+        parsed_schema = _resolve_schema(schema)
+        constraints_artifact, constraints_sha256 = _resolve_constraints(constraints)
+        if is_table_store_uri(path):
+            if constraints_artifact is not None:
+                raise typer.BadParameter(
+                    "Reviewed constraints are not yet accepted for warehouse URI repair."
+                )
+            store = store_from_uri(path, row_ids=tuple(row_id or ()))
+            store_result = run_table_store_repair(
+                store,
+                mode="apply" if apply else "dry_run",
+                schema=parsed_schema,
+                allow_llm=allow_llm,
+                model=llm_model,
+                allow_pii=allow_pii,
+                confirm_pii=confirm_pii,
+                confirm_escalations=confirm_escalations,
+            )
+            if json_output:
+                typer.echo(
+                    json.dumps(store_result.model_dump(mode="json"), indent=2, sort_keys=True)
+                )
+                return
+            output_console = Console()
+            output_console.print(
+                Panel(
+                    (
+                        f"[bold]{store_result.backend}[/bold] patch plan "
+                        f"{store_result.patch_plan.plan_id}\n"
+                        f"Operations: {len(store_result.patch_plan.operations)}\n"
+                        f"Apply supported: {store_result.patch_plan.apply_supported}\n"
+                        f"Reason: {store_result.patch_plan.reason}"
+                    ),
+                    title="Warehouse Repair Plan",
+                    style="green" if store_result.patch_plan.apply_supported else "yellow",
+                )
+            )
+            return
+
+        resolved_path = resolve_cli_path(Path(path))
+        if not resolved_path.exists():
+            raise typer.BadParameter(f"CSV file '{path}' does not exist.")
+    except TableStoreError as exc:
+        _print_error(str(exc))
+        raise typer.Exit(code=1 if apply else 2) from exc
+    except Exception as exc:
+        _print_error(str(exc))
+        raise typer.Exit(code=2) from exc
+
+    try:
+        from dataforge.engine.repair import RepairPipelineRequest, run_repair_pipeline
+
+        result = run_repair_pipeline(
+            RepairPipelineRequest(
+                source_path=resolved_path,
+                mode="apply" if apply else "dry_run",
+                schema=parsed_schema,
+                constraints=constraints_artifact,
+                constraints_artifact_sha256=constraints_sha256,
+                allow_llm=allow_llm,
+                model=llm_model,
+                allow_pii=allow_pii,
+                confirm_pii=confirm_pii,
+                confirm_escalations=confirm_escalations,
+                interactive=apply,
+            )
+        )
+    except Exception as exc:
+        _print_error(
+            f"Failed to apply repairs: {exc}" if apply else f"Failed to repair: {exc}",
+            hint="The source file was restored to its pre-apply bytes." if apply else None,
+        )
+        raise typer.Exit(code=1 if apply else 2) from exc
+
+    if json_output:
+        typer.echo(_json_result(result))
+        raise typer.Exit(code=0 if result.fixes else 1)
+
+    output_console = Console()
+    render_repair_diff(result.fixes, output_console, file_path=str(resolved_path))
+    failed_issue_count = _render_failure_summary(result, output_console)
+
+    if not result.fixes and failed_issue_count == 0:
+        if result.receipt.reason != "No accepted fixes were produced.":
+            output_console.print(
+                Panel(
+                    f"[yellow]{result.receipt.reason}[/yellow]",
+                    title="Repair Summary",
+                    style="yellow",
+                )
+            )
+        raise typer.Exit(code=1)
+
+    if dry_run:
+        raise typer.Exit(code=0 if result.fixes else 1)
+
+    if not result.fixes or not result.receipt.applied:
+        raise typer.Exit(code=1)
+
+    output_console.print(
+        Panel(
+            f"[green]Applied {len(result.fixes)} fix(es).[/green]\n"
+            f"Transaction ID: [bold]{result.receipt.txn_id}[/bold]",
+            title="Repair Applied",
+            style="green",
+        )
+    )
+    if failed_issue_count:
+        output_console.print(
+            Panel(
+                f"[yellow]{failed_issue_count} issue(s) were attempted but not fixed.[/yellow]",
+                title="Week 3 Summary",
+                style="yellow",
+            )
+        )