dataforge-07 0.1.0__py3-none-any.whl

dataforge/cli/constraints.py

@@ -0,0 +1,407 @@
+"""CLI subcommands for reviewing inferred constraint artifacts."""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from textual import on
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Horizontal, Vertical
+from textual.widgets import DataTable, Footer, Header, Input, Static
+
+from dataforge.cli.common import resolve_cli_path
+from dataforge.schema_inference import (
+    REPAIR_SUPPORTED_CONSTRAINT_KINDS,
+    ConstraintDecision,
+    ConstraintReviewArtifact,
+    ConstraintReviewError,
+    load_constraint_review_artifact,
+    update_constraint_review_artifact,
+    write_constraint_review_artifact_atomic,
+)
+
+constraints_app = typer.Typer(
+    help="Review inferred profile constraints before repair can use them.",
+    no_args_is_help=True,
+)
+_console = Console(stderr=True)
+
+
+def _candidate_target(candidate: Any) -> str:
+    """Return a compact target description for one candidate."""
+    columns = ", ".join(candidate.columns)
+    if candidate.dependent:
+        return f"{columns} -> {candidate.dependent}"
+    return columns
+
+
+def _candidate_summary(reviewed: Any) -> dict[str, Any]:
+    """Return a machine-readable review summary for one candidate."""
+    candidate = reviewed.candidate
+    return {
+        "candidate_id": reviewed.candidate_id,
+        "decision": reviewed.decision,
+        "kind": candidate.kind,
+        "target": _candidate_target(candidate),
+        "confidence": candidate.confidence,
+        "repair_supported": candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS,
+        "evidence": candidate.evidence,
+        "review_note": reviewed.review_note,
+    }
+
+
+def _artifact_summary(
+    artifact: ConstraintReviewArtifact,
+    *,
+    path: Path,
+    sha256: str | None = None,
+) -> dict[str, Any]:
+    """Return a stable summary payload for CLI and CI consumers."""
+    decision_counts = Counter(reviewed.decision for reviewed in artifact.candidates)
+    repair_supported_count = sum(
+        1
+        for reviewed in artifact.candidates
+        if reviewed.candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS
+    )
+    return {
+        "path": str(path),
+        "schema_version": artifact.schema_version,
+        "source_path": artifact.source_path,
+        "source_sha256": artifact.source_sha256,
+        "row_count": artifact.row_count,
+        "candidate_count": len(artifact.candidates),
+        "repair_supported_count": repair_supported_count,
+        "decision_counts": {
+            "accepted": decision_counts.get("accepted", 0),
+            "pending": decision_counts.get("pending", 0),
+            "rejected": decision_counts.get("rejected", 0),
+        },
+        "sha256": sha256,
+        "candidates": [_candidate_summary(reviewed) for reviewed in artifact.candidates],
+    }
+
+
+def _parse_notes(raw_notes: list[str] | None) -> dict[str, str | None]:
+    """Parse repeated ``--note cnd-id=text`` options."""
+    parsed: dict[str, str | None] = {}
+    for raw_note in raw_notes or []:
+        candidate_id, separator, note = raw_note.partition("=")
+        if not separator or not candidate_id:
+            raise typer.BadParameter("--note must use the form cnd-...=text")
+        parsed[candidate_id] = note or None
+    return parsed
+
+
+def _print_review_table(artifact: ConstraintReviewArtifact) -> None:
+    """Render a compact non-interactive review table."""
+    table = Table(title="Constraint Review")
+    table.add_column("Candidate ID", overflow="fold")
+    table.add_column("Decision")
+    table.add_column("Kind")
+    table.add_column("Target", overflow="fold")
+    table.add_column("Confidence", justify="right")
+    table.add_column("Repair")
+    table.add_column("Evidence", overflow="fold")
+    for reviewed in artifact.candidates:
+        candidate = reviewed.candidate
+        table.add_row(
+            reviewed.candidate_id,
+            reviewed.decision,
+            candidate.kind,
+            _candidate_target(candidate),
+            f"{candidate.confidence:.4f}",
+            "yes" if candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS else "review-only",
+            candidate.evidence,
+        )
+    Console().print(table)
+
+
+class ConstraintReviewApp(App[ConstraintReviewArtifact]):
+    """Textual review UI for a constraint artifact."""
+
+    CSS = """
+    DataTable {
+        height: 1fr;
+    }
+
+    #detail {
+        width: 45%;
+        height: 1fr;
+        overflow-y: auto;
+        border: solid $accent;
+        padding: 1;
+    }
+
+    #note {
+        height: 3;
+    }
+    """
+
+    BINDINGS = [
+        Binding("a", "accept", "Accept"),
+        Binding("r", "reject", "Reject"),
+        Binding("p", "pending", "Pending"),
+        Binding("n", "focus_note", "Note"),
+        Binding("s", "save", "Save"),
+        Binding("q", "quit_without_save", "Quit"),
+    ]
+
+    def __init__(self, artifact: ConstraintReviewArtifact) -> None:
+        """Create a review application for an already validated artifact."""
+        super().__init__()
+        self.artifact = artifact
+        self.saved = False
+        self.selected_candidate_id = (
+            artifact.candidates[0].candidate_id if artifact.candidates else None
+        )
+
+    def compose(self) -> ComposeResult:
+        """Compose the review screen."""
+        yield Header()
+        with Vertical():
+            with Horizontal():
+                yield DataTable(id="candidates")
+                yield Static(id="detail")
+            yield Input(
+                placeholder="Review note for selected candidate; press Enter to save note",
+                id="note",
+            )
+        yield Footer()
+
+    def on_mount(self) -> None:
+        """Populate the table when the TUI starts."""
+        table = self.query_one("#candidates", DataTable)
+        table.cursor_type = "row"
+        table.add_columns("ID", "Decision", "Kind", "Target", "Conf", "Repair")
+        for reviewed in self.artifact.candidates:
+            candidate = reviewed.candidate
+            table.add_row(
+                reviewed.candidate_id,
+                reviewed.decision,
+                candidate.kind,
+                _candidate_target(candidate),
+                f"{candidate.confidence:.4f}",
+                "yes" if candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS else "review-only",
+                key=reviewed.candidate_id,
+            )
+        table.focus()
+        self._refresh_detail()
+
+    @on(DataTable.RowHighlighted)
+    def _on_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
+        """Track the currently highlighted candidate."""
+        self.selected_candidate_id = str(event.row_key.value)
+        self._refresh_detail()
+
+    @on(Input.Submitted, "#note")
+    def _on_note_submitted(self, event: Input.Submitted) -> None:
+        """Save a note for the selected candidate."""
+        if self.selected_candidate_id is None:
+            return
+        self.artifact = update_constraint_review_artifact(
+            self.artifact,
+            notes={self.selected_candidate_id: event.value},
+        )
+        event.input.value = ""
+        self._refresh_table()
+        self._refresh_detail()
+
+    def action_accept(self) -> None:
+        """Accept the selected candidate."""
+        self._set_selected_decision("accepted")
+
+    def action_reject(self) -> None:
+        """Reject the selected candidate."""
+        self._set_selected_decision("rejected")
+
+    def action_pending(self) -> None:
+        """Reset the selected candidate to pending."""
+        self._set_selected_decision("pending")
+
+    def action_focus_note(self) -> None:
+        """Focus the note editor."""
+        self.query_one("#note", Input).focus()
+
+    def action_save(self) -> None:
+        """Exit the TUI with the reviewed artifact."""
+        self.saved = True
+        self.exit(self.artifact)
+
+    def action_quit_without_save(self) -> None:
+        """Exit the TUI without saving."""
+        self.saved = False
+        self.exit(self.artifact)
+
+    def _set_selected_decision(self, decision: ConstraintDecision) -> None:
+        """Apply a decision to the selected candidate."""
+        if self.selected_candidate_id is None:
+            return
+        if decision == "accepted":
+            self.artifact = update_constraint_review_artifact(
+                self.artifact,
+                accept_ids=(self.selected_candidate_id,),
+            )
+        elif decision == "rejected":
+            self.artifact = update_constraint_review_artifact(
+                self.artifact,
+                reject_ids=(self.selected_candidate_id,),
+            )
+        else:
+            self.artifact = update_constraint_review_artifact(
+                self.artifact,
+                pending_ids=(self.selected_candidate_id,),
+            )
+        self._refresh_table()
+        self._refresh_detail()
+
+    def _selected_candidate(self) -> Any | None:
+        """Return the selected reviewed candidate."""
+        for reviewed in self.artifact.candidates:
+            if reviewed.candidate_id == self.selected_candidate_id:
+                return reviewed
+        return None
+
+    def _refresh_table(self) -> None:
+        """Refresh table rows after a decision change."""
+        table = self.query_one("#candidates", DataTable)
+        table.clear(columns=True)
+        table.add_columns("ID", "Decision", "Kind", "Target", "Conf", "Repair")
+        for reviewed in self.artifact.candidates:
+            candidate = reviewed.candidate
+            table.add_row(
+                reviewed.candidate_id,
+                reviewed.decision,
+                candidate.kind,
+                _candidate_target(candidate),
+                f"{candidate.confidence:.4f}",
+                "yes" if candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS else "review-only",
+                key=reviewed.candidate_id,
+            )
+
+    def _refresh_detail(self) -> None:
+        """Refresh the detail pane for the selected candidate."""
+        reviewed = self._selected_candidate()
+        detail = self.query_one("#detail", Static)
+        if reviewed is None:
+            detail.update("No constraint candidates.")
+            return
+        candidate = reviewed.candidate
+        repair_note = (
+            "Repair-supported in v1."
+            if candidate.kind in REPAIR_SUPPORTED_CONSTRAINT_KINDS
+            else "Review-only in v1; repair ignores this kind."
+        )
+        payload = json.dumps(reviewed.model_dump(mode="json"), indent=2, sort_keys=True)
+        detail.update(
+            "\n".join(
+                [
+                    f"Candidate: {reviewed.candidate_id}",
+                    f"Decision: {reviewed.decision}",
+                    f"Source: {self.artifact.source_path}",
+                    f"Source SHA-256: {self.artifact.source_sha256}",
+                    f"Repair: {repair_note}",
+                    "",
+                    "Candidate JSON:",
+                    payload,
+                ]
+            )
+        )
+
+
+@constraints_app.command(name="review")
+def review_constraints(
+    path: Annotated[
+        Path,
+        typer.Argument(help="Path to a constraint_review_v1 JSON artifact."),
+    ],
+    accept: Annotated[
+        list[str] | None,
+        typer.Option("--accept", help="Mark a candidate id accepted. Repeatable."),
+    ] = None,
+    reject: Annotated[
+        list[str] | None,
+        typer.Option("--reject", help="Mark a candidate id rejected. Repeatable."),
+    ] = None,
+    pending: Annotated[
+        list[str] | None,
+        typer.Option("--pending", help="Reset a candidate id to pending. Repeatable."),
+    ] = None,
+    note: Annotated[
+        list[str] | None,
+        typer.Option("--note", help="Set a review note with cnd-...=text. Repeatable."),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", help="Write the reviewed artifact to a separate path."),
+    ] = None,
+    dry_run: Annotated[
+        bool,
+        typer.Option("--dry-run", help="Preview changes without writing an artifact."),
+    ] = False,
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print review state as JSON."),
+    ] = False,
+    no_tui: Annotated[
+        bool,
+        typer.Option("--no-tui", help="Run deterministic non-interactive review mode."),
+    ] = False,
+) -> None:
+    """Review profile-inferred constraint candidates before repair uses them."""
+    resolved_path = resolve_cli_path(path)
+    try:
+        artifact, artifact_sha256 = load_constraint_review_artifact(resolved_path)
+        parsed_notes = _parse_notes(note)
+        updated = update_constraint_review_artifact(
+            artifact,
+            accept_ids=tuple(accept or ()),
+            reject_ids=tuple(reject or ()),
+            pending_ids=tuple(pending or ()),
+            notes=parsed_notes,
+        )
+    except (ConstraintReviewError, typer.BadParameter) as exc:
+        _console.print(Panel(f"[bold red]{exc}[/bold red]", title="Constraint Review Error"))
+        raise typer.Exit(code=2) from exc
+
+    non_interactive = no_tui or bool(
+        accept or reject or pending or note or output or dry_run or json_output
+    )
+    if not non_interactive:
+        app = ConstraintReviewApp(updated)
+        tui_result = app.run()
+        if tui_result is None or not app.saved:
+            _console.print("[yellow]Constraint review cancelled; no file was changed.[/yellow]")
+            raise typer.Exit(code=1)
+        updated = tui_result
+
+    target_path = resolve_cli_path(output) if output is not None else resolved_path
+    written_sha256: str | None = artifact_sha256
+    if dry_run:
+        written_sha256 = None
+    elif updated != artifact or target_path != resolved_path:
+        try:
+            written_sha256 = write_constraint_review_artifact_atomic(target_path, updated)
+        except OSError as exc:
+            _console.print(Panel(f"[bold red]{exc}[/bold red]", title="Constraint Review Error"))
+            raise typer.Exit(code=2) from exc
+
+    if json_output:
+        typer.echo(
+            json.dumps(
+                _artifact_summary(updated, path=target_path, sha256=written_sha256),
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        _print_review_table(updated)
+        if dry_run:
+            _console.print("[yellow]Dry run: no constraint artifact was written.[/yellow]")

dataforge/cli/profile.py

@@ -0,0 +1,147 @@
+"""CLI subcommand: ``dataforge profile <path> [--schema <yaml>]``.
+
+Reads a CSV file, runs all detectors, and renders detected issues as a
+rich-formatted terminal table. Diagnostics exit 0 by default; use
+``--fail-on`` for CI gating.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from hashlib import sha256
+from pathlib import Path
+from typing import Annotated, Literal
+
+import typer
+from rich.console import Console
+
+from dataforge.cli.common import load_schema, read_csv, resolve_cli_path
+from dataforge.detectors import run_all_detectors
+from dataforge.detectors.base import Issue, Schema, Severity
+from dataforge.schema_inference import (
+    build_constraint_review_artifact,
+    dump_constraint_review_artifact,
+    infer_schema,
+)
+from dataforge.ui.profile_view import render_profile_table
+
+_console = Console(stderr=True)
+
+FailOn = Literal["never", "unsafe", "review", "any"]
+
+
+def _should_fail(issues: Sequence[Issue], fail_on: FailOn) -> bool:
+    """Return whether profile findings should trip the requested CI gate."""
+    if fail_on == "never":
+        return False
+    if fail_on == "any":
+        return bool(issues)
+    severities = [issue.severity for issue in issues]
+    if fail_on == "unsafe":
+        return any(severity == Severity.UNSAFE for severity in severities)
+    return any(severity >= Severity.REVIEW for severity in severities)
+
+
+def profile(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Path to the CSV file to profile.",
+        ),
+    ],
+    schema: Annotated[
+        Path | None,
+        typer.Option(
+            "--schema",
+            help="Path to a YAML schema file with column types and FDs.",
+        ),
+    ] = None,
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Print profile results as JSON."),
+    ] = False,
+    constraints_out: Annotated[
+        Path | None,
+        typer.Option(
+            "--constraints-out",
+            help="Write inferred constraints as a pending review artifact.",
+        ),
+    ] = None,
+    fail_on: Annotated[
+        FailOn,
+        typer.Option(
+            "--fail-on",
+            help="Exit 1 when findings meet this threshold: never, unsafe, review, any.",
+        ),
+    ] = "never",
+) -> None:
+    """Profile a CSV file for data-quality issues.
+
+    Reads the CSV, runs all detectors (type_mismatch, decimal_shift,
+    fd_violation), and renders a rich-formatted table of detected issues.
+
+    Exit code 0 unless ``--fail-on`` is set and matching findings are present.
+    """
+    resolved_path = resolve_cli_path(path)
+    if not resolved_path.exists():
+        _console.print(f"[bold red]CSV file not found:[/bold red] {path}")
+        raise typer.Exit(code=2)
+
+    # Load the CSV with dtype=str to avoid pandas type-coercion artifacts.
+    try:
+        df = read_csv(resolved_path)
+    except Exception as exc:
+        _console.print(f"[bold red]Error reading CSV:[/bold red] {exc}")
+        raise typer.Exit(code=2) from exc
+
+    # Optionally load schema.
+    parsed_schema: Schema | None = None
+    if schema is not None:
+        resolved_schema = resolve_cli_path(schema)
+        if not resolved_schema.exists():
+            _console.print(f"[bold red]Schema file not found:[/bold red] {schema}")
+            raise typer.Exit(code=2)
+        parsed_schema = load_schema(resolved_schema)
+
+    # Run all detectors.
+    issues = run_all_detectors(df, parsed_schema)
+    schema_inference = infer_schema(df)
+    source_sha256 = sha256(resolved_path.read_bytes()).hexdigest()
+    if constraints_out is not None:
+        try:
+            artifact = build_constraint_review_artifact(
+                schema_inference,
+                source_path=resolved_path,
+                source_sha256=source_sha256,
+            )
+            constraints_out.parent.mkdir(parents=True, exist_ok=True)
+            constraints_out.write_text(
+                dump_constraint_review_artifact(artifact),
+                encoding="utf-8",
+            )
+        except Exception as exc:
+            _console.print(f"[bold red]Error writing constraints artifact:[/bold red] {exc}")
+            raise typer.Exit(code=2) from exc
+
+    # Render the results.
+    if json_output:
+        typer.echo(
+            json.dumps(
+                {
+                    "path": str(resolved_path),
+                    "issues_count": len(issues),
+                    "fail_on": fail_on,
+                    "issues": [issue.model_dump(mode="json") for issue in issues],
+                    "schema_inference": schema_inference.model_dump(mode="json"),
+                },
+                indent=2,
+                sort_keys=True,
+            )
+        )
+    else:
+        output_console = Console()
+        render_profile_table(issues, output_console, file_path=str(resolved_path))
+
+    if _should_fail(issues, fail_on):
+        raise typer.Exit(code=1)