querycleaner 0.1.0__py3-none-any.whl

qc/__init__.py

@@ -0,0 +1,4 @@
+"""QueryCleaner/QC - local-first data profiling, sanitization, and synthetic data."""
+
+__version__ = "0.1.0"
+TOOL_VERSION = __version__

qc/application/__init__.py

@@ -0,0 +1 @@
+"""QC application package."""

qc/application/cancellation.py

@@ -0,0 +1,54 @@
+"""Process-wide registry of cancellable jobs (SPEC 12.6, 15).
+
+Maps job ids to :class:`~qc.domain.cancellation.CancellationToken` so
+MCP ``cancel_job`` and CLI signal handlers can stop a running operation
+without holding a reference to its coroutine.
+"""
+
+from __future__ import annotations
+
+import threading
+
+from qc.domain.cancellation import CancellationToken
+
+
+class CancellationRegistry:
+    """Thread-safe registry; a cancellation from any thread is observed."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._tokens: dict[str, CancellationToken] = {}
+
+    def register(self, job_id: str) -> CancellationToken:
+        token = CancellationToken()
+        with self._lock:
+            self._tokens[job_id] = token
+        return token
+
+    def get(self, job_id: str) -> CancellationToken | None:
+        with self._lock:
+            return self._tokens.get(job_id)
+
+    def cancel(self, job_id: str, reason: str | None = None) -> bool:
+        """Cancel a registered job; returns False if it is not running."""
+        with self._lock:
+            token = self._tokens.get(job_id)
+        if token is None:
+            return False
+        token.cancel(reason)
+        return True
+
+    def unregister(self, job_id: str) -> None:
+        with self._lock:
+            self._tokens.pop(job_id, None)
+
+    def running(self) -> list[str]:
+        with self._lock:
+            return [jid for jid, tok in self._tokens.items() if not tok.cancelled]
+
+
+#: Process-wide registry shared by the CLI and MCP server.
+REGISTRY = CancellationRegistry()
+
+
+__all__ = ["REGISTRY", "CancellationRegistry"]

qc/application/exports.py

@@ -0,0 +1,128 @@
+"""Database destination export (SPEC 10.3).
+
+Separate source/destination configs; approved destinations only;
+create-new-table by default; append/replace require explicit mode.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from qc.domain.enums import OperationKind
+from qc.domain.errors import PolicyDeniedError, ValidationError, WriteError
+from qc.security.policy import DecisionContext, Operation
+
+if TYPE_CHECKING:
+    from qc.application.runtime import Runtime
+
+
+async def export_to_database(
+    rt: Runtime,
+    *,
+    output: Path,
+    destination: str,
+    table: str,
+    policy_name: str | None = None,
+    mode: str = "create",
+) -> dict[str, Any]:
+    """Write a JSONL/CSV file to an explicitly approved destination."""
+    # argument validation first (cheap, no I/O), then policy gates
+    if mode not in ("create", "append", "replace"):
+        raise ValidationError(f"invalid mode '{mode}'; use create|append|replace")
+    if not table.isidentifier() and not all(p.isidentifier() for p in table.split(".")):
+        raise ValidationError(f"invalid table name: {table}")
+
+    engine = rt.engine(policy_name)
+    policy = engine.policy
+    ctx = DecisionContext(source=destination, dataset=table)
+    engine.require(Operation.WRITE, ctx)
+    engine.require(Operation.EXPORT, ctx)
+
+    if not policy.write_allowed:
+        raise PolicyDeniedError("policy does not allow writes (write_allowed=false)")
+
+    dest_cfg = rt.source(destination)
+    rows = _read_rows(output)
+    if not rows:
+        raise ValidationError("output file contains no rows")
+
+    # destination role is a *separate* writer (SPEC 6, 10.3): source
+    # connectors never gain write_rows by configuration
+    writer = _destination_writer(dest_cfg)
+    try:
+        columns = list(rows[0].keys())
+        written = await writer.write_rows(dest_cfg, table, columns, rows, mode=mode)
+    finally:
+        await writer.close()
+
+    rt.audit.record(
+        OperationKind.EXPORT_DATABASE,
+        source_id=dest_cfg.id,
+        dataset_id=table,
+        policy_version=policy.version,
+        decision="allow",
+        rows_affected=written,
+        result_status="ok",
+        detail={"mode": mode},
+    )
+    return {
+        "destination": destination,
+        "table": table,
+        "rows": written,
+        "mode": mode,
+        "columns": len(rows[0]),
+    }
+
+
+def _destination_writer(config: Any) -> Any:
+    """Return the writable destination role for a source config."""
+    kind = config.connector
+    if kind == "postgres":
+        from qc.exports.db_writer import PostgresDestination
+
+        return PostgresDestination()
+    if kind == "mysql":
+        from qc.exports.db_writer import MySQLDestination
+
+        return MySQLDestination()
+    raise WriteError(
+        f"connector '{kind}' does not support database destination writes",
+        detail={"destination": config.name},
+    )
+
+
+def _read_rows(path: Path) -> list[dict[str, Any]]:
+    if not path.is_file():
+        raise WriteError(f"output file not found: {path.name}")
+    rows: list[dict[str, Any]] = []
+    if path.suffix == ".csv":
+        import csv
+
+        with path.open(encoding="utf-8", newline="") as fh:
+            for raw in csv.DictReader(fh):
+                rows.append({k: _coerce(v) for k, v in raw.items()})
+    else:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if line:
+                obj = json.loads(line)
+                if isinstance(obj, dict):
+                    rows.append(obj)
+    return rows
+
+
+def _coerce(v: str) -> Any:
+    if v == "\\N":
+        return None
+    if v in ("true", "false"):
+        return v == "true"
+    try:
+        return int(v)
+    except ValueError:
+        pass
+    try:
+        return float(v)
+    except ValueError:
+        return v

qc/application/generation.py

@@ -0,0 +1,439 @@
+"""Generation use cases: plan, run, validate (SPEC sections 9, 3.3)."""
+
+from __future__ import annotations
+
+import json
+import random
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from qc.artifacts.profiles import find_profile, load_profile
+from qc.domain.cancellation import CancellationToken
+from qc.domain.enums import OperationKind
+from qc.domain.errors import GenerationError, NotFoundError, ValidationError
+from qc.domain.models import GenerationPlan, GenerationReport, StrategyOverride
+from qc.generation.strategies import GenerationContext, execute, resolve_strategy
+from qc.generation.validation import validate_generation
+from qc.observability.metrics import OperationMetrics
+from qc.security.policy import DecisionContext, Operation
+
+if TYPE_CHECKING:
+    from qc.application.runtime import Runtime
+
+
+def create_plan(
+    rt: Runtime,
+    *,
+    profile_ref: str,
+    rows: int,
+    seed: int = 0,
+    locale: str = "en_US",
+    policy_name: str | None = None,
+    output: Path | None = None,
+    allow_fallbacks: bool = False,
+    strategy_overrides: dict[str, str] | None = None,
+) -> GenerationPlan:
+    """Build a GenerationPlan from a saved profile after policy checks.
+
+    ``strategy_overrides`` maps column -> strategy name (SPEC 9.1 step 1:
+    explicit user-defined strategies win over the registry).
+    """
+    engine = rt.engine(policy_name)
+    path = rt.state.find_profile_index(profile_ref) or find_profile(rt.paths.profiles, profile_ref)
+    artifact = load_profile(path)
+    # policy keys are source *names*; artifacts store opaque ids
+    src_cfg = rt.sources.get_by_id(artifact.source_id)
+    if src_cfg is None:
+        raise NotFoundError(
+            "profile's source is not configured in this project "
+            "(sources may have been re-created); re-run profiling"
+        )
+    ctx = DecisionContext(source=src_cfg.name, dataset=artifact.dataset.qualified_name)
+    engine.require(Operation.DATASET_READ, ctx)
+
+    if rows > rt.ceiling.max_rows:
+        raise ValidationError(
+            f"requested {rows} rows exceeds ceiling {rt.ceiling.max_rows}",
+        )
+    known_columns = {c.name for c in artifact.columns}
+    overrides = [
+        StrategyOverride(column=col, strategy=strat)
+        for col, strat in (strategy_overrides or {}).items()
+    ]
+    for ov in overrides:
+        if ov.column not in known_columns:
+            raise ValidationError(
+                f"strategy override names unknown column '{ov.column}'",
+                detail={"column": ov.column},
+            )
+    plan = GenerationPlan(
+        profile_ids=[artifact.profile_id],
+        dataset=artifact.dataset,
+        rows=rows,
+        seed=seed,
+        locale=locale,
+        output_format=(output.suffix.lstrip(".") if output else "jsonl"),
+        allow_fallbacks=allow_fallbacks,
+        strategy_overrides=overrides,
+        budget=artifact.budget,
+    )
+    plan_path = rt.paths.reports / f"{plan.plan_id}.plan.json"
+    plan_path.parent.mkdir(parents=True, exist_ok=True)
+    plan_path.write_text(json.dumps(plan.json_safe(), indent=2), encoding="utf-8")
+    rt.audit.record(
+        OperationKind.GENERATE,
+        source_id=artifact.source_id,
+        dataset_id=artifact.dataset.qualified_name,
+        policy_version=engine.policy.version,
+        decision="allow",
+        result_status="ok",
+        detail={"plan_id": plan.plan_id, "rows": str(rows)},
+    )
+    return plan
+
+
+def plan_path(rt: Runtime, plan: GenerationPlan) -> Path:
+    return rt.paths.reports / f"{plan.plan_id}.plan.json"
+
+
+def _ensure_unique(
+    value: Any,
+    column: str,
+    seen: set[str],
+    strat_plan: Any,
+    context: Any,
+    max_attempts: int = 100,
+) -> Any:
+    """Bounded retry for unique columns; raises when the strategy cannot
+    satisfy uniqueness (SPEC 9.6: never silently weaken a constraint)."""
+    key = f"{type(value).__name__}:{value}"
+    attempts = 0
+    while key in seen and attempts < max_attempts:
+        value = execute(strat_plan, context)
+        key = f"{type(value).__name__}:{value}"
+        attempts += 1
+    if key in seen:
+        raise GenerationError(
+            f"strategy '{strat_plan.strategy}' could not produce a unique "
+            f"value for column '{column}' within {max_attempts} attempts; "
+            "define a strategy override or reduce row count",
+            detail={"column": column, "strategy": strat_plan.strategy},
+        )
+    seen.add(key)
+    return value
+
+
+def _check_unique_feasibility(resolved: dict[str, Any], unique_cols: set[str], rows: int) -> None:
+    """Reject impossible uniqueness before generating a single row.
+
+    A bounded numeric range that cannot yield ``rows`` distinct values
+    fails immediately with an actionable error instead of exhausting
+    per-row retries (SPEC 9.6).
+    """
+    for name in sorted(unique_cols):
+        plan = resolved[name]
+        col = plan.profile
+        if col is None or col.numeric is None:
+            continue
+        if plan.strategy != "numeric_distribution":
+            continue
+        if col.numeric.minimum is None or col.numeric.maximum is None:
+            continue
+        try:
+            lo = int(float(col.numeric.minimum))
+            hi = int(float(col.numeric.maximum))
+        except (TypeError, ValueError):
+            continue
+        space = hi - lo + 1
+        if space < rows:
+            raise GenerationError(
+                f"column '{name}' requires {rows} unique values but its "
+                f"observed range [{col.numeric.minimum}, {col.numeric.maximum}] "
+                f"holds only {space}; reduce --rows, override the strategy, "
+                "or drop the uniqueness requirement",
+                detail={
+                    "column": name,
+                    "value_space": str(space),
+                    "rows_requested": str(rows),
+                },
+            )
+
+
+async def run_generation(
+    rt: Runtime,
+    *,
+    plan_file: Path,
+    output: Path | None = None,
+    report_path: Path | None = None,
+    seed: int | None = None,
+    strict: bool = False,
+    allow_fallbacks: bool = False,
+    cancel: CancellationToken | None = None,
+) -> dict[str, Any]:
+    """Execute a generation plan: synthesize, validate, report.
+
+    ``cancel`` is polled between generated rows and before writing output
+    so a cancelled job never leaves a partial artifact marked complete.
+    """
+    from faker import Faker
+
+    from qc.exports.csv_writer import write_csv_rows
+
+    def check_cancel() -> None:
+        if cancel is not None:
+            cancel.raise_if_cancelled()
+
+    if not plan_file.is_file():
+        raise NotFoundError(f"plan not found: {plan_file.name}")
+    plan = GenerationPlan.model_validate(json.loads(plan_file.read_text(encoding="utf-8")))
+    if seed is not None:
+        plan.seed = seed
+    if allow_fallbacks:
+        plan.allow_fallbacks = True
+
+    # Load all referenced profiles
+    profiles: dict[str, Any] = {}
+    for pid in plan.profile_ids:
+        p = rt.state.find_profile_index(pid) or find_profile(rt.paths.profiles, pid)
+        artifact = load_profile(p)
+        profiles[artifact.dataset.qualified_name] = artifact
+    if not profiles:
+        raise NotFoundError("plan references no loadable profiles")
+    primary = next(iter(profiles.values()))
+
+    # Policy check: artifacts store opaque source ids, policy keys are names
+    src_cfg = rt.sources.get_by_id(primary.source_id)
+    engine = rt.engine()
+    engine.require(
+        Operation.DATASET_READ,
+        DecisionContext(
+            source=src_cfg.name if src_cfg else plan.dataset.source_id,
+            dataset=plan.dataset.qualified_name,
+        ),
+    )
+
+    started = time.monotonic()
+    metrics = OperationMetrics(operation="generate")
+    rng = random.Random(plan.seed)
+    fake = Faker(plan.locale)
+    Faker.seed(plan.seed)
+    context = GenerationContext(plan=plan, profiles=profiles, faker=fake, rng=rng)
+
+    # Resolve strategies per column (fail-closed before generating rows)
+    overrides = {o.column: o.strategy for o in plan.strategy_overrides}
+    resolved: dict[str, Any] = {}
+    fallbacks: list[str] = []
+    for col in primary.columns:
+        strat_plan = resolve_strategy(col, context, overrides)
+        strat_plan.column_profile = col
+        resolved[col.name] = strat_plan
+        if strat_plan.fallback:
+            fallbacks.append(col.name)
+
+    # Uniqueness requirements (SPEC 9.2 "declared and *approved* inferred
+    # uniqueness"): enforce exact uniqueness only where it is actually a
+    # key — a declared PK, an identifier-like type, or an id-ish name.
+    # Sample-uniqueness on measure columns (e.g. a qty column that
+    # happens to be distinct in a small table) is evidence, not a key
+    # constraint, and must not make generation fail (SPEC 9.2, 9.6).
+    declared_pk = set(primary.dataset_schema.primary_key)
+    unique_cols: set[str] = set()
+    unenforced_sample_unique: list[str] = []
+    for name, sp in resolved.items():
+        col = sp.profile
+        if col is None or col.unique is not True:
+            continue
+        if col.unique_measurement.value != "exact":
+            continue
+        lowered = name.lower()
+        keyish = (
+            name in declared_pk
+            or col.logical_type.value in ("identifier", "uuid")
+            or lowered in ("id", "uuid", "guid")
+            or lowered.endswith("_id")
+            or lowered.endswith("_uuid")
+        )
+        if keyish:
+            unique_cols.add(name)
+        else:
+            unenforced_sample_unique.append(name)
+    seen: dict[str, set[str]] = {c: set() for c in unique_cols}
+    pre_warnings: list[str] = []
+    if unenforced_sample_unique:
+        pre_warnings.append(
+            "sample-uniqueness not enforced for non-key columns: "
+            + ", ".join(sorted(unenforced_sample_unique))
+        )
+
+    # Fail fast when a unique key's value space cannot fit the request
+    # (SPEC 9.6: stop and report; never silently weaken the constraint).
+    _check_unique_feasibility(resolved, unique_cols, plan.rows)
+
+    # Observed null rates (SPEC 9.2): preserve within tolerance using the
+    # seeded RNG so output stays reproducible. Rate comes from the profile,
+    # not from re-reading the source. Unique columns never emit nulls here
+    # (their profile null_count is 0 by construction).
+    null_rates: dict[str, float] = {}
+    for col in primary.columns:
+        total = col.row_count_seen or 0
+        nulls = col.falsy.null_count
+        if total and nulls and col.name not in unique_cols:
+            null_rates[col.name] = nulls / total
+
+    rows: list[dict[str, Any]] = []
+    warnings: list[str] = list(pre_warnings)
+    for i in range(plan.rows):
+        check_cancel()
+        context.row_index = i
+        row: dict[str, Any] = {}
+        for col_name, strat_plan in resolved.items():
+            rate = null_rates.get(col_name, 0.0)
+            if rate > 0 and rng.random() < rate:
+                row[col_name] = None
+                continue
+            value = execute(strat_plan, context)
+            if col_name in unique_cols:
+                value = _ensure_unique(value, col_name, seen[col_name], strat_plan, context)
+            row[col_name] = value
+        rows.append(row)
+
+    # Validate against profile (SPEC 9.5)
+    fidelity = validate_generation(rows, primary, tolerances=None)
+    hard_failures = [m for m in fidelity if m.hard and not m.passed]
+    soft_failures = [m for m in fidelity if not m.hard and not m.passed]
+    metrics.rows_generated = len(rows)
+    metrics.validation_failures = len(hard_failures) + len(soft_failures)
+
+    if plan.preserve_relationships:
+        for rel in primary.relationships:
+            if rel.kind == "foreign_key" and rel.verified:
+                warnings.append(
+                    f"relationship {rel.from_columns}->{rel.to_columns} "
+                    "preservation is declarative in v1"
+                )
+
+    metrics.stop()
+    from qc.observability.metrics import RECORDER
+
+    RECORDER.record(metrics)
+    warnings.extend(
+        [
+            "synthetic data is NOT anonymous; empirical-distribution synthesis "
+            "can leak information (no differential privacy claimed)",
+            "phone-like values use fictional 555 ranges but are not guaranteed unassigned",
+        ]
+    )
+    if fallbacks:
+        warnings.append(f"fallback strategies used for: {', '.join(fallbacks)}")
+
+    # Write output
+    check_cancel()
+    out_path = output or (rt.paths.output / f"synthetic.{plan.output_format}")
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    if plan.output_format == "csv":
+        write_csv_rows(out_path, rows)
+    else:
+        from qc.exports.jsonl import write_jsonl
+
+        write_jsonl(out_path, rows)
+
+    report = GenerationReport(
+        plan_id=plan.plan_id,
+        rows_requested=plan.rows,
+        rows_generated=len(rows),
+        seed=plan.seed,
+        metrics=fidelity,
+        constraint_violations=[
+            f"{m.name}: generated {m.generated} vs observed {m.observed}" for m in hard_failures
+        ],
+        strategy_fallbacks=fallbacks,
+        warnings=warnings,
+        preserved_properties=[
+            m.name
+            for m in fidelity
+            if m.passed and m.name.endswith(("null_rate", "categorical_tvd"))
+        ],
+        approximated_properties=[m.name for m in soft_failures],
+        runtime_seconds=time.monotonic() - started,
+        status="failed" if hard_failures or (strict and soft_failures) else "completed",
+        output_path=str(out_path),
+    )
+    rpath = report_path or (rt.paths.reports / f"{report.report_id}.report.json")
+    rpath.parent.mkdir(parents=True, exist_ok=True)
+    rpath.write_text(json.dumps(report.json_safe(), indent=2), encoding="utf-8")
+
+    rt.audit.record(
+        OperationKind.GENERATE,
+        source_id=primary.source_id,
+        dataset_id=plan.dataset.qualified_name,
+        policy_version=engine.policy.version,
+        decision="allow",
+        rows_affected=len(rows),
+        duration_seconds=report.runtime_seconds,
+        result_status=report.status,
+        detail={"report_id": report.report_id, "hard_failures": str(len(hard_failures))},
+    )
+    ok = not hard_failures and not (strict and soft_failures)
+    return {
+        "ok": ok,
+        "report": report,
+        "output": str(out_path),
+        "report_path": str(rpath),
+        "hard_failures": len(hard_failures),
+        "soft_failures": len(soft_failures),
+        "metrics": metrics.as_dict(),
+    }
+
+
+async def validate_output(rt: Runtime, *, output: Path, profile_ref: str) -> GenerationReport:
+    """Validate an existing generated file against a profile."""
+    if not output.is_file():
+        raise NotFoundError(f"output not found: {output.name}")
+    path = rt.state.find_profile_index(profile_ref) or find_profile(rt.paths.profiles, profile_ref)
+    artifact = load_profile(path)
+
+    rows: list[dict[str, Any]] = []
+    if output.suffix == ".csv":
+        import csv
+
+        with output.open(encoding="utf-8", newline="") as fh:
+            for raw in csv.DictReader(fh):
+                rows.append({k: _maybe_number(v) for k, v in raw.items()})
+    else:
+        for line in output.read_text(encoding="utf-8").splitlines():
+            if line.strip():
+                rows.append(json.loads(line))
+
+    metrics = validate_generation(rows, artifact)
+    hard = [m for m in metrics if m.hard and not m.passed]
+    soft = [m for m in metrics if not m.hard and not m.passed]
+    from qc import TOOL_VERSION
+
+    return GenerationReport(
+        plan_id="validation",
+        rows_requested=len(rows),
+        rows_generated=len(rows),
+        metrics=metrics,
+        constraint_violations=[m.name for m in hard],
+        approximated_properties=[m.name for m in soft],
+        status="failed" if hard else "completed",
+        tool_version=TOOL_VERSION,
+        output_path=str(output),
+        warnings=["validation run against saved profile; no source data accessed"],
+    )
+
+
+def _maybe_number(v: str) -> Any:
+    if v == "\\N" or v == "":
+        return None
+    if v in ("true", "false"):
+        return v == "true"
+    try:
+        return int(v)
+    except ValueError:
+        pass
+    try:
+        return float(v)
+    except ValueError:
+        return v