floe-python 0.8.0__py3-none-any.whl

floe/report/entity.py

@@ -0,0 +1,245 @@
+"""Entity `run.json` (Rust: `report/{entity,build,mod}.rs`). Dicts in Rust's field order, serialised like serde."""
+
+import json
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from floe.checks.mismatch import MismatchOutcome
+from floe.checks.normalize import resolve_strategy, source_names
+from floe.checks.rows import RowCheck
+from floe.checks.unique import UniqueResult
+from floe.config.types import EntityConfig, SchemaEvolutionConfig, WriteMode
+from floe.storage.inputs import ResolvedInputs
+from floe.write.outputs import AcceptedWrite
+
+MAX_RESOLVED_INPUTS = 50
+_RULES = ("not_null", "cast_error", "unique", "schema_error")
+# `serde_json::Map` is a BTreeMap, so the echoed options and metadata come out key-sorted.
+_OPTION_KEYS = ("header", "separator", "encoding", "null_values", "recursive", "glob",
+                "json_mode", "row_tag", "namespace", "value_tag")  # fmt: skip
+_META_KEYS = ("data_product", "domain", "owner", "description", "tags")
+_UNIQUE_EFFECT = {
+    "warn": ("warn", "warning"),
+    "reject": ("reject_rows", "rows_rejected"),
+    "abort": ("abort_run", "run_aborted"),
+}
+
+
+def serialize(report: dict[str, Any]) -> str:
+    """`serde_json::to_string_pretty`: two-space indent, raw UTF-8, no trailing newline."""
+    return json.dumps(report, indent=2, ensure_ascii=False)
+
+
+def summarize_rules(
+    checks: Sequence[RowCheck], entity: EntityConfig
+) -> list[dict[str, Any]]:
+    """Rust's `summarize_validation_exprs`: per rule (fixed order), per column (sorted) violation counts."""
+    columns = entity.schema_.columns
+    names = source_names(columns, resolve_strategy(entity.schema_))
+    types = {n: c.column_type for n, c in zip(names, columns, strict=True)}
+    sources = {n: c.source for n, c in zip(names, columns, strict=True) if c.source}
+    rules: list[dict[str, Any]] = []
+    for rule in _RULES:
+        counts: dict[str, int] = {}
+        for c in checks:
+            if c.rule == rule or (rule == "schema_error" and c.rule not in _RULES):
+                counts[c.column] = counts.get(c.column, 0) + int(c.mask.sum())
+        cols: list[dict[str, Any]] = []
+        for name, n in sorted(counts.items()):
+            if n:
+                col: dict[str, Any] = {"column": name, "violations": n}
+                if rule == "cast_error" and name in types:
+                    col["target_type"] = types[name]
+                if name in sources:
+                    col["source"] = sources[name]
+                cols.append(col)
+        if cols:
+            total = sum(c["violations"] for c in cols)
+            rules.append(
+                {"rule": rule, "severity": entity.policy.severity, "violations": total, "columns": cols}
+            )  # fmt: skip
+    return rules
+
+
+def file_report(
+    entity: EntityConfig,
+    input_file: str,
+    row_count: int,
+    rejected_count: int,
+    checks: Sequence[RowCheck],
+    mismatch: MismatchOutcome,
+    rejected_path: str | None = None,
+    archived_path: str | None = None,
+    errors_path: str | None = None,
+    forced_rejects: int = 0,
+) -> dict[str, Any]:
+    """One `files[]` entry. `rejected_count` is the split's; severity decides status and counts (Rust `validate_split`)."""
+    # a rejected/aborted mismatch decides the file before any row check runs (Rust `precheck.rs`)
+    precheck = mismatch.aborted or mismatch.rejected
+    rules = [] if precheck else summarize_rules(checks, entity)
+    violations = sum(r["violations"] for r in rules)
+    severity = entity.policy.severity
+    if precheck:
+        status = "aborted" if mismatch.aborted else "rejected"
+        accepted, rejected = 0, row_count
+    elif (
+        severity == "warn" or not violations
+    ):  # warn still rejects merge-key duplicates
+        status, accepted, rejected = (
+            "success",
+            row_count - forced_rejects,
+            forced_rejects,
+        )
+    elif severity == "reject":
+        status, accepted, rejected = (
+            "rejected",
+            row_count - rejected_count,
+            rejected_count,
+        )
+    else:
+        status, accepted, rejected = "aborted", 0, row_count
+    warnings = violations if severity == "warn" else 0
+    return {
+        "input_file": input_file,
+        "status": status,
+        "row_count": row_count,
+        "accepted_count": accepted,
+        "rejected_count": rejected,
+        "mismatch": mismatch.report,
+        "output": {
+            "accepted_path": None,
+            "rejected_path": rejected_path,
+            "errors_path": errors_path,
+            "archived_path": archived_path,
+        },
+        "validation": {
+            "errors": violations - warnings + mismatch.errors,
+            "warnings": warnings + mismatch.warnings,
+            "rules": rules,
+        },
+    }  # fmt: skip
+
+
+def _unique_report(severity: str, r: UniqueResult) -> dict[str, Any]:
+    action, effect = (
+        _UNIQUE_EFFECT[severity] if r.duplicates_count else ("none", "none")
+    )
+    out: dict[str, Any] = {"columns": r.columns, "duplicates_count": r.duplicates_count}
+    if batch := r.duplicates_count - r.target_duplicates_count:
+        out["batch_duplicates_count"] = batch
+    if r.target_duplicates_count:
+        out["target_duplicates_count"] = r.target_duplicates_count
+    out |= {
+        "affected_rows_count": r.duplicates_count,
+        "action": action,
+        "status_effect": effect,
+    }
+    if r.samples:
+        out["samples"] = [
+            {"values": dict(sorted(s.values.items())), "count": s.count}
+            for s in r.samples
+        ]
+    return out  # fmt: skip
+
+
+def build_report(
+    entity: EntityConfig,
+    version: str,
+    inputs: ResolvedInputs,
+    files: list[dict[str, Any]],
+    write: AcceptedWrite,
+    *,
+    source_uri: str,
+    accepted_uri: str,
+    rejected_uri: str | None,
+    unique: Sequence[UniqueResult] = (),
+    write_mode: WriteMode | None = None,
+    table_root_uri: str | None = None,
+    table: Mapping[str, Any] | None = None,
+    merge: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """The entity's `run.json`; `files` are the `file_report`s, in input order.
+
+    `table` holds a table sink's fields (Iceberg: version, snapshot, catalog), whose metrics are reported even for no files.
+    """
+    src, sink, severity = entity.source, entity.sink, entity.policy.severity
+    if write.files_written:
+        for f in files:
+            if f["status"] != "skipped":
+                f["output"]["accepted_path"] = accepted_uri
+    totals: dict[str, int] = {
+        "files_total": len(files),
+        "files_skipped": sum(f["status"] == "skipped" for f in files),
+        "rows_total": sum(f["row_count"] for f in files),
+        "accepted_total": sum(f["accepted_count"] for f in files),
+        "rejected_total": sum(f["rejected_count"] for f in files),
+        "warnings_total": sum(f["validation"]["warnings"] for f in files),
+        "errors_total": sum(f["validation"]["errors"] for f in files),
+    }
+    entity_echo: dict[str, Any] = {"name": entity.id}
+    if entity.metadata:
+        entity_echo["metadata"] = {
+            k: v
+            for k in sorted(_META_KEYS)
+            if (v := getattr(entity.metadata, k)) is not None
+        }
+    sink_echo: dict[str, Any] = {
+        "accepted": {"format": sink.accepted.format, "path": accepted_uri}
+    }
+    if sink.rejected:
+        sink_echo["rejected"] = {
+            "format": sink.rejected.format,
+            "path": rejected_uri or sink.rejected.path,
+        }
+    archive: dict[str, Any] = {
+        "enabled": (entity.incremental_mode in ("none", "archive"))
+        and sink.archive is not None
+    }
+    if sink.archive:
+        archive["path"] = sink.archive.path
+    sink_echo["archive"] = archive
+    evolution = entity.schema_.schema_evolution or SchemaEvolutionConfig()
+    report: dict[str, Any] = {
+        "spec_version": version,
+        "entity": entity_echo,
+        "source": {
+            "format": src.format,
+            "path": source_uri,
+            "options": {
+                k: v for k in sorted(_OPTION_KEYS) if (v := getattr(src.options, k)) is not None
+            },
+            "cast_mode": src.cast_mode,
+            "read_plan": "raw_and_typed",
+            "resolved_inputs": {
+                "mode": inputs.mode,
+                "file_count": len(inputs.listed),
+                "files": inputs.listed[:MAX_RESOLVED_INPUTS],
+            },
+        },
+        "sink": sink_echo,
+        "policy": {"severity": severity},
+        "accepted_output": {
+            "path": accepted_uri,
+            "table_root_uri": table_root_uri or accepted_uri,
+            "write_mode": write_mode or sink.write_mode,
+            "accepted_rows": totals["accepted_total"],
+            "files_written": write.files_written,
+            "parts_written": write.files_written,
+            **({"part_files": write.part_files} if write.part_files else {}),
+            **(table or {}),
+            # Rust's default output (nothing written) has no byte metrics
+            "total_bytes_written": write.total_bytes_written if write.files_written or table else None,
+            "avg_file_size_mb": write.avg_file_size_mb if write.files_written or table else None,
+            "small_files_count": write.small_files_count if write.files_written or table else None,
+            **(merge or {}),
+        },
+        "schema_evolution": {
+            "enabled": False,
+            "mode": evolution.mode,
+            "applied": False,
+            "incompatible_changes_detected": False,
+        },
+    }  # fmt: skip
+    if unique:
+        report["unique_constraints"] = [_unique_report(severity, r) for r in unique]
+    return report | {"results": totals, "files": files}

floe/report/summary.py

@@ -0,0 +1,114 @@
+"""Run `run.summary.json`, run status and exit code, report layout (Rust: `report/{mod,output}.rs`, `run/mod.rs`)."""
+
+from collections.abc import Sequence
+from typing import Any
+
+from floe import __version__
+from floe.config.project import ProjectMetadata
+from floe.config.types import EntityConfig
+from floe.report.entity import serialize
+from floe.storage.types import StorageClient
+
+_TOTALS = ("files_total", "files_skipped", "rows_total", "accepted_total",
+           "rejected_total", "warnings_total", "errors_total")  # fmt: skip
+
+
+def id_path(entity: EntityConfig) -> str:
+    """Report sub-directory of an entity: `<domain>/<name>` when the id is qualified."""
+    if entity.domain and entity.id != entity.name:
+        return f"{entity.domain}/{entity.name}"
+    return entity.id
+
+
+def run_domain(entities: Sequence[EntityConfig]) -> str | None:
+    """D37: the domain every entity of the run belongs to, else None; a pre-0.3 domain that is no safe directory name (`..`, `a/b`) is None too."""
+    domains = {e.domain for e in entities}
+    domain = domains.pop() if len(domains) == 1 else None
+    return domain if domain and not any(c in domain for c in "./\\") else None
+
+
+def run_dir(report_path: str, run_id: str, domain: str | None) -> str:
+    """D37: `<report>/<domain>/run_<id>` for a single-domain run, else `<report>/run_<id>`."""
+    return "/".join([report_path.rstrip("/"), *([domain] if domain else []), f"run_{run_id}"])  # fmt: skip
+
+
+def run_outcome(file_statuses: Sequence[str], warnings: int = 0) -> tuple[str, int]:
+    """Rust's `compute_run_outcome` then `upgrade_status_for_warnings`: (status, exit code)."""
+    if "failed" in file_statuses:
+        return "failed", 1
+    if "aborted" in file_statuses:
+        return "aborted", 2
+    if "rejected" in file_statuses:
+        return "rejected", 0
+    return ("success_with_warnings" if warnings else "success"), 0
+
+
+def build_summary(
+    version: str,
+    reports: Sequence[tuple[str, dict[str, Any]]],
+    *,
+    run_id: str,
+    started_at: str,
+    finished_at: str,
+    duration_ms: int,
+    config_path: str,
+    report_path: str,
+    metadata: ProjectMetadata | None = None,
+    domain: str | None = None,
+) -> dict[str, Any]:
+    """`reports` are the entity `run.json` dicts in run order, each with its `id_path`; `domain` is the run's one domain (D37)."""
+    run_path = run_dir(report_path, run_id, domain)
+    totals = {k: sum(r["results"][k] for _, r in reports) for k in _TOTALS}
+    statuses = [f["status"] for _, r in reports for f in r["files"]]
+    status, exit_code = run_outcome(statuses, totals["warnings_total"])
+    entities = [
+        {
+            "name": r["entity"]["name"],
+            "status": run_outcome(
+                [f["status"] for f in r["files"]], r["results"]["warnings_total"]
+            )[0],
+            "results": r["results"],
+            "report_file": f"{run_path}/{path}/run.json",
+        }
+        for path, r in reports
+    ]
+    return {
+        "spec_version": version,
+        "tool": {"name": "floe", "version": __version__},
+        "run": {
+            "run_id": run_id,
+        }
+        | ({"domain": domain} if domain else {})
+        | {
+            "started_at": started_at,
+            "finished_at": finished_at,
+            "duration_ms": duration_ms,
+            "status": status,
+            "exit_code": exit_code,
+        },
+        "config": {"path": config_path, "version": version}
+        | ({"metadata": metadata.model_dump(exclude_none=True)} if metadata else {}),
+        "report": {"path": report_path, "report_file": f"{run_path}/run.summary.json"},
+        "results": totals,
+        "entities": entities,
+    }
+
+
+def write_reports(
+    summary: dict[str, Any],
+    reports: Sequence[tuple[str, dict[str, Any]]],
+    client: StorageClient,
+) -> None:
+    """Write `<run_dir>/<id_path>/run.json` per entity, then `run.summary.json` (overwriting, as Rust)."""
+    summary_file = summary["report"]["report_file"]
+    run_dir = summary_file.removesuffix("/run.summary.json")
+    for path, report in reports:
+        write_entity_report(run_dir, path, report, client)
+    client.write(summary_file, serialize(summary).encode())
+
+
+def write_entity_report(
+    run_dir: str, path: str, report: dict[str, Any], client: StorageClient
+) -> None:
+    """One entity's `run.json`, written as the entity finishes (Rust)."""
+    client.write(f"{run_dir}/{path}/run.json", serialize(report).encode())

floe/run/__init__.py

@@ -0,0 +1,175 @@
+"""A run: entity selection, run context, sequential or parallel entities, dry run, full refresh, events (Rust: `run/mod.rs`, `run/context.rs`)."""
+
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, cast
+
+from floe.checks.rows import Formatter
+from floe.config.project import RootConfig
+from floe.config.types import EntityConfig
+from floe.errors import FloeConfigError, FloeError
+from floe.report.summary import (
+    build_summary,
+    id_path,
+    run_dir,
+    run_domain,
+    run_outcome,
+    write_entity_report,
+    write_reports,
+)
+from floe.run.entity import EntityRun, list_source, resolve_source, run_entity
+from floe.run.events import Emitter, Observer
+from floe.storage.inputs import ResolvedInputs
+from floe.storage.paths import normalize_local_path
+from floe.storage.resolve import resolve_path, storage_client
+
+
+@dataclass(frozen=True)
+class RunOutcome:
+    run_id: str
+    summary: dict[str, Any]
+    dry_run: dict[str, list[str]] | None  # entity id -> scanned files
+    report_base_path: str
+    run_dir: str  # D37: `<report>/<domain>/run_<id>` for a single-domain run
+    entity_reports: list[dict[str, Any]]
+
+
+def select_entities(root: RootConfig, selectors: Sequence[str]) -> list[EntityConfig]:
+    """Config order; a selector is an entity id, else a name (Rust `select_entities`)."""
+    if not selectors:
+        return list(root.entities)
+    picked: set[int] = set()
+    missing: list[str] = []
+    for sel in selectors:
+        hits = [i for i, e in enumerate(root.entities) if e.id == sel] or [
+            i for i, e in enumerate(root.entities) if e.name == sel
+        ]
+        if len(hits) > 1:
+            ids = ", ".join(root.entities[i].id for i in hits)
+            raise FloeConfigError(f"entity {sel} is ambiguous; use one of: {ids}")
+        if hits:
+            picked.add(hits[0])
+        else:
+            missing.append(sel)
+    if missing:
+        raise FloeConfigError(f"entities not found: {', '.join(missing)}")
+    return [e for i, e in enumerate(root.entities) if i in picked]
+
+
+def _rfc3339() -> str:
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def _refresh(entity: EntityConfig) -> EntityConfig:
+    """Full refresh: an append sink overwrites; merge sinks cannot refresh."""
+    mode = entity.sink.write_mode
+    if mode in ("merge_scd1", "merge_scd2"):
+        raise FloeConfigError(f"entity '{entity.name}': --full-refresh is not supported with write_mode '{mode}'")  # fmt: skip
+    if mode != "append":
+        return entity
+    return entity.model_copy(update={"sink": entity.sink.model_copy(update={"write_mode": "overwrite"})})  # fmt: skip
+
+
+def _finished(emit: Emitter, name: str, report: dict[str, Any] | None) -> None:
+    """`entity_finished`; a failed entity (no report) counts zeros."""
+    keys = ("files", "files_skipped", "rows", "accepted", "rejected", "warnings", "errors")  # fmt: skip
+    results: dict[str, int] = report["results"] if report else {}
+    totals = {k: results.get(k if k == "files_skipped" else f"{k}_total", 0) for k in keys}  # fmt: skip
+    status = "failed"
+    if report:
+        status = run_outcome([f["status"] for f in report["files"]], totals["warnings"])[0]  # fmt: skip
+    emit("entity_finished", name=name, status=status, **totals)
+
+
+def run(
+    path: Path,
+    root: RootConfig,
+    *,
+    run_id: str | None = None,
+    entities: Sequence[str] = (),
+    dry_run: bool = False,
+    full_refresh: bool = False,
+    observer: Observer | None = None,
+    config_uri: str | None = None,
+) -> RunOutcome:
+    """Run the selected entities in config order, writing each entity's report as it finishes. A failing entity raises: no summary, as Rust."""
+    started, timer = _rfc3339(), time.monotonic()
+    selected = select_entities(root, entities)
+    if full_refresh:
+        selected = [_refresh(e) for e in selected]
+    # a symlinked manifest or config: paths are relative to its target
+    config_dir = normalize_local_path(path.resolve().parent)
+    # a remote manifest is reported as typed
+    config = config_uri or str(normalize_local_path(path))
+    report_uri, report_storage = resolve_path(root.storages, config_dir, root.report.storage, root.report.path, "report.storage")  # fmt: skip
+    report_path = report_uri.removeprefix("local://")
+    run_id = run_id or started.replace(":", "-")
+    domain = run_domain(selected)
+    run_path = run_dir(report_path, run_id, domain)
+
+    def inputs(e: EntityConfig) -> ResolvedInputs:
+        return list_source(
+            e, config_dir, root.storages, *resolve_source(e, config_dir, root.storages), full_refresh
+        )  # fmt: skip
+
+    if dry_run:  # Rust: no events, no reports; the inputs are only listed
+        scanned = {e.id: [f.source_uri for f in inputs(e).files] for e in selected}
+        summary = build_summary(
+            root.version, [], run_id=run_id, started_at=started, finished_at=_rfc3339(),
+            duration_ms=0, config_path=config, report_path=report_path,
+            metadata=root.metadata, domain=domain,
+        )  # fmt: skip
+        summary["report"]["report_file"] = "disabled (dry-run)"
+        return RunOutcome(run_id, summary, scanned, report_path, run_path, [])
+    emit = Emitter(observer, run_id)
+    formatter = cast(
+        Formatter, root.report.formatter or "json"
+    )  # validated with the config
+
+    def one(entity: EntityConfig) -> EntityRun:
+        emit("entity_started", name=entity.id)
+        try:
+            return run_entity(
+                entity, config_dir, root.version, formatter, root.storages, emit, full_refresh,
+                root.catalogs, listed.get(entity.id),
+            )  # fmt: skip
+        except FloeError:
+            _finished(emit, entity.id, None)
+            raise
+
+    emit("run_started", config=config, report_base=report_path)
+    # Rust resolves every entity's inputs before the first entity_started (#864); run_entity reuses them
+    listed: dict[str, ResolvedInputs] = {}
+    for e in selected:
+        # D24: a full refresh may rebuild from the archive alone, so run_entity decides
+        archive = full_refresh and e.sink.archive and e.incremental_mode in ("none", "archive")  # fmt: skip
+        if not archive:
+            listed[e.id] = inputs(e)
+    reports: list[tuple[str, dict[str, Any]]] = []
+    client = storage_client(report_storage)
+    # D36: a policy abort fails only its entity; a FloeError still stops the run
+    for e in selected:
+        report = one(e).report
+        reports.append((id_path(e), report))
+        write_entity_report(run_path, id_path(e), report, client)
+        _finished(emit, e.id, report)
+    summary = build_summary(
+        root.version, reports,
+        run_id=run_id, started_at=started, finished_at=_rfc3339(),
+        duration_ms=int((time.monotonic() - timer) * 1000),
+        config_path=config, report_path=report_path,
+        metadata=root.metadata, domain=domain,
+    )  # fmt: skip
+    write_reports(summary, [], client)
+    r, t = summary["run"], summary["results"]
+    emit("run_finished", status=r["status"], exit_code=r["exit_code"], files=t["files_total"],
+         files_skipped=t["files_skipped"], rows=t["rows_total"], accepted=t["accepted_total"], rejected=t["rejected_total"],
+         warnings=t["warnings_total"], errors=t["errors_total"],
+         summary_uri=summary["report"]["report_file"], report_base=report_path,
+         entity_report_uris={e["name"]: e["report_file"] for e in summary["entities"]})  # fmt: skip
+    return RunOutcome(
+        run_id, summary, None, report_path, run_path, [r for _, r in reports]
+    )