focus-data-toolkit 0.11.0__py3-none-any.whl

focus_data_toolkit/provenance.py

@@ -0,0 +1,106 @@
+"""Value provenance (lineage) for converted FOCUS 1.4 columns.
+
+Every produced column is classified by *how* its value was obtained. The headline
+classification is a property of the conversion **rule** for a column, so it is
+deterministic and drives both the manifest and the strict-mode gating: in ``STRICT``
+mode a dataset is produced only when every Mandatory non-nullable column has a
+**factual** lineage (not assumed / unavailable).
+
+Some rules act differently per row (e.g. a backfill only touches null source
+values). For those columns the headline rule stays the *weakest* lineage the rule
+can produce (conservative for gating), and a :class:`LineageCounters` accumulator
+records how many values actually took each lineage — surfaced in the manifest as
+``lineage_summary`` so a column-level label never hides the per-value mix.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class Lineage(StrEnum):
+    OBSERVED = "OBSERVED"        # value present directly in the source
+    RENAMED = "RENAMED"          # copied from an equivalent column, name change only
+    DERIVED = "DERIVED"          # computed exactly and verifiably from source values
+    ENRICHED = "ENRICHED"        # from a complementary authoritative source/context
+    ASSUMED = "ASSUMED"          # hypothetical value (not a source fact)
+    UNAVAILABLE = "UNAVAILABLE"  # absent and not derivable (emitted null)
+
+
+# Lineages that represent real, non-fabricated data. A Mandatory non-nullable column
+# whose lineage is NOT factual blocks strict production of its dataset.
+FACTUAL_LINEAGES: frozenset[Lineage] = frozenset(
+    {Lineage.OBSERVED, Lineage.RENAMED, Lineage.DERIVED, Lineage.ENRICHED}
+)
+
+
+@dataclass(frozen=True)
+class ColumnRule:
+    """How one target column's value is obtained during conversion."""
+
+    lineage: Lineage
+    source: str | None = None
+    note: str | None = None
+
+    @property
+    def is_factual(self) -> bool:
+        return self.lineage in FACTUAL_LINEAGES
+
+    @property
+    def is_assumed(self) -> bool:
+        return self.lineage is Lineage.ASSUMED
+
+    def as_dict(self) -> dict[str, str]:
+        out: dict[str, str] = {"lineage": self.lineage.value}
+        if self.source:
+            out["source"] = self.source
+        if self.note:
+            out["note"] = self.note
+        return out
+
+
+def strict_blockers(provenance: dict[str, ColumnRule], columns: dict[str, dict]) -> list[str]:
+    """Return the Mandatory non-nullable columns that block strict production.
+
+    ``columns`` is the model's column spec map for the dataset. A column blocks when it
+    is Mandatory and non-nullable and its lineage is not factual (assumed, unavailable,
+    or missing from ``provenance``).
+    """
+    blockers: list[str] = []
+    for col, spec in columns.items():
+        if spec.get("feature_level") != "Mandatory" or spec.get("allows_nulls", True):
+            continue
+        rule = provenance.get(col)
+        if rule is None or not rule.is_factual:
+            blockers.append(col)
+    return sorted(blockers)
+
+
+def has_assumptions(provenance: dict[str, ColumnRule]) -> bool:
+    return any(rule.is_assumed for rule in provenance.values())
+
+
+class LineageCounters:
+    """Per-column counts of the lineage each emitted value actually took.
+
+    Bounded (columns x lineage categories) and deterministic, so the eager and
+    streaming pipelines produce identical summaries for the same input.
+    """
+
+    def __init__(self) -> None:
+        self._counts: defaultdict[str, Counter[str]] = defaultdict(Counter)
+
+    def record(self, column: str, lineage: Lineage, n: int = 1) -> None:
+        self._counts[column][lineage.value] += n
+
+    def summary(self) -> dict[str, dict[str, int]]:
+        """Sorted ``{column: {lineage: count}}`` (deterministic manifest payload)."""
+        return {
+            column: {lineage: count for lineage, count in sorted(counts.items())}
+            for column, counts in sorted(self._counts.items())
+        }
+
+    def __bool__(self) -> bool:
+        return bool(self._counts)

focus_data_toolkit/py.typed

@@ -0,0 +1 @@
+# PEP 561 marker: focus-data-toolkit ships inline type annotations.

focus_data_toolkit/runtime.py

@@ -0,0 +1,243 @@
+"""Runtime configuration: working directory + disk budgets, read once from the environment.
+
+A streaming conversion uses **two filesystems**, budgeted separately:
+
+* the **work** filesystem — scratch state (the SQLite aggregation index and the bundle-validation
+  spill), relocatable via ``FOCUS_TOOLKIT_WORK_DIR`` to a faster/larger disk;
+* the **output** filesystem — the atomic staging directory and the final files, pinned to the
+  parent of ``--out`` by the publish rename (same-``st_dev`` requirement), so it cannot be moved.
+
+Environment variables:
+
+* ``FOCUS_TOOLKIT_WORK_DIR`` — directory for scratch state (default: alongside the output).
+* ``FOCUS_TOOLKIT_MAX_WORK_BYTES`` — cap on scratch bytes; exceeding it fails the run (FDT-IO-006).
+* ``FOCUS_TOOLKIT_MIN_WORK_FREE_BYTES`` — refuse/abort if the work filesystem free space drops below.
+* ``FOCUS_TOOLKIT_MIN_OUTPUT_FREE_BYTES`` — refuse/abort if the output filesystem free space drops below.
+* ``FOCUS_TOOLKIT_LOG_LEVEL`` — level for the ``focus_data_toolkit`` logger (default: WARNING).
+
+Sizes accept ``128MB`` / ``512KB`` / ``2GB`` style suffixes or a plain byte count. The pre-flight
+estimate is **best-effort with a safety margin** derived from the input size — never a precise
+prediction of the space a conversion will consume.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+
+from focus_data_toolkit.errors import Diagnostic, Severity
+
+_InputPaths = Sequence["str | os.PathLike[str] | None"]
+
+_ENV_WORK_DIR = "FOCUS_TOOLKIT_WORK_DIR"
+_ENV_MAX_WORK = "FOCUS_TOOLKIT_MAX_WORK_BYTES"
+_ENV_MIN_WORK_FREE = "FOCUS_TOOLKIT_MIN_WORK_FREE_BYTES"
+_ENV_MIN_OUTPUT_FREE = "FOCUS_TOOLKIT_MIN_OUTPUT_FREE_BYTES"
+_ENV_LOG_LEVEL = "FOCUS_TOOLKIT_LOG_LEVEL"
+
+# CSV 1.4 output can be somewhat larger than the source, so the pre-flight uses a margin on the
+# input size. Parquet output is smaller, making the estimate a conservative upper bound there.
+_OUTPUT_ESTIMATE_MARGIN = 1.3
+
+_SIZE_SUFFIXES = (("KB", 1000), ("MB", 1000**2), ("GB", 1000**3), ("TB", 1000**4), ("B", 1))
+
+
+class ResourceLimitError(Exception):
+    """Raised when a disk budget / free-space check fails (the CLI maps it to exit code 5).
+
+    Carries the structured :class:`~focus_data_toolkit.errors.Diagnostic` (``FDT-IO-005`` for the
+    output filesystem, ``FDT-IO-006`` for the work filesystem / temp budget) so callers can
+    render it uniformly. It is **not** a ``ConversionError``: raising it inside the atomic output
+    context still removes the staging directory (cleanup keys on "not committed", not on the
+    exception type), so nothing partial is ever published.
+    """
+
+    def __init__(self, diagnostic: Diagnostic) -> None:
+        super().__init__(diagnostic.message)
+        self.diagnostic = diagnostic
+
+
+def parse_size(value: str | None) -> int | None:
+    """Parse a byte size (``128MB`` / ``512KB`` / ``2GB`` / plain count). ``None``/empty -> ``None``.
+
+    Raises ``ValueError`` on a malformed value.
+    """
+    if value is None:
+        return None
+    text = value.strip().upper()
+    if not text:
+        return None
+    for suffix, mult in _SIZE_SUFFIXES:
+        if text.endswith(suffix):
+            return int(float(text[: -len(suffix)]) * mult)
+    return int(text)
+
+
+def _size_from(env: Mapping[str, str], name: str) -> int | None:
+    """Parse a size env var, ignoring (not crashing on) a malformed value."""
+    try:
+        return parse_size(env.get(name))
+    except ValueError:
+        return None
+
+
+@dataclass(frozen=True)
+class RuntimeConfig:
+    """Resolved runtime configuration (see the module docstring for the env vars)."""
+
+    work_dir: Path | None = None
+    max_work_bytes: int | None = None
+    min_work_free_bytes: int | None = None
+    min_output_free_bytes: int | None = None
+    log_level: str | None = None
+
+    @classmethod
+    def from_env(cls, env: Mapping[str, str] | None = None) -> RuntimeConfig:
+        env = os.environ if env is None else env
+        work_dir = env.get(_ENV_WORK_DIR)
+        return cls(
+            work_dir=Path(work_dir) if work_dir else None,
+            max_work_bytes=_size_from(env, _ENV_MAX_WORK),
+            min_work_free_bytes=_size_from(env, _ENV_MIN_WORK_FREE),
+            min_output_free_bytes=_size_from(env, _ENV_MIN_OUTPUT_FREE),
+            log_level=env.get(_ENV_LOG_LEVEL) or None,
+        )
+
+    def apply_logging(self) -> None:
+        """Configure the ``focus_data_toolkit`` logger from ``FOCUS_TOOLKIT_LOG_LEVEL`` (if set)."""
+        if not self.log_level:
+            return
+        level = logging.getLevelName(self.log_level.strip().upper())
+        if isinstance(level, int):
+            logging.getLogger("focus_data_toolkit").setLevel(level)
+
+
+def _free(path: Path) -> int | None:
+    try:
+        return shutil.disk_usage(path).free
+    except OSError:
+        return None
+
+
+def _estimate_output_bytes(input_paths: _InputPaths) -> int:
+    total = 0
+    for raw in input_paths:
+        if not raw:
+            continue
+        try:
+            total += Path(raw).stat().st_size
+        except OSError:
+            continue
+    return int(total * _OUTPUT_ESTIMATE_MARGIN)
+
+
+def _io_error(code: str, message: str, path: Path, **context: str) -> ResourceLimitError:
+    ctx = {"path": str(path), **context}
+    return ResourceLimitError(
+        Diagnostic(code=code, severity=Severity.ERROR, message=message, context=ctx)
+    )
+
+
+def work_run_dir(config: RuntimeConfig, run_id: str) -> Path | None:
+    """Per-run scratch subdirectory under ``WORK_DIR`` (created), or ``None`` when unset.
+
+    Relocated scratch is scoped by the conversion's run id (``WORK_DIR/fdt-<run_id>/``) so that
+    concurrent runs sharing a single ``FOCUS_TOOLKIT_WORK_DIR`` never collide on the same SQLite
+    files. The directory lives outside the atomic staging dir, so the caller removes it on exit.
+    """
+    if config.work_dir is None:
+        return None
+    run_dir = config.work_dir / f"fdt-{run_id}"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    return run_dir
+
+
+def preflight(config: RuntimeConfig, out_parent: Path, input_paths: _InputPaths) -> None:
+    """Best-effort pre-flight run *before* any staging; raises on a clear shortfall.
+
+    Estimates the output need from the input size (with a margin) and checks both the output and
+    work filesystems against their free space and configured minimums. The estimate is deliberately
+    rough — it guards against the obvious "nowhere near enough disk" case, not against every run.
+    """
+    out_parent = Path(out_parent)
+    free_out = _free(out_parent)
+    if free_out is not None:
+        # Require room for the bytes we expect to write AND the reserve that must remain free
+        # afterwards — not the larger of the two (which would let a run drive free space below
+        # the configured minimum and only fail later during an in-run check).
+        need = _estimate_output_bytes(input_paths) + (config.min_output_free_bytes or 0)
+        if need and free_out < need:
+            raise _io_error(
+                "FDT-IO-005",
+                f"insufficient free space on the output filesystem at {out_parent}: "
+                f"need ~{need} bytes, {free_out} free",
+                out_parent,
+                needed=str(need),
+                free=str(free_out),
+            )
+    work_dir = config.work_dir or out_parent
+    free_work = _free(Path(work_dir))
+    if (
+        free_work is not None
+        and config.min_work_free_bytes is not None
+        and free_work < config.min_work_free_bytes
+    ):
+        raise _io_error(
+            "FDT-IO-006",
+            f"insufficient free space on the work filesystem at {work_dir}: "
+            f"need {config.min_work_free_bytes} bytes, {free_work} free",
+            Path(work_dir),
+            needed=str(config.min_work_free_bytes),
+            free=str(free_work),
+        )
+
+
+def enforce_limits(
+    config: RuntimeConfig, out_parent: Path, work_dir: Path, scratch_bytes: int
+) -> None:
+    """In-run enforcement (called periodically): min free space on both filesystems + work budget."""
+    if config.min_output_free_bytes is not None:
+        free_out = _free(Path(out_parent))
+        if free_out is not None and free_out < config.min_output_free_bytes:
+            raise _io_error(
+                "FDT-IO-005",
+                f"output filesystem free space fell below the configured minimum at {out_parent}: "
+                f"{free_out} < {config.min_output_free_bytes} bytes",
+                Path(out_parent),
+                free=str(free_out),
+                minimum=str(config.min_output_free_bytes),
+            )
+    if config.min_work_free_bytes is not None:
+        free_work = _free(Path(work_dir))
+        if free_work is not None and free_work < config.min_work_free_bytes:
+            raise _io_error(
+                "FDT-IO-006",
+                f"work filesystem free space fell below the configured minimum at {work_dir}: "
+                f"{free_work} < {config.min_work_free_bytes} bytes",
+                Path(work_dir),
+                free=str(free_work),
+                minimum=str(config.min_work_free_bytes),
+            )
+    if config.max_work_bytes is not None and scratch_bytes > config.max_work_bytes:
+        raise _io_error(
+            "FDT-IO-006",
+            f"temporary work budget exceeded: {scratch_bytes} > {config.max_work_bytes} bytes "
+            "(FOCUS_TOOLKIT_MAX_WORK_BYTES)",
+            Path(work_dir),
+            used=str(scratch_bytes),
+            budget=str(config.max_work_bytes),
+        )
+
+
+__all__ = [
+    "ResourceLimitError",
+    "RuntimeConfig",
+    "enforce_limits",
+    "parse_size",
+    "preflight",
+    "work_run_dir",
+]

focus_data_toolkit/schema/__init__.py

@@ -0,0 +1,17 @@
+"""FOCUS schema knowledge: per-(dataset, version) column registry and detection.
+
+The registry (:mod:`focus_data_toolkit.schema.registry`) derives the normative column
+set of each FOCUS dataset at each supported version from the committed FOCUS 1.4 model
+(every column carries its introduction ``version``) plus a small table of columns removed
+by 1.4. The detector (:mod:`focus_data_toolkit.schema.detection`) uses it to identify the
+dataset and version of an arbitrary header row, with a confidence assessment.
+"""
+
+from __future__ import annotations
+
+from focus_data_toolkit.schema.detection import (
+    SchemaDetectionResult,
+    detect_focus_schema,
+)
+
+__all__ = ["SchemaDetectionResult", "detect_focus_schema"]

focus_data_toolkit/schema/detection.py

@@ -0,0 +1,274 @@
+"""Identify the FOCUS dataset and version of an arbitrary header row.
+
+The previous approach tested a header against four marker columns and returned only
+``"1.2"``/``"1.3"``. That mis-detects a 1.3 export missing an optional 1.3 column as 1.2,
+never recognises 1.4, and cannot tell the four datasets apart.
+
+:func:`detect_focus_schema` instead scores the header against every ``(dataset, version)``
+schema in the registry (present columns, absent-but-expected columns, and FOCUS columns
+that belong to the dataset but a *different* version — the hybrid signal), and reports a
+confidence plus the exact discrepancies. ``x_``-prefixed extension columns never count
+against a match; unknown non-``x_`` columns are surfaced separately.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from focus_data_toolkit.schema import registry
+
+CONF_HIGH = "HIGH"
+CONF_MEDIUM = "MEDIUM"
+CONF_LOW = "LOW"
+
+# A runner-up candidate whose Jaccard similarity is within this of the best is "ambiguous".
+_AMBIGUITY_DELTA = 0.04
+# Below this best-similarity floor the header is treated as "not FOCUS" (no dataset).
+_DATASET_FLOOR = 0.20
+# When a version is forced, the user's choice is respected down to a much lower floor (a
+# sparse-but-real projection is valid); only near-zero overlap (a non-FOCUS file) is rejected.
+_FORCED_OVERLAP_FLOOR = 0.05
+
+
+@dataclass(frozen=True)
+class SchemaDetectionResult:
+    """Outcome of detecting the FOCUS dataset/version of a header row."""
+
+    dataset: str | None
+    detected_version: str | None
+    confidence: str
+    exact_match: bool
+    score: float
+    missing_columns: tuple[str, ...]
+    additional_focus_columns: tuple[str, ...]
+    extension_columns: tuple[str, ...]
+    unknown_columns: tuple[str, ...]
+    ambiguous_candidates: tuple[tuple[str, str], ...]
+    forced: bool = False
+    notes: tuple[str, ...] = ()
+
+    @property
+    def ok(self) -> bool:
+        """A confident, unambiguous identification of a supported schema."""
+        return (
+            self.dataset is not None
+            and self.detected_version is not None
+            and self.confidence == CONF_HIGH
+        )
+
+    def as_dict(self) -> dict:
+        """JSON-serialisable view for the manifest."""
+        return {
+            "dataset": self.dataset,
+            "detected_version": self.detected_version,
+            "confidence": self.confidence,
+            "exact_match": self.exact_match,
+            "score": round(self.score, 4),
+            "forced": self.forced,
+            "missing_columns": list(self.missing_columns),
+            "additional_focus_columns": list(self.additional_focus_columns),
+            "extension_columns": list(self.extension_columns),
+            "unknown_columns": list(self.unknown_columns),
+            "ambiguous_candidates": [list(c) for c in self.ambiguous_candidates],
+            "notes": list(self.notes),
+        }
+
+
+@dataclass(frozen=True)
+class _Candidate:
+    dataset: str
+    version: str
+    jaccard: float
+    mandatory_coverage: float
+    missing: tuple[str, ...]
+    additional_focus: tuple[str, ...]
+
+
+def _score(dataset: str, version: str, headers: frozenset[str]) -> _Candidate:
+    expected = registry.version_columns(dataset, version)
+    dataset_focus = registry.all_dataset_columns(dataset)
+    present = expected & headers
+    missing = expected - headers
+    additional_focus = (headers & dataset_focus) - expected
+    union = expected | (headers & dataset_focus)
+    jaccard = len(present) / len(union) if union else 0.0
+    mandatory = registry.mandatory_columns(dataset, version)
+    mandatory_coverage = len(mandatory & headers) / len(mandatory) if mandatory else 1.0
+    return _Candidate(
+        dataset=dataset,
+        version=version,
+        jaccard=jaccard,
+        mandatory_coverage=mandatory_coverage,
+        missing=tuple(sorted(missing)),
+        additional_focus=tuple(sorted(additional_focus)),
+    )
+
+
+def detect_focus_schema(
+    headers: Iterable[str],
+    *,
+    dataset: str | None = None,
+    version: str | None = None,
+) -> SchemaDetectionResult:
+    """Detect the FOCUS dataset and version of ``headers``.
+
+    ``dataset`` and/or ``version`` force the corresponding dimension (they still get scored,
+    so a bad forced choice yields a low confidence the caller can reject). Unknown forced
+    values raise ``ValueError``.
+    """
+    all_headers = list(headers)
+    # A malformed CSV row with surplus fields makes ``csv.DictReader`` emit a ``None`` key;
+    # drop non-string header names (and record it) rather than crashing on ``.startswith``.
+    header_list = [h for h in all_headers if isinstance(h, str)]
+    malformed = len(all_headers) - len(header_list)
+    header_set = frozenset(header_list)
+    extension = tuple(sorted(h for h in header_set if h.startswith("x_")))
+    focus_all = registry.all_focus_columns()
+    unknown = tuple(sorted(h for h in header_set if not h.startswith("x_") and h not in focus_all))
+
+    forced_dataset = registry.resolve_dataset_name(dataset) if dataset is not None else None
+    forced_version = registry.normalize_version(version) if version is not None else None
+    forced = forced_dataset is not None or forced_version is not None
+
+    candidates = registry.candidate_schemas()
+    if forced_dataset is not None:
+        candidates = [c for c in candidates if c[0] == forced_dataset]
+    if forced_version is not None:
+        candidates = [c for c in candidates if c[1] == forced_version]
+
+    if not candidates:
+        note = "forced (dataset, version) does not exist in FOCUS"
+        return SchemaDetectionResult(
+            dataset=forced_dataset,
+            detected_version=forced_version,
+            confidence=CONF_LOW,
+            exact_match=False,
+            score=0.0,
+            missing_columns=(),
+            additional_focus_columns=(),
+            extension_columns=extension,
+            unknown_columns=unknown,
+            ambiguous_candidates=(),
+            forced=forced,
+            notes=(note,),
+        )
+
+    scored = sorted(
+        (_score(d, v, header_set) for d, v in candidates),
+        key=lambda c: (c.jaccard, c.mandatory_coverage),
+        reverse=True,
+    )
+    best = scored[0]
+
+    # Ambiguity: any other candidate whose similarity is within delta of the best.
+    ambiguous = tuple(
+        (c.dataset, c.version)
+        for c in scored[1:]
+        if c.jaccard > 0 and best.jaccard - c.jaccard < _AMBIGUITY_DELTA
+    )
+
+    notes: list[str] = []
+    dataset_name: str | None = best.dataset
+    detected_version: str | None = best.version
+
+    if not forced and best.jaccard < _DATASET_FLOOR:
+        # Header does not resemble any FOCUS schema.
+        return SchemaDetectionResult(
+            dataset=None,
+            detected_version=None,
+            confidence=CONF_LOW,
+            exact_match=False,
+            score=round(best.jaccard, 4),
+            missing_columns=(),
+            additional_focus_columns=(),
+            extension_columns=extension,
+            unknown_columns=unknown,
+            ambiguous_candidates=(),
+            forced=forced,
+            notes=("header does not match any known FOCUS dataset/version",),
+        )
+
+    # A FOCUS column that belongs to a *different* dataset (e.g. PaymentTerms on a Cost and
+    # Usage header) is neither "additional focus of this dataset" nor "unknown"; count it as a
+    # mismatch so it breaks exact_match and caps confidence (strict must not silently drop it).
+    foreign_focus = tuple(
+        sorted((header_set & focus_all) - registry.all_dataset_columns(best.dataset))
+    )
+    additional_focus = tuple(sorted(set(best.additional_focus) | set(foreign_focus)))
+
+    exact_match = not best.missing and not additional_focus and not unknown and not malformed
+
+    # Confidence.
+    if forced_version is not None:
+        # The version is locked by the user. Only columns that belong to *another* version of
+        # this dataset (``additional_focus``) make the forced version genuinely impossible;
+        # merely missing columns are a completeness issue the lint reports, not a reason to
+        # override an explicit choice. But a header with essentially no overlap (e.g. a
+        # non-FOCUS file) must still be rejected even when a version is forced.
+        if best.jaccard < _FORCED_OVERLAP_FLOOR:
+            confidence = CONF_LOW
+            notes.append(
+                f"header has no meaningful overlap with the forced schema "
+                f"{best.dataset} {forced_version}"
+            )
+        elif additional_focus:
+            confidence = CONF_LOW
+            notes.append(
+                f"header is incompatible with forced version {forced_version} "
+                f"(columns not in this schema: {', '.join(additional_focus)})"
+            )
+        elif best.missing or best.mandatory_coverage < 0.999:
+            confidence = CONF_MEDIUM  # compatible but incomplete (lint will flag specifics)
+        else:
+            confidence = CONF_HIGH
+    elif (
+        best.jaccard >= 0.9
+        and best.mandatory_coverage >= 0.999
+        and not ambiguous
+        and not unknown
+        and not additional_focus
+        and not malformed
+    ):
+        confidence = CONF_HIGH
+    elif best.jaccard >= 0.6 and best.mandatory_coverage >= 0.8:
+        confidence = CONF_MEDIUM
+    else:
+        confidence = CONF_LOW
+
+    if malformed:
+        notes.append(f"{malformed} malformed (non-string) header name(s) ignored")
+    if foreign_focus:
+        notes.append(
+            "FOCUS columns from another dataset present: " + ", ".join(foreign_focus)
+        )
+    if ambiguous and confidence == CONF_HIGH:
+        confidence = CONF_MEDIUM
+    if unknown and confidence == CONF_HIGH:
+        confidence = CONF_MEDIUM
+        notes.append(f"{len(unknown)} unknown non-x_ column(s) present")
+    elif unknown:
+        notes.append(f"{len(unknown)} unknown non-x_ column(s) present")
+    if best.additional_focus:
+        notes.append(
+            "columns from another FOCUS version present: " + ", ".join(best.additional_focus)
+        )
+    if ambiguous:
+        notes.append(
+            "close alternative schema(s): " + ", ".join(f"{d} {v}" for d, v in ambiguous)
+        )
+
+    return SchemaDetectionResult(
+        dataset=dataset_name,
+        detected_version=detected_version,
+        confidence=confidence,
+        exact_match=exact_match,
+        score=round(best.jaccard, 4),
+        missing_columns=best.missing,
+        additional_focus_columns=additional_focus,
+        extension_columns=extension,
+        unknown_columns=unknown,
+        ambiguous_candidates=ambiguous,
+        forced=forced,
+        notes=tuple(notes),
+    )