kiri-cli 1.0.0b3__py3-none-any.whl

kiri_dbt_bridge/silver_kimball/fact_generator.py

@@ -0,0 +1,320 @@
+"""Silver-Kimball FACT generator (ADR 0099).
+
+Mirrors :mod:`kiri_dbt_bridge.gold.fact_generator` with two differences:
+
+- reads ``contract.kiri.silver`` instead of ``contract.kiri.gold``;
+- pulls FROM bronze via :func:`silver_kimball.lineage.from_clause`.
+
+Silver-Kimball facts have **no** ``dimension_refs`` — at the
+conformed-master layer the silver facts ARE the atomic facts, joining
+silver dims by their PK directly. There is no separate dim-FK
+validation step; the projection emits each silver column from its
+own lineage and authors point FK columns at the right bronze PK.
+
+Lineage-driven projection emits each silver column as
+``<bronze_alias>.<bronze_col> AS <silver_col>``.
+
+Every silver-fact column must declare a non-empty
+``kiri.lineage.from_columns``. Empty-lineage on facts means we'd
+silently drop the column from emission — fail closed.
+
+Measure validation walks lineage to the bronze source and rejects
+SUM / AVG over non-numeric bronze columns.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from kiri_odcs.kiri_ext import GoldKind, MeasureAggregation
+
+from kiri_dbt_bridge.silver_kimball.lineage import (
+    SilverLineagePlan,
+    build_silver_lineage_plan,
+    from_clause,
+    parse_bronze_lineage,
+    render_property_projection,
+)
+from kiri_dbt_bridge.silver_kimball.watermark_window import (
+    render_pre_hook,
+    render_window_predicate,
+)
+from kiri_dbt_bridge.silver_kimball.zone import resolve_silver_schema
+
+if TYPE_CHECKING:
+    from kiri_core import CanonicalContract, Project
+    from kiri_core.lifecycle import WatermarkWindow
+
+    from kiri_dbt_bridge.gold.dialects import GoldDialect
+
+
+_NUMERIC_LOGICAL_TYPES = {"integer", "long", "number", "decimal", "float", "double"}
+_NUMERIC_REQUIRED_AGGS = {MeasureAggregation.SUM, MeasureAggregation.AVG}
+
+
+def _check_all_columns_have_lineage(contract: CanonicalContract) -> None:
+    """Raise if any silver-fact column has no or empty lineage."""
+    schema = contract.primary_schema
+    if schema is None:
+        raise ValueError(f"Silver-Kimball fact {contract.contract.name!r} has no schema.")
+    missing: list[str] = []
+    empty: list[str] = []
+    for prop in schema.properties:
+        key = f"{schema.name}.{prop.name}"
+        col_meta = contract.kiri.properties.get(key)
+        if col_meta is None or col_meta.lineage is None:
+            missing.append(prop.name)
+            continue
+        if not col_meta.lineage.from_columns:
+            empty.append(prop.name)
+    if missing:
+        raise ValueError(
+            f"Silver-Kimball fact {contract.contract.name!r}: column(s) "
+            f"{missing} have no kiri.lineage.from_columns declared. Every "
+            "fact column must declare lineage; empty list ('originates "
+            "here') is only valid on dimensions."
+        )
+    if empty:
+        raise ValueError(
+            f"Silver-Kimball fact {contract.contract.name!r}: column(s) "
+            f"{empty} have empty kiri.lineage.from_columns. Facts have no "
+            "'originates here' columns — declare the bronze source for "
+            "every column or remove it from the schema."
+        )
+
+
+def _validate_measures(contract: CanonicalContract, project: Project) -> None:
+    """Reject measures whose lineage-resolved bronze source type can't
+    sustain the declared aggregation.
+    """
+    schema = contract.primary_schema
+    if schema is None:
+        return
+    bronze_by_name: dict[str, CanonicalContract] = {c.contract.name: c for c in project.contracts}
+    for measure in contract.kiri.silver.measures:
+        if measure.aggregation not in _NUMERIC_REQUIRED_AGGS:
+            continue
+        key = f"{schema.name}.{measure.column}"
+        col_meta = contract.kiri.properties.get(key)
+        if col_meta is None or col_meta.lineage is None or not col_meta.lineage.from_columns:
+            continue
+        first_urn = col_meta.lineage.from_columns[0]
+        try:
+            ref = parse_bronze_lineage(first_urn)
+        except ValueError:
+            continue
+        bronze = bronze_by_name.get(ref.contract_name)
+        if bronze is None or bronze.primary_schema is None:
+            continue
+        bronze_col_type: str | None = None
+        for prop in bronze.primary_schema.properties:
+            if prop.name == ref.column:
+                bronze_col_type = (prop.logical_type or "").lower()
+                break
+        if bronze_col_type and bronze_col_type not in _NUMERIC_LOGICAL_TYPES:
+            raise ValueError(
+                f"Silver-Kimball fact {contract.contract.name!r}: measure "
+                f"{measure.name!r} declares aggregation "
+                f"{measure.aggregation.value!r} over silver column "
+                f"{measure.column!r}, whose lineage resolves to bronze "
+                f"{ref.contract_name}::{ref.column} with "
+                f"logicalType={bronze_col_type!r}. "
+                f"{measure.aggregation.value.upper()} requires a numeric "
+                f"bronze source ({sorted(_NUMERIC_LOGICAL_TYPES)}). "
+                "Either change the aggregation (e.g. COUNT) or "
+                "pre-compute a numeric column in bronze."
+            )
+
+
+def generate_fact_model(
+    contract: CanonicalContract,
+    project: Project,
+    dialect: GoldDialect,
+    window: WatermarkWindow | None = None,
+) -> str:
+    """Render the full ``models/silver/<fact_name>.sql`` body.
+
+    Backwards compatible (ADR 0080 "backwards compat"): ``window=None`` — or a
+    contract that does not declare ``kiri.watermark`` — yields the byte-identical
+    ``materialized='table'`` full-build SQL this generator always produced. When a
+    ``window`` is supplied AND the contract declares ``kiri.watermark``, the model
+    becomes ``materialized='incremental'`` with ``incremental_strategy='append'``
+    (a fact is an append of new fact rows) paired with a ``pre_hook`` window-delete
+    — EXACTLY the flat shape (ADR 0080 §3/§4 Slice D). A fact carries no dbt
+    ``unique_key`` (its grain is not a merge key), and in the installed
+    dbt-duckdb/dbt-databricks macros a keyless ``delete+insert`` does NOT delete,
+    so the keyless ``append`` + a ``pre_hook`` that clears exactly the
+    ``(lower, upper]`` window before the append is the portable idempotent path
+    (see :func:`kiri_dbt_bridge.silver_kimball.watermark_window.render_pre_hook`).
+
+    Load-bearing (ADR 0080 §4): you must NOT add a bare source-window WHERE to the
+    current full-rebuild ``materialized='table'`` model — that would replace the
+    table with only the windowed rows (data loss). The model becomes genuinely
+    incremental; the source WHERE is guarded by ``{% if is_incremental() %}`` so
+    the first run builds full up to ``upper``, and the post-success advance reads
+    ``max(<field>)`` over the materialised SILVER OUTPUT — so the watermark field
+    MUST be a projected output column (:func:`render_window_predicate` fails closed
+    when it is not).
+    """
+    silver = contract.kiri.silver
+    if silver.kind not in (
+        GoldKind.FACT,
+        GoldKind.FACTLESS_FACT,
+        GoldKind.SNAPSHOT_FACT,
+        GoldKind.ACCUMULATING_SNAPSHOT,
+    ):
+        raise ValueError(
+            f"generate_fact_model called with kind={silver.kind!r}; "
+            "this generator handles the four FACT-family kinds: "
+            "fact / factless_fact / snapshot_fact / accumulating_snapshot."
+        )
+
+    if not silver.grain:
+        raise ValueError(
+            f"Silver-Kimball fact {contract.contract.name!r} has no grain — "
+            "set kiri.silver.grain to the column(s) that identify a row."
+        )
+
+    coarser_grain = len(silver.grain) > 1
+
+    _check_all_columns_have_lineage(contract)
+    _validate_measures(contract, project)
+
+    plan = build_silver_lineage_plan(contract, project)
+    schema = resolve_silver_schema(contract, project)
+
+    # Incremental ONLY when a window is supplied AND the contract declares a
+    # watermark. Otherwise the byte-identical full-table build (backwards compat).
+    incremental = window is not None and contract.kiri.watermark is not None
+
+    if incremental:
+        assert window is not None  # narrowed by `incremental`
+        pre_hook = render_pre_hook(contract, window, dialect)
+        config = (
+            f"{{{{ config(materialized='incremental', incremental_strategy='append'"
+            f"{pre_hook}, "
+            f"schema='{schema}') }}}}\n"
+            f"-- Generated from silver-Kimball contract {contract.contract.name!r} "
+            f"({silver.kind.value}, grain={silver.grain}, incremental over a "
+            f"store-derived watermark window, ADR 0080 §3/§4). append + pre_hook "
+            f"window-delete (idempotent re-run: the pre_hook deletes exactly the "
+            f"(lower, upper] window from this before the append — no duplicate "
+            f"facts; a keyless delete+insert would NOT delete). Target-native SQL "
+            f"emitted via the {dialect.name!r} dialect helper (ADR 0099).\n"
+        )
+        predicate = render_window_predicate(contract, window, dialect, plan, shape="fact")
+    else:
+        config = (
+            f"{{{{ config(materialized='table', schema='{schema}') }}}}\n"
+            f"-- Generated from silver-Kimball contract {contract.contract.name!r} "
+            f"({silver.kind.value}, grain={silver.grain}). Target-native SQL "
+            f"emitted via the {dialect.name!r} dialect helper (ADR 0099).\n"
+        )
+        predicate = ""
+
+    body = (
+        _coarser_grain_sql(contract, plan, predicate)
+        if coarser_grain
+        else _row_grain_sql(contract, plan, predicate)
+    )
+
+    return config + "\n" + body
+
+
+def _row_grain_sql(contract: CanonicalContract, plan: SilverLineagePlan, predicate: str) -> str:
+    """Row-grain fact: plain SELECT projection + FROM/JOIN [+ window WHERE]."""
+    projection_lines = [
+        render_property_projection(contract, silver_col, ref, plan)
+        for silver_col, ref in plan.column_sources
+    ]
+    if not projection_lines:
+        raise ValueError("Row-grain silver fact produced no projection lines.")
+    body = "select\n    " + ",\n    ".join(projection_lines) + "\n" + from_clause(plan) + "\n"
+    if predicate:
+        body += predicate + "\n"
+    return body
+
+
+def _coarser_grain_sql(contract: CanonicalContract, plan: SilverLineagePlan, predicate: str) -> str:
+    """Coarser-grain fact: GROUP BY the grain columns + aggregate measures.
+
+    Each grain column is projected as ``<alias>.<bronze_col> AS <grain_col>``;
+    each declared ``kiri.silver.measures`` entry emits
+    ``<AGG>(<alias>.<bronze_col>) AS <measure_name>``; every OTHER
+    silver column (degenerate dimensions, FK columns reached via the
+    grain) is projected raw — they must be functionally determined by
+    the grain (Kimball invariant).
+    """
+    silver = contract.kiri.silver
+    grain_set = set(silver.grain)
+    measure_source_cols = {m.column for m in silver.measures}
+
+    sources: dict[str, object] = dict(plan.column_sources)
+    aliases = plan.aliases
+
+    select_lines: list[str] = []
+    group_by_lines: list[str] = []
+
+    for grain_col in silver.grain:
+        ref = sources.get(grain_col)
+        if ref is None:
+            raise ValueError(
+                f"Silver-Kimball fact {contract.contract.name!r}: grain "
+                f"column {grain_col!r} has no lineage source."
+            )
+        qualified = f"{aliases[ref.contract_name]}.{ref.column}"  # type: ignore[attr-defined]
+        select_lines.append(f"{qualified} as {grain_col}")
+        group_by_lines.append(qualified)
+
+    for measure in silver.measures:
+        m_ref = sources.get(measure.column)
+        if m_ref is None:
+            raise ValueError(
+                f"Silver-Kimball fact {contract.contract.name!r}: measure "
+                f"{measure.name!r} references column "
+                f"{measure.column!r} which has no lineage source."
+            )
+        bronze_alias = aliases[m_ref.contract_name]  # type: ignore[attr-defined]
+        bronze_col = m_ref.column  # type: ignore[attr-defined]
+        operand = f"{bronze_alias}.{bronze_col}"
+        agg = measure.aggregation.value.upper()
+        expr = f"COUNT(DISTINCT {operand})" if agg == "COUNT_DISTINCT" else f"{agg}({operand})"
+        select_lines.append(f"{expr} as {measure.name}")
+
+    for silver_col, ref in plan.column_sources:
+        if silver_col in grain_set or silver_col in measure_source_cols:
+            continue
+        qualified_ref = f"{aliases[ref.contract_name]}.{ref.column}"
+        select_lines.append(f"{qualified_ref} as {silver_col}")
+        group_by_lines.append(qualified_ref)
+
+    # The store-derived window WHERE (guarded by {% if is_incremental() %})
+    # filters the bronze SOURCE rows BEFORE the aggregation, so it slots between
+    # FROM and GROUP BY. Empty (no window / non-incremental) → unchanged SQL.
+    where = f"{predicate}\n" if predicate else ""
+    return (
+        "select\n    "
+        + ",\n    ".join(select_lines)
+        + "\n"
+        + from_clause(plan)
+        + "\n"
+        + where
+        + "group by "
+        + ", ".join(group_by_lines)
+        + "\n"
+    )
+
+
+def write_fact_model(
+    contract: CanonicalContract,
+    project: Project,
+    dialect: GoldDialect,
+    output_dir: Path,
+    window: WatermarkWindow | None = None,
+) -> Path:
+    """Write the model file and return its path."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / f"{contract.contract.name}.sql"
+    path.write_text(generate_fact_model(contract, project, dialect, window), encoding="utf-8")
+    return path

kiri_dbt_bridge/silver_kimball/flat_generator.py

@@ -0,0 +1,156 @@
+"""Silver-Kimball FLAT generator (ADR 0099).
+
+Explicit opt-out from Kimball star-schema modelling
+(``kiri.silver.kind=flat``). The silver contract just projects columns
+from one or more bronze sources — no grain, no business keys, no
+SCD semantics. lint's dimensional rules skip flat contracts.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from kiri_odcs.kiri_ext import GoldKind
+
+from kiri_dbt_bridge.silver_kimball.lineage import (
+    build_silver_lineage_plan,
+    from_clause,
+    render_property_projection,
+)
+from kiri_dbt_bridge.silver_kimball.watermark_window import (
+    render_pre_hook,
+    render_window_predicate,
+)
+from kiri_dbt_bridge.silver_kimball.zone import resolve_silver_schema
+
+if TYPE_CHECKING:
+    from kiri_core import CanonicalContract, Project
+    from kiri_core.lifecycle import WatermarkWindow
+
+    from kiri_dbt_bridge.gold.dialects import GoldDialect
+
+
+def _check_all_columns_have_lineage(contract: CanonicalContract) -> None:
+    """Same fail-closed check as fact_generator: every column must
+    declare a non-empty lineage."""
+    schema = contract.primary_schema
+    if schema is None:
+        raise ValueError(f"Silver-Kimball flat {contract.contract.name!r} has no schema.")
+    missing: list[str] = []
+    empty: list[str] = []
+    for prop in schema.properties:
+        key = f"{schema.name}.{prop.name}"
+        col_meta = contract.kiri.properties.get(key)
+        if col_meta is None or col_meta.lineage is None:
+            missing.append(prop.name)
+            continue
+        if not col_meta.lineage.from_columns:
+            empty.append(prop.name)
+    if missing:
+        raise ValueError(
+            f"Silver-Kimball flat {contract.contract.name!r}: column(s) "
+            f"{missing} have no kiri.lineage.from_columns declared."
+        )
+    if empty:
+        raise ValueError(
+            f"Silver-Kimball flat {contract.contract.name!r}: column(s) "
+            f"{empty} have empty kiri.lineage.from_columns — flat tables "
+            "have no 'originates here' columns."
+        )
+
+
+def generate_flat_model(
+    contract: CanonicalContract,
+    project: Project,
+    dialect: GoldDialect,
+    window: WatermarkWindow | None = None,
+) -> str:
+    """Render the flat silver dbt model SQL.
+
+    Backwards compatible (ADR 0080 "backwards compat"): ``window=None`` — or a
+    contract that does not declare ``kiri.watermark`` — yields the byte-identical
+    ``materialized='table'`` full-build SQL this generator always produced. When a
+    ``window`` is supplied AND the contract declares ``kiri.watermark``, the model
+    becomes ``materialized='incremental'`` with the store-derived window injected
+    into a ``{% if is_incremental() %}`` WHERE predicate (ADR 0080 §3/§4, Slice B).
+
+    Incremental strategy is ``append`` paired with a ``pre_hook`` window-delete
+    (NOT a keyless ``delete+insert``). Flat silver carries no business/unique key
+    (the validator forbids ``business_keys`` on ``kind=flat``), and in the
+    installed dbt-duckdb/dbt-databricks macros ``delete+insert``/``merge`` only
+    delete when a ``unique_key`` is set — a keyless ``delete+insert`` silently
+    degrades to a plain append → duplicates on a held-on-failure re-run or a
+    ``--reset`` backfill, violating ADR 0017's safe-re-run guarantee. Instead the
+    model appends and a ``pre_hook`` unconditionally deletes exactly the
+    ``(lower, upper]`` window from ``{{ this }}`` (guarded by
+    ``{% if is_incremental() %}``) before the append, so a re-run is a true
+    delete-then-reinsert (no duplicates). ``pre_hook`` + ``append`` behave
+    identically on dbt-databricks and dbt-duckdb (ADR 0130); see
+    :func:`kiri_dbt_bridge.silver_kimball.watermark_window.render_pre_hook`.
+    """
+    silver = contract.kiri.silver
+    if silver.kind is not GoldKind.FLAT:
+        raise ValueError(
+            f"generate_flat_model called with kind={silver.kind!r}; "
+            "this generator only handles kiri.silver.kind=flat."
+        )
+    _check_all_columns_have_lineage(contract)
+    plan = build_silver_lineage_plan(contract, project)
+    schema = resolve_silver_schema(contract, project)
+
+    projection_lines = [
+        render_property_projection(contract, silver_col, ref, plan)
+        for silver_col, ref in plan.column_sources
+    ]
+    if not projection_lines:
+        raise ValueError(
+            f"Silver-Kimball flat {contract.contract.name!r} produced no projection lines."
+        )
+
+    # Incremental ONLY when a window is supplied AND the contract declares a
+    # watermark. Otherwise the byte-identical full-table build (backwards compat).
+    incremental = window is not None and contract.kiri.watermark is not None
+
+    if incremental:
+        assert window is not None  # narrowed by `incremental`
+        pre_hook = render_pre_hook(contract, window, dialect)
+        config = (
+            f"{{{{ config(materialized='incremental', incremental_strategy='append'"
+            f"{pre_hook}, "
+            f"schema='{schema}') }}}}\n"
+            f"-- Generated from silver-Kimball contract {contract.contract.name!r} "
+            f"(flat — Kimball opt-out, incremental over a store-derived watermark "
+            f"window, ADR 0080 §3/§4). append + pre_hook window-delete (idempotent "
+            f"re-run: the pre_hook deletes exactly the (lower, upper] window from "
+            f"this before the append — no duplicate append; a keyless delete+insert "
+            f"would NOT delete). Target-native SQL via the "
+            f"{dialect.name!r} dialect helper (ADR 0099).\n"
+        )
+        predicate = render_window_predicate(contract, window, dialect, plan, shape="flat")
+    else:
+        config = (
+            f"{{{{ config(materialized='table', schema='{schema}') }}}}\n"
+            f"-- Generated from silver-Kimball contract {contract.contract.name!r} "
+            f"(flat — Kimball opt-out). Target-native SQL via the "
+            f"{dialect.name!r} dialect helper (ADR 0099).\n"
+        )
+        predicate = ""
+
+    body = "select\n    " + ",\n    ".join(projection_lines) + "\n" + from_clause(plan) + "\n"
+    if predicate:
+        body += predicate + "\n"
+    return config + "\n" + body
+
+
+def write_flat_model(
+    contract: CanonicalContract,
+    project: Project,
+    dialect: GoldDialect,
+    output_dir: Path,
+    window: WatermarkWindow | None = None,
+) -> Path:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / f"{contract.contract.name}.sql"
+    path.write_text(generate_flat_model(contract, project, dialect, window), encoding="utf-8")
+    return path