flextool 4.0.0__py3-none-any.whl

flextool/engine_polars/autoscale/_report.py

@@ -0,0 +1,336 @@
+"""YAML audit report for the autoscaler.
+
+Layer 1 writes the four ranges + trigger flag.  Layer 2 / Layer 3 will
+extend the same file with their decisions (column scalers per quantity,
+``user_bound_scale`` value applied, etc.) — the structure is kept flat
+and self-documenting so the operator can read it without consulting
+schema docs.
+"""
+from __future__ import annotations
+
+import math
+from pathlib import Path
+from typing import Any, Mapping
+
+from polar_high.autoscale import Layer3Plan, RangeReport
+
+from ._layer2 import Layer2Plan
+
+
+def _decades(span: tuple[float, float]) -> Any:
+    """Return the spread of a range as decades (log10(hi/lo)).
+
+    ``(nan, nan)`` (empty group) returns ``None`` — the caller renders
+    that as ``"empty"`` in the console line.
+    """
+    lo, hi = span
+    if math.isnan(lo) or math.isnan(hi):
+        return None
+    if lo <= 0.0:
+        return None
+    if hi <= 0.0:
+        return None
+    return math.log10(hi / lo)
+
+
+def _fmt_decades(value: Any) -> str:
+    """Render a decade spread as ``"7.1d"`` or ``"empty"``."""
+    if value is None:
+        return "empty"
+    return f"{value:.1f}d"
+
+
+def _coerce_range(span: tuple[float, float]) -> dict[str, Any]:
+    """Render a ``(lo, hi)`` tuple as a small YAML dict.
+
+    NaN entries (the "no finite non-zero entries" sentinel) become the
+    string ``"empty"`` so the YAML is still valid (YAML disallows bare
+    NaN in most loaders) and the meaning is unambiguous in a manual read.
+    """
+    lo, hi = span
+    if math.isnan(lo) or math.isnan(hi):
+        return {"status": "empty"}
+    return {
+        "min": float(lo),
+        "max": float(hi),
+        "ratio": float(hi) / float(lo) if lo != 0 else None,
+    }
+
+
+def _coerce_cross_ratio(value: float) -> Any:
+    """Render the cross-group ratio, mapping NaN → ``None``."""
+    if math.isnan(value):
+        return None
+    return float(value)
+
+
+def _render_layer1(report: RangeReport) -> dict[str, Any]:
+    """Build the ``layer1`` section of the autoscaler audit YAML.
+
+    Kept as a small pure function so Layer 2 / Layer 3 sections can be
+    composed alongside it without :func:`write_report` growing branches.
+    """
+    return {
+        "ranges": {
+            "matrix": _coerce_range(report.matrix),
+            "cost": _coerce_range(report.cost),
+            "bound": _coerce_range(report.bound),
+            "rhs": _coerce_range(report.rhs),
+        },
+        "cross_group_max_ratio": _coerce_cross_ratio(report.cross_group_max_ratio),
+        "trigger": bool(report.trigger),
+    }
+
+
+def _dump_simple_yaml(obj: Any, indent: int = 0) -> str:
+    """Tiny dependency-free YAML emitter.
+
+    Avoids a hard PyYAML dependency for this stub — the autoscaler only
+    ever writes scalar / mapping / list trees with no anchors, references
+    or tags, so the surface stays trivial.  Keys are emitted in insertion
+    order (Python 3.7+ dict semantics) so the operator-visible layout is
+    stable.  Floats are rendered with ``repr`` so round-tripping into a
+    real YAML loader recovers the bit-pattern.
+
+    If a future phase needs richer YAML (e.g. multi-line strings),
+    swap to :mod:`yaml`; the API of :func:`write_report` does not change.
+    """
+    pad = " " * indent
+    if isinstance(obj, Mapping):
+        if not obj:
+            return "{}"
+        lines: list[str] = []
+        for k, v in obj.items():
+            if isinstance(v, (Mapping, list)):
+                rendered = _dump_simple_yaml(v, indent + 2)
+                if isinstance(v, Mapping) and v:
+                    lines.append(f"{pad}{k}:")
+                    lines.append(rendered)
+                elif isinstance(v, list) and v:
+                    lines.append(f"{pad}{k}:")
+                    lines.append(rendered)
+                else:
+                    lines.append(f"{pad}{k}: {rendered}")
+            else:
+                lines.append(f"{pad}{k}: {_scalar(v)}")
+        return "\n".join(lines)
+    if isinstance(obj, list):
+        if not obj:
+            return "[]"
+        return "\n".join(
+            f"{pad}- {_dump_simple_yaml(item, indent + 2).lstrip() if isinstance(item, (Mapping, list)) else _scalar(item)}"
+            for item in obj
+        )
+    return _scalar(obj)
+
+
+def _scalar(v: Any) -> str:
+    """Render a scalar leaf for the YAML emitter."""
+    if v is None:
+        return "null"
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if isinstance(v, float):
+        if math.isnan(v):
+            return ".nan"
+        if math.isinf(v):
+            return ".inf" if v > 0 else "-.inf"
+        return repr(v)
+    if isinstance(v, int):
+        return str(v)
+    s = str(v)
+    if any(c in s for c in (":", "#", "\n", "{", "}", "[", "]", "&", "*")) or s.strip() != s:
+        # Escape via single-quoted scalar — YAML's safest form.
+        return "'" + s.replace("'", "''") + "'"
+    return s
+
+
+def render_layer2(plan: Layer2Plan) -> dict[str, Any]:
+    """Build the ``layer2`` section of the autoscaler audit YAML.
+
+    Captures the per-type exponents, the count of skipped rows /
+    integer columns, and the per-type before/after magnitude ranges.
+    Also surfaces the post-scale RHS range of the
+    ``ladder_tier_cap_annual_roll`` family for the H2_trade sanity
+    check — the family the regression handoff identifies as the
+    Layer-2 trigger.
+    """
+    exponents = {t.value: int(e) for t, e in plan.type_exponents.items()}
+    type_ranges_before = {
+        t.value: _coerce_range(r) for t, r in plan.type_buckets_before.items()
+    }
+    type_ranges_after = {
+        t.value: _coerce_range(r) for t, r in plan.type_buckets_after.items()
+    }
+    return {
+        "type_exponents": exponents,
+        "skipped_rows_count": len(plan.skipped_rows),
+        "skipped_rows": list(plan.skipped_rows),
+        "skipped_integer_cols_count": len(plan.skipped_integer_cols),
+        "n_cols": int(plan.col_factors.shape[0]),
+        "n_rows": int(plan.row_factors.shape[0]),
+        "type_ranges_before": type_ranges_before,
+        "type_ranges_after": type_ranges_after,
+    }
+
+
+def render_layer3(plan: Layer3Plan) -> dict[str, Any]:
+    """Build the ``layer3`` section of the autoscaler audit YAML.
+
+    Surfaces the three HiGHS options Layer 3 set and the reasoning
+    string so the operator can correlate the YAML entry with the
+    one-line log emitted at apply time.
+    """
+    return {
+        "user_objective_scale": int(plan.user_objective_scale),
+        "user_bound_scale": int(plan.user_bound_scale),
+        "simplex_scale_strategy": int(plan.simplex_scale_strategy),
+        "reasoning": str(plan.reasoning),
+    }
+
+
+def write_report(result: Mapping[str, Any], path: Path | str) -> Path:
+    """Serialise the autoscaler result tree to ``path`` as YAML.
+
+    ``result`` is a mapping whose top-level keys are layer names
+    (``"layer1"``, later ``"layer2"`` / ``"layer3"``) and whose values
+    are the per-layer payloads.  Layer 1 hands a :class:`RangeReport`;
+    we render it via :func:`_render_layer1`.  Unknown payload types pass
+    through to the generic dumper — keeps the door open for later
+    phases without coupling the layers via shared schema.
+
+    Returns the resolved :class:`Path` that was written, for caller
+    bookkeeping (the wire-in logs it).
+    """
+    rendered: dict[str, Any] = {}
+    for key, payload in result.items():
+        if isinstance(payload, RangeReport):
+            rendered[key] = _render_layer1(payload)
+        else:
+            rendered[key] = payload
+
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    text = _dump_simple_yaml(rendered)
+    path.write_text(text + "\n", encoding="utf-8")
+    return path
+
+
+def format_console_summary(
+    *,
+    ranges_pre: RangeReport,
+    ranges_post: "RangeReport | None",
+    layer2_plan: "Layer2Plan | None",
+    layer3_plan: "Layer3Plan | None",
+    threshold_decades: float,
+) -> str:
+    """Compose the one-line console summary surfaced when autoscale fires.
+
+    Renders the four pre-Layer-1 spreads in decades, the per-type Layer 2
+    exponents, the Layer 3 ``user_*_scale`` + ``simplex_scale_strategy``
+    values, and the post-action four-range spreads so the operator sees
+    "before → after" at a glance.
+
+    When ``ranges_pre.trigger`` is False the function returns the quieter
+    "within comfort zone" confirmation — callers can branch on that path
+    by inspecting the report themselves, but exposing the message here
+    keeps the format definition in one place.
+    """
+    if not ranges_pre.trigger:
+        return (
+            "Autoscale by polar-high: LP within HiGHS comfort zone "
+            f"(spreads <= {threshold_decades:g} decades) — no scaling applied"
+        )
+
+    pre_matrix = _fmt_decades(_decades(ranges_pre.matrix))
+    pre_cost = _fmt_decades(_decades(ranges_pre.cost))
+    pre_bound = _fmt_decades(_decades(ranges_pre.bound))
+    pre_rhs = _fmt_decades(_decades(ranges_pre.rhs))
+
+    parts: list[str] = [
+        f"Autoscale by polar-high: ranges pre Matrix={pre_matrix} Cost={pre_cost} "
+        f"Bound={pre_bound} RHS={pre_rhs}"
+    ]
+
+    if layer2_plan is not None:
+        exps = ", ".join(
+            f"{t.value}:{e:+d}" for t, e in layer2_plan.type_exponents.items()
+        )
+        skipped = len(layer2_plan.skipped_rows)
+        parts.append(
+            f"L2 exponents {{{exps}}}"
+            + (f" skipped_rows={skipped}" if skipped else "")
+        )
+    else:
+        parts.append("L2 skipped")
+
+    if layer3_plan is not None:
+        parts.append(
+            f"L3 user_obj={layer3_plan.user_objective_scale} "
+            f"user_bnd={layer3_plan.user_bound_scale} "
+            f"simplex={layer3_plan.simplex_scale_strategy}"
+        )
+    else:
+        parts.append("L3 skipped")
+
+    if ranges_post is not None:
+        post_matrix = _fmt_decades(_decades(ranges_post.matrix))
+        post_cost = _fmt_decades(_decades(ranges_post.cost))
+        post_bound = _fmt_decades(_decades(ranges_post.bound))
+        post_rhs = _fmt_decades(_decades(ranges_post.rhs))
+        parts.append(
+            f"ranges post Matrix={post_matrix} Cost={post_cost} "
+            f"Bound={post_bound} RHS={post_rhs}"
+        )
+
+    flat = " -> ".join(parts)
+    if len(flat) <= 100:
+        return flat
+    return parts[0] + "".join(f"\n  -> {p}" for p in parts[1:])
+
+
+def format_nonoptimal_hint(ranges: RangeReport) -> str:
+    """Compose the scaling-related *possibility* hint for a rejected solve.
+
+    Returns the empty string unless ``ranges.trigger`` is ``True``.  The
+    caller must pass the ranges of the *actually solved* (post-autoscale)
+    LP — a still-tripped trigger there means the autoscaler could NOT tame
+    the range spread, so scaling is a plausible (not proven) contributor to
+    the failure.  Framed as a possibility with remediations, never as the
+    certified cause: a wide range does not by itself explain a non-optimal
+    status, and passing the raw pre-autoscale ranges here (which are always
+    wide for FlexTool's commodity ladders) would make the hint fire on
+    essentially every failure regardless of cause.
+    """
+    if not ranges.trigger:
+        return ""
+
+    def _span_decades(span: tuple[float, float]) -> str:
+        d = _decades(span)
+        if d is None:
+            return "empty"
+        return f"{d:.1f} decades"
+
+    rhs_span = _span_decades(ranges.rhs)
+    cost_span = _span_decades(ranges.cost)
+
+    return (
+        "The solved (post-autoscale) LP is still wide-ranged "
+        f"(RHS spans {rhs_span}, Cost spans {cost_span}), so scaling MAY be "
+        "a contributing factor. Possible remediations:\n"
+        "  - Check unit conventions on commodity-ladder entities "
+        "(set unitsize so quantity/unitsize <= 1e+6)\n"
+        "  - Re-run with --highs-threads 1 to bypass parallel-mode "
+        "brittleness\n"
+        "  - Re-run with --scaling=solver_only if you suspect the "
+        "autoscaler interferes (HiGHS' internal scaling still runs)"
+    )
+
+
+__all__ = [
+    "format_console_summary",
+    "format_nonoptimal_hint",
+    "render_layer2",
+    "render_layer3",
+    "write_report",
+]

flextool/engine_polars/chain.py

@@ -0,0 +1,259 @@
+"""Thin compat shim around the native cascade — Δ.12e.
+
+The legacy file-symlink driver (loading per-sub-solve
+``solve_data_<sub>/`` snapshots directly from a work folder) retired
+in Δ.12e once the native cascade in
+:mod:`flextool.engine_polars._orchestration` reached feature parity:
+warm-LP across structurally-compatible iterations (Δ.12d), in-memory
+handoff carriers between sub-solves (Δ.11/Δ.12), full output writer
+coverage (Δ.1, Δ.12c-fix), and override-chain authority for Direct /
+Derived / Projection params (Δ.12-drop, Δ.12c, Δ.12c-fix2).
+
+What's left here is a thin convenience wrapper that converts a
+work-folder path (containing a ``tests.sqlite`` / ``input.sqlite``)
+into the canonical native call
+:func:`flextool.engine_polars._orchestration.run_chain_from_db` and
+adapts the result to :class:`ChainStep` shape for backwards
+compatibility with the few external callers that still import
+``run_chain`` directly.
+
+The warm-update primitives (structural fingerprint, Param classification,
+:func:`_apply_warm_updates`) live in :mod:`flextool.engine_polars._warm`
+and are re-exported here as module-level names for backwards
+compatibility (e.g. ``from flextool.engine_polars.chain import
+_MUTABLE_PARAMS`` in ``test_warm_param_autoupdate``).
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from polar_high import Solution
+
+# Backwards-compat re-exports of warm-update primitives that now live in
+# :mod:`flextool.engine_polars._warm`.  Documented in the module
+# docstring above; consumed by ``tests/engine_polars/test_warm_param_autoupdate.py``
+# (and any external caller that still does ``from
+# flextool.engine_polars.chain import _MUTABLE_PARAMS``).  Listed in
+# ``__all__`` so ruff F401 does not strip them as unused.
+from flextool.engine_polars._warm import (  # noqa: F401
+    _STRUCTURAL_FIELDS,
+    _WARM_PARAMS,
+    _MUTABLE_PARAMS,
+    _WARM_PARAMS_DEFERRED,
+    _WARM_PARAMS_NO_OP,
+    _WARM_PARAM_GATES,
+    _IncompatibleUpdate,
+    _fingerprint,
+    _param_frame_equal,
+    _param_values_position_equal,
+    _gate_active,
+    _apply_warm_updates,
+    _build_warm_problem,
+)
+
+
+__all__ = [
+    "run_chain",
+    "ChainStep",
+    # Re-exported warm-update primitives (see docstring).
+    "_STRUCTURAL_FIELDS",
+    "_WARM_PARAMS",
+    "_MUTABLE_PARAMS",
+    "_WARM_PARAMS_DEFERRED",
+    "_WARM_PARAMS_NO_OP",
+    "_WARM_PARAM_GATES",
+    "_IncompatibleUpdate",
+    "_fingerprint",
+    "_param_frame_equal",
+    "_param_values_position_equal",
+    "_gate_active",
+    "_apply_warm_updates",
+    "_build_warm_problem",
+]
+
+
+class ChainStep:
+    """Per-sub-solve result of :func:`run_chain`.
+
+    Attributes
+    ----------
+    solve_name : str
+        The sub-solve identifier (e.g. ``"y2025_5week"``).
+    solution : polar_high.Solution
+        The HiGHS solution for this sub-solve.
+    handoff : flextool.SolveHandoff
+        The handoff carriers derived from the polar_high solution
+        (see :func:`flextool.input.build_handoff_from_solution`).
+    warm_used : bool
+        True if this sub-solve was solved by warm-updating the prior
+        sub-solve's :class:`WarmProblem` instance; False if it was a
+        cold rebuild.  Always False for the first sub-solve and for
+        ``warm=False`` runs.
+    """
+
+    __slots__ = ("solve_name", "solution", "handoff", "warm_used")
+
+    def __init__(self, solve_name: str, solution: Solution, handoff,
+                 warm_used: bool = False):
+        self.solve_name = solve_name
+        self.solution = solution
+        self.handoff = handoff
+        self.warm_used = warm_used
+
+    def __repr__(self) -> str:  # pragma: no cover — debug-only
+        return (f"ChainStep(solve_name={self.solve_name!r}, "
+                f"obj={self.solution.obj!r}, "
+                f"warm_used={self.warm_used}, "
+                f"handoff_empty={self.handoff.is_empty()})")
+
+
+# Same overrides used by the parity-sweep fixtures — see
+# tests/engine_polars/test_solve_config_parity._discover_fixtures.  The
+# keys are work_folder dirnames; values are the scenario names that
+# produced those snapshots.
+_NATIVE_SCENARIO_OVERRIDES: dict[str, str] = {
+    "work_2day_stochastic_dispatch_full_storage": "2_day_stochastic_dispatch",
+    "work_commodity_ladder_annual": "coal_ladder_annual",
+    "work_commodity_ladder_cumulative": "coal_ladder_cumulative",
+    "work_delay_source_coef": "water_pump_delayed",
+    "work_inflation_check": "wind_battery_invest_lifetime_renew",
+}
+
+
+def _resolve_native_scenario(db_path: Path, work: Path) -> str | None:
+    """Map ``work/`` dirname → scenario name using the same convention
+    as the parity tests.  Returns ``None`` when no rule matches.
+    """
+    import re
+    import spinedb_api as api
+
+    if work.name in _NATIVE_SCENARIO_OVERRIDES:
+        return _NATIVE_SCENARIO_OVERRIDES[work.name]
+
+    scen_target = work.name.removeprefix("work_") if work.name.startswith("work_") else None
+    if scen_target is None:
+        return None
+
+    try:
+        with api.DatabaseMapping("sqlite:///" + str(db_path)) as db:
+            scenarios = sorted(s.name for s in db.query(db.scenario_sq).all())
+    except Exception:
+        return None
+
+    candidates = [scen_target]
+    candidates.append(re.sub(r"(^|_)(\d+)([a-z])", r"\1\2_\3", scen_target))
+    candidates.append(re.sub(r"(\d+)_([a-z])", r"\1\2", scen_target))
+    if scen_target.endswith("_full_storage"):
+        base = scen_target[: -len("_full_storage")]
+        candidates.append(re.sub(r"(^|_)(\d+)([a-z])", r"\1\2_\3", base))
+        candidates.append(base)
+    for cand in candidates:
+        if cand in scenarios:
+            return cand
+    return None
+
+
+def run_chain(
+    work_folder: Path | str,
+    *,
+    warm: bool = True,
+    scenario: str | None = None,
+) -> dict[str, ChainStep]:
+    """Run a flextool multi-solve scenario end-to-end via the native
+    cascade — convenience adapter that takes a work-folder path.
+
+    Δ.12e — this function is now a thin shim that:
+
+    1. Looks up ``tests.sqlite`` / ``input.sqlite`` under
+       *work_folder*.
+    2. Resolves the scenario name from the dir-name convention (see
+       :data:`_NATIVE_SCENARIO_OVERRIDES`) or the explicit
+       ``scenario`` kwarg / ``FLEXTOOL_NATIVE_SCENARIO`` env var.
+    3. Delegates to
+       :func:`flextool.engine_polars._orchestration.run_chain_from_db`.
+    4. Adapts the result from
+       :class:`flextool.engine_polars.OrchestrationStep` to
+       :class:`ChainStep` shape (the legacy return type a few
+       external callers still import).
+
+    The previous file-symlink driver (loading
+    ``solve_data_<sub>/`` snapshots directly from the work folder)
+    and the ``use_handoff_overlay`` / ``native`` / ``chain`` kwargs
+    retired in Δ.12e once the native cascade reached feature parity.
+    Callers driving DB scenarios should prefer
+    :func:`run_chain_from_db` directly.
+
+    Parameters
+    ----------
+    work_folder : Path | str
+        Directory containing a Spine SQLite (``tests.sqlite`` or
+        ``input.sqlite``).  The native cascade re-runs flextool's
+        preprocessing into a private tempdir; the work folder is
+        consulted only for the DB.
+    warm : bool, default False
+        When True, attempt warm LP updates between consecutive
+        structurally-compatible sub-solves using
+        :class:`polar_high.WarmProblem`.  See
+        :func:`flextool.engine_polars._orchestration.run_orchestration`
+        for the per-iteration semantics.
+    scenario : str | None, default None
+        Explicit scenario name override.  ``None`` consults the
+        ``FLEXTOOL_NATIVE_SCENARIO`` env var, then falls back to the
+        ``work_<scenario>`` dir-name convention.
+
+    Returns
+    -------
+    dict[str, ChainStep]
+        Mapping ``solve_name → ChainStep(solve_name, solution, handoff,
+        warm_used)``.
+
+    Raises
+    ------
+    ValueError
+        If no DB is found under *work_folder* or the scenario can't
+        be determined.
+    """
+    import os
+    work = Path(work_folder)
+    db_path = None
+    for cand in ("tests.sqlite", "input.sqlite"):
+        p = work / cand
+        if p.exists():
+            db_path = p
+            break
+    if db_path is None:
+        raise ValueError(
+            f"run_chain: no DB found under {work} "
+            f"(looked for tests.sqlite, input.sqlite).  The native "
+            f"cascade requires a DB scenario."
+        )
+
+    # Late import to avoid a build-time cycle between chain and _orchestration.
+    from flextool.engine_polars._orchestration import run_chain_from_db
+
+    if scenario is None:
+        scenario = os.environ.get("FLEXTOOL_NATIVE_SCENARIO") or None
+    if scenario is None:
+        scenario = _resolve_native_scenario(db_path, work)
+    if scenario is None:
+        raise ValueError(
+            f"run_chain: cannot determine which scenario to run for "
+            f"{work}.  Pass scenario= explicitly, set "
+            f"FLEXTOOL_NATIVE_SCENARIO, or rename the work directory to "
+            f"match the scenario name (work_<scenario>).  "
+            f"work.name={work.name!r}"
+        )
+
+    # Phase C.5 — ``ChainStep`` keeps the per-step ``solution``
+    # contract for legacy callers; opt into the full per-step state on
+    # the underlying cascade.
+    steps = run_chain_from_db(db_path, scenario, warm=warm, keep_solutions=True)
+    out: dict[str, ChainStep] = {}
+    for name, step in steps.items():
+        out[name] = ChainStep(
+            solve_name=name,
+            solution=step.solution,
+            handoff=step.handoff,
+            warm_used=getattr(step, "warm_used", False),
+        )
+    return out