flextool 4.0.0__py3-none-any.whl

flextool/calibrate/_readers.py

@@ -0,0 +1,223 @@
+"""Pure post-solve readers for the adequacy calibrator.
+
+Each function takes the directory that holds a solve's result parquets
+(``<out_root>/output_parquet/<scenario>/``) and returns plain Python
+dicts.  Nothing here solves, mutates the DB, or touches the network — they
+are pure readers, so the loop can call them once per iteration and the
+tests can point them straight at real parquets.
+
+Why :func:`flextool.lean_parquet.read_lean_parquet` and not
+``pd.read_parquet``
+-------------------------------------------------------------------
+FlexTool writes these tables with a compact custom footer that records the
+MultiIndex level names; a raw ``pd.read_parquet`` returns flat
+string-tuple column names and loses the level structure the group-bys here
+rely on.  :func:`read_lean_parquet` reconstructs the real row/column
+MultiIndex, so ``groupby(level=...)`` works.
+
+Signals
+-------
+* ``node_slack_up_d_e`` — per-node unserved-energy up-slack (annual MWh,
+  ≥0).  Dense and always present on a valid solve → the robust adequacy
+  signal the loop steers on.
+* ``unit_curtailment_outputNode_d_ee`` — per-(unit, sink) curtailment
+  (MWh); OPTIONAL (absent when nothing curtails).  Provided now for the
+  over-build guard (C1c) to consume.
+* ``cost_node_discounted_d_ec`` — discounted per-(node, category) node
+  cost (M€); its ``"upward slack penalty"`` category is the monetised
+  slack.  This table is legitimately ABSENT on a zero-slack solve
+  (out_costs.py skips an empty category) → treated as zero penalty, never
+  an error.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from flextool.lean_parquet import read_lean_parquet
+
+# Registry keys of the parquet tables read here.  Resolved to on-disk
+# basenames through P2's registry helper so a schema rename breaks loudly
+# here rather than silently missing a file.
+_SLACK_KEY = "node_slack_up_d_e"
+_SLACK_KEY_DT = "node_slack_up_dt_e"
+_CURTAILMENT_KEY = "unit_curtailment_outputNode_d_ee"
+_COST_KEY = "cost_node_discounted_d_ec"
+_UNIT_CAPACITY_KEY = "unit_capacity_ed_p"
+_TOTAL_COST_KEY = "costs_discounted_p_"
+
+_SLACK_PENALTY_CATEGORY = "upward slack penalty"
+
+
+def _resolve_filename(key: str) -> str:
+    """Resolve an output *key* to its ``<key>.parquet`` basename.
+
+    Prefers P2's registry helper
+    (``flextool.calibrate._solve_status._registry_filename``) so an
+    OUTPUT_TRANSFORM rename surfaces as a clear error; falls back to the
+    ``<key>.parquet`` basename (which matches the OUTPUT_TRANSFORM keys) if
+    that private helper is ever removed.
+    """
+    try:
+        from flextool.calibrate._solve_status import _registry_filename
+
+        return _registry_filename(key)
+    except (ImportError, AttributeError):
+        return f"{key}.parquet"
+
+
+def read_residual_unserved(assess_dir: Path) -> dict[str, float]:
+    """Return ``{node: total_unserved_MWh}`` from ``node_slack_up_d_e``.
+
+    Sums each node's up-slack over all periods and drops the ``scenario``
+    column level, so the result is keyed by node.  This is the robust,
+    always-present adequacy signal the loop converges on.
+    """
+    path = Path(assess_dir) / _resolve_filename(_SLACK_KEY)
+    df = read_lean_parquet(path)
+    # columns: (scenario, node); index: period.  Sum over periods, then
+    # collapse any scenario level so the key is the node name.
+    totals = df.sum(axis=0).groupby(level="node").sum()
+    return {str(node): float(val) for node, val in totals.items()}
+
+
+def read_residual_unserved_dt(assess_dir: Path) -> dict[str, "object"]:
+    """Return ``{node: DataFrame[period, time, value]}`` from ``node_slack_up_dt_e``.
+
+    The per-``(period, time)`` companion to :func:`read_residual_unserved`:
+    where that reader collapses the up-slack to one annual MWh per node, this
+    one keeps the FULL per-timestep profile UNFOLDED onto the base timeline —
+    the stress SHAPE the ``timed`` sizer redistributes the additive margin
+    over.  ``node_slack_up_dt_e`` has row index ``(period, time)`` and column
+    levels ``(scenario, node)``; per node the scenario level is collapsed
+    (summed) and the result returned as a tidy pandas frame with columns
+    ``period``, ``time``, ``value`` (one row per non-null base cell).
+
+    Read with :func:`read_lean_parquet` (never ``pd.read_parquet``) so the
+    ``(period, time)`` / ``(scenario, node)`` MultiIndex is reconstructed.
+    A node absent from the table (no slack rows) simply has no key.
+    """
+    path = Path(assess_dir) / _resolve_filename(_SLACK_KEY_DT)
+    df = read_lean_parquet(path)
+    out: dict[str, object] = {}
+    node_level = df.columns.get_level_values("node")
+    for node in dict.fromkeys(node_level):  # ordered-unique node names
+        sub = df.loc[:, node_level == node]
+        # Collapse any scenario level → one value per (period, time) row.
+        series = sub.sum(axis=1)
+        frame = series.reset_index()
+        frame.columns = ["period", "time", "value"]
+        out[str(node)] = frame
+    return out
+
+
+def read_curtailment_by_sink(assess_dir: Path) -> dict[str, float]:
+    """Return ``{sink_node: total_curtailment_MWh}`` or ``{}`` if absent.
+
+    Reads ``unit_curtailment_outputNode_d_ee`` (columns
+    ``(scenario, unit, sink)``), sums each column over periods and groups
+    the totals by ``sink`` node.  The file is OPTIONAL (no curtailment ⇒ no
+    table); a missing file yields ``{}``.  Consumed by the over-build guard
+    (C1c).
+    """
+    path = Path(assess_dir) / _resolve_filename(_CURTAILMENT_KEY)
+    if not path.is_file():
+        return {}
+    df = read_lean_parquet(path)
+    totals = df.sum(axis=0).groupby(level="sink").sum()
+    return {str(sink): float(val) for sink, val in totals.items()}
+
+
+def read_slack_penalty(assess_dir: Path) -> tuple[float, dict[str, float]]:
+    """Return ``(total_Meur, {node: penalty_Meur})`` from the node-cost table.
+
+    Reads ``cost_node_discounted_d_ec`` (row index ``(period, node)``,
+    columns ``(scenario, category)``), selects the
+    ``"upward slack penalty"`` category and sums it per node (over periods
+    and scenarios).  This table is legitimately ABSENT on a zero-slack
+    solve (out_costs.py skips an empty cost category) → returns
+    ``(0.0, {})`` rather than raising.
+    """
+    path = Path(assess_dir) / _resolve_filename(_COST_KEY)
+    if not path.is_file():
+        return 0.0, {}
+    df = read_lean_parquet(path)
+    is_penalty = df.columns.get_level_values("category") == _SLACK_PENALTY_CATEGORY
+    penalty = df.loc[:, is_penalty]
+    if penalty.shape[1] == 0:
+        return 0.0, {}
+    # One value per (period, node) row across the selected penalty
+    # column(s); collapse periods (and any scenario level) per node.
+    per_row = penalty.sum(axis=1)
+    per_node = per_row.groupby(level="node").sum()
+    by_node = {str(node): float(val) for node, val in per_node.items()}
+    total = float(per_row.sum())
+    return total, by_node
+
+
+def read_unit_capacity_total(assess_dir: Path) -> dict[str, float]:
+    """Return ``{unit: total_capacity_MW}`` from ``unit_capacity_ed_p``.
+
+    Reads the post-invest fleet the solve actually committed to: the ``total``
+    column of ``unit_capacity_ed_p`` (row index ``(unit, period)``, column
+    levels ``(scenario, parameter)`` with ``parameter`` in
+    ``{existing, invested, divested, total}`` — see
+    ``flextool.process_outputs.out_capacity.unit_capacity``).  ``total`` is the
+    CUMULATIVE ``entity_all_capacity`` (existing + carried-forward invest), so
+    it is non-decreasing across periods.
+
+    Per unit the capacity is taken as the MAX over periods — the *mature*
+    (fully-built) fleet the net-load signal must be sized against; for a
+    single-invest-period solve this is simply that period's total, and for a
+    multi-period invest solve it is the final built-out capacity.  Any
+    ``scenario`` column level is collapsed (summed; a single-scenario solve
+    dir has exactly one) before the per-unit reduction.  Units whose ``total``
+    is entirely null (never realized) are dropped, so a unit absent from the
+    result cleanly falls back to its existing cap in
+    :func:`flextool.representative_periods.netload.build_group_capacities`.
+    Sorted by unit name for determinism.
+
+    This is a plain reader (no solving / DB / network), used by the net-load
+    solve-iteration driver to feed each iteration's invested caps back into the
+    representative-period selection.
+    """
+    path = Path(assess_dir) / _resolve_filename(_UNIT_CAPACITY_KEY)
+    df = read_lean_parquet(path)
+    # Select the ``total`` parameter column, collapsing any scenario level so
+    # the result is a Series indexed by (unit, period).
+    total = df.xs("total", level="parameter", axis=1).sum(axis=1)
+    per_unit = total.groupby(level="unit").max().dropna()
+    return {str(unit): float(val) for unit, val in sorted(per_unit.items())}
+
+
+def read_total_system_cost(assess_dir: Path) -> float:
+    """Return the discounted total system cost (M€) from ``costs_discounted_p_``.
+
+    ``costs_discounted_p_`` is the authoritative full-horizon cost summary
+    (``flextool.process_outputs.out_costs.cost_summaries``): every investment
+    AND dispatch cost category summed over all realized periods, discounted and
+    years-represented-weighted exactly as the LP objective is.  It INCLUDES the
+    ``upward slack penalty`` / ``downward slack penalty`` categories (the
+    monetised unserved-energy penalties) and the negative ``commodity_sales``
+    revenue term, so summing every category cell yields the same signed total
+    the objective minimises — the correct comparability metric for the
+    keep-best selection (same full timeline + same penalty prices each
+    iteration).
+
+    On disk the table is the per-scenario cost Series tagged with a single
+    ``scenario`` column level (row index ``category``); every numeric cell is
+    summed to the scalar total.
+    """
+    path = Path(assess_dir) / _resolve_filename(_TOTAL_COST_KEY)
+    df = read_lean_parquet(path)
+    return float(df.to_numpy().sum())
+
+
+__all__ = [
+    "read_curtailment_by_sink",
+    "read_residual_unserved",
+    "read_residual_unserved_dt",
+    "read_slack_penalty",
+    "read_total_system_cost",
+    "read_unit_capacity_total",
+]

flextool/calibrate/_report.py

@@ -0,0 +1,263 @@
+"""Pure rendering of a calibration run's outcome (C2).
+
+This module turns an in-memory :class:`~flextool.calibrate._loop.CalibResult`
+into (a) machine-readable CSV artifacts and (b) a compact human-readable
+stdout summary.  It does **no** solving and re-reads **no** parquet — it
+operates purely on the trajectory already carried by ``CalibResult`` — so it
+is cheap, deterministic, and safe to call anywhere.
+
+Two CSVs are written by :func:`write_report`:
+
+* a per-``(iteration, node)`` **long** table — one row per node observed in
+  each iteration, carrying that iteration's residual unserved energy, the
+  adder snapshot that was solved, the per-sink curtailment, the monetised
+  per-node slack, and whether the node ended the run flagged
+  resource-capped;
+* a per-iteration **summary** table — total unserved energy, total slack
+  penalty, the count of shedding nodes, the run's flagged count, and the
+  convergence flag.
+
+Both are emitted in a fully deterministic order: iterations ascending,
+nodes sorted lexicographically.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from flextool.calibrate._loop import CalibResult, IterRecord
+
+# CSV column orders — pinned here so both the writer and any downstream
+# consumer (and the tests) agree on the exact schema.
+LONG_COLUMNS = [
+    "iteration",
+    "node",
+    "residual_mwh",
+    "adder_mwh",
+    "curtailment_mwh",
+    "penalty_meur",
+    "flagged",
+]
+SUMMARY_COLUMNS = [
+    "iteration",
+    "total_unserved_mwh",
+    "total_penalty_meur",
+    "n_shedding",
+    "n_flagged",
+    "solve_seconds",
+    "converged",
+]
+
+LONG_FILENAME = "calibration_by_iteration_node.csv"
+SUMMARY_FILENAME = "calibration_summary.csv"
+
+
+def _record_nodes(record: IterRecord) -> list[str]:
+    """Sorted union of every node observed in one iteration's signals."""
+    nodes: set[str] = set()
+    nodes.update(record.residual)
+    nodes.update(record.adders)
+    nodes.update(record.curtailment)
+    nodes.update(record.penalty_by_node)
+    return sorted(nodes)
+
+
+def _n_shedding(record: IterRecord) -> int:
+    """Count of nodes with strictly-positive residual unserved energy."""
+    return sum(1 for v in record.residual.values() if v > 0.0)
+
+
+def _adder_scalar(adder: "float | dict") -> float:
+    """Collapse a node's adder to one scalar MWh for reporting.
+
+    A ``uniform`` adder is already the scalar per-timestep value; a ``timed``
+    adder is a ``{(period, time): float}`` map — reported as the SUM of its
+    per-cell values (its total per-timestep adder mass), so the CSV/summary
+    stay single-valued and sortable across both modes.
+    """
+    if isinstance(adder, dict):
+        return float(sum(adder.values()))
+    return float(adder)
+
+
+def write_report(result: CalibResult, *, out_dir: Path) -> list[Path]:
+    """Write the two machine-readable CSV artifacts for *result*.
+
+    Emits :data:`LONG_FILENAME` (per-``(iteration, node)``) and
+    :data:`SUMMARY_FILENAME` (per-iteration) into *out_dir*, creating it if
+    needed.  Rows are ordered iterations ascending then nodes sorted, so the
+    output is byte-deterministic for a given result.
+
+    Returns the two written paths in ``[long, summary]`` order.
+    """
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    flagged = set(result.guard_flagged_nodes)
+
+    long_path = out_dir / LONG_FILENAME
+    with long_path.open("w", newline="", encoding="utf-8") as fh:
+        writer = csv.writer(fh)
+        writer.writerow(LONG_COLUMNS)
+        for record in sorted(result.trajectory, key=lambda r: r.iteration):
+            for node in _record_nodes(record):
+                writer.writerow(
+                    [
+                        record.iteration,
+                        node,
+                        record.residual.get(node, 0.0),
+                        _adder_scalar(record.adders.get(node, 0.0)),
+                        record.curtailment.get(node, 0.0),
+                        record.penalty_by_node.get(node, 0.0),
+                        node in flagged,
+                    ]
+                )
+
+    summary_path = out_dir / SUMMARY_FILENAME
+    with summary_path.open("w", newline="", encoding="utf-8") as fh:
+        writer = csv.writer(fh)
+        writer.writerow(SUMMARY_COLUMNS)
+        ordered = sorted(result.trajectory, key=lambda r: r.iteration)
+        last_iteration = ordered[-1].iteration if ordered else None
+        for record in ordered:
+            # ``converged`` is a run-level outcome; it is true only on the
+            # final iteration of a converged run (the one that met the
+            # threshold), false on every earlier row.
+            row_converged = result.converged and record.iteration == last_iteration
+            writer.writerow(
+                [
+                    record.iteration,
+                    record.total_unserved,
+                    record.penalty_total,
+                    _n_shedding(record),
+                    len(result.guard_flagged_nodes),
+                    record.solve_seconds,
+                    row_converged,
+                ]
+            )
+
+    return [long_path, summary_path]
+
+
+def format_summary(result: CalibResult) -> str:
+    """Render a compact human-readable summary of *result* for stdout.
+
+    Covers: convergence status and iterations run; initial→final total
+    unserved energy (and slack penalty, when non-zero); the final per-node
+    adders sorted descending with each node's final residual; and a clearly
+    marked resource-capped section listing the guard-flagged nodes (the ones
+    that need firm capacity / imports / storage rather than more demand
+    margin).  Pure — reads only the in-memory *result*.
+    """
+    lines: list[str] = []
+    lines.append("=" * 60)
+    lines.append("Adequacy-margin calibration summary")
+    lines.append("=" * 60)
+
+    n_adjust = result.iterations_run - 1
+    if result.stop_reason == "converged":
+        lines.append(
+            f"Status: CONVERGED after {result.iterations_run} solve(s) "
+            f"({n_adjust} adjustment iteration(s))."
+        )
+    elif result.stop_reason == "stalled":
+        lines.append(
+            f"Status: CONVERGED MODULO RESOURCE-CAPPED NODES after "
+            f"{result.iterations_run} solve(s) ({n_adjust} adjustment "
+            f"iteration(s))."
+        )
+        lines.append(
+            "The energy-margin lever is exhausted: every remaining shedding "
+            "node is resource-capped, so no further demand margin can be "
+            "added and another solve would only reproduce this one."
+        )
+    else:  # "budget_exhausted"
+        lines.append(
+            f"Status: NOT CONVERGED — ran the full budget of "
+            f"{result.iterations_run} solve(s) without reaching the slack "
+            f"threshold."
+        )
+
+    trajectory = sorted(result.trajectory, key=lambda r: r.iteration)
+    if trajectory:
+        first = trajectory[0]
+        last = trajectory[-1]
+        lines.append("")
+        lines.append(
+            f"Total unserved energy: {first.total_unserved:,.1f} MWh "
+            f"(baseline) -> {last.total_unserved:,.1f} MWh (final)."
+        )
+        if first.penalty_total or last.penalty_total:
+            lines.append(
+                f"Slack penalty:         {first.penalty_total:,.3f} M-EUR "
+                f"(baseline) -> {last.penalty_total:,.3f} M-EUR (final)."
+            )
+        if any(r.solve_seconds for r in trajectory):
+            per_iter = ", ".join(
+                f"#{r.iteration}:{r.solve_seconds:,.1f}s" for r in trajectory
+            )
+            total_s = sum(r.solve_seconds for r in trajectory)
+            lines.append(
+                f"Solve time:            {total_s:,.1f} s total "
+                f"({per_iter})."
+            )
+
+    # Final per-node adders, largest first; pair each with its final residual.
+    final_residual = trajectory[-1].residual if trajectory else {}
+    lines.append("")
+    if result.final_adders:
+        lines.append("Final energy-margin adders (MWh/timestep), largest first:")
+        for node, adder in sorted(
+            result.final_adders.items(),
+            key=lambda kv: (-_adder_scalar(kv[1]), kv[0]),
+        ):
+            resid = final_residual.get(node, 0.0)
+            lines.append(
+                f"  {node:<24} adder={_adder_scalar(adder):>14,.4f}   "
+                f"final residual={resid:>14,.1f} MWh"
+            )
+    else:
+        lines.append("Final energy-margin adders: none (no node was bumped).")
+
+    # Resource-capped section — nodes the over-build guard froze.
+    lines.append("")
+    lines.append(
+        "Resource-capped nodes (need firm capacity / imports / storage, "
+        "not more demand margin):"
+    )
+    if result.guard_flagged_nodes:
+        for node in result.guard_flagged_nodes:
+            resid = final_residual.get(node, 0.0)
+            lines.append(f"  {node:<24} final residual={resid:>14,.1f} MWh")
+    else:
+        lines.append("  none")
+
+    if result.stop_reason == "stalled" and trajectory:
+        lines.append("")
+        lines.append(
+            f"Residual unserved energy still {trajectory[-1].total_unserved:,.1f} "
+            f"MWh at exit, but the calibration is CONVERGED MODULO the "
+            f"resource-capped nodes above: that residual sits on the "
+            f"guard-flagged nodes, which need firm capacity / imports / "
+            f"storage — not more demand margin. Raising --iterations will not "
+            f"help; the demand-margin lever is exhausted."
+        )
+    elif result.stop_reason == "budget_exhausted" and trajectory:
+        lines.append("")
+        lines.append(
+            f"Residual unserved energy still {trajectory[-1].total_unserved:,.1f} "
+            f"MWh at exit — raise --iterations or investigate the flagged nodes."
+        )
+
+    lines.append("=" * 60)
+    return "\n".join(lines)
+
+
+__all__ = [
+    "LONG_COLUMNS",
+    "LONG_FILENAME",
+    "SUMMARY_COLUMNS",
+    "SUMMARY_FILENAME",
+    "format_summary",
+    "write_report",
+]