flextool 4.0.0__py3-none-any.whl

flextool/calibrate/_db_alt.py

@@ -0,0 +1,166 @@
+"""Idempotent write of the per-scenario adequacy-calibration alternative.
+
+Each calibrator iteration needs the current per-node
+``energy_margin_adder`` (and its enabling ``energy_margin_method =
+inflow_adder``) applied to the model *without editing the fixture's own
+alternatives*.  We do that by writing a dedicated calibration alternative
+named ``<scenario>_adeq_calib`` and appending it to the scenario's
+alternative stack at the TOP rank, so its values WIN over any baseline the
+scenario already sets while leaving that baseline untouched.
+
+The write is idempotent by construction: :func:`spinedb_api.import_data`
+defaults to ``on_conflict='merge'`` (an UPDATE in place), so re-writing a
+changed adder updates the single existing row rather than creating a
+duplicate.  Re-writing an *identical* state leaves nothing to commit, which
+:meth:`DatabaseMapping.commit_session` signals by raising
+:class:`NothingToCommit` — caught and treated as success.
+
+The scenario link is wired SEPARATELY from the parameter import, by adding
+only the single new ``(scenario, alt)`` row at the top rank.  We must NOT
+route it through ``import_data(scenario_alternatives=...)``: that path
+re-derives the scenario's ENTIRE ordered stack and re-yields every existing
+link with freshly recomputed *1-based* ranks.  On a database whose stack is
+stored with a different rank base (real FlexTool models use *0-based* ranks),
+every existing link's rank then shifts by one and collides on the
+``(scenario, rank)`` unique key — surfacing as spurious "already a
+scenario_alternative" errors for the untouched baseline alternatives.  The
+single-row append below is base-agnostic and idempotent (mirrors
+``representative_periods.scenario_stack.add_alternative_to_scenario``).
+
+This module never solves and never touches the network beyond the target
+SpineDB.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from spinedb_api import DatabaseMapping, import_data
+from spinedb_api.exception import NothingToCommit
+from spinedb_api.parameter_value import Map
+
+# Suffix appended to a scenario name to form its calibration alternative.
+_CALIB_ALT_SUFFIX = "_adeq_calib"
+
+
+def _adder_map(cells: dict[tuple[str, str], float]) -> Map:
+    """Build a 2-D ``(period → time → float)`` Map from per-cell adders.
+
+    *cells* is ``{(period, time): adder}`` (the ``timed`` sizer's output for
+    one node).  The nested ``Map`` round-trips through ``import_data`` into the
+    ``pdt_energy_margin_adder.csv`` spec the emitter reads, placing each cell's
+    value at exactly that invest ``(period, time)``.  Periods and times are
+    emitted in sorted order for a deterministic, idempotent write.
+    """
+    by_period: dict[str, dict[str, float]] = defaultdict(dict)
+    for (period, time), value in cells.items():
+        by_period[str(period)][str(time)] = float(value)
+    periods = sorted(by_period)
+    inner_maps = []
+    for period in periods:
+        times = sorted(by_period[period])
+        inner_maps.append(
+            Map(times, [by_period[period][t] for t in times], index_name="time")
+        )
+    return Map(periods, inner_maps, index_name="period")
+
+
+def calib_alt_name(scenario: str) -> str:
+    """Return the calibration alternative name for *scenario*.
+
+    A pure naming helper (``f"{scenario}{_CALIB_ALT_SUFFIX}"``) so callers
+    and tests agree on the alternative the calibrator writes into without
+    duplicating the string literal.
+    """
+    return f"{scenario}{_CALIB_ALT_SUFFIX}"
+
+
+def _normalise_url(url: str) -> str:
+    """Accept either a bare filesystem path or a full SQLAlchemy URL.
+
+    A bare path (no ``"://"`` scheme) is promoted to a ``sqlite:///`` URL;
+    anything already carrying a scheme is passed through verbatim.
+    """
+    return url if "://" in url else f"sqlite:///{url}"
+
+
+def write_calib_alt(
+    url: str,
+    scenario: str,
+    per_node_adder: "dict[str, float | dict[tuple[str, str], float]]",
+) -> None:
+    """Write (or update) the calibration alternative for *scenario*.
+
+    For every ``node -> adder`` in *per_node_adder*, set both
+    ``energy_margin_method = inflow_adder`` and ``energy_margin_adder`` on that
+    node under the ``<scenario>_adeq_calib`` alternative, and append that
+    alternative to *scenario*'s stack at the top rank (higher rank wins; the
+    existing stack is left intact).
+
+    The adder value is EITHER a scalar float (the ``uniform`` sizer — a
+    constant per-timestep margin) OR a ``{(period, time): float}`` map (the
+    ``timed`` sizer — per-cell margin), written as a 2-D
+    ``period → time → float`` :class:`spinedb_api.parameter_value.Map` that
+    the emitter ingests as ``pdt_energy_margin_adder.csv``.  Both share the
+    idempotent-overwrite path.
+
+    Idempotent: re-writing a changed adder UPDATEs the single row in place
+    (``import_data`` merges on conflict); re-writing an identical state
+    commits nothing (:class:`NothingToCommit` is swallowed).  An empty
+    *per_node_adder* still materialises the alternative and its scenario
+    link — so a baseline (k=0) iteration is solved *through* the calibration
+    alternative from the start, keeping the alternative stack constant
+    across every iteration.
+
+    Parameters
+    ----------
+    url:
+        Target SpineDB — a bare path (promoted to ``sqlite:///``) or a full
+        SQLAlchemy URL.
+    scenario:
+        The model scenario whose stack the calibration alternative joins.
+    per_node_adder:
+        ``{node_name: adder_MWh}``.  May be empty (baseline iteration).
+    """
+    alt = calib_alt_name(scenario)
+
+    pvs: list[tuple] = []
+    for node, adder in per_node_adder.items():
+        pvs.append(("node", node, "energy_margin_method", "inflow_adder", alt))
+        value = _adder_map(adder) if isinstance(adder, dict) else float(adder)
+        pvs.append(("node", node, "energy_margin_adder", value, alt))
+
+    with DatabaseMapping(_normalise_url(url)) as db:
+        # 1. Materialise the calibration alternative and its per-node adders.
+        #    Deliberately NO ``scenario_alternatives`` here — see the module
+        #    docstring: routing the link through import_data re-ranks the whole
+        #    stack (1-based) and collides on 0-based real-model stacks.
+        _count, errors = import_data(
+            db,
+            alternatives=[alt],
+            parameter_values=pvs,
+        )
+        assert not errors, errors
+        try:
+            db.commit_session("adeq_calib iteration")
+        except NothingToCommit:
+            # Re-writing an identical state changes nothing to persist.
+            pass
+
+        # 2. Append the calibration alternative to the scenario's stack at the
+        #    TOP rank (max existing rank + 1, so its values WIN over the
+        #    baseline).  Idempotent + base-agnostic: touch ONLY the single new
+        #    (scenario, alt) row, leaving every existing link's rank untouched.
+        existing = db.get_scenario_alternative_items(scenario_name=scenario)
+        if not any(sa["alternative_name"] == alt for sa in existing):
+            next_rank = max((sa["rank"] for sa in existing), default=0) + 1
+            db.add_scenario_alternative(
+                scenario_name=scenario, alternative_name=alt, rank=next_rank
+            )
+            try:
+                db.commit_session("adeq_calib scenario link")
+            except NothingToCommit:
+                pass
+
+
+__all__ = ["calib_alt_name", "write_calib_alt"]

flextool/calibrate/_final_outputs.py

@@ -0,0 +1,110 @@
+"""Regenerate the non-parquet output formats from the final surviving parquet.
+
+The calibration loop solves each iteration with ``--write-methods parquet``
+only (fast; the loop reads its signals straight from the parquet), and clears
+the tree between iterations, so when :func:`~flextool.calibrate._loop.run_calibration`
+returns the LAST iteration's full results survive at
+``<out_root>/output_parquet/<scenario>/``.
+
+This module turns that surviving parquet tree into the other regular output
+formats (csv / excel / spinedb / plot) **without re-solving** — it drives the
+engine's own disk-replay path (:func:`flextool.process_outputs.write_outputs.write_outputs`
+with ``read_parquet_dir=True``), the same code the ``flextool-write-outputs``
+CLI and the Toolbox re-plot flow use.  So the calibrated model's results are
+available "just like regular results", but produced by re-reading parquet
+rather than paying for another solve.
+
+Known limitations of the replay path (inherent to reading back from parquet,
+not to this module):
+
+* ``spinedb`` replay cannot recover the two investment/operations
+  discount-factor parameters (the writer needs the live ``par`` frame) nor the
+  ordering of a two-way connection's (source, sink);
+* ``csv`` replay writes the per-table frames only — it does not regenerate the
+  native run's ``summary.csv``.
+
+parquet is intentionally excluded from the accepted methods: it is already on
+disk (re-writing it would be a no-op the engine guards against anyway).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+# The formats this post-loop step can regenerate from parquet. ``parquet`` is
+# excluded on purpose (already present); the rest are the regular output
+# formats a normal run would emit.
+FINAL_WRITE_METHOD_CHOICES = ("csv", "excel", "spinedb", "plot")
+
+
+def write_final_outputs(
+    out_root: Path,
+    scenario: str,
+    write_methods: "list[str] | tuple[str, ...]",
+    *,
+    results_db_url: str | None = None,
+) -> list[str]:
+    """Replay the final parquet tree into *write_methods*; return what was written.
+
+    Reads ``<out_root>/output_parquet/<scenario>/`` (the surviving output of the
+    last calibration solve) and emits each requested format alongside it —
+    ``output_csv/<scenario>/``, ``output_excel/output_<scenario>.xlsx``,
+    ``output_plots/<scenario>/`` and/or a results SpineDB — with **no** solve.
+
+    Parameters
+    ----------
+    out_root:
+        The calibration ``--output-location`` root: the parent that holds
+        ``output_parquet/`` (and gains the sibling ``output_csv/`` … trees).
+    scenario:
+        Scenario name; also the ``output_parquet`` sub-folder to read back.
+    write_methods:
+        Which formats to regenerate (a subset of
+        :data:`FINAL_WRITE_METHOD_CHOICES`).  Empty ⇒ nothing to do.
+    results_db_url:
+        Target SpineDB URL for the ``spinedb`` method; defaults inside the
+        engine to ``<out_root>/results.sqlite`` when omitted.
+
+    Returns
+    -------
+    The list of methods actually written (empty when *write_methods* is empty).
+
+    Raises
+    ------
+    FileNotFoundError
+        If the expected ``output_parquet/<scenario>/`` directory is absent —
+        i.e. there is no final solve to replay.
+    """
+    methods = [m for m in write_methods if m]
+    if not methods:
+        return []
+
+    out_root = Path(out_root)
+    parquet_dir = out_root / "output_parquet" / scenario
+    if not parquet_dir.is_dir():
+        raise FileNotFoundError(
+            f"no parquet output to replay for scenario '{scenario}': "
+            f"expected {parquet_dir}/ (did the final solve write outputs?)"
+        )
+
+    # Deferred import: write_outputs pulls in the plotting/Excel stack, which we
+    # do not want to load unless the operator actually asked for final outputs.
+    from flextool.process_outputs.write_outputs import write_outputs
+
+    write_outputs(
+        scenario_name=scenario,
+        output_location=str(out_root),
+        # subdir is the folder name under output_parquet/ — the scenario — so
+        # the replay reads exactly the tree the calibration loop left behind.
+        subdir=scenario,
+        read_parquet_dir=True,
+        write_methods=methods,
+        # Belt-and-suspenders: keep output_location non-empty even if the engine
+        # ever tries to fall back to settings resolution on the replay path.
+        fallback_output_location=str(out_root),
+        results_db_url=results_db_url,
+    )
+    return methods
+
+
+__all__ = ["FINAL_WRITE_METHOD_CHOICES", "write_final_outputs"]

flextool/calibrate/_guard.py

@@ -0,0 +1,151 @@
+"""Over-build guard for the adequacy calibrator (C1c) — a PURE decision.
+
+The sizer (C1b) keeps bumping ``energy_margin_adder`` on every node that
+still sheds unserved energy.  On a node whose shortfall is genuinely
+resource-capped — no more firm capacity, imports, or storage can be built to
+serve it — bumping the margin further just injects demand the solve cannot
+serve, and (worse) concentrating demand on an unresponsive node drives the
+solve toward infeasibility.  This module decides, each correction, which
+shedding nodes to KEEP bumping and which to FREEZE and FLAG as
+resource-capped so the operator surfaces them for firm capacity / imports /
+storage instead of more demand margin.
+
+The observable that IS present
+------------------------------
+Earlier revisions gated the freeze on margin-induced CURTAILMENT rising AT
+the shedding node (``C_k > C_0``).  That is the WRONG observable for the
+calibrator's real targets: demand nodes never curtail (VRE spill is upstream),
+so ``C_k > C_0`` is permanently FALSE at a demand node and the guard could
+NEVER flag it — while it kept concentrating demand there until the solve went
+infeasible.  The signature that IS present at a resource-capped demand node is
+a residual that stays FLAT despite its adder rising: the margin is buying no
+adequacy.  So the freeze keys off the residual RESPONSE to the bump, not
+curtailment.
+
+The rule, per node ``n``, comparing the just-solved iteration ``k`` against
+the prior iteration ``k-1``:
+
+* ``ΔSlack(n) = S_{k-1}(n) − S_k(n)``  — unserved energy REDUCED (MWh) by the
+  bump ``n`` received last round.
+
+Freeze ``n`` (and add it to the flagged set) when the PRECONDITION and the
+STALL condition both hold:
+
+* PRECONDITION — ``S_{k-1}(n) > shed_tol``: the node was ALREADY shedding in
+  the prior iteration, so it WAS bumped last round.  The freeze rule asks "did
+  MY prior bump to this node fail to buy adequacy?", which is only meaningful
+  for a node that was actually shedding (and therefore bumped) last round.  A
+  node with ``S_{k-1}(n) ≤ shed_tol`` was NOT shedding last round (so the sizer
+  proposed no bump for it) and only STARTS shedding this round — because margin
+  added to OTHER nodes shifted dispatch onto it.  Such a genuinely short,
+  never-yet-bumped node must get its FIRST bump, never be frozen; without this
+  precondition its ``ΔSlack < 0`` (gap grew from zero) would trip the stall
+  condition (``ΔSlack < stall_fraction · 0 = 0``) and freeze it forever,
+  exactly backwards.  ``shed_tol`` is the SAME per-node shedding tolerance the
+  sizer uses (:func:`._sizing.sized_increments`), threaded in so the two agree
+  on "was this node shedding?".
+
+* STALL — ``ΔSlack(n) < stall_fraction · S_{k-1}(n)``: the residual barely
+  moved (or grew) in response to last round's bump, i.e. the margin bought
+  essentially no adequacy.  ``stall_fraction`` is the CLI ``--stall-fraction``
+  knob (default :data:`_STALLED_GAP_FRACTION` = 0.05): a HIGHER value freezes a
+  stalled node SOONER (demands a larger relative residual drop to keep bumping).
+
+The precondition plus the stall condition are required — deliberately
+conservative, biased to trip one iteration LATE rather than early (every node
+gets at least one bump before it can be flagged, and a little idle capacity
+beats leaving real unserved energy on a node margin CAN still help).  A
+legitimately resource-capped node still gets frozen: it sheds every round (so
+``S_{k-1} > shed_tol`` once it has been bumped) and its residual stalls, so the
+condition trips after it has had its bump(s), and it never runs away to
+infeasibility.  The guard needs a prior iteration to diff, so it can only act
+from the SECOND correction onward; the loop calls it only when a prior record
+exists.  Frozen nodes stay frozen for the rest of the run — the loop persists
+the flagged set and never bumps a flagged node again.
+
+Curtailment plays NO part in the freeze decision (the loop still reads it for
+reporting).  Everything here is pure arithmetic over plain dicts — no I/O — so
+the rule is unit-testable in isolation, which is the point of C1c.
+"""
+
+from __future__ import annotations
+
+# Default fraction of the prior gap below which ΔSlack counts as "stopped
+# moving" (the STALL condition).  Exposed on the CLI as ``--stall-fraction``;
+# this constant is the default the loop threads in.
+_STALLED_GAP_FRACTION = 0.05
+
+# Default per-node shedding tolerance (MWh) for the freeze PRECONDITION —
+# mirrors the sizer's ``_sizing._SHED_TOL_MWH`` so guard and sizer agree on
+# "was this node shedding last round?".  The loop threads the sizer's actual
+# constant in; this default keeps the pure function standalone-testable.
+_DEFAULT_SHED_TOL_MWH = 1e-6
+
+
+def guard_freeze(
+    candidate: dict[str, float],
+    *,
+    residual: dict[str, float],
+    prev_residual: dict[str, float],
+    stall_fraction: float = _STALLED_GAP_FRACTION,
+    shed_tol: float = _DEFAULT_SHED_TOL_MWH,
+) -> tuple[dict[str, float], set[str]]:
+    """Freeze resource-capped nodes out of *candidate* increments.
+
+    Applies the freeze rule (module docstring) to each node *candidate*
+    proposes to bump, using the current iteration ``k`` residual (*residual*)
+    and the prior iteration ``k-1`` residual (*prev_residual*).  A node is
+    frozen (and flagged) when it was shedding last round
+    (``prev_residual[node] > shed_tol``) AND its residual failed to respond to
+    that bump (``ΔSlack < stall_fraction · prev_residual[node]``).
+    Curtailment is NOT consulted — a demand node never curtails, so keying the
+    freeze on curtailment could never flag it (the bug this replaces).
+
+    Parameters
+    ----------
+    candidate
+        ``{node: increment}`` the sizer proposes for this correction (already
+        stripped of persistently-flagged nodes by the caller).
+    residual, prev_residual
+        This iteration's and the prior iteration's per-node residual unserved
+        energy (MWh).  A node absent from either contributes ``0.0``.
+    stall_fraction
+        Fraction of the prior gap below which ΔSlack counts as stalled
+        (``config.stall_fraction`` / ``--stall-fraction``).  HIGHER freezes a
+        stalled node sooner.
+    shed_tol
+        Per-node shedding tolerance (MWh).  A node is eligible for freezing
+        only if it was shedding LAST round (``prev_residual[node] > shed_tol``);
+        a node at/below it was not bumped last round and must get its first
+        bump.  Threaded from the sizer so the two agree (default matches
+        :data:`._sizing._SHED_TOL_MWH`).
+
+    Returns
+    -------
+    tuple[dict[str, float], set[str]]
+        ``(kept, newly_flagged)`` — *candidate* with the frozen nodes removed,
+        and the set of nodes flagged this round.
+    """
+    kept: dict[str, float] = {}
+    newly_flagged: set[str] = set()
+    for node, inc in candidate.items():
+        s_k = residual.get(node, 0.0)
+        s_prev = prev_residual.get(node, 0.0)
+        delta_slack = s_prev - s_k
+
+        # PRECONDITION: only a node that was shedding last round was bumped
+        # last round, so the freeze question is only meaningful for it.  A
+        # newly-shedding node (s_prev <= shed_tol) must get its first bump.
+        cond_was_shedding = s_prev > shed_tol
+        # STALL: the residual barely moved despite the bump → margin buys no
+        # adequacy at this (resource-capped) node.
+        cond_stalled = delta_slack < stall_fraction * s_prev
+
+        if cond_was_shedding and cond_stalled:
+            newly_flagged.add(node)
+        else:
+            kept[node] = inc
+    return kept, newly_flagged
+
+
+__all__ = ["guard_freeze"]