flextool 4.0.0__py3-none-any.whl

flextool/engine_polars/_provider_keys.py

@@ -0,0 +1,173 @@
+"""Canonical Provider key constants.
+
+Centralises the parent-qualified Provider keys
+(``"solve_data/<basename>"``, ``"input/<basename>"``, etc.) used by
+emit_* writers, cascade readers, and orchestration code.
+
+Why this module exists
+----------------------
+
+Phase 0a of ``specs/provider_consolidation.md`` retired the dual-key
+registration in :func:`_emit_provider_io._emit`.  Every frame lives
+under exactly one Provider key — the parent-qualified form.  Stringly-
+typed lookups (``provider.get("solve_data/active_timelines")``) are
+typo-prone: a rename in one site without the other becomes a silent
+``None`` return that the cascade tolerates by emitting an empty frame,
+which then surfaces as wrong numbers downstream.
+
+This module defines named Python constants for every key the cascade
+emits or consumes.  Producers do ``provider.put(K.SOLVE_DATA_ACTIVE_TIMELINES, df)``;
+consumers do ``provider.get(K.SOLVE_DATA_ACTIVE_TIMELINES)``.  IDE
+rename / Find-Usages stays sound; a typo is a ``NameError`` at import
+time, not a silent runtime miss.
+
+Naming convention
+-----------------
+
+* The canonical form is **``"<parent>/<basename>"``** — no ``.csv``
+  suffix.  Phase 3b retired the ``.csv`` suffix from key names: it
+  reinforced a misleading "this is a file" mental model, even though
+  the Provider stores frames in memory and strips ``.csv`` at lookup
+  time anyway.  ``snapshot_processed_inputs`` adds the ``.csv``
+  suffix on disk dump.
+* Constant names are uppercased, underscore-separated, prefixed with
+  the parent: ``SOLVE_DATA_ACTIVE_TIMELINES``, ``INPUT_TIMELINE``,
+  ``DERIVED_CT_METHOD_OVERRIDES``, ``HANDOFF_REALIZED_INVEST``.
+* Group constants by parent dir, then alphabetically within the group.
+* New keys: add the constant here first, then use it from the call sites.
+
+Coverage scope
+--------------
+
+This module currently covers the **cross-solve carrier set**, the
+**handoff carriers** (Phase 2 translator), and other high-traffic keys
+touched by orchestration and the ``_solve_context`` loaders.  The
+single-use ``solve_data/*`` emit-and-forget keys (most of
+:func:`_flex_data_accumulator.expected_basenames`) still appear as
+inline string literals at their producer call sites — migrating them
+to constants is a mechanical follow-up that can land in batches
+without changing semantics.
+"""
+from __future__ import annotations
+
+
+SOLVE_DATA_FIX_STORAGE_QUANTITY = "solve_data/fix_storage_quantity"
+SOLVE_DATA_FIX_STORAGE_PRICE = "solve_data/fix_storage_price"
+SOLVE_DATA_FIX_STORAGE_USAGE = "solve_data/fix_storage_usage"
+SOLVE_DATA_P_ENTITY_PRE_EXISTING = "solve_data/p_entity_pre_existing"
+SOLVE_DATA_P_ENTITY_INVESTED = "solve_data/p_entity_invested"
+SOLVE_DATA_P_ENTITY_DIVESTED = "solve_data/p_entity_divested"
+SOLVE_DATA_P_ENTITY_PERIOD_EXISTING_CAPACITY = (
+    "solve_data/p_entity_period_existing_capacity"
+)
+SOLVE_DATA_P_ROLL_CONTINUE_STATE = "solve_data/p_roll_continue_state"
+SOLVE_DATA_CO2_CUM_REALIZED_TONNES = "solve_data/co2_cum_realized_tonnes"
+SOLVE_DATA_LADDER_CUM_SIM_HOURS = "solve_data/ladder_cum_sim_hours"
+SOLVE_DATA_LADDER_CUM_REALIZED_MWH = "solve_data/ladder_cum_realized_mwh"
+SOLVE_DATA_ED_HISTORY_REALIZED = "solve_data/ed_history_realized"
+SOLVE_DATA_ED_HISTORY_REALIZED_FIRST = "solve_data/ed_history_realized_first"
+SOLVE_DATA_EDD_HISTORY = "solve_data/edd_history"
+
+
+# ---------------------------------------------------------------------------
+# Frequently-referenced solve_data/ keys — touched by multiple cascade
+# modules.  Add to this section as call sites accumulate.
+# ---------------------------------------------------------------------------
+
+SOLVE_DATA_ACTIVE_TIMELINES = "solve_data/active_timelines"
+SOLVE_DATA_INVEST_PERIODS_OF_CURRENT_SOLVE = (
+    "solve_data/invest_periods_of_current_solve"
+)
+SOLVE_DATA_PROCESS_SOURCE_TOSINK = "solve_data/process_source_toSink"
+SOLVE_DATA_REGION_COUPLING = "solve_data/region_coupling"
+
+
+# ---------------------------------------------------------------------------
+# RP-blended-weights keys — populated by ``_emit_solve_writers._compute_rp_frames``
+# (the six derived sets/params) plus the node-state RP subset (``_emit_leaf_sets``)
+# and the per-solve base-period set (``_emit_per_solve``).  Consumed by the
+# storage-binding-method machinery once the model-side wiring lands.
+# ---------------------------------------------------------------------------
+
+SOLVE_DATA_NODE_STATE_RP = "solve_data/nodeState_rp"
+SOLVE_DATA_RP_BASE_CHAIN = "solve_data/rp_base_chain"
+SOLVE_DATA_RP_BASE_FIRST = "solve_data/rp_base_first"
+SOLVE_DATA_RP_BASE_LAST = "solve_data/rp_base_last"
+SOLVE_DATA_RP_BASE_PERIOD_SET = "solve_data/rp_base_period_set"
+SOLVE_DATA_RP_BLOCK_FIRST = "solve_data/rp_block_first"
+SOLVE_DATA_RP_BLOCK_LAST = "solve_data/rp_block_last"
+SOLVE_DATA_RP_BLOCK_START_LAST = "solve_data/rp_block_start_last"
+SOLVE_DATA_RP_REP_PERIOD_SET = "solve_data/rp_rep_period_set"
+SOLVE_DATA_RP_WEIGHTS = "solve_data/rp_weights"
+
+
+# ---------------------------------------------------------------------------
+# input/ keys — populated by ``seed_provider_from_dir(kind="input")`` and
+# the input_derivation modules.  Subset listed; expand as call sites grow.
+# ---------------------------------------------------------------------------
+
+INPUT_COMMODITY_LADDER_ANNUAL = "input/commodity_ladder_annual"
+INPUT_COMMODITY_LADDER_CUMULATIVE = "input/commodity_ladder_cumulative"
+INPUT_CONNECTION_DC_POWER_FLOW = "input/connection_dc_power_flow"
+INPUT_GROUP__CO2_METHOD = "input/group__co2_method"
+INPUT_NODE_DC_POWER_FLOW = "input/node_dc_power_flow"
+INPUT_NODE_REFERENCE_ANGLE = "input/node_reference_angle"
+INPUT_P_CONNECTION_SUSCEPTANCE = "input/p_connection_susceptance"
+INPUT_PERIODS_AVAILABLE = "input/periods_available"
+INPUT_PROCESS_METHOD = "input/process_method"
+
+
+# ---------------------------------------------------------------------------
+# derived/ keys — populated by input_derivation between Spine read and
+# cascade build.  Distinct from input/ so derivation intermediates don't
+# collide with raw user input.
+# ---------------------------------------------------------------------------
+
+DERIVED_CT_METHOD_OVERRIDES = "derived/ct_method_overrides"
+
+
+# ---------------------------------------------------------------------------
+# handoff/ keys — populated by ``_provider_translators.translate_handoff_to_provider``
+# at iteration start, from the previous sub-solve's ``SolveHandoff``.
+# One key per consumed handoff field; empty header-only frame written
+# when the corresponding handoff field is ``None`` (first sub-solve, or
+# carrier not active for the previous solve).  Consumers should treat
+# ``provider.get(K.HANDOFF_X).height == 0`` as the "no prior carrier"
+# signal instead of checking the SolveHandoff object directly.
+# ---------------------------------------------------------------------------
+
+HANDOFF_REALIZED_INVEST = "handoff/realized_invest"
+HANDOFF_REALIZED_EXISTING = "handoff/realized_existing"
+HANDOFF_DIVEST_CUMULATIVE = "handoff/divest_cumulative"
+HANDOFF_ROLL_END_STATE = "handoff/roll_end_state"
+HANDOFF_UPWARD_ROLL_END_STATE = "handoff/upward_roll_end_state"
+HANDOFF_CUMULATIVE_CO2 = "handoff/cumulative_co2"
+HANDOFF_CUMULATIVE_COMMODITY = "handoff/cumulative_commodity"
+HANDOFF_CUM_SIM_HOURS = "handoff/cum_sim_hours"
+HANDOFF_FIX_STORAGE_QUANTITY = "handoff/fix_storage_quantity"
+HANDOFF_FIX_STORAGE_PRICE = "handoff/fix_storage_price"
+HANDOFF_FIX_STORAGE_USAGE = "handoff/fix_storage_usage"
+
+
+# ---------------------------------------------------------------------------
+# override/ keys — parallel to handoff/* and populated by
+# ``_provider_translators.translate_overrides_to_provider`` from an
+# external overrides dict (Phase 5 of specs/provider_consolidation.md).
+# When present, ``read_handoff_frame`` returns the override frame in
+# preference to the natural handoff carrier, so existing consumers pick
+# up overrides without code changes.  Each override key shadows the
+# correspondingly-named ``HANDOFF_*`` key; only the existing handoff
+# carriers are overridable.
+# ---------------------------------------------------------------------------
+
+OVERRIDE_REALIZED_INVEST = "override/realized_invest"
+OVERRIDE_REALIZED_EXISTING = "override/realized_existing"
+OVERRIDE_DIVEST_CUMULATIVE = "override/divest_cumulative"
+OVERRIDE_ROLL_END_STATE = "override/roll_end_state"
+OVERRIDE_UPWARD_ROLL_END_STATE = "override/upward_roll_end_state"
+OVERRIDE_CUMULATIVE_CO2 = "override/cumulative_co2"
+OVERRIDE_CUMULATIVE_COMMODITY = "override/cumulative_commodity"
+OVERRIDE_CUM_SIM_HOURS = "override/cum_sim_hours"
+OVERRIDE_FIX_STORAGE_QUANTITY = "override/fix_storage_quantity"
+OVERRIDE_FIX_STORAGE_PRICE = "override/fix_storage_price"
+OVERRIDE_FIX_STORAGE_USAGE = "override/fix_storage_usage"

flextool/engine_polars/_provider_translators.py

@@ -0,0 +1,225 @@
+"""Translators that fan typed orchestrator carriers into Provider entries.
+
+Phase 2 of ``specs/provider_consolidation.md`` retires the
+``prior_handoff`` parameter threading through preprocessing.  Instead of
+passing the typed :class:`SolveHandoff` dataclass alongside the Provider
+into every preprocessing call site, the orchestrator translates each
+consumed handoff field into a dedicated Provider key
+(``handoff/<field>.csv``) at iteration start.  Preprocessing consumers
+then read via ``provider.get(K.HANDOFF_X)`` — one read interface, no
+parameter threading.
+
+The translator writes an **empty header-only frame** when a handoff
+field is ``None`` so consumers can use a single check
+(``provider.get(K.X).height > 0``) to distinguish "prior carrier
+populated" from "no prior carrier".  This matches the
+``_read_csv``-style empty-frame contract used elsewhere in the cascade.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import polars as pl
+
+from flextool.engine_polars import _provider_keys as K
+
+if TYPE_CHECKING:
+    from flextool.engine_polars._flex_data_provider import FlexDataProvider
+    from flextool.engine_polars._solve_handoff import SolveHandoff
+
+
+# Schemas for each handoff carrier's empty-frame fallback.  Column names
+# match what consumers iter_rows(named=True) keys; Utf8 dtype matches
+# the canonical CSV-roundtrip shape (consumers cast to float on read).
+_HANDOFF_EMPTY_SCHEMAS: dict[str, tuple[str, ...]] = {
+    K.HANDOFF_REALIZED_INVEST:    ("entity", "period", "value"),
+    K.HANDOFF_REALIZED_EXISTING:  ("entity", "period", "value"),
+    K.HANDOFF_DIVEST_CUMULATIVE:  ("entity", "value"),
+    K.HANDOFF_ROLL_END_STATE:     ("node", "value"),
+    K.HANDOFF_UPWARD_ROLL_END_STATE: ("node", "value"),
+    K.HANDOFF_CUMULATIVE_CO2:     ("group", "period", "value"),
+    K.HANDOFF_CUMULATIVE_COMMODITY: ("commodity", "tier", "period",
+                                      "p_ladder_cum_realized_mwh"),
+    K.HANDOFF_CUM_SIM_HOURS:      ("period", "p_ladder_cum_sim_hours"),
+    K.HANDOFF_FIX_STORAGE_QUANTITY: ("node", "period", "step", "p_fix_storage_quantity"),
+    K.HANDOFF_FIX_STORAGE_PRICE:    ("node", "period", "step", "p_fix_storage_price"),
+    K.HANDOFF_FIX_STORAGE_USAGE:    ("node", "period", "step", "p_fix_storage_usage"),
+}
+
+
+def _empty_handoff_frame(key: str) -> pl.DataFrame:
+    cols = _HANDOFF_EMPTY_SCHEMAS[key]
+    return pl.DataFrame(
+        {c: [] for c in cols},
+        schema={c: pl.Utf8 for c in cols},
+    )
+
+
+def translate_handoff_to_provider(
+    handoff: "SolveHandoff | None", provider,
+) -> None:
+    """Write each consumed ``SolveHandoff`` field as a Provider entry.
+
+    Called at iteration start *before* preprocessing runs.  When
+    *handoff* is ``None`` (first sub-solve, or no prior carrier
+    available) every key receives an empty header-only frame so
+    consumers can read unconditionally and check ``height > 0`` to
+    decide whether prior-carrier data is available.
+
+    The translated keys are listed in
+    ``flextool.engine_polars._provider_keys`` under the ``HANDOFF_*``
+    constants.  See :data:`_HANDOFF_EMPTY_SCHEMAS` for column shapes.
+    """
+    field_to_key = (
+        ("realized_invest",     K.HANDOFF_REALIZED_INVEST),
+        ("realized_existing",   K.HANDOFF_REALIZED_EXISTING),
+        ("divest_cumulative",   K.HANDOFF_DIVEST_CUMULATIVE),
+        ("roll_end_state",      K.HANDOFF_ROLL_END_STATE),
+        ("upward_roll_end_state", K.HANDOFF_UPWARD_ROLL_END_STATE),
+        ("cumulative_co2",      K.HANDOFF_CUMULATIVE_CO2),
+        ("cumulative_commodity", K.HANDOFF_CUMULATIVE_COMMODITY),
+        ("cum_sim_hours",       K.HANDOFF_CUM_SIM_HOURS),
+        ("fix_storage_quantity", K.HANDOFF_FIX_STORAGE_QUANTITY),
+        ("fix_storage_price",    K.HANDOFF_FIX_STORAGE_PRICE),
+        ("fix_storage_usage",    K.HANDOFF_FIX_STORAGE_USAGE),
+    )
+    for field_name, key in field_to_key:
+        frame = getattr(handoff, field_name, None) if handoff is not None else None
+        if frame is None:
+            frame = _empty_handoff_frame(key)
+        provider.put(key, frame)
+
+
+# Mapping from user-facing handoff Provider keys to their override
+# siblings.  Phase 5 of specs/provider_consolidation.md: external code
+# supplies overrides keyed by ``K.HANDOFF_X`` (the user-facing identity);
+# ``translate_overrides_to_provider`` writes each frame under the
+# corresponding ``K.OVERRIDE_X`` key, and ``read_handoff_frame`` checks
+# the override slot before falling back to the natural handoff carrier.
+# Only the existing handoff carriers are overridable.
+_HANDOFF_TO_OVERRIDE: dict[str, str] = {
+    K.HANDOFF_REALIZED_INVEST:      K.OVERRIDE_REALIZED_INVEST,
+    K.HANDOFF_REALIZED_EXISTING:    K.OVERRIDE_REALIZED_EXISTING,
+    K.HANDOFF_DIVEST_CUMULATIVE:    K.OVERRIDE_DIVEST_CUMULATIVE,
+    K.HANDOFF_ROLL_END_STATE:       K.OVERRIDE_ROLL_END_STATE,
+    K.HANDOFF_UPWARD_ROLL_END_STATE: K.OVERRIDE_UPWARD_ROLL_END_STATE,
+    K.HANDOFF_CUMULATIVE_CO2:       K.OVERRIDE_CUMULATIVE_CO2,
+    K.HANDOFF_CUMULATIVE_COMMODITY: K.OVERRIDE_CUMULATIVE_COMMODITY,
+    K.HANDOFF_CUM_SIM_HOURS:        K.OVERRIDE_CUM_SIM_HOURS,
+    K.HANDOFF_FIX_STORAGE_QUANTITY: K.OVERRIDE_FIX_STORAGE_QUANTITY,
+    K.HANDOFF_FIX_STORAGE_PRICE:    K.OVERRIDE_FIX_STORAGE_PRICE,
+    K.HANDOFF_FIX_STORAGE_USAGE:    K.OVERRIDE_FIX_STORAGE_USAGE,
+}
+
+
+def translate_overrides_to_provider(
+    overrides: "dict[str, pl.DataFrame] | None", provider,
+) -> None:
+    """Write external overrides under ``override/<field>`` Provider keys.
+
+    Phase 5 of ``specs/provider_consolidation.md``.  External code provides
+    overrides keyed by ``K.HANDOFF_X`` constants (the user-facing identity);
+    this translator writes each frame under the corresponding ``K.OVERRIDE_X``
+    Provider key.  Existing :func:`read_handoff_frame` consumers automatically
+    pick up overrides because the helper checks ``override/<field>`` before
+    falling back to ``handoff/<field>``.
+
+    When *overrides* is ``None`` or empty, no writes are made — the override
+    layer simply isn't present.
+
+    Raises
+    ------
+    ValueError
+        If *overrides* contains a key that is not one of the whitelisted
+        ``K.HANDOFF_*`` constants.  Arbitrary Provider-key writes through
+        this translator are rejected so the override surface stays
+        explicit.
+    """
+    if not overrides:
+        return
+    for handoff_key, frame in overrides.items():
+        override_key = _HANDOFF_TO_OVERRIDE.get(handoff_key)
+        if override_key is None:
+            raise ValueError(
+                f"translate_overrides_to_provider: key {handoff_key!r} is not "
+                f"a whitelisted handoff carrier; only K.HANDOFF_* keys are "
+                f"overridable.  Allowed keys: "
+                f"{sorted(_HANDOFF_TO_OVERRIDE.keys())!r}"
+            )
+        # Phase 6a — tag every override write so the audit dump can
+        # distinguish externally-supplied keys from natural-cascade
+        # writes.  The override key itself is the discriminator at the
+        # dump site, so a single stable tag is sufficient.
+        provider.put(override_key, frame, source="external_override")
+
+
+def read_handoff_frame(provider, key: str) -> "pl.DataFrame | None":
+    """Return a populated handoff frame from the Provider, or ``None``.
+
+    Phase 5: checks ``override/<field>`` first, falls back to
+    ``handoff/<field>``.  External overrides written via
+    :func:`translate_overrides_to_provider` shadow the natural handoff
+    carrier; consumers stay key-stable.
+
+    The translator writes an empty header-only frame when the
+    corresponding handoff field is ``None``; this helper collapses the
+    ``height == 0`` empty back to ``None`` so consumers can preserve
+    their existing ``if frame is not None`` guard pattern.  The same
+    ``height == 0 → None`` semantics apply to the override slot: an
+    empty override frame is treated as "no override present" and we
+    fall through to the handoff carrier.
+    """
+    if provider is None:
+        return None
+    override_key = _HANDOFF_TO_OVERRIDE.get(key)
+    if override_key is not None:
+        frame = provider.get(override_key)
+        if frame is not None and frame.height > 0:
+            return frame
+    frame = provider.get(key)
+    if frame is None or frame.height == 0:
+        return None
+    return frame
+
+
+def dump_provider_sources(
+    provider: "FlexDataProvider",
+    dump_path: "str | Path",
+    solve_name: str,
+) -> None:
+    """Append every source-tagged Provider key to *dump_path*.
+
+    Phase 6b of ``specs/provider_consolidation.md``.  Iterates the
+    Provider's keys; for each key whose :meth:`get_source` returns a
+    non-``None`` tag, appends one tab-separated line in the format::
+
+        <solve_name>\\t<key>\\t<source>\\n
+
+    The file is opened in append mode so multiple sub-solves
+    accumulate in a single log.  The parent directory is created if
+    missing.
+
+    Untagged (natural-cascade) keys are skipped — the dump is a
+    minimal audit trail of writes whose origin differs from the
+    default preprocessing chain, not a snapshot of the full Provider.
+    """
+    path = Path(dump_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # Sorted iteration keeps the log deterministic across runs so the
+    # audit trail diffs cleanly between cascades that differ only in
+    # dict-iteration order.
+    with path.open("a", encoding="utf-8") as fh:
+        for key in sorted(provider.keys()):
+            source = provider.get_source(key)
+            if source is None:
+                continue
+            fh.write(f"{solve_name}\t{key}\t{source}\n")
+
+
+__all__ = [
+    "translate_handoff_to_provider",
+    "translate_overrides_to_provider",
+    "read_handoff_frame",
+    "dump_provider_sources",
+]