flextool 4.0.0__py3-none-any.whl

flextool/engine_polars/_solve_handoff.py

@@ -0,0 +1,164 @@
+"""In-memory carrier of state passed between solves.
+
+Home of the canonical :class:`SolveHandoff` dataclass — the typed
+record of "what one solve produced that the next solve(s) consume".
+Built post-solve by :func:`flextool.engine_polars.input.build_handoff_from_solution`
+directly from the polar_high ``Solution`` object; consumed by the next
+sub-solve via the orchestrator's iteration-start translator
+(:func:`_provider_translators.translate_handoff_to_provider`) which
+fans each field into the Provider under a ``handoff/<field>`` key.
+
+Phase 3 of ``specs/provider_consolidation.md`` retired the legacy
+``capture_post_solve()`` disk-read constructor — the cascade was
+already using ``build_handoff_from_solution`` exclusively; the
+capture-from-disk function was dead code and has been removed,
+together with the three carrier fields it was the only populator
+for (``fix_storage_timesteps``, ``ed_history_realized_first``,
+``edd_history``).  Consumers of those fields already fell through
+to Provider/CSV reads when the carrier was ``None`` (the universal
+state in the cascade path).
+
+Carrier schemas (each is the on-disk equivalent's columns, renamed
+for in-memory uniformity — string keys + ``value`` for single-value
+carriers, named metric columns for multi-value):
+
+    realized_invest        [entity, period, value]
+    realized_existing      [entity, period, value]
+    divest_cumulative      [entity, value]
+    roll_end_state         [node, value]
+    fix_storage_quantity   [node, period, step, p_fix_storage_quantity]
+    fix_storage_price      [node, period, step, p_fix_storage_price]
+    fix_storage_usage      [node, period, step, p_fix_storage_usage]
+    cumulative_co2         [group, period, value]
+    cumulative_commodity   [commodity, tier, period, p_ladder_cum_realized_mwh]
+    cum_sim_hours          [period, p_ladder_cum_sim_hours]
+
+Phase 4.1a moved ``cumulative_commodity`` and ``cum_sim_hours`` to their
+canonical column names (matching the ``solve_data/`` Provider key
+schemas) so the iteration-start handoff translator can route the
+frames straight through to ``handoff/cumulative_commodity`` /
+``handoff/cum_sim_hours`` without a per-iteration rename.
+
+(Δ.1 — ``periods_already_emitted`` was previously listed here; it
+moved to ``_output_writer.OutputWriterState`` since it gates writer-
+side emission and isn't a true solver-handoff carrier.)
+
+The three ``fix_storage_*`` narrow carriers replaced the legacy
+single wide ``fix_storage`` field (retired in Phase 4.1l).  Each
+metric is independent, so a separate per-metric carrier matches the
+producer/consumer reality without NULL-padding columns.
+
+``realized_invest`` and ``realized_existing`` together cover the two
+columns of ``solve_data/p_entity_period_existing_capacity.csv``:
+``realized_invest`` is what was *built this solve*, ``realized_existing``
+is the resolved existing-capacity history (pre-existing decay +
+divest enter this column and aren't reconstructible from
+``realized_invest`` alone).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import polars as pl
+
+
+
+@dataclass
+class SolveHandoff:
+    """Per-solve output → dependent-solve input carrier.
+
+    Each field is a polars DataFrame in the schema documented in this
+    module's docstring, or ``None`` when that carrier kind isn't active
+    for this handoff.  Solves are identified by full solve name; the
+    handoff represents the *output* of one solve becoming the *input*
+    to its dependent (child / next-roll) solves.
+    """
+
+    # Realized invest *built this solve* in absolute units (post-unitsize).
+    # Producer: any solve with v_invest > 0.
+    # Consumer: subsequent solves' preprocessing.
+    # File equivalent: ``p_entity_period_invested_capacity`` column of
+    #   solve_data/p_entity_period_existing_capacity.csv.
+    realized_invest: pl.DataFrame | None = None
+
+    # Realized existing-capacity history per (entity, period) — captures
+    # pre-existing decay + divest that ``realized_invest`` doesn't.
+    # File equivalent: ``p_entity_period_existing_capacity`` column of
+    #   solve_data/p_entity_period_existing_capacity.csv.
+    realized_existing: pl.DataFrame | None = None
+
+    # Cumulative divested capacity per entity (scalar, not per-period).
+    # Carries pre-existing decay forward across solves.
+    # File equivalent: solve_data/p_entity_divested.csv.
+    divest_cumulative: pl.DataFrame | None = None
+
+    # End-of-roll storage state for ``bind_forward_only`` carry-over.
+    # Producer: prior roll's v_state at its last (d, t).
+    # Consumer: next roll's nodeBalance_eq first-timestep term.
+    # File equivalent: solve_data/p_roll_continue_state.csv.
+    roll_end_state: pl.DataFrame | None = None
+
+    # Upward feedback carrier: nested dispatch sub-solve's realized
+    # end-of-horizon v_state routed UPWARD to its parent storage
+    # solve's next roll.  When a dispatch sub-solve completes, the
+    # parent storage solve's next roll prefers this realized state
+    # over its own previously-predicted state — closing the loop
+    # between the lower-information storage plan and the higher-
+    # information dispatch result.  Always-on for any storage→dispatch
+    # nesting; no opt-in flag (per specs/feature_fixes.md §1).
+    # Schema mirrors ``roll_end_state``: ``[node, value]``.
+    # Producer: dispatch sub-solve's roll_end_state (copy at handoff
+    # capture; no separate v_state extraction needed).
+    # Consumer: parent storage's continuation-roll
+    # ``p_roll_continue_state`` (preferred over sequential prior when
+    # available).
+    # No file equivalent (in-memory only).
+    upward_roll_end_state: pl.DataFrame | None = None
+
+    # Storage quota imposed by parent solve on child solve — narrow
+    # per-metric carriers in canonical column schema.  The trio is
+    # independent (parent may set quantity-only, price-only, or any
+    # combination); each metric travels in its own field.
+    # Producer: parent solve's v_state + cost duals.
+    # Consumer: child's fix_storage_* constraints.
+    # File equivalents: solve_data/fix_storage_{quantity,price,usage}.csv
+    # Each schema: ``[node, period, step, p_fix_storage_<metric>]``.
+    fix_storage_quantity: pl.DataFrame | None = None
+    fix_storage_price: pl.DataFrame | None = None
+    fix_storage_usage: pl.DataFrame | None = None
+
+    # Running CO2 totals carried across rolls for cumulative-cap constraint.
+    # File equivalent: solve_data/co2_cum_realized_tonnes.csv.
+    cumulative_co2: pl.DataFrame | None = None
+
+    # Running per-tier commodity consumption for cumulative ladder pricing.
+    # File equivalent: solve_data/commodity_ladder_cumulative.csv (per-tier mwh).
+    cumulative_commodity: pl.DataFrame | None = None
+
+    # Running simulated-hour total per period.  Independent 1-D
+    # carrier shared by ladder + CO2-cap constraints.
+    # File equivalent: solve_data/ladder_cum_sim_hours.csv.
+    cum_sim_hours: pl.DataFrame | None = None
+
+    _FIELDS = (
+        "realized_invest", "realized_existing", "divest_cumulative",
+        "roll_end_state", "upward_roll_end_state",
+        "fix_storage_quantity", "fix_storage_price", "fix_storage_usage",
+        "cumulative_co2", "cumulative_commodity", "cum_sim_hours",
+    )
+
+    def is_empty(self) -> bool:
+        """True when no carrier is populated."""
+        return all(getattr(self, f) is None for f in self._FIELDS)
+
+
+# Phase 4.1i — ``write_fix_storage_files_from_handoff`` was retired
+# once all readers of ``solve_data/fix_storage_*`` migrated to the
+# per-metric ``handoff/*`` Provider keys seeded by the
+# iteration-start translator (Phases 4.1f–4.1h).  The wide → narrow
+# CSV fan-out has no consumers.
+
+
+__all__ = [
+    "SolveHandoff",
+]

flextool/engine_polars/_solve_state.py

@@ -0,0 +1,232 @@
+"""Solve-orchestration state types.
+
+Foundation module: every downstream orchestration module (timeline,
+recursive solve, stochastic, orchestration loop) needs the exception
+classes, ``ActiveTimeEntry`` namedtuple, ``SolveResult`` dataclass and
+a slim ``RunnerState`` carrier.  Runs natively on HiGHS via
+``polar_high``; per-CLI flags live on the CLI wrapper rather than in
+shared state.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, NamedTuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import polars as pl
+
+    from flextool.cli._timing import TimingRecorder
+    from flextool.engine_polars._solve_config import SolveConfig
+    from flextool.engine_polars._solve_handoff import SolveHandoff
+
+
+# ---------------------------------------------------------------------------
+# Exception types
+# ---------------------------------------------------------------------------
+
+
+class FlexToolError(Exception):
+    """Base exception for FlexTool runner errors."""
+
+
+class FlexToolConfigError(FlexToolError):
+    """Raised for configuration / input data errors."""
+
+
+class FlexToolSolveError(FlexToolError):
+    """Raised for solver execution errors."""
+
+
+# ---------------------------------------------------------------------------
+# Lightweight value types
+# ---------------------------------------------------------------------------
+
+
+class ActiveTimeEntry(NamedTuple):
+    """A single timestep in an active time list.
+
+    Backwards-compatible with the previous ``(timestep, index, duration)``
+    tuples — ``entry[0]`` still works, but ``entry.timestep`` is preferred.
+    """
+
+    timestep: str
+    index: int
+    duration: str
+
+
+@dataclass
+class SolveResult:
+    """Result container for the recursive solve structure builder.
+
+    Populated by the recursive solve builder (Γ.8.C) and consumed by the
+    orchestration loop (Γ.8.D).
+    """
+
+    solves: list = field(default_factory=list)
+    complete_solves: dict = field(default_factory=dict)
+    active_time_lists: dict = field(default_factory=dict)
+    fix_storage_time_lists: dict = field(default_factory=dict)
+    realized_time_lists: dict = field(default_factory=dict)
+    parent_roll_lists: dict = field(default_factory=dict)
+
+
+@dataclass
+class PathConfig:
+    """Directory layout for a FlexTool run.
+
+    Carries the work folder plus the optional ancillary directories
+    :class:`FlexToolRunner` resolves (a package data dir, a
+    CLI-overrideable ``solver_config/`` for ``highs.opt`` and a project
+    root).  Native engine_polars callers only populate ``work_folder``;
+    the rest are ``None``.
+    """
+
+    work_folder: Path
+    output_path: Path | None = None
+    flextool_dir: Path | None = None
+    solver_config_dir: Path | None = None
+    root_dir: Path | None = None
+
+
+@dataclass
+class RunnerState:
+    """Cross-cutting state for a native polar_high solve run.
+
+    Only the fields needed by Γ.8.A are populated.  Timeline + handoff
+    fields will be filled in Γ.8.B / Γ.8.D as those modules land.
+    """
+
+    paths: PathConfig
+    solve: "SolveConfig"
+    logger: logging.Logger
+    # Filled by Γ.8.B (timeline module).  Typed as ``object`` so that
+    # importing :class:`RunnerState` doesn't pull a non-existent module
+    # into the import graph.
+    timeline: object | None = None
+    # Agent 8 (LP-scaling): opt-in flag — when True the Python
+    # ScaleAnalyzer's recommendations are auto-applied.  Batch C.10
+    # removed the DB-stored ``use_row_scaling`` knob; the per-solve
+    # row-scaling toggle is now driven entirely by --scaling CLI +
+    # this auto_scale flag (or FLEXTOOL_FORCE_ROW_SCALING test hook).
+    # Always-False in the default path preserves pre-Agent-8 behaviour.
+    auto_scale: bool = False
+    # Roll-loop scratch — set by the orchestration loop just before
+    # ``solver.run`` so per-roll diagnostics / handoff capture can find
+    # the correct row.  ``None`` outside an active solve iteration.
+    current_roll_index: int | None = None
+    # Agent 18c (LP-scaling): the orchestration loop sets this to the
+    # ``ScaleTable`` for the currently-active solve just before calling
+    # ``solver.run``.  ``_run_highs`` uses it to update bound-scaling
+    # diagnostics in the right cache entry even when the roll name
+    # differs from the parent (complete) solve name passed to the
+    # solver.  ``None`` outside an active solve iteration.
+    current_scale_solve_name: str | None = None
+    # Name of the most-recent solve whose post-solve hook deposited a
+    # ``SolveHandoff`` into ``handoffs``.  Set by ``orchestration.run_model``
+    # after each capture; consulted by post-solve writers (e.g. the
+    # cumulative-handoff writers in ``solver_runner._run_highs``) to
+    # source prior-roll state from the in-memory dict instead of disk.
+    # ``None`` outside an active solve loop and on the first solve of
+    # any loop.
+    last_captured_solve: str | None = None
+    # In-memory solve-to-solve handoff.  ``None`` keeps file-based
+    # behaviour; opt-in by setting ``state.handoffs = {}``.  See
+    # ``audit/handoff_csv_retirement.md`` for the migration plan.
+    handoffs: "dict[str, SolveHandoff] | None" = None
+    # Phase 5b — external override provider.  When set, the runner
+    # invokes this callable at iteration start (after the sequential
+    # + parent handoff translators) and fans the returned dict into
+    # the ``override/*`` Provider namespace via
+    # :func:`flextool.engine_polars._provider_translators.translate_overrides_to_provider`.
+    # The callable is owned by external code wrapping the runner
+    # (e.g. file-watch, ZeroMQ bridge); ``None`` means no overrides.
+    override_provider: Callable[[], "dict[str, pl.DataFrame]"] | None = None
+    # Per-CLI-invocation phase timing recorder.  The CLI constructs one
+    # very early in ``cmd_run_flextool.main`` and assigns it onto
+    # ``state.timing_recorder``; callers using :class:`FlexToolRunner`
+    # directly (without going through the CLI) bootstrap their own in
+    # ``FlexToolRunner.__init__``.  Always non-None inside an active run.
+    timing_recorder: "TimingRecorder | None" = None
+    # HiGHS thread count (CLI override; solver_runner defaults to 4 when None).
+    highs_threads: int | None = None
+    # Gates ``data.dump_csvs`` from inside the cascade — set by
+    # ``run_orchestration`` from its ``csv_dump`` argument.
+    csv_dump: bool = False
+    # Seeded cascade-input Provider — set by ``write_input`` / by the
+    # orchestration entry point so per-sub-solve Providers can clone
+    # the ``input/<class>`` frames.  Typed as ``object`` to avoid
+    # pulling :class:`FlexDataProvider` into the import graph.
+    cascade_input_provider: object | None = None
+    # Per-sub-solve Provider currently driving the cascade.  Set by
+    # ``_native_run_model`` immediately before each ``solver.run``
+    # invocation; consumed by post-solve writers that need a Provider
+    # handle but were called without one.  ``None`` outside an active
+    # solve iteration.
+    current_provider: object | None = None
+    # Per-level Provider cache — keyed by :func:`compute_level_key`.
+    # The orchestration loop populates this so sub-solves at the same
+    # "level" (matching LP matrix shape) share a :class:`FlexDataProvider`.
+    # Typed as ``dict | None``; ``None`` means "not yet initialised".
+    _level_providers: dict | None = None
+
+
+# ---------------------------------------------------------------------------
+# Per-level Provider — level-key helper (Design A, step A1).
+# ---------------------------------------------------------------------------
+
+
+def compute_level_key(
+    *,
+    solve_name: str,
+    complete_solve_name: str,
+    solve_config,
+    timeline_config,
+) -> tuple:
+    """Compute a cheap level identifier for a sub-solve.
+
+    Two sub-solves with the same level_key share LP matrix shape and
+    can share a :class:`FlexDataProvider` (per the user's per-level
+    Provider intent).  Two sub-solves with different keys must not
+    share.
+
+    Composition (in order):
+
+    1. Tuple of timesets used by ``complete_solve_name``
+       (``solve_config.timesets_used_by_solves[complete_solve_name]``,
+       sorted for determinism).
+    2. ``timeline_config.new_step_durations.get(complete_solve_name)``
+       — the explicit step-size override (1h, 3h, …).  ``None`` when
+       absent.
+    3. ``solve_config.rolling_times.get(complete_solve_name)`` — the
+       rolling-window triple ``[jump, horizon, duration]``, coerced to
+       a tuple.  ``None`` when not rolling.
+    4. ``solve_config.solve_modes.get(complete_solve_name)``.
+
+    Returns a hashable tuple.  Same key on consecutive iterations
+    means "same level — reuse Provider"; different key means "level
+    transition — fresh Provider".
+    """
+    timesets = solve_config.timesets_used_by_solves.get(
+        complete_solve_name, ()
+    )
+    timesets = tuple(sorted(timesets)) if timesets else ()
+    step_dur = timeline_config.new_step_durations.get(complete_solve_name)
+    rolling = solve_config.rolling_times.get(complete_solve_name)
+    if rolling is not None:
+        rolling = tuple(rolling)
+    mode = solve_config.solve_modes.get(complete_solve_name)
+    return (timesets, step_dur, rolling, mode)
+
+
+__all__ = [
+    "FlexToolError",
+    "FlexToolConfigError",
+    "FlexToolSolveError",
+    "ActiveTimeEntry",
+    "SolveResult",
+    "PathConfig",
+    "RunnerState",
+    "compute_level_key",
+]

flextool/engine_polars/_solver_base.py

@@ -0,0 +1,36 @@
+"""SolverRunner shell — base class for the native cascade's solver subclasses.
+
+The cascade's ``_PolarHighCascadeSolver`` / ``_NoOpSolver`` in
+:mod:`flextool.engine_polars._orchestration` extend :class:`SolverRunner`
+and override ``run`` to drive the polar-high LP build/solve in-process.
+Direct instantiation + ``run()`` is not supported.
+"""
+from __future__ import annotations
+
+
+class SolverRunner:
+    """Minimal shell that the native cascade's solver subclasses extend.
+
+    ``__init__`` stores ``state`` and ``logger`` for the subclasses,
+    matching the contract those subclasses already rely on via
+    ``super().__init__(runner_state)``.  ``run`` is unimplemented by
+    design — the cascade subclasses override it.
+    """
+
+    def __init__(self, state) -> None:
+        self.state = state
+        self.logger = state.logger
+
+    def run(self, current_solve: str) -> int:  # noqa: ARG002
+        """Unimplemented — subclasses override this method.
+
+        Direct invocation on the base class is not supported.
+        """
+        raise NotImplementedError(
+            "SolverRunner.run is unimplemented on the base class.  "
+            "The cascade subclasses this class and overrides run(); "
+            "direct invocation is not supported."
+        )
+
+
+__all__ = ["SolverRunner"]