flextool 4.0.0__py3-none-any.whl

flextool/engine_polars/_flex_data_accumulator.py

@@ -0,0 +1,388 @@
+"""Per-sub-solve frame capture coverage manifest.
+
+This module hosts :func:`expected_basenames` — the public-contract list
+of CSV basenames the cascade's emit_* functions push into the
+in-memory :class:`FlexDataProvider`.  Tests cross-check this manifest
+against the disk-resident frames produced by ``--csv-dump``.
+
+Prior to the writer→emitter refactor this module also exposed a
+``capture_frames`` context manager that monkey-patched every
+participating writer's ``_write(df, path)`` helper to redirect frames
+into a Provider.  Phase 3 of the refactor migrated every call site to
+``emit_*(..., provider=...)``; the monkey-patch (and the ``_PATCH_MODULES``
+tuple it consumed) is gone.  Only the basename manifest remains.
+"""
+from __future__ import annotations
+
+
+__all__ = [
+    "expected_basenames",
+]
+
+
+# ---------------------------------------------------------------------------
+# Coverage manifest
+# ---------------------------------------------------------------------------
+#
+# Basenames of CSVs the cascade's emit_* functions push into the
+# in-memory :class:`FlexDataProvider`.  This list is the public contract
+# Phase D / E-a consumers read against to know which solve_data/*.csv
+# files the cascade captures.
+#
+# When a new emit_* lands that adds a previously-uncaptured frame, add
+# its target basename below.  The matching test in
+# ``tests/engine_polars/test_phase_c_flex_data_accumulator.py`` cross-
+# checks the in-Provider-vs-disk frames for each basename present in
+# the cascade run.
+
+_THIN_WRAPPER_BASENAMES: tuple[str, ...] = (
+    # _emit_leaf_sets — 27 thin writers
+    "period_group.csv",
+    "period_node.csv",
+    "period_commodity.csv",
+    "period_process.csv",
+    "entityInvest.csv",
+    "entityDivest.csv",
+    "group_invest.csv",
+    "group_divest.csv",
+    "group_co2_price.csv",
+    "group_co2_max_period.csv",
+    "group_co2_max_total.csv",
+    "optional_yes.csv",
+    "reserve__upDown__group.csv",
+    "group_loss_share.csv",
+    "def_optional_yes.csv",
+    "process_delayed.csv",
+    "period_solve.csv",
+    "time.csv",
+    "enable_optional_outputs.csv",
+    "nodeState_rp.csv",
+    "nodeStateBlock.csv",
+    "timeline.csv",
+    "commodity__tier_ann.csv",
+    "solve_period.csv",
+    # _emit_mid_sets — thin writers.
+    #
+    # Phase 4 audit (2026-05-21) corrected the spelling of seven
+    # entries below to match the actual emit keys produced by the
+    # cascade.  The legacy single-underscore names
+    # (``entity_lifetime_method.csv`` etc.) had been stale in this
+    # manifest since pre-Phase-1; the corresponding writers have
+    # always emitted the double-underscore (or camelCase) variant.
+    # Tests never consulted this manifest until Phase 4 introduced
+    # ``test_csv_dump_post_refactor`` so the drift was undetected.
+    "group_entity.csv",
+    "process_delayed__duration.csv",
+    "process__sink_nonSync.csv",
+    "entity__lifetime_method.csv",
+    "process__ct_method.csv",
+    "process__startup_method.csv",
+    "node__inflow_method.csv",
+    "node__storage_binding_method.csv",
+    "node__storage_nested_fix_method.csv",
+    "node__penalty_method.csv",
+    "commodity_node_co2.csv",
+    "process__group_inside_group_nonSync.csv",
+    "process_source_coeff_zero.csv",
+    "process_sink_coeff_zero.csv",
+    # — invest_total_sets
+    "e_invest_total.csv",
+    "e_divest_total.csv",
+    "g_invest_total.csv",
+    "g_divest_total.csv",
+    "g_invest_cumulative.csv",
+    "ci_ladder_cumulative.csv",
+    # _emit_calc_params — thin writers
+    "process_VRE.csv",
+    "process_online_linear.csv",
+    "process_online_integer.csv",
+    "process__method_indirect.csv",
+    # _emit_arc_unions — thin writers + Phase E-b lifted streamed writers
+    "node__TimeParam_in_use.csv",
+    "process_source_delayed.csv",
+    "process_source_undelayed.csv",
+    "process__source__sink__profile__profile_method_connection.csv",
+    "ed_history_realized_first.csv",
+    "process_source_sink_coeff_zero.csv",
+    "process_source_sink_delayed.csv",
+    "process_source_sink_undelayed.csv",
+    "nodeGroupDispatch__process_fully_inside.csv",
+    # — ramp family (Phase E-b promoted)
+    "process_source_sink_ramp_limit_source_up.csv",
+    "process_source_sink_ramp_limit_sink_up.csv",
+    "process_source_sink_ramp_limit_source_down.csv",
+    "process_source_sink_ramp_limit_sink_down.csv",
+    "process_source_sink_ramp_cost.csv",
+    # — param_in_use family
+    "node__PeriodParam_in_use.csv",
+    "process__PeriodParam_in_use.csv",
+    "process_TimeParam_in_use.csv",
+    "process_source_sourceSinkTimeParam_in_use.csv",
+    "process_sink_sourceSinkTimeParam_in_use.csv",
+    # — Phase E-b lifted streamed writers
+    "peedt.csv",
+    "p_process_delay_weight.csv",
+    "p_flow_max.csv",
+    "p_storage_state_reference_price.csv",
+    "process__source__sink__profile__profile_method.csv",
+    # — 12-CSV nodeGroupDispatch family
+    "nodeGroupDispatch__process__unit__to_node_Not_in_aggregate.csv",
+    "nodeGroupDispatch__process__node__to_unit_Not_in_aggregate.csv",
+    "nodeGroupDispatch__group_aggregate__process__unit__to_node.csv",
+    "nodeGroupDispatch__group_aggregate__process__node__to_unit.csv",
+    "nodeGroupDispatch__process__node__to_connection_Not_in_aggregate.csv",
+    "nodeGroupDispatch__process__connection__to_node_Not_in_aggregate.csv",
+    "nodeGroupDispatch__connection_Not_in_aggregate.csv",
+    "nodeGroupDispatch__group_aggregate__process__connection__to_node.csv",
+    "nodeGroupDispatch__group_aggregate__process__node__to_connection.csv",
+    "nodeGroupDispatch__group_aggregate_Connection.csv",
+    "nodeGroupDispatch__group_aggregate_Unit_to_group.csv",
+    "nodeGroupDispatch__group_aggregate_Group_to_unit.csv",
+    # _emit_chain_params — Phase E-b lifted streamed writers
+    "p_entity_pre_existing.csv",
+    # — 2-CSV existing chain (count/integer_count/later_solves emits
+    #   pruned — dead)
+    "p_entity_all_existing.csv",
+    "p_entity_previously_invested_capacity.csv",
+    # — 2-CSV capacity max chain (max_capacity/invest_cumulative_max pruned — dead)
+    "p_entity_max_units.csv",
+    "p_entity_dispatch_capacity_max.csv",
+    # _emit_co2_accumulators — Phase E-b lifted.  ``co2_cum_realized_tonnes.csv``
+    # used to live here as a guaranteed empty-seed contract; Phase 1b of
+    # specs/provider_consolidation.md retired the empty seed.  The
+    # post-solve accumulator computed by
+    # ``compute_co2_rolling_accumulator`` still flows through the
+    # handoff/Provider plumbing when it fires, but single-solve
+    # scenarios never run that path — so the basename is conditional,
+    # not part of the expected_basenames contract.
+    # _emit_pdt_params — Phase E-b lifted streamed writers
+    "pdtProcess.csv",
+    "pdtNode.csv",
+    "pdtProcess_source.csv",
+    "pdtProcess_sink.csv",
+    # _emit_period_params — Phase E-b lifted streamed writers
+    "pdtNodeInflow.csv",
+    "pdtProfile.csv",
+    "pdtProcess_section.csv",
+    "pdtProcess_slope.csv",
+    "pdGroup.csv",
+    "pdtGroup.csv",
+    "pdtCommodity.csv",
+    "p_positive_inflow.csv",
+    "p_negative_inflow.csv",
+    "pdtProcess__source__sink__dt_varCost.csv",
+    "pssdt_varCost_noEff.csv",
+    "pssdt_varCost_eff_unit_source.csv",
+    "pssdt_varCost_eff_unit_sink.csv",
+    "pssdt_varCost_eff_connection.csv",
+    "ed_invest_max_period.csv",
+    "ed_invest_min_period.csv",
+    "ed_divest_max_period.csv",
+    "ed_divest_min_period.csv",
+    "ed_cumulative_max_capacity.csv",
+    "ed_cumulative_min_capacity.csv",
+    # _emit_calc_params — Phase E-b lifted streamed writers
+    # — process_arc_method_joins (10 CSVs, methodgated arc joins)
+    "process_sink_toProcess.csv",
+    "process_process_toSource.csv",
+    "process_source_toSink.csv",
+    "process_source_toProcess_direct.csv",
+    "process_process_toSink_direct.csv",
+    "process_sink_toProcess_direct.csv",
+    "process_sink_toSource.csv",
+    "process_process_toSink_noConversion.csv",
+    "process_source_toProcess_noConversion.csv",
+    "process_process_toSource_direct.csv",
+    # — process_profile_method_joins (2 CSVs)
+    "process__profileProcess__toSink__profile__profile_method.csv",
+    "process__source__toProfileProcess__profile__profile_method.csv",
+    # _emit_dispatchers — Phase E-b lifted entity_period_calc_params
+    "pdProcess.csv",
+    "pdNode.csv",
+    "edEntity_lifetime.csv",
+    "ed_fixed_cost.csv",
+    "p_entity_unitsize.csv",
+    # _emit_dispatchers — Phase E-b lifted process_arc_unions monolith
+    "process_source_sink_eff.csv",
+    "process_source_sink_noEff.csv",
+    "process_online.csv",
+    "process_minload.csv",
+    "process_source_sink.csv",
+    "process_source_sink_alwaysProcess.csv",
+    "process__source__sink__profile__profile_method_direct.csv",
+    # _emit_entity_annual — Phase E-b lifted (6-CSV monolith)
+    "ed_entity_annual.csv",
+    "ed_entity_annual_discounted.csv",
+    "ed_entity_annual_divest.csv",
+    "ed_entity_annual_divest_discounted.csv",
+    "ed_lifetime_fixed_cost.csv",
+    "ed_lifetime_fixed_cost_divest.csv",
+    # _emit_inflow_scaling — only the 6 CONSUMED outputs are emitted; the 9
+    # internal middle parameters (orig_flow_sum, period_share_of_annual_flow,
+    # new_peak_sign, old_peak_max, old_peak_min, old_peak_sign,
+    # new_peak_divided_by_old_peak, new_peak_inflow_sum, new_old_multiplier)
+    # have no external consumer and are no longer written.
+    "ptNode_inflow.csv",
+    "_node_cap_inflow_fallback.csv",
+    "period_flow_annual_multiplier.csv",
+    "period_flow_proportional_multiplier.csv",
+    "new_old_slope.csv",
+    "new_old_section.csv",
+    # _emit_lp_scaling — only the 3 CONSUMED outputs are emitted; the 6
+    # stage-to-stage middle-products (_node_cap_unitsize_sum, _node_cap_raw,
+    # _node_cap_pow10, inv_node_cap, _group_cap_raw, _group_cap_pow10) have
+    # no functional reader and are no longer written.
+    "node_capacity_for_scaling.csv",
+    "group_capacity_for_scaling.csv",
+    "inv_group_cap.csv",
+    # _emit_solve_writers — Phase E-b7 (34 small per-solve CSVs)
+    "steps_in_timeline.csv",
+    "steps_in_use.csv",
+    "steps_complete_solve.csv",
+    "step_previous.csv",
+    "period_block_time.csv",
+    "period_block_succ.csv",
+    "p_years_represented.csv",
+    "period_with_history.csv",
+    "p_discount_years.csv",
+    "realized_invest_periods_of_current_solve.csv",
+    "period_last.csv",
+    "period_first_of_solve.csv",
+    "period_first.csv",
+    "p_model.csv",
+    "p_nested_model.csv",
+    "solve_current.csv",
+    "first_timesteps.csv",
+    "last_timesteps.csv",
+    "realized_dispatch.csv",
+    "fix_storage_timesteps.csv",
+    "period__branch.csv",
+    "solve_branch_weight.csv",
+    "solve_branch__time_branch.csv",
+    "p_entity_invested.csv",
+    "p_entity_divested.csv",
+    "p_entity_period_existing_capacity.csv",
+    # NB ladder_cum_realized_mwh.csv, ladder_cum_sim_hours.csv, and
+    # co2_cum_realized_tonnes.csv used to live here as header-only
+    # empty seeds emitted by ``emit_empty_cumulative_files``.  Phase 1b
+    # of specs/provider_consolidation.md dropped the empty-seed writer
+    # — consumers fall through to ``_read_csv``'s empty-frame default
+    # on first-roll Provider miss, and subsequent rolls receive
+    # populated frames via ``_fan_out_ladder_accumulators`` /
+    # ``compute_co2_rolling_accumulator`` (via the handoff plumbing).
+    # Listing those basenames in ``expected_basenames`` would re-
+    # introduce the contract surface we just retired.
+    "fix_storage_price.csv",
+    "fix_storage_quantity.csv",
+    "fix_storage_usage.csv",
+    "p_roll_continue_state.csv",
+    "costs_discounted.csv",
+    "co2.csv",
+    "period_capacity.csv",
+    "timesets_in_use.csv",
+    "timesets__timeline.csv",
+    "p_use_row_scaling.csv",
+    "scale_the_objective.csv",
+    "delay_duration.csv",
+    "dtt__delay_duration.csv",
+    "rp_weights.csv",
+    "rp_base_chain.csv",
+    "rp_base_first.csv",
+    "rp_base_last.csv",
+    "rp_block_first.csv",
+    "rp_block_last.csv",
+    "rp_block_start_last.csv",
+    "timestep_weight.csv",
+    # _emit_period_calc — Phase E-b8 (12 + 2 CSVs)
+    "p_inflation_factor_operations_yearly.csv",
+    "complete_period_share_of_year_calc.csv",
+    "p_timeline_duration_in_years.csv",
+    "hours_in_period.csv",
+    "period_share_of_year.csv",
+    "p_years_d.csv",
+    "complete_hours_in_period.csv",
+    "p_years_until_invest.csv",
+    "p_years_until_dispatch.csv",
+    "p_inflation_factor_investment_yearly.csv",
+    "f_d_k.csv",
+    "pd_branch_weight.csv",
+    "pdt_branch_weight.csv",
+    # _emit_per_solve — Phase E-b8 (30 + 19 + 1 CSVs)
+    "branch_set.csv",
+    "year_set.csv",
+    "period_from_period_time_set.csv",
+    "period_in_use_set.csv",
+    "time_in_use_set.csv",
+    "complete_time_in_use_set.csv",
+    "rp_base_period_set.csv",
+    "rp_rep_period_set.csv",
+    "period_block_set.csv",
+    "dtt_set.csv",
+    "d_fix_storage_period_set.csv",
+    "period_set.csv",
+    "periodAll_set.csv",
+    "block_set.csv",
+    "period__timeline_set.csv",
+    "dt_realize_dispatch_set.csv",
+    "d_realized_period_set.csv",
+    "d_realize_dispatch_or_invest_set.csv",
+    "dt_non_anticipativity_set.csv",
+    "cnd_ladder_set.csv",
+    "cndi_ladder_cum_set.csv",
+    "cndi_ladder_ann_set.csv",
+    "cndi_ladder_set.csv",
+    "dtdt_next_set.csv",
+    "n_fix_storage_quantity_set.csv",
+    "n_fix_storage_price_set.csv",
+    "n_fix_storage_usage_set.csv",
+    "p_online_dt_set.csv",
+    "ed_invest.csv",
+    "ed_divest.csv",
+    "ed_invest_period.csv",
+    "ed_divest_period.csv",
+    "ed_invest_cumulative.csv",
+    "pd_invest.csv",
+    "nd_invest.csv",
+    "pd_divest.csv",
+    "nd_divest.csv",
+    "edd_history_choice.csv",
+    "edd_history_automatic.csv",
+    "edd_history_no_investment.csv",
+    "edd_history.csv",
+    "edd_history_invest.csv",
+    "edd_invest.csv",
+    "gd_invest.csv",
+    "gd_divest.csv",
+    "gd_invest_period.csv",
+    "gd_divest_period.csv",
+    "ed_invest_forbidden_no_investment.csv",
+    # _emit_reserve — Phase E-b8 (1 + 2 + 4 CSVs)
+    "pdtReserve_upDown_group.csv",
+    "process_reserve_upDown_node_active.csv",
+    "prundt.csv",
+    "p_process_reserve_upDown_node_reliability.csv",
+    "process_reserve_upDown_node_increase_reserve_ratio.csv",
+    "process_reserve_upDown_node_large_failure_ratio.csv",
+    "process_large_failure.csv",
+    # Per-solve block frames populated by emit_block_data into the
+    # Provider.
+    "entity_block.csv",
+    "process_side_block.csv",
+    "process_block.csv",
+    "block_step_duration.csv",
+    "overlap_set.csv",
+    "block_step_previous.csv",
+    "block_period_time_first.csv",
+    "block_period_time_last.csv",
+)
+
+
+def expected_basenames() -> tuple[str, ...]:
+    """Return the basenames the cascade is expected to capture.
+
+    The list comes from the Phase B writer audit (every ``OK_thin_wrapper``
+    entry, plus the streamed writers lifted into the canonical pattern by
+    Phase E-b).  Tests use this to cross-check disk-vs-Provider parity
+    without re-enumerating the list inline.
+    """
+    return _THIN_WRAPPER_BASENAMES