flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,885 @@
|
|
|
1
|
+
"""Per-solve in-memory state — the typed replacement for ``solve_data/``.
|
|
2
|
+
|
|
3
|
+
Δ.12a — port per-solve preprocessing metadata into engine_polars.
|
|
4
|
+
|
|
5
|
+
This module hosts :class:`SolveContext`, the typed dataclass the
|
|
6
|
+
:doc:`audit/native_data_path_design_solve_context` schematic specifies.
|
|
7
|
+
It wraps the per-solve preprocessing artefacts that today live as CSVs
|
|
8
|
+
under ``workdir/solve_data/`` and exposes them as in-memory polars
|
|
9
|
+
frames so the override helpers (``apply_derived_a..g``) and the
|
|
10
|
+
``_load_*`` family in :mod:`flextool.engine_polars.input` no longer have
|
|
11
|
+
to re-read the same files dozens of times per solve.
|
|
12
|
+
|
|
13
|
+
:class:`SolveContext` reads the ``solve_data/*.csv`` files written by
|
|
14
|
+
the per-solve preprocessing. The point of going through this object
|
|
15
|
+
is:
|
|
16
|
+
|
|
17
|
+
1. **Single funnel** — every workdir CSV read goes through one place,
|
|
18
|
+
so the gross :func:`flextool.engine_polars._input_source._read_csv_file`
|
|
19
|
+
call count drops dramatically (helpers consume cached frames instead
|
|
20
|
+
of re-issuing ``pl.read_csv`` every call).
|
|
21
|
+
2. **Typed entry points** — the most-consumed metadata
|
|
22
|
+
(``solveFirst``, ``realized_periods``, ``period_in_use``,
|
|
23
|
+
``period_branch``, ``edd_history``, etc.) is parsed once into typed
|
|
24
|
+
Python state and exposed as named attributes. Helpers that today
|
|
25
|
+
re-derive these from raw CSVs each call (e.g.
|
|
26
|
+
``_read_active_solve``, ``_read_realize_invest_periods``) consume the
|
|
27
|
+
typed fields.
|
|
28
|
+
3. **Future cutover seam** — when Δ.12c lands, the typed-fields
|
|
29
|
+
constructor swaps to populate from
|
|
30
|
+
:class:`~flextool.engine_polars._solve_state.RunnerState` directly
|
|
31
|
+
(``state.solve.realized_periods[solve_name]``,
|
|
32
|
+
``state.timeline.dt_for_solve(solve_name)``). Helpers don't change.
|
|
33
|
+
|
|
34
|
+
See :doc:`audit/native_data_path_design_solve_context` for the design
|
|
35
|
+
rationale and :doc:`audit/handoff_csv_retirement.md` for the broader
|
|
36
|
+
CSV-retirement plan.
|
|
37
|
+
"""
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
|
|
43
|
+
import polars as pl
|
|
44
|
+
|
|
45
|
+
from ._axis_enums import schema_dtype
|
|
46
|
+
from ._input_source import _install_csv_cache, _read_csv_file
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
# SolveContext
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class SolveContext:
|
|
56
|
+
"""Typed in-memory carrier of per-solve preprocessing state.
|
|
57
|
+
|
|
58
|
+
Construct via :meth:`from_workdir` to populate the typed fields from
|
|
59
|
+
a flextool workdir (the CSV-bridge path used by Δ.12a) or via the
|
|
60
|
+
plain constructor to hand-build for tests.
|
|
61
|
+
|
|
62
|
+
All ``DataFrame`` fields are eager (``pl.DataFrame``) — helpers can
|
|
63
|
+
convert via ``df.lazy()`` at the call site to keep the rest of the
|
|
64
|
+
pipeline lazy per the project invariant.
|
|
65
|
+
|
|
66
|
+
Attributes
|
|
67
|
+
----------
|
|
68
|
+
workdir : Path
|
|
69
|
+
The flextool workdir. Kept around so :meth:`read_csv` can
|
|
70
|
+
service helpers that still need a file the typed fields don't
|
|
71
|
+
cover (e.g. flexible-shape "user-defined" CSVs that genuinely
|
|
72
|
+
have no Spine analogue). Each such read is cached so repeated
|
|
73
|
+
calls hit memory.
|
|
74
|
+
solve_name : str | None
|
|
75
|
+
Active solve as written to ``solve_data/solve_current.csv``.
|
|
76
|
+
``None`` for fixtures without per-solve preprocessing.
|
|
77
|
+
solveFirst : bool
|
|
78
|
+
``True`` iff this is the first sub-solve in a multi-solve cascade
|
|
79
|
+
(per ``solve_data/p_model.csv:solveFirst``). Default ``True``
|
|
80
|
+
(single-solve fixtures behave as first-of-chain).
|
|
81
|
+
realized_periods : set[str]
|
|
82
|
+
Periods realized this solve (``realized_dispatch.csv`` distinct
|
|
83
|
+
``period`` column). Empty when the file is missing.
|
|
84
|
+
realized_invest_periods : set[str]
|
|
85
|
+
Periods where invest was realized this solve
|
|
86
|
+
(``realized_invest_periods_of_current_solve.csv``).
|
|
87
|
+
period_in_use : pl.DataFrame
|
|
88
|
+
``[d]`` distinct frame from ``period_in_use_set.csv``. This is
|
|
89
|
+
the authoritative active-period set INCLUDING any stochastic-
|
|
90
|
+
branch periods (per ``per_solve_sets.py:95-101``). Empty frame
|
|
91
|
+
when the file is missing.
|
|
92
|
+
period_branch : pl.DataFrame
|
|
93
|
+
``[d_anchor, b]`` from ``period__branch.csv`` — full unfiltered
|
|
94
|
+
anchor → sibling map. Empty frame when the file is missing.
|
|
95
|
+
edd_history : pl.DataFrame
|
|
96
|
+
``[entity, d_decided, d_apply]`` from ``edd_history.csv`` — the
|
|
97
|
+
invest-decision-date → apply-date map used by handoff-derived
|
|
98
|
+
``p_entity_previously_invested_capacity``. Empty frame when the
|
|
99
|
+
file is missing.
|
|
100
|
+
p_entity_pre_existing : pl.DataFrame
|
|
101
|
+
``[entity, period, value]`` from ``p_entity_pre_existing.csv``.
|
|
102
|
+
Empty when missing.
|
|
103
|
+
|
|
104
|
+
The remaining workdir CSVs (block-layout, dtttdt, etc.) are still
|
|
105
|
+
accessed via :meth:`read_csv` — they're either covered by other
|
|
106
|
+
typed objects (:class:`BlockLayout`) or read few enough times that
|
|
107
|
+
string-keyed access is fine.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
workdir: Path
|
|
111
|
+
solve_name: str | None = None
|
|
112
|
+
solveFirst: bool = True
|
|
113
|
+
realized_periods: set[str] = field(default_factory=set)
|
|
114
|
+
realized_invest_periods: set[str] = field(default_factory=set)
|
|
115
|
+
# Phase 3 — Provider stashed at construction so the lazy DataFrame
|
|
116
|
+
# property loaders can consult it without re-plumbing the kwarg
|
|
117
|
+
# through every caller. When ``provider`` is None the legacy disk
|
|
118
|
+
# path remains active (test-only fallback that goes away with
|
|
119
|
+
# Phase 4 activation).
|
|
120
|
+
provider: "object | None" = field(default=None, repr=False)
|
|
121
|
+
# ``period_in_use`` / ``period_branch`` / ``edd_history`` /
|
|
122
|
+
# ``p_entity_pre_existing`` are populated on first access via the
|
|
123
|
+
# descriptor machinery below — they're DataFrame-shaped fields and
|
|
124
|
+
# each requires a CSV read. Most
|
|
125
|
+
# parity tests only consume one or two of them, so deferring the
|
|
126
|
+
# reads until an attribute access asks for them avoids paying the
|
|
127
|
+
# IO cost upfront.
|
|
128
|
+
_period_in_use_loaded: bool = field(default=False, repr=False)
|
|
129
|
+
_period_in_use: pl.DataFrame = field(
|
|
130
|
+
default_factory=lambda: pl.DataFrame(
|
|
131
|
+
schema={"d": schema_dtype(None, "d")}
|
|
132
|
+
),
|
|
133
|
+
repr=False,
|
|
134
|
+
)
|
|
135
|
+
_period_branch_loaded: bool = field(default=False, repr=False)
|
|
136
|
+
_period_branch: pl.DataFrame = field(
|
|
137
|
+
default_factory=lambda: pl.DataFrame(
|
|
138
|
+
schema={
|
|
139
|
+
"d_anchor": schema_dtype(None, "d_anchor"),
|
|
140
|
+
"b": schema_dtype(None, "b"),
|
|
141
|
+
}
|
|
142
|
+
),
|
|
143
|
+
repr=False,
|
|
144
|
+
)
|
|
145
|
+
_edd_history_loaded: bool = field(default=False, repr=False)
|
|
146
|
+
_edd_history: pl.DataFrame = field(
|
|
147
|
+
default_factory=lambda: pl.DataFrame(schema={}), repr=False,
|
|
148
|
+
)
|
|
149
|
+
_ppe_loaded: bool = field(default=False, repr=False)
|
|
150
|
+
_ppe: pl.DataFrame = field(
|
|
151
|
+
default_factory=lambda: pl.DataFrame(schema={}), repr=False,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# ── Path B Cat B (WriterSnapshot top-7) extensions ──────────────────
|
|
155
|
+
# Per-solve preprocessing artefacts written by ``_emit_solve_writers``
|
|
156
|
+
# / ``_emit_period_calc`` and consumed by the cascade (audit
|
|
157
|
+
# `specs/in_cascade_csv_audit.md` §Category B). Lazy-loaded on first
|
|
158
|
+
# attribute access — same pattern as ``period_in_use`` /
|
|
159
|
+
# ``period_branch`` above. Schemas mirror the renamed canonical form
|
|
160
|
+
# the cascade helpers themselves construct, so callers can swap a
|
|
161
|
+
# ``_read_csv_file + rename + select + unique`` chain for a single
|
|
162
|
+
# attribute access.
|
|
163
|
+
_steps_in_use_loaded: bool = field(default=False, repr=False)
|
|
164
|
+
_steps_in_use: pl.DataFrame = field(
|
|
165
|
+
default_factory=lambda: pl.DataFrame(
|
|
166
|
+
schema={
|
|
167
|
+
"d": schema_dtype(None, "d"),
|
|
168
|
+
"t": schema_dtype(None, "t"),
|
|
169
|
+
"step_duration": pl.Float64,
|
|
170
|
+
}
|
|
171
|
+
),
|
|
172
|
+
repr=False,
|
|
173
|
+
)
|
|
174
|
+
_period_share_loaded: bool = field(default=False, repr=False)
|
|
175
|
+
_period_share: pl.DataFrame = field(
|
|
176
|
+
default_factory=lambda: pl.DataFrame(
|
|
177
|
+
schema={"d": schema_dtype(None, "d"), "value": pl.Float64}
|
|
178
|
+
),
|
|
179
|
+
repr=False,
|
|
180
|
+
)
|
|
181
|
+
_p_entity_all_existing_loaded: bool = field(default=False, repr=False)
|
|
182
|
+
_p_entity_all_existing: pl.DataFrame = field(
|
|
183
|
+
default_factory=lambda: pl.DataFrame(schema={}), repr=False,
|
|
184
|
+
)
|
|
185
|
+
_solve_branch_weight_loaded: bool = field(default=False, repr=False)
|
|
186
|
+
_solve_branch_weight: pl.DataFrame = field(
|
|
187
|
+
default_factory=lambda: pl.DataFrame(
|
|
188
|
+
schema={
|
|
189
|
+
"b": schema_dtype(None, "b"),
|
|
190
|
+
"p_branch_weight_input": pl.Float64,
|
|
191
|
+
}
|
|
192
|
+
),
|
|
193
|
+
repr=False,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
# Internal: ad-hoc CSV cache. Keyed by ``str(path)`` (no syscall).
|
|
197
|
+
# Values are ``None`` when the file is absent so we don't re-stat
|
|
198
|
+
# repeatedly.
|
|
199
|
+
_csv_cache: dict[str, pl.DataFrame | None] = field(
|
|
200
|
+
default_factory=dict, repr=False
|
|
201
|
+
)
|
|
202
|
+
# Internal: process-level cache view installed by ``activate`` —
|
|
203
|
+
# a strict subset of ``_csv_cache`` (None-valued entries excluded
|
|
204
|
+
# because the ``_read_csv_file`` cache only stores successfully-
|
|
205
|
+
# read frames). ``None`` means caching is not currently active.
|
|
206
|
+
_active_cache: "dict[str, pl.DataFrame] | None" = field(
|
|
207
|
+
default=None, repr=False
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# ------------------------------------------------------------------
|
|
211
|
+
# Factories
|
|
212
|
+
# ------------------------------------------------------------------
|
|
213
|
+
|
|
214
|
+
@classmethod
|
|
215
|
+
def from_workdir(cls, workdir: Path | str,
|
|
216
|
+
*, provider: "object | None" = None) -> "SolveContext":
|
|
217
|
+
"""Construct a SolveContext from a flextool workdir.
|
|
218
|
+
|
|
219
|
+
Eagerly reads only the cheap typed scalars
|
|
220
|
+
(``solve_name``, ``solveFirst``, ``realized_periods``,
|
|
221
|
+
``realized_invest_periods``) — these come from small single-
|
|
222
|
+
column CSVs and are consumed by the apply_derived_* boundary
|
|
223
|
+
checks that gate the rest of the cascade.
|
|
224
|
+
|
|
225
|
+
The DataFrame-shaped fields (``period_in_use``, ``period_branch``,
|
|
226
|
+
``edd_history``, ``p_entity_pre_existing``) are loaded **lazily**
|
|
227
|
+
on first attribute access. Most parity tests only consume one or
|
|
228
|
+
two,
|
|
229
|
+
so paying the IO cost upfront for a bag of frames the test
|
|
230
|
+
never touches dominates the cache savings on small fixtures.
|
|
231
|
+
|
|
232
|
+
Step 1-g-4b — *provider* threads the live
|
|
233
|
+
:class:`FlexDataProvider` through to the four eager scalar
|
|
234
|
+
readers so they hit the in-memory frames before falling back
|
|
235
|
+
to the seed funnel / disk.
|
|
236
|
+
"""
|
|
237
|
+
wd = Path(workdir)
|
|
238
|
+
sd = wd / "solve_data"
|
|
239
|
+
ctx = cls(workdir=wd, provider=provider)
|
|
240
|
+
ctx.solve_name = _read_active_solve(wd, provider=provider)
|
|
241
|
+
ctx.solveFirst = _read_solve_first(wd, provider=provider)
|
|
242
|
+
ctx.realized_periods = _read_realized_dispatch_periods(
|
|
243
|
+
sd / "realized_dispatch.csv", provider=provider,
|
|
244
|
+
)
|
|
245
|
+
ctx.realized_invest_periods = _read_period_set(
|
|
246
|
+
sd / "realized_invest_periods_of_current_solve.csv",
|
|
247
|
+
provider=provider,
|
|
248
|
+
)
|
|
249
|
+
return ctx
|
|
250
|
+
|
|
251
|
+
# ------------------------------------------------------------------
|
|
252
|
+
# Cached CSV reader
|
|
253
|
+
# ------------------------------------------------------------------
|
|
254
|
+
|
|
255
|
+
def read_csv(
|
|
256
|
+
self,
|
|
257
|
+
relative: str | Path,
|
|
258
|
+
*,
|
|
259
|
+
kind: str = "solve_data",
|
|
260
|
+
) -> pl.DataFrame | None:
|
|
261
|
+
"""Cached read for an ad-hoc workdir CSV.
|
|
262
|
+
|
|
263
|
+
Parameters
|
|
264
|
+
----------
|
|
265
|
+
relative : str | Path
|
|
266
|
+
Filename relative to the directory selected by *kind*. May
|
|
267
|
+
be a bare ``foo.csv`` or a relative subpath.
|
|
268
|
+
kind : str
|
|
269
|
+
``"solve_data"`` (default) or ``"input"`` — selects the
|
|
270
|
+
subdirectory under :attr:`workdir`. Anything else is
|
|
271
|
+
interpreted as a workdir-rooted relative path.
|
|
272
|
+
|
|
273
|
+
Returns
|
|
274
|
+
-------
|
|
275
|
+
pl.DataFrame | None
|
|
276
|
+
The frame, or ``None`` when the file is missing. The result
|
|
277
|
+
is cached by absolute path so subsequent calls hit memory.
|
|
278
|
+
"""
|
|
279
|
+
rel = Path(relative)
|
|
280
|
+
if kind == "solve_data":
|
|
281
|
+
path = self.workdir / "solve_data" / rel
|
|
282
|
+
elif kind == "input":
|
|
283
|
+
path = self.workdir / "input" / rel
|
|
284
|
+
else:
|
|
285
|
+
path = self.workdir / rel
|
|
286
|
+
# Cache key = str(path). Same key as ``_read_csv_file``'s active-
|
|
287
|
+
# cache so SolveContext.read_csv and direct ``_read_csv_file``
|
|
288
|
+
# calls share the cache when ``activate`` is in effect. Avoids
|
|
289
|
+
# the per-call ``Path.resolve`` syscall.
|
|
290
|
+
key = str(path)
|
|
291
|
+
if key in self._csv_cache:
|
|
292
|
+
return self._csv_cache[key]
|
|
293
|
+
if not path.exists():
|
|
294
|
+
self._csv_cache[key] = None
|
|
295
|
+
return None
|
|
296
|
+
try:
|
|
297
|
+
df = _read_csv_file(path)
|
|
298
|
+
except pl.exceptions.NoDataError:
|
|
299
|
+
df = pl.DataFrame()
|
|
300
|
+
self._csv_cache[key] = df
|
|
301
|
+
return df
|
|
302
|
+
|
|
303
|
+
# ------------------------------------------------------------------
|
|
304
|
+
# Cache activation (process-level, scoped via context manager)
|
|
305
|
+
# ------------------------------------------------------------------
|
|
306
|
+
|
|
307
|
+
def activate(self) -> None:
|
|
308
|
+
"""Δ.12a — install the read-cache so helper modules' direct
|
|
309
|
+
``_read_csv_file`` calls hit this context's cache on repeat.
|
|
310
|
+
|
|
311
|
+
Idempotent: calling ``activate`` twice on the same context is
|
|
312
|
+
a no-op. Pair with :meth:`deactivate` to clear, or use the
|
|
313
|
+
context-manager protocol.
|
|
314
|
+
"""
|
|
315
|
+
# Build the active-cache view (DataFrame-only; absent-file
|
|
316
|
+
# entries from ``read_csv`` are dropped because
|
|
317
|
+
# ``_read_csv_file`` doesn't track absences). Subsequent
|
|
318
|
+
# cache misses populate ``cache_view`` directly through
|
|
319
|
+
# ``_read_csv_file``; ``read_csv`` continues to populate
|
|
320
|
+
# ``_csv_cache`` with the broader (None-included) shape.
|
|
321
|
+
cache_view: dict[str, pl.DataFrame] = {
|
|
322
|
+
k: v for k, v in self._csv_cache.items() if v is not None
|
|
323
|
+
}
|
|
324
|
+
_install_csv_cache(cache_view)
|
|
325
|
+
self._active_cache = cache_view
|
|
326
|
+
|
|
327
|
+
def deactivate(self) -> None:
|
|
328
|
+
"""Δ.12a — uninstall the read-cache."""
|
|
329
|
+
_install_csv_cache(None)
|
|
330
|
+
self._active_cache = None
|
|
331
|
+
|
|
332
|
+
def __enter__(self) -> "SolveContext":
|
|
333
|
+
self.activate()
|
|
334
|
+
return self
|
|
335
|
+
|
|
336
|
+
def __exit__(self, *exc) -> None:
|
|
337
|
+
self.deactivate()
|
|
338
|
+
|
|
339
|
+
# ------------------------------------------------------------------
|
|
340
|
+
# Lazy DataFrame fields
|
|
341
|
+
# ------------------------------------------------------------------
|
|
342
|
+
|
|
343
|
+
@property
|
|
344
|
+
def period_in_use(self) -> pl.DataFrame:
|
|
345
|
+
if not self._period_in_use_loaded:
|
|
346
|
+
self._period_in_use = _load_period_in_use(
|
|
347
|
+
self.solve_data_dir / "period_in_use_set.csv",
|
|
348
|
+
provider=self.provider,
|
|
349
|
+
)
|
|
350
|
+
self._period_in_use_loaded = True
|
|
351
|
+
return self._period_in_use
|
|
352
|
+
|
|
353
|
+
@property
|
|
354
|
+
def period_branch(self) -> pl.DataFrame:
|
|
355
|
+
if not self._period_branch_loaded:
|
|
356
|
+
self._period_branch = _load_period_branch(
|
|
357
|
+
self.solve_data_dir / "period__branch.csv",
|
|
358
|
+
provider=self.provider,
|
|
359
|
+
)
|
|
360
|
+
self._period_branch_loaded = True
|
|
361
|
+
return self._period_branch
|
|
362
|
+
|
|
363
|
+
@property
|
|
364
|
+
def edd_history(self) -> pl.DataFrame:
|
|
365
|
+
if not self._edd_history_loaded:
|
|
366
|
+
self._edd_history = _load_edd_history(
|
|
367
|
+
self.solve_data_dir / "edd_history.csv",
|
|
368
|
+
provider=self.provider,
|
|
369
|
+
)
|
|
370
|
+
self._edd_history_loaded = True
|
|
371
|
+
return self._edd_history
|
|
372
|
+
|
|
373
|
+
@property
|
|
374
|
+
def p_entity_pre_existing(self) -> pl.DataFrame:
|
|
375
|
+
if not self._ppe_loaded:
|
|
376
|
+
self._ppe = _maybe_read(
|
|
377
|
+
self.solve_data_dir / "p_entity_pre_existing.csv",
|
|
378
|
+
provider=self.provider,
|
|
379
|
+
)
|
|
380
|
+
self._ppe_loaded = True
|
|
381
|
+
return self._ppe
|
|
382
|
+
|
|
383
|
+
# ── Path B Cat B (WriterSnapshot top-7) accessors ─────────────────
|
|
384
|
+
@property
|
|
385
|
+
def steps_in_use(self) -> pl.DataFrame:
|
|
386
|
+
"""``[d, t, step_duration]`` from ``steps_in_use.csv`` —
|
|
387
|
+
canonical column names. Empty when the file is missing.
|
|
388
|
+
|
|
389
|
+
Path B / audit §Category B: cascade helpers that previously
|
|
390
|
+
re-read the CSV (``_derived_params.py:5421, 7974``;
|
|
391
|
+
``_derived_branch.py:285``) should prefer this typed accessor.
|
|
392
|
+
"""
|
|
393
|
+
if not self._steps_in_use_loaded:
|
|
394
|
+
self._steps_in_use = _load_steps_in_use(
|
|
395
|
+
self.solve_data_dir / "steps_in_use.csv",
|
|
396
|
+
provider=self.provider,
|
|
397
|
+
)
|
|
398
|
+
self._steps_in_use_loaded = True
|
|
399
|
+
return self._steps_in_use
|
|
400
|
+
|
|
401
|
+
@property
|
|
402
|
+
def period_share_of_year(self) -> pl.DataFrame:
|
|
403
|
+
"""``[d, value]`` from ``complete_period_share_of_year_calc.csv``
|
|
404
|
+
(canonical form: ``period`` → ``d``). Falls back to the non-
|
|
405
|
+
``_calc`` variant when only that file exists. Empty when neither
|
|
406
|
+
is present.
|
|
407
|
+
|
|
408
|
+
Path B / audit §Category B: cascade helpers that previously
|
|
409
|
+
re-read the CSV (``_derived_params.py:7990``) should prefer this
|
|
410
|
+
typed accessor.
|
|
411
|
+
"""
|
|
412
|
+
if not self._period_share_loaded:
|
|
413
|
+
self._period_share = _load_period_share(
|
|
414
|
+
self.solve_data_dir, provider=self.provider,
|
|
415
|
+
)
|
|
416
|
+
self._period_share_loaded = True
|
|
417
|
+
return self._period_share
|
|
418
|
+
|
|
419
|
+
@property
|
|
420
|
+
def p_entity_all_existing(self) -> pl.DataFrame:
|
|
421
|
+
"""``p_entity_all_existing.csv`` — preserved schema (canonical
|
|
422
|
+
column names already present at writer side). Empty when the
|
|
423
|
+
file is missing.
|
|
424
|
+
|
|
425
|
+
Path B / audit §Category B: cascade helpers that previously
|
|
426
|
+
re-read the CSV (``_derived_params.py:4940``) should prefer this
|
|
427
|
+
typed accessor.
|
|
428
|
+
"""
|
|
429
|
+
if not self._p_entity_all_existing_loaded:
|
|
430
|
+
self._p_entity_all_existing = _maybe_read(
|
|
431
|
+
self.solve_data_dir / "p_entity_all_existing.csv",
|
|
432
|
+
provider=self.provider,
|
|
433
|
+
)
|
|
434
|
+
self._p_entity_all_existing_loaded = True
|
|
435
|
+
return self._p_entity_all_existing
|
|
436
|
+
|
|
437
|
+
@property
|
|
438
|
+
def solve_branch_weight(self) -> pl.DataFrame:
|
|
439
|
+
"""``[b, p_branch_weight_input]`` from
|
|
440
|
+
``solve_branch_weight.csv`` (rename ``branch`` → ``b``). Empty
|
|
441
|
+
when the file is missing.
|
|
442
|
+
|
|
443
|
+
Path B / audit §Category B: cascade helpers that previously
|
|
444
|
+
re-read the CSV (``_derived_params.py:7623``) should prefer
|
|
445
|
+
this typed accessor.
|
|
446
|
+
"""
|
|
447
|
+
if not self._solve_branch_weight_loaded:
|
|
448
|
+
self._solve_branch_weight = _load_solve_branch_weight(
|
|
449
|
+
self.solve_data_dir / "solve_branch_weight.csv",
|
|
450
|
+
provider=self.provider,
|
|
451
|
+
)
|
|
452
|
+
self._solve_branch_weight_loaded = True
|
|
453
|
+
return self._solve_branch_weight
|
|
454
|
+
|
|
455
|
+
# ------------------------------------------------------------------
|
|
456
|
+
# Typed-field convenience accessors
|
|
457
|
+
# ------------------------------------------------------------------
|
|
458
|
+
|
|
459
|
+
@property
|
|
460
|
+
def solve_data_dir(self) -> Path:
|
|
461
|
+
return self.workdir / "solve_data"
|
|
462
|
+
|
|
463
|
+
@property
|
|
464
|
+
def input_dir(self) -> Path:
|
|
465
|
+
return self.workdir / "input"
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# ---------------------------------------------------------------------------
|
|
469
|
+
# Loaders for the typed fields
|
|
470
|
+
# ---------------------------------------------------------------------------
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _carrier_miss(path: Path, consumer: str) -> "FlexDataError":
|
|
474
|
+
"""Build the strict-Provider miss error for a SolveContext loader.
|
|
475
|
+
|
|
476
|
+
The producer-side writer should have populated the carrier before
|
|
477
|
+
the loader runs. A miss here is a cascade bug — likely the writer
|
|
478
|
+
didn't run, or its frame was emitted under a non-canonical key.
|
|
479
|
+
"""
|
|
480
|
+
return FlexDataError(
|
|
481
|
+
f"FlexDataProvider has no carrier for '{path.name}' "
|
|
482
|
+
f"(consumer: {consumer}). This per-solve frame must be "
|
|
483
|
+
f"populated by its emitter before SolveContext consumes it. "
|
|
484
|
+
f"Check that the producer ran and registered under the canonical "
|
|
485
|
+
f"key '{_provider_key_for(path)}'."
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _provider_key_for(path: Path) -> str:
|
|
490
|
+
"""Return the canonical Provider key for *path* (parent/stem)."""
|
|
491
|
+
parent = path.parent.name
|
|
492
|
+
stem = path.stem
|
|
493
|
+
if parent:
|
|
494
|
+
return f"{parent}/{stem}"
|
|
495
|
+
return stem
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
class FlexDataError(RuntimeError):
|
|
499
|
+
"""Raised when the Provider lacks a carrier the cascade requires.
|
|
500
|
+
|
|
501
|
+
This is a programming/wiring error — surfaced rather than worked
|
|
502
|
+
around so the broken edge is visible. See ``specs/enum_dtype_refactor_plan.md``
|
|
503
|
+
§Phase 3 for the rationale (no silent disk-fallback arms).
|
|
504
|
+
"""
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _provider_fetch_or_raise(
|
|
508
|
+
provider: "object", path: Path, consumer: str,
|
|
509
|
+
) -> pl.DataFrame:
|
|
510
|
+
"""Fetch *path*'s carrier from *provider* strictly; raise on miss."""
|
|
511
|
+
from ._emit_provider_io import _provider_key
|
|
512
|
+
key = _provider_key(path)
|
|
513
|
+
if provider.has(key):
|
|
514
|
+
df = provider.get(key)
|
|
515
|
+
if df is not None:
|
|
516
|
+
return df
|
|
517
|
+
raise _carrier_miss(path, consumer)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _maybe_read(path: Path,
|
|
521
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
522
|
+
"""Eager read returning empty frame on missing / empty CSV.
|
|
523
|
+
|
|
524
|
+
Provider-first: when *provider* is supplied, the carrier is fetched
|
|
525
|
+
strictly from the Provider; a missing key raises :class:`FlexDataError`.
|
|
526
|
+
Empty frames (height 0) are returned as ``pl.DataFrame()`` so callers
|
|
527
|
+
that ``if df.height == 0`` continue to short-circuit cleanly.
|
|
528
|
+
|
|
529
|
+
Disk fallback only runs when *provider* is None (the test-only path
|
|
530
|
+
that remains until Phase 4 activation makes provider mandatory).
|
|
531
|
+
"""
|
|
532
|
+
if provider is not None:
|
|
533
|
+
df = _provider_fetch_or_raise(provider, path, "SolveContext._maybe_read")
|
|
534
|
+
if df.height == 0:
|
|
535
|
+
return pl.DataFrame()
|
|
536
|
+
return df
|
|
537
|
+
if not path.exists():
|
|
538
|
+
return pl.DataFrame()
|
|
539
|
+
try:
|
|
540
|
+
return _read_csv_file(path)
|
|
541
|
+
except pl.exceptions.NoDataError:
|
|
542
|
+
return pl.DataFrame()
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _read_active_solve(workdir: Path,
|
|
546
|
+
*, provider: "object | None" = None) -> str | None:
|
|
547
|
+
"""Mirror of ``_derived_params._read_active_solve``.
|
|
548
|
+
|
|
549
|
+
Provider-first read; falls back to disk when the Provider is absent
|
|
550
|
+
or doesn't carry the frame.
|
|
551
|
+
"""
|
|
552
|
+
from flextool.engine_polars._emit_provider_io import _provider_key
|
|
553
|
+
p = workdir / "solve_data" / "solve_current.csv"
|
|
554
|
+
if provider is not None and provider.has(_provider_key(p)):
|
|
555
|
+
df = provider.get(_provider_key(p))
|
|
556
|
+
if df is None or df.height == 0:
|
|
557
|
+
return None
|
|
558
|
+
col = df.columns[0]
|
|
559
|
+
return df[col][0]
|
|
560
|
+
if not p.exists():
|
|
561
|
+
return None
|
|
562
|
+
try:
|
|
563
|
+
df = _read_csv_file(p)
|
|
564
|
+
except pl.exceptions.NoDataError:
|
|
565
|
+
return None
|
|
566
|
+
if df.height == 0:
|
|
567
|
+
return None
|
|
568
|
+
col = df.columns[0]
|
|
569
|
+
return df[col][0]
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _read_solve_first(work_folder: Path,
|
|
573
|
+
*, provider: "object | None" = None) -> bool:
|
|
574
|
+
"""Mirror of ``input._read_solve_first``.
|
|
575
|
+
|
|
576
|
+
Reads ``modelParam == 'solveFirst'`` from ``p_model.csv``. Resolves
|
|
577
|
+
in order: ``solve_data/p_model.csv`` → ``input/p_model.csv`` → True.
|
|
578
|
+
"""
|
|
579
|
+
from flextool.engine_polars._emit_provider_io import _provider_key
|
|
580
|
+
|
|
581
|
+
def _cell_str(value: "object | None") -> str:
|
|
582
|
+
return "" if value is None else str(value)
|
|
583
|
+
|
|
584
|
+
for cand in ("solve_data/p_model.csv", "input/p_model.csv"):
|
|
585
|
+
path = work_folder / cand
|
|
586
|
+
key = _provider_key(path)
|
|
587
|
+
if not provider.has(key):
|
|
588
|
+
continue
|
|
589
|
+
df = provider.get(key)
|
|
590
|
+
if "modelParam" not in df.columns or "p_model" not in df.columns:
|
|
591
|
+
return True
|
|
592
|
+
for r in df.iter_rows(named=True):
|
|
593
|
+
if _cell_str(r["modelParam"]) == "solveFirst":
|
|
594
|
+
try:
|
|
595
|
+
return bool(int(_cell_str(r["p_model"])))
|
|
596
|
+
except (ValueError, TypeError):
|
|
597
|
+
return True
|
|
598
|
+
# Frame existed but didn't contain the flag — treat as default.
|
|
599
|
+
return True
|
|
600
|
+
return True
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _read_period_set(path: Path,
|
|
604
|
+
*, provider: "object | None" = None) -> set[str]:
|
|
605
|
+
"""Read a single-column period CSV (header row, then one period per row)."""
|
|
606
|
+
from flextool.engine_polars._emit_provider_io import _provider_key
|
|
607
|
+
|
|
608
|
+
def _cell_str(value: "object | None") -> str:
|
|
609
|
+
return "" if value is None else str(value)
|
|
610
|
+
|
|
611
|
+
key = _provider_key(path)
|
|
612
|
+
if not provider.has(key):
|
|
613
|
+
return set()
|
|
614
|
+
df = provider.get(key)
|
|
615
|
+
out: set[str] = set()
|
|
616
|
+
if df.width == 0:
|
|
617
|
+
return out
|
|
618
|
+
col0 = df.columns[0]
|
|
619
|
+
for value in df.get_column(col0):
|
|
620
|
+
c0 = _cell_str(value)
|
|
621
|
+
if c0:
|
|
622
|
+
out.add(c0)
|
|
623
|
+
return out
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _read_realized_dispatch_periods(path: Path,
|
|
627
|
+
*, provider: "object | None" = None) -> set[str]:
|
|
628
|
+
"""Read distinct periods from ``realized_dispatch.csv``."""
|
|
629
|
+
from flextool.engine_polars._emit_provider_io import _provider_key
|
|
630
|
+
|
|
631
|
+
def _cell_str(value: "object | None") -> str:
|
|
632
|
+
return "" if value is None else str(value)
|
|
633
|
+
|
|
634
|
+
key = _provider_key(path)
|
|
635
|
+
if not provider.has(key):
|
|
636
|
+
return set()
|
|
637
|
+
df = provider.get(key)
|
|
638
|
+
if "period" not in df.columns:
|
|
639
|
+
return set()
|
|
640
|
+
out: set[str] = set()
|
|
641
|
+
for value in df.get_column("period"):
|
|
642
|
+
c = _cell_str(value)
|
|
643
|
+
if c:
|
|
644
|
+
out.add(c)
|
|
645
|
+
return out
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _read_frame(path: Path,
|
|
649
|
+
*, provider: "object | None" = None,
|
|
650
|
+
consumer: str = "SolveContext") -> pl.DataFrame | None:
|
|
651
|
+
"""Provider-first frame fetch with strict semantics.
|
|
652
|
+
|
|
653
|
+
Returns the frame when present (Provider lookup if *provider* is
|
|
654
|
+
supplied; disk read when *provider* is None — the legacy test-only
|
|
655
|
+
path). Returns ``None`` when the source genuinely yields an empty
|
|
656
|
+
body (height 0 or NoDataError). Raises :class:`FlexDataError` when
|
|
657
|
+
a Provider is supplied but lacks the carrier.
|
|
658
|
+
"""
|
|
659
|
+
if provider is not None:
|
|
660
|
+
df = _provider_fetch_or_raise(provider, path, consumer)
|
|
661
|
+
if df.height == 0:
|
|
662
|
+
return None
|
|
663
|
+
return df
|
|
664
|
+
if not path.exists():
|
|
665
|
+
return None
|
|
666
|
+
try:
|
|
667
|
+
df = _read_csv_file(path)
|
|
668
|
+
except pl.exceptions.NoDataError:
|
|
669
|
+
return None
|
|
670
|
+
if df.height == 0:
|
|
671
|
+
return None
|
|
672
|
+
return df
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _load_period_in_use(path: Path,
|
|
676
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
677
|
+
"""Load ``period_in_use_set.csv`` and rename to canonical ``[d]``.
|
|
678
|
+
|
|
679
|
+
Preserves CSV row order (``unique(maintain_order=True)``) so callers
|
|
680
|
+
that consume the frame as an ordered list of periods (e.g. the
|
|
681
|
+
canonical-order reorder in ``_dt_period_active_steps``) get the
|
|
682
|
+
same active_time_list ordering the workdir CSV exposes.
|
|
683
|
+
|
|
684
|
+
Phase 4.6 — cast the renamed ``d`` column to the canonical axis
|
|
685
|
+
Enum when activation is on so downstream joins against cascade
|
|
686
|
+
frames (which are Enum-typed) match dtypes.
|
|
687
|
+
"""
|
|
688
|
+
from flextool.engine_polars._axis_enums import (
|
|
689
|
+
get_global_axis_enums,
|
|
690
|
+
cast_frame_axes,
|
|
691
|
+
)
|
|
692
|
+
_live = get_global_axis_enums()
|
|
693
|
+
_empty_d_dtype = (
|
|
694
|
+
_live.get("d", pl.Utf8) if _live is not None else pl.Utf8
|
|
695
|
+
)
|
|
696
|
+
empty = pl.DataFrame(schema={"d": _empty_d_dtype})
|
|
697
|
+
df = _read_frame(path, provider=provider,
|
|
698
|
+
consumer="SolveContext.period_in_use")
|
|
699
|
+
if df is None:
|
|
700
|
+
return empty
|
|
701
|
+
df = df.rename({df.columns[0]: "d"})
|
|
702
|
+
df = df.select("d").unique(maintain_order=True)
|
|
703
|
+
if _live is not None:
|
|
704
|
+
df = cast_frame_axes(df, _live)
|
|
705
|
+
return df
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _load_period_branch(path: Path,
|
|
709
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
710
|
+
"""Load ``period__branch.csv`` as ``[d_anchor, b]``.
|
|
711
|
+
|
|
712
|
+
Phase 4.6 — under activation cast the dim columns to the canonical
|
|
713
|
+
axis enums so downstream joins see Enum dtypes consistently.
|
|
714
|
+
"""
|
|
715
|
+
from flextool.engine_polars._axis_enums import (
|
|
716
|
+
get_global_axis_enums,
|
|
717
|
+
cast_frame_axes,
|
|
718
|
+
)
|
|
719
|
+
_live = get_global_axis_enums()
|
|
720
|
+
_d_anchor = _live.get("d_anchor", pl.Utf8) if _live is not None else pl.Utf8
|
|
721
|
+
_b = _live.get("branch", pl.Utf8) if _live is not None else pl.Utf8
|
|
722
|
+
empty = pl.DataFrame(schema={"d_anchor": _d_anchor, "b": _b})
|
|
723
|
+
df = _read_frame(path, provider=provider,
|
|
724
|
+
consumer="SolveContext.period_branch")
|
|
725
|
+
if df is None:
|
|
726
|
+
return empty
|
|
727
|
+
rename = {}
|
|
728
|
+
if "period" in df.columns:
|
|
729
|
+
rename["period"] = "d_anchor"
|
|
730
|
+
if "branch" in df.columns:
|
|
731
|
+
rename["branch"] = "b"
|
|
732
|
+
df = df.rename(rename)
|
|
733
|
+
cols = [c for c in ("d_anchor", "b") if c in df.columns]
|
|
734
|
+
df = df.select(cols).unique(maintain_order=True) if cols else empty
|
|
735
|
+
if _live is not None:
|
|
736
|
+
df = cast_frame_axes(df, _live)
|
|
737
|
+
return df
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def _load_edd_history(path: Path,
|
|
741
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
742
|
+
"""Load ``edd_history.csv`` — schema preserved as-is."""
|
|
743
|
+
df = _read_frame(path, provider=provider,
|
|
744
|
+
consumer="SolveContext.edd_history")
|
|
745
|
+
if df is None:
|
|
746
|
+
return pl.DataFrame()
|
|
747
|
+
return df
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _load_steps_in_use(path: Path,
|
|
751
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
752
|
+
"""Load ``steps_in_use.csv`` and rename to canonical ``[d, t,
|
|
753
|
+
step_duration]``. Empty frame when missing.
|
|
754
|
+
|
|
755
|
+
Audit §Category B (WriterSnapshot top-7): the cascade helpers
|
|
756
|
+
consume this frame after renaming ``period`` → ``d`` and
|
|
757
|
+
``step`` / ``time`` → ``t``; this helper centralises that rename
|
|
758
|
+
so the cascade sees the canonical form directly.
|
|
759
|
+
|
|
760
|
+
Phase 4.6 — cast ``d``/``t`` to canonical axis enums when
|
|
761
|
+
activation is on so downstream joins (against cascade frames that
|
|
762
|
+
are Enum-typed) match dtypes.
|
|
763
|
+
"""
|
|
764
|
+
from flextool.engine_polars._axis_enums import (
|
|
765
|
+
get_global_axis_enums,
|
|
766
|
+
cast_frame_axes,
|
|
767
|
+
)
|
|
768
|
+
_live = get_global_axis_enums()
|
|
769
|
+
_d_dt = _live.get("d", pl.Utf8) if _live is not None else pl.Utf8
|
|
770
|
+
_t_dt = _live.get("t", pl.Utf8) if _live is not None else pl.Utf8
|
|
771
|
+
empty = pl.DataFrame(
|
|
772
|
+
schema={"d": _d_dt, "t": _t_dt, "step_duration": pl.Float64}
|
|
773
|
+
)
|
|
774
|
+
df = _read_frame(path, provider=provider,
|
|
775
|
+
consumer="SolveContext.steps_in_use")
|
|
776
|
+
if df is None:
|
|
777
|
+
return empty
|
|
778
|
+
period_col = next((c for c in ("period", "d") if c in df.columns), None)
|
|
779
|
+
step_col = next(
|
|
780
|
+
(c for c in ("step", "t", "time") if c in df.columns), None
|
|
781
|
+
)
|
|
782
|
+
if period_col is None or step_col is None:
|
|
783
|
+
return empty
|
|
784
|
+
out = df.rename({period_col: "d", step_col: "t"})
|
|
785
|
+
if "step_duration" in out.columns:
|
|
786
|
+
out = out.with_columns(
|
|
787
|
+
pl.col("step_duration").cast(pl.Float64, strict=False)
|
|
788
|
+
)
|
|
789
|
+
cols = [c for c in ("d", "t", "step_duration") if c in out.columns]
|
|
790
|
+
out = out.select(cols)
|
|
791
|
+
if _live is not None:
|
|
792
|
+
out = cast_frame_axes(out, _live)
|
|
793
|
+
return out
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _load_period_share(solve_data_dir: Path,
|
|
797
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
798
|
+
"""Load ``complete_period_share_of_year_calc.csv`` (preferred) or
|
|
799
|
+
its non-``_calc`` variant — rename ``period`` → ``d``.
|
|
800
|
+
|
|
801
|
+
Audit §Category B (WriterSnapshot top-7): the cascade currently
|
|
802
|
+
falls through both names at its read sites (e.g.
|
|
803
|
+
``_derived_params.py:7984-7988``). Centralising the fallback here
|
|
804
|
+
gives every consumer the same semantics.
|
|
805
|
+
"""
|
|
806
|
+
empty = pl.DataFrame(schema={"d": pl.Utf8, "value": pl.Float64})
|
|
807
|
+
cand_paths = (
|
|
808
|
+
solve_data_dir / "complete_period_share_of_year_calc.csv",
|
|
809
|
+
solve_data_dir / "complete_period_share_of_year.csv",
|
|
810
|
+
)
|
|
811
|
+
if provider is not None:
|
|
812
|
+
# Provider-strict: at least one of the canonical variants must
|
|
813
|
+
# carry the frame. We probe ``_calc`` first (the producer's
|
|
814
|
+
# default), then the legacy non-``_calc`` name as a fallback.
|
|
815
|
+
from ._emit_provider_io import _provider_key
|
|
816
|
+
for path in cand_paths:
|
|
817
|
+
key = _provider_key(path)
|
|
818
|
+
if provider.has(key):
|
|
819
|
+
df = provider.get(key)
|
|
820
|
+
if df is None or df.height == 0:
|
|
821
|
+
continue
|
|
822
|
+
out = df
|
|
823
|
+
if "period" in out.columns:
|
|
824
|
+
out = out.rename({"period": "d"})
|
|
825
|
+
if "value" in out.columns:
|
|
826
|
+
out = out.with_columns(
|
|
827
|
+
pl.col("value").cast(pl.Float64, strict=False)
|
|
828
|
+
)
|
|
829
|
+
cols = [c for c in ("d", "value") if c in out.columns]
|
|
830
|
+
if cols:
|
|
831
|
+
return out.select(cols)
|
|
832
|
+
# Neither variant carried a usable frame — treat as empty.
|
|
833
|
+
# ``period_share_of_year`` is genuinely optional (some fixtures
|
|
834
|
+
# skip the inflation-factor pipeline entirely); empty is the
|
|
835
|
+
# legacy semantics.
|
|
836
|
+
return empty
|
|
837
|
+
# Legacy disk path — kept until Phase 4 makes provider mandatory.
|
|
838
|
+
for path in cand_paths:
|
|
839
|
+
if path.exists():
|
|
840
|
+
try:
|
|
841
|
+
df = _read_csv_file(path)
|
|
842
|
+
except pl.exceptions.NoDataError:
|
|
843
|
+
continue
|
|
844
|
+
if df.height == 0:
|
|
845
|
+
continue
|
|
846
|
+
out = df
|
|
847
|
+
if "period" in out.columns:
|
|
848
|
+
out = out.rename({"period": "d"})
|
|
849
|
+
if "value" in out.columns:
|
|
850
|
+
out = out.with_columns(
|
|
851
|
+
pl.col("value").cast(pl.Float64, strict=False)
|
|
852
|
+
)
|
|
853
|
+
cols = [c for c in ("d", "value") if c in out.columns]
|
|
854
|
+
if cols:
|
|
855
|
+
return out.select(cols)
|
|
856
|
+
return empty
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
def _load_solve_branch_weight(path: Path,
|
|
860
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
861
|
+
"""Load ``solve_branch_weight.csv`` and rename to canonical
|
|
862
|
+
``[b, p_branch_weight_input]``.
|
|
863
|
+
|
|
864
|
+
Audit §Category B (WriterSnapshot top-7).
|
|
865
|
+
"""
|
|
866
|
+
empty = pl.DataFrame(
|
|
867
|
+
schema={"b": pl.Utf8, "p_branch_weight_input": pl.Float64}
|
|
868
|
+
)
|
|
869
|
+
df = _read_frame(path, provider=provider,
|
|
870
|
+
consumer="SolveContext.solve_branch_weight")
|
|
871
|
+
if df is None:
|
|
872
|
+
return empty
|
|
873
|
+
ren = {}
|
|
874
|
+
if "branch" in df.columns:
|
|
875
|
+
ren["branch"] = "b"
|
|
876
|
+
out = df.rename(ren)
|
|
877
|
+
if "p_branch_weight_input" in out.columns:
|
|
878
|
+
out = out.with_columns(
|
|
879
|
+
pl.col("p_branch_weight_input").cast(pl.Float64, strict=False)
|
|
880
|
+
)
|
|
881
|
+
cols = [c for c in ("b", "p_branch_weight_input") if c in out.columns]
|
|
882
|
+
return out.select(cols) if cols else empty
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
__all__ = ["SolveContext", "FlexDataError"]
|