flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,769 @@
|
|
|
1
|
+
"""Cluster D — multi-branch / stochastic propagation (Δ.8).
|
|
2
|
+
|
|
3
|
+
Lazy-polars port of flextool's stochastic-branch derived helpers.
|
|
4
|
+
Cluster D is the fourth of six derived-helper port phases per
|
|
5
|
+
``audit/native_data_path_design_derived_clusters.md``.
|
|
6
|
+
|
|
7
|
+
Cluster D fields (per
|
|
8
|
+
``audit/native_data_path_design_derived_clusters.md``):
|
|
9
|
+
|
|
10
|
+
* ``pd_branch_weight`` — per-period branch probability weight. In
|
|
11
|
+
deterministic / non-stochastic fixtures, defaults to 1.0 per
|
|
12
|
+
realised period. In multi-branch fixtures: normalises
|
|
13
|
+
``solve_branch_weight[d]`` against the sum across sibling branches
|
|
14
|
+
that share the same ``(d2, b)`` parent and the same first-step.
|
|
15
|
+
* ``pdt_branch_weight`` — per-(d, t) variant. Same normalisation but
|
|
16
|
+
iterating over ``dt`` rows instead of period × first-step pairs.
|
|
17
|
+
Output is dense over ``dt`` (mirroring the .mod's ``param
|
|
18
|
+
pdt_branch_weight {(d, t) in dt}`` declaration).
|
|
19
|
+
* ``dt_non_anticipativity`` — (d, t) where the four
|
|
20
|
+
``non_anticipativity_*`` constraints fire. Built as
|
|
21
|
+
``realized_dispatch ∪ fix_storage_timesteps``. Empty when
|
|
22
|
+
stochastics are inactive.
|
|
23
|
+
* ``period_branch_full`` — full ``period__branch.csv`` (anchor →
|
|
24
|
+
sibling). Used by the model layer's storage / online / reserve
|
|
25
|
+
non-anticipativity coupling.
|
|
26
|
+
* ``period_in_use_set`` — periods active in the active solve.
|
|
27
|
+
``realized_periods ∪ stochastic_branches ∪ invest_periods ∪
|
|
28
|
+
fix_storage_periods``.
|
|
29
|
+
|
|
30
|
+
All helpers are lazy ``pl.LazyFrame`` chains; the public
|
|
31
|
+
``apply_branch_cluster`` entry collects once per emitted Param.
|
|
32
|
+
|
|
33
|
+
Algorithm reference: flextool's
|
|
34
|
+
``preprocessing/period_calculated_params.py:write_branch_weights:364-451``
|
|
35
|
+
and ``preprocessing/per_solve_sets.py:96-101, 267-276``.
|
|
36
|
+
|
|
37
|
+
R-O6 invariant (per ``audit/a6_b_dim_alternative.md``): branches stay
|
|
38
|
+
realised-only for invest. This module does NOT introduce per-branch
|
|
39
|
+
``v_invest`` variables; it only emits the *operational* probability
|
|
40
|
+
weights and the non-anticipativity gate that pin storage / online /
|
|
41
|
+
reserve dispatch across siblings.
|
|
42
|
+
|
|
43
|
+
Workdir CSV reads (deferred to in-memory ``SolveContext`` per Δ.9+):
|
|
44
|
+
|
|
45
|
+
* ``solve_data/period__branch.csv`` — anchor → sibling pairs.
|
|
46
|
+
* ``solve_data/solve_branch_weight.csv`` — branch → input weight.
|
|
47
|
+
* ``solve_data/first_timesteps.csv`` — period → first step.
|
|
48
|
+
* ``solve_data/period_in_use_set.csv`` — output domain (active set).
|
|
49
|
+
* ``solve_data/realized_dispatch.csv`` — realised dispatch (d, t).
|
|
50
|
+
* ``solve_data/fix_storage_timesteps.csv`` — fix-storage (d, t).
|
|
51
|
+
* ``input/groupIncludeStochastics.csv`` — stochastic-coupling groups.
|
|
52
|
+
"""
|
|
53
|
+
from __future__ import annotations
|
|
54
|
+
|
|
55
|
+
from pathlib import Path
|
|
56
|
+
from typing import TYPE_CHECKING
|
|
57
|
+
|
|
58
|
+
import polars as pl
|
|
59
|
+
|
|
60
|
+
from polar_high import Param
|
|
61
|
+
|
|
62
|
+
from ._axis_enums import (
|
|
63
|
+
alias_to_axis,
|
|
64
|
+
cast_dim,
|
|
65
|
+
rename_to_axis,
|
|
66
|
+
schema_dtype,
|
|
67
|
+
)
|
|
68
|
+
from ._emit_provider_io import _provider_key
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _provider_get(provider, path: "Path") -> "pl.DataFrame | None":
|
|
72
|
+
"""Provider-only fetch. Returns ``None`` when the Provider is
|
|
73
|
+
missing or doesn't carry *path*'s canonical key.
|
|
74
|
+
"""
|
|
75
|
+
if provider is None:
|
|
76
|
+
return None
|
|
77
|
+
key = _provider_key(path)
|
|
78
|
+
if not provider.has(key):
|
|
79
|
+
return None
|
|
80
|
+
return provider.get(key)
|
|
81
|
+
|
|
82
|
+
# Substrate handle for the cascade-wide axis enum vocabulary.
|
|
83
|
+
# Bare ``None`` here; ``cast_dim`` / ``schema_dtype`` in
|
|
84
|
+
# ``_axis_enums`` fall back to ``_LIVE_AXIS_ENUMS_CTX`` (the live
|
|
85
|
+
# ContextVar) when this is ``None``, so substrate sites pick up
|
|
86
|
+
# activation set by ``load_flextool`` automatically.
|
|
87
|
+
_enums: "dict | None" = None
|
|
88
|
+
|
|
89
|
+
if TYPE_CHECKING:
|
|
90
|
+
from flextool.engine_polars._input_source import InputSource
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
# Workdir CSV readers — lazy frames
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _empty_lf(schema: dict[str, pl.DataType]) -> pl.LazyFrame:
|
|
99
|
+
"""Return an empty :class:`pl.LazyFrame` with the given schema."""
|
|
100
|
+
return pl.DataFrame(schema=schema).lazy()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _maybe_provider_lf(provider, path: Path,
|
|
104
|
+
rename: dict[str, str] | None = None,
|
|
105
|
+
) -> pl.LazyFrame | None:
|
|
106
|
+
"""Return a lazy frame from the Provider with optional rename.
|
|
107
|
+
|
|
108
|
+
Returns ``None`` when the Provider is missing or doesn't carry the
|
|
109
|
+
canonical key for *path*.
|
|
110
|
+
"""
|
|
111
|
+
df = _provider_get(provider, path)
|
|
112
|
+
if df is None or df.height == 0:
|
|
113
|
+
return None
|
|
114
|
+
lf = df.lazy()
|
|
115
|
+
if rename:
|
|
116
|
+
# Only rename columns that actually exist (defensive against
|
|
117
|
+
# column-name drift across fixture vintages).
|
|
118
|
+
cols = df.columns
|
|
119
|
+
applied = {k: v for k, v in rename.items() if k in cols}
|
|
120
|
+
if applied:
|
|
121
|
+
lf = lf.pipe(rename_to_axis, applied)
|
|
122
|
+
return lf
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def period_branch_pairs_lf(
|
|
126
|
+
workdir: Path | None,
|
|
127
|
+
*,
|
|
128
|
+
ctx: "object | None" = None,
|
|
129
|
+
provider: "object | None" = None,
|
|
130
|
+
) -> pl.LazyFrame:
|
|
131
|
+
"""Read ``solve_data/period__branch.csv`` as a lazy ``(d, b)`` frame.
|
|
132
|
+
|
|
133
|
+
Returns an empty frame (with the correct schema) when the file is
|
|
134
|
+
absent / empty — non-stochastic fixtures' "deterministic" path.
|
|
135
|
+
|
|
136
|
+
The ``b`` column is the *sibling branch* name (in non-stochastic
|
|
137
|
+
fixtures, ``d == b`` for every row).
|
|
138
|
+
|
|
139
|
+
Path B Cat B (WriterSnapshot top-7): when ``ctx`` is supplied, the
|
|
140
|
+
cached ``ctx.period_branch`` (canonical ``[d_anchor, b]``) is
|
|
141
|
+
rewritten to ``[d, b]`` lazily — avoids re-reading the CSV on every
|
|
142
|
+
cascade pass.
|
|
143
|
+
"""
|
|
144
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
145
|
+
"b": schema_dtype(_enums, "b")}
|
|
146
|
+
if ctx is not None:
|
|
147
|
+
pb_df = getattr(ctx, "period_branch", None)
|
|
148
|
+
if pb_df is not None and pb_df.height > 0:
|
|
149
|
+
return (pb_df.lazy()
|
|
150
|
+
.select(alias_to_axis("d_anchor", "d"),
|
|
151
|
+
cast_dim(pl.col("b"), _enums, "b"))
|
|
152
|
+
.unique())
|
|
153
|
+
if workdir is None:
|
|
154
|
+
return _empty_lf(schema)
|
|
155
|
+
p = Path(workdir) / "solve_data" / "period__branch.csv"
|
|
156
|
+
lf = _maybe_provider_lf(provider, p, rename={"period": "d", "branch": "b"})
|
|
157
|
+
if lf is None:
|
|
158
|
+
return _empty_lf(schema)
|
|
159
|
+
return lf.select(
|
|
160
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
161
|
+
cast_dim(pl.col("b"), _enums, "b"),
|
|
162
|
+
).unique()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def solve_branch_weights_lf(
|
|
166
|
+
workdir: Path | None,
|
|
167
|
+
*,
|
|
168
|
+
ctx: "object | None" = None,
|
|
169
|
+
provider: "object | None" = None,
|
|
170
|
+
) -> pl.LazyFrame:
|
|
171
|
+
"""Read ``solve_data/solve_branch_weight.csv`` as ``(b, w)`` lazy.
|
|
172
|
+
|
|
173
|
+
The flextool CSV header is either ``branch,p_branch_weight_input``
|
|
174
|
+
(stochastic fixtures) or ``branch,value`` (deterministic). We
|
|
175
|
+
normalise to ``(b, w)``.
|
|
176
|
+
|
|
177
|
+
Defaults to the empty frame (schema only) when the file is absent.
|
|
178
|
+
|
|
179
|
+
Path B Cat B (WriterSnapshot top-7): when ``ctx`` is supplied, the
|
|
180
|
+
cached ``ctx.solve_branch_weight`` (canonical
|
|
181
|
+
``[b, p_branch_weight_input]``) is used directly.
|
|
182
|
+
"""
|
|
183
|
+
schema = {"b": schema_dtype(_enums, "b"), "w": pl.Float64}
|
|
184
|
+
if ctx is not None:
|
|
185
|
+
sbw = getattr(ctx, "solve_branch_weight", None)
|
|
186
|
+
if sbw is not None and sbw.height > 0:
|
|
187
|
+
return (sbw.lazy()
|
|
188
|
+
.select(cast_dim(pl.col("b"), _enums, "b"),
|
|
189
|
+
pl.col("p_branch_weight_input").alias("w")))
|
|
190
|
+
if workdir is None:
|
|
191
|
+
return _empty_lf(schema)
|
|
192
|
+
p = Path(workdir) / "solve_data" / "solve_branch_weight.csv"
|
|
193
|
+
df = _provider_get(provider, p)
|
|
194
|
+
if df is None or df.height == 0:
|
|
195
|
+
return _empty_lf(schema)
|
|
196
|
+
cols = df.columns
|
|
197
|
+
b_col = "branch" if "branch" in cols else cols[0]
|
|
198
|
+
if "p_branch_weight_input" in cols:
|
|
199
|
+
v_col = "p_branch_weight_input"
|
|
200
|
+
elif "value" in cols:
|
|
201
|
+
v_col = "value"
|
|
202
|
+
elif len(cols) >= 2:
|
|
203
|
+
v_col = cols[1]
|
|
204
|
+
else:
|
|
205
|
+
return _empty_lf(schema)
|
|
206
|
+
return (df.lazy()
|
|
207
|
+
.select(alias_to_axis(b_col, "b"),
|
|
208
|
+
pl.col(v_col).cast(pl.Float64, strict=False).alias("w")))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def first_timesteps_lf(workdir: Path | None,
|
|
212
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
213
|
+
"""Read ``solve_data/first_timesteps.csv`` as ``(d, ts)`` lazy.
|
|
214
|
+
|
|
215
|
+
Maps each period (or branch period) to its first timestep — the
|
|
216
|
+
discriminator the ``pd_branch_weight`` algorithm uses for grouping
|
|
217
|
+
sibling branches.
|
|
218
|
+
"""
|
|
219
|
+
schema = {"d": schema_dtype(_enums, "d"), "ts": pl.Utf8}
|
|
220
|
+
if workdir is None:
|
|
221
|
+
return _empty_lf(schema)
|
|
222
|
+
p = Path(workdir) / "solve_data" / "first_timesteps.csv"
|
|
223
|
+
df = _provider_get(provider, p)
|
|
224
|
+
if df is None or df.height == 0:
|
|
225
|
+
return _empty_lf(schema)
|
|
226
|
+
cols = df.columns
|
|
227
|
+
d_col = "period" if "period" in cols else cols[0]
|
|
228
|
+
s_col = ("step" if "step" in cols else
|
|
229
|
+
("time" if "time" in cols else cols[1]))
|
|
230
|
+
return (df.lazy()
|
|
231
|
+
.select(alias_to_axis(d_col, "d"),
|
|
232
|
+
pl.col(s_col).cast(pl.Utf8, strict=False).alias("ts")))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def period_in_use_set_lf(workdir: Path | None,
|
|
236
|
+
source: "InputSource | None" = None,
|
|
237
|
+
active_solve: str | None = None,
|
|
238
|
+
*,
|
|
239
|
+
ctx: "object | None" = None,
|
|
240
|
+
provider: "object | None" = None,
|
|
241
|
+
) -> pl.LazyFrame:
|
|
242
|
+
"""Read ``solve_data/period_in_use_set.csv`` as ``(d,)`` lazy.
|
|
243
|
+
|
|
244
|
+
When the workdir CSV is absent (single-solve / no chain runner),
|
|
245
|
+
fall back to ``realized_periods ∪ invest_periods`` from the
|
|
246
|
+
``solve`` parameters. This is sufficient for non-stochastic
|
|
247
|
+
fixtures.
|
|
248
|
+
|
|
249
|
+
Path B Cat B (WriterSnapshot top-7): when ``ctx`` is supplied, the
|
|
250
|
+
cached ``ctx.period_in_use`` (canonical ``[d]``) is used directly.
|
|
251
|
+
"""
|
|
252
|
+
schema = {"d": schema_dtype(_enums, "d")}
|
|
253
|
+
if ctx is not None:
|
|
254
|
+
piu_df = getattr(ctx, "period_in_use", None)
|
|
255
|
+
if piu_df is not None and piu_df.height > 0:
|
|
256
|
+
return piu_df.lazy().select("d").unique()
|
|
257
|
+
if workdir is not None:
|
|
258
|
+
p = Path(workdir) / "solve_data" / "period_in_use_set.csv"
|
|
259
|
+
df = _provider_get(provider, p)
|
|
260
|
+
if df is not None and df.height > 0 and df.columns:
|
|
261
|
+
col = df.columns[0]
|
|
262
|
+
return (df.lazy()
|
|
263
|
+
.select(alias_to_axis(col, "d"))
|
|
264
|
+
.unique())
|
|
265
|
+
# Fall back to source-derived realized + invest.
|
|
266
|
+
if source is None or active_solve is None:
|
|
267
|
+
return _empty_lf(schema)
|
|
268
|
+
parts: list[pl.LazyFrame] = []
|
|
269
|
+
for ec, par in (("solve", "realized_periods"),
|
|
270
|
+
("solve", "invest_periods")):
|
|
271
|
+
try:
|
|
272
|
+
df = source.parameter(ec, par)
|
|
273
|
+
except KeyError:
|
|
274
|
+
continue
|
|
275
|
+
if df.height == 0:
|
|
276
|
+
continue
|
|
277
|
+
parts.append(df.lazy()
|
|
278
|
+
.filter(pl.col("name") == active_solve)
|
|
279
|
+
.select(alias_to_axis("value", "d")))
|
|
280
|
+
if not parts:
|
|
281
|
+
return _empty_lf(schema)
|
|
282
|
+
return pl.concat(parts).unique()
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def realized_dispatch_lf(workdir: Path | None,
|
|
286
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
287
|
+
"""Read ``solve_data/realized_dispatch.csv`` as ``(d, t)`` lazy."""
|
|
288
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
289
|
+
"t": schema_dtype(_enums, "t")}
|
|
290
|
+
if workdir is None:
|
|
291
|
+
return _empty_lf(schema)
|
|
292
|
+
p = Path(workdir) / "solve_data" / "realized_dispatch.csv"
|
|
293
|
+
df = _provider_get(provider, p)
|
|
294
|
+
if df is None or df.height == 0:
|
|
295
|
+
return _empty_lf(schema)
|
|
296
|
+
cols = df.columns
|
|
297
|
+
d_col = "period" if "period" in cols else cols[0]
|
|
298
|
+
t_col = ("step" if "step" in cols else
|
|
299
|
+
("time" if "time" in cols else cols[1]))
|
|
300
|
+
return (df.lazy()
|
|
301
|
+
.select(alias_to_axis(d_col, "d"),
|
|
302
|
+
alias_to_axis(t_col, "t"))
|
|
303
|
+
.unique())
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def fix_storage_timesteps_lf(workdir: Path | None,
|
|
307
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
308
|
+
"""Read ``solve_data/fix_storage_timesteps.csv`` as ``(d, t)`` lazy."""
|
|
309
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
310
|
+
"t": schema_dtype(_enums, "t")}
|
|
311
|
+
if workdir is None:
|
|
312
|
+
return _empty_lf(schema)
|
|
313
|
+
p = Path(workdir) / "solve_data" / "fix_storage_timesteps.csv"
|
|
314
|
+
df = _provider_get(provider, p)
|
|
315
|
+
if df is None or df.height == 0:
|
|
316
|
+
return _empty_lf(schema)
|
|
317
|
+
cols = df.columns
|
|
318
|
+
d_col = "period" if "period" in cols else cols[0]
|
|
319
|
+
t_col = ("step" if "step" in cols else
|
|
320
|
+
("time" if "time" in cols else cols[1]))
|
|
321
|
+
return (df.lazy()
|
|
322
|
+
.select(alias_to_axis(d_col, "d"),
|
|
323
|
+
alias_to_axis(t_col, "t"))
|
|
324
|
+
.unique())
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def steps_in_use_lf(workdir: Path | None,
|
|
328
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
329
|
+
"""Read ``solve_data/steps_in_use.csv`` as ``(d, t)`` lazy."""
|
|
330
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
331
|
+
"t": schema_dtype(_enums, "t")}
|
|
332
|
+
if workdir is None:
|
|
333
|
+
return _empty_lf(schema)
|
|
334
|
+
p = Path(workdir) / "solve_data" / "steps_in_use.csv"
|
|
335
|
+
df = _provider_get(provider, p)
|
|
336
|
+
if df is None or df.height == 0:
|
|
337
|
+
return _empty_lf(schema)
|
|
338
|
+
cols = df.columns
|
|
339
|
+
d_col = "period" if "period" in cols else cols[0]
|
|
340
|
+
t_col = ("step" if "step" in cols else
|
|
341
|
+
("time" if "time" in cols else cols[1]))
|
|
342
|
+
return (df.lazy()
|
|
343
|
+
.select(alias_to_axis(d_col, "d"),
|
|
344
|
+
alias_to_axis(t_col, "t"))
|
|
345
|
+
.unique())
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# ---------------------------------------------------------------------------
|
|
349
|
+
# pd_branch_weight / pdt_branch_weight — lazy normalisation
|
|
350
|
+
# ---------------------------------------------------------------------------
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def pd_branch_weight_lf(
|
|
354
|
+
workdir: Path | None,
|
|
355
|
+
source: "InputSource | None" = None,
|
|
356
|
+
active_solve: str | None = None,
|
|
357
|
+
*,
|
|
358
|
+
ctx: "object | None" = None,
|
|
359
|
+
provider: "object | None" = None,
|
|
360
|
+
) -> pl.LazyFrame:
|
|
361
|
+
"""Per-period branch weight (lazy) — full multi-branch cascade.
|
|
362
|
+
|
|
363
|
+
Mirrors flextool's
|
|
364
|
+
``preprocessing/period_calculated_params.py:write_branch_weights:364-451``
|
|
365
|
+
pd loop::
|
|
366
|
+
|
|
367
|
+
pd_branch_weight[d] = w[d] / sum w[b] over branches b such that
|
|
368
|
+
(d2, b) ∈ period__branch
|
|
369
|
+
AND (b, ts) ∈ period__time_first
|
|
370
|
+
AND (d, ts) ∈ period__time_first (same first-step as d)
|
|
371
|
+
AND (d2, d) ∈ period__branch (same parent)
|
|
372
|
+
|
|
373
|
+
The (d2, b) pair iteration deliberately counts a branch ``b`` once
|
|
374
|
+
per parent ``d2`` it shares with ``d`` (so multi-parent siblings
|
|
375
|
+
contribute to the denominator multiple times — matching the .mod's
|
|
376
|
+
``sum {d2, b in period__branch}`` semantics).
|
|
377
|
+
|
|
378
|
+
Defaults to ``1.0`` per realised period when ``period__branch`` is
|
|
379
|
+
silent (deterministic fixtures).
|
|
380
|
+
|
|
381
|
+
Returns a lazy ``(d, value)`` frame. Empty frame when no
|
|
382
|
+
``period_in_use`` rows exist.
|
|
383
|
+
"""
|
|
384
|
+
pb = period_branch_pairs_lf(workdir, ctx=ctx, provider=provider).collect()
|
|
385
|
+
piu = period_in_use_set_lf(workdir, source, active_solve, ctx=ctx,
|
|
386
|
+
provider=provider).collect()
|
|
387
|
+
if piu.height == 0:
|
|
388
|
+
return _empty_lf({"d": pl.Utf8, "value": pl.Float64})
|
|
389
|
+
if pb.height == 0:
|
|
390
|
+
# Deterministic fallback: 1.0 per realised period.
|
|
391
|
+
return (piu.lazy()
|
|
392
|
+
.select(pl.col("d"))
|
|
393
|
+
.with_columns(value=pl.lit(1.0))
|
|
394
|
+
.sort("d"))
|
|
395
|
+
bw = solve_branch_weights_lf(workdir, ctx=ctx, provider=provider).collect()
|
|
396
|
+
ft = first_timesteps_lf(workdir, provider=provider).collect()
|
|
397
|
+
# weights mapping with fallback 1.0 (mirrors flextool's
|
|
398
|
+
# ``branch_weight.get(b, 1.0)``).
|
|
399
|
+
w_lookup = {row["b"]: float(row["w"]) for row in bw.iter_rows(named=True)
|
|
400
|
+
if row["w"] is not None}
|
|
401
|
+
|
|
402
|
+
# Build the parent set: pb_set = frozenset((d2, b)).
|
|
403
|
+
pb_set = {(row["d"], row["b"]) for row in pb.iter_rows(named=True)}
|
|
404
|
+
# times_with_first[ts] = {b : (b, ts) ∈ first_ts}
|
|
405
|
+
first_ts_map = {row["d"]: row["ts"] for row in ft.iter_rows(named=True)}
|
|
406
|
+
times_with_first: dict[str, set[str]] = {}
|
|
407
|
+
for d, ts in first_ts_map.items():
|
|
408
|
+
times_with_first.setdefault(ts, set()).add(d)
|
|
409
|
+
|
|
410
|
+
def w(b: str) -> float:
|
|
411
|
+
return w_lookup.get(b, 1.0)
|
|
412
|
+
|
|
413
|
+
rows: list[tuple[str, float]] = []
|
|
414
|
+
period_list = piu["d"].to_list()
|
|
415
|
+
for d in period_list:
|
|
416
|
+
ts = first_ts_map.get(d)
|
|
417
|
+
if ts is None:
|
|
418
|
+
continue
|
|
419
|
+
branches_at_ts = times_with_first.get(ts, set())
|
|
420
|
+
denom = 0.0
|
|
421
|
+
# Iterate (d2, b) pairs (mirrors mod's ``sum {d2, b in pb}``)
|
|
422
|
+
for row in pb.iter_rows(named=True):
|
|
423
|
+
d2, b = row["d"], row["b"]
|
|
424
|
+
if b not in branches_at_ts:
|
|
425
|
+
continue
|
|
426
|
+
if (d2, d) not in pb_set:
|
|
427
|
+
continue
|
|
428
|
+
denom += w(b)
|
|
429
|
+
if denom == 0.0:
|
|
430
|
+
continue
|
|
431
|
+
rows.append((d, w(d) / denom))
|
|
432
|
+
if not rows:
|
|
433
|
+
return _empty_lf({"d": pl.Utf8, "value": pl.Float64})
|
|
434
|
+
out = pl.DataFrame(rows, schema={"d": schema_dtype(_enums, "d"),
|
|
435
|
+
"value": pl.Float64},
|
|
436
|
+
orient="row").sort("d")
|
|
437
|
+
return out.lazy()
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def pdt_branch_weight_lf(
|
|
441
|
+
workdir: Path | None,
|
|
442
|
+
source: "InputSource | None" = None,
|
|
443
|
+
active_solve: str | None = None,
|
|
444
|
+
dt: pl.DataFrame | None = None,
|
|
445
|
+
*,
|
|
446
|
+
ctx: "object | None" = None,
|
|
447
|
+
provider: "object | None" = None,
|
|
448
|
+
) -> pl.LazyFrame:
|
|
449
|
+
"""Per-(d, t) branch weight (lazy) — full multi-branch cascade.
|
|
450
|
+
|
|
451
|
+
Mirrors flextool's
|
|
452
|
+
``preprocessing/period_calculated_params.py:write_branch_weights:364-451``
|
|
453
|
+
pdt loop::
|
|
454
|
+
|
|
455
|
+
pdt_branch_weight[d, t] = w[d] / sum w[b] over branches b such that
|
|
456
|
+
(d2, b) ∈ period__branch
|
|
457
|
+
AND (b, t) ∈ dt
|
|
458
|
+
AND (d2, d) ∈ period__branch
|
|
459
|
+
|
|
460
|
+
The output is dense over ``dt`` (the ``input.py`` cascade
|
|
461
|
+
overrides any helper output with a dt × value left-join, so we
|
|
462
|
+
must supply at least every (d, t) in dt). When no
|
|
463
|
+
``period__branch`` rows are present, defaults to 1.0 per (d, t).
|
|
464
|
+
|
|
465
|
+
The denominator-zero rows fall through to 1.0 so dense semantics
|
|
466
|
+
are preserved (matching ``input.py``'s coalesce default).
|
|
467
|
+
|
|
468
|
+
Returns a lazy ``(d, t, value)`` frame.
|
|
469
|
+
"""
|
|
470
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
471
|
+
"t": schema_dtype(_enums, "t"),
|
|
472
|
+
"value": pl.Float64}
|
|
473
|
+
if dt is None or dt.height == 0:
|
|
474
|
+
# Fall back to steps_in_use as the dense domain.
|
|
475
|
+
dt_lf = steps_in_use_lf(workdir, provider=provider).collect()
|
|
476
|
+
if dt_lf.height == 0:
|
|
477
|
+
return _empty_lf(schema)
|
|
478
|
+
dt_pairs_df = dt_lf
|
|
479
|
+
else:
|
|
480
|
+
dt_pairs_df = dt.select("d", "t").unique()
|
|
481
|
+
pb = period_branch_pairs_lf(workdir, ctx=ctx, provider=provider).collect()
|
|
482
|
+
if pb.height == 0:
|
|
483
|
+
# Default 1.0 per (d, t).
|
|
484
|
+
return (dt_pairs_df.lazy()
|
|
485
|
+
.select(cast_dim(pl.col("d"), _enums, "d"),
|
|
486
|
+
cast_dim(pl.col("t"), _enums, "t"))
|
|
487
|
+
.with_columns(value=pl.lit(1.0))
|
|
488
|
+
.sort("d", "t"))
|
|
489
|
+
bw = solve_branch_weights_lf(workdir, ctx=ctx, provider=provider).collect()
|
|
490
|
+
w_lookup = {row["b"]: float(row["w"]) for row in bw.iter_rows(named=True)
|
|
491
|
+
if row["w"] is not None}
|
|
492
|
+
pb_set = {(row["d"], row["b"]) for row in pb.iter_rows(named=True)}
|
|
493
|
+
# branches_for_t[t] = {b : (b, t) ∈ dt} — derived from the dt set.
|
|
494
|
+
# Mirrors flextool's ``branches_for_t`` from steps_in_use rows.
|
|
495
|
+
branches_for_t: dict[str, set[str]] = {}
|
|
496
|
+
dt_pairs: list[tuple[str, str]] = []
|
|
497
|
+
for r in dt_pairs_df.iter_rows(named=True):
|
|
498
|
+
d, t = str(r["d"]), str(r["t"])
|
|
499
|
+
dt_pairs.append((d, t))
|
|
500
|
+
branches_for_t.setdefault(t, set()).add(d)
|
|
501
|
+
|
|
502
|
+
def w(b: str) -> float:
|
|
503
|
+
return w_lookup.get(b, 1.0)
|
|
504
|
+
|
|
505
|
+
rows: list[tuple[str, str, float]] = []
|
|
506
|
+
pb_iter = list(pb.iter_rows(named=True))
|
|
507
|
+
for d, t in dt_pairs:
|
|
508
|
+
branches_with_t = branches_for_t.get(t, set())
|
|
509
|
+
denom = 0.0
|
|
510
|
+
for row in pb_iter:
|
|
511
|
+
d2, b = row["d"], row["b"]
|
|
512
|
+
if b not in branches_with_t:
|
|
513
|
+
continue
|
|
514
|
+
if (d2, d) not in pb_set:
|
|
515
|
+
continue
|
|
516
|
+
denom += w(b)
|
|
517
|
+
if denom == 0.0:
|
|
518
|
+
# Dense semantics — fall through to 1.0.
|
|
519
|
+
rows.append((d, t, 1.0))
|
|
520
|
+
else:
|
|
521
|
+
rows.append((d, t, w(d) / denom))
|
|
522
|
+
if not rows:
|
|
523
|
+
return _empty_lf(schema)
|
|
524
|
+
out = pl.DataFrame(rows, schema=schema, orient="row").sort("d", "t")
|
|
525
|
+
return out.lazy()
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
# ---------------------------------------------------------------------------
|
|
529
|
+
# dt_non_anticipativity — (d, t) realised-dispatch ∪ fix-storage-timesteps
|
|
530
|
+
# ---------------------------------------------------------------------------
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def dt_non_anticipativity_lf(workdir: Path | None,
|
|
534
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
535
|
+
"""Compute ``dt_non_anticipativity`` lazily as ``(d, t)``.
|
|
536
|
+
|
|
537
|
+
Mirrors flextool's
|
|
538
|
+
``preprocessing/per_solve_sets.py:267-276``::
|
|
539
|
+
|
|
540
|
+
dt_non_anticipativity = realized_dispatch ∪ fix_storage_timesteps
|
|
541
|
+
|
|
542
|
+
The four ``non_anticipativity_*`` constraints fire on this set
|
|
543
|
+
(storage_use, online_int, online_lin, reserve).
|
|
544
|
+
|
|
545
|
+
Returns the empty frame (schema only) when no stochastic / chain
|
|
546
|
+
activity is present — which keeps the model layer's
|
|
547
|
+
non-anticipativity constraints disabled by default.
|
|
548
|
+
"""
|
|
549
|
+
schema = {"d": schema_dtype(_enums, "d"),
|
|
550
|
+
"t": schema_dtype(_enums, "t")}
|
|
551
|
+
rd = realized_dispatch_lf(workdir, provider=provider).collect()
|
|
552
|
+
fs = fix_storage_timesteps_lf(workdir, provider=provider).collect()
|
|
553
|
+
if rd.height == 0 and fs.height == 0:
|
|
554
|
+
return _empty_lf(schema)
|
|
555
|
+
parts: list[pl.LazyFrame] = []
|
|
556
|
+
if rd.height > 0:
|
|
557
|
+
parts.append(rd.lazy())
|
|
558
|
+
if fs.height > 0:
|
|
559
|
+
parts.append(fs.lazy())
|
|
560
|
+
return pl.concat(parts).unique().sort("d", "t")
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
# ---------------------------------------------------------------------------
|
|
564
|
+
# period_branch_full / period_in_use_set — exposed as Param-side frames
|
|
565
|
+
# ---------------------------------------------------------------------------
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def period_branch_full_lf(workdir: Path | None,
|
|
569
|
+
*, provider: "object | None" = None) -> pl.LazyFrame:
|
|
570
|
+
"""The unfiltered ``period__branch.csv`` as ``(d, b)`` lazy.
|
|
571
|
+
|
|
572
|
+
Distinct from the existing ``period_branch`` rolling-handoff field
|
|
573
|
+
(which renames columns to ``d_upper`` / ``d``). This is the raw
|
|
574
|
+
anchor → sibling map consumed by the model layer's
|
|
575
|
+
non-anticipativity constraints.
|
|
576
|
+
"""
|
|
577
|
+
return period_branch_pairs_lf(workdir, provider=provider).select("d", "b")
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
# ---------------------------------------------------------------------------
|
|
581
|
+
# Public Param helpers (collect-at-boundary)
|
|
582
|
+
# ---------------------------------------------------------------------------
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def pd_branch_weight_param(
|
|
586
|
+
workdir: Path | None,
|
|
587
|
+
source: "InputSource | None" = None,
|
|
588
|
+
active_solve: str | None = None,
|
|
589
|
+
*,
|
|
590
|
+
ctx: "object | None" = None,
|
|
591
|
+
provider: "object | None" = None,
|
|
592
|
+
) -> Param | None:
|
|
593
|
+
"""Public ``pd_branch_weight`` :class:`Param` builder.
|
|
594
|
+
|
|
595
|
+
Returns ``None`` when no realised periods are available (matches
|
|
596
|
+
the ``apply_derived_g`` skip-on-None contract).
|
|
597
|
+
"""
|
|
598
|
+
df = pd_branch_weight_lf(workdir, source, active_solve,
|
|
599
|
+
ctx=ctx, provider=provider).collect()
|
|
600
|
+
if df.height == 0:
|
|
601
|
+
return None
|
|
602
|
+
return Param(("d",), df)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def pdt_branch_weight_param(
|
|
606
|
+
workdir: Path | None,
|
|
607
|
+
source: "InputSource | None" = None,
|
|
608
|
+
active_solve: str | None = None,
|
|
609
|
+
dt: pl.DataFrame | None = None,
|
|
610
|
+
*,
|
|
611
|
+
ctx: "object | None" = None,
|
|
612
|
+
provider: "object | None" = None,
|
|
613
|
+
) -> Param | None:
|
|
614
|
+
"""Public ``pdt_branch_weight`` :class:`Param` builder."""
|
|
615
|
+
df = pdt_branch_weight_lf(workdir, source, active_solve, dt,
|
|
616
|
+
ctx=ctx, provider=provider).collect()
|
|
617
|
+
if df.height == 0:
|
|
618
|
+
return None
|
|
619
|
+
return Param(("d", "t"), df)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def dt_non_anticipativity_df(workdir: Path | None,
|
|
623
|
+
*, provider: "object | None" = None,
|
|
624
|
+
) -> pl.DataFrame | None:
|
|
625
|
+
"""Public ``dt_non_anticipativity`` plain-DataFrame builder."""
|
|
626
|
+
df = dt_non_anticipativity_lf(workdir, provider=provider).collect()
|
|
627
|
+
if df.height == 0:
|
|
628
|
+
return None
|
|
629
|
+
return df
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def period_branch_full_df(workdir: Path | None,
|
|
633
|
+
*, provider: "object | None" = None,
|
|
634
|
+
) -> pl.DataFrame | None:
|
|
635
|
+
"""Public ``period_branch_full`` plain-DataFrame builder."""
|
|
636
|
+
df = period_branch_full_lf(workdir, provider=provider).collect()
|
|
637
|
+
if df.height == 0:
|
|
638
|
+
return None
|
|
639
|
+
return df
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def period_in_use_set_df(
|
|
643
|
+
workdir: Path | None,
|
|
644
|
+
source: "InputSource | None" = None,
|
|
645
|
+
active_solve: str | None = None,
|
|
646
|
+
*,
|
|
647
|
+
ctx: "object | None" = None,
|
|
648
|
+
provider: "object | None" = None,
|
|
649
|
+
) -> pl.DataFrame | None:
|
|
650
|
+
"""Public ``period_in_use_set`` plain-DataFrame builder."""
|
|
651
|
+
df = period_in_use_set_lf(workdir, source, active_solve,
|
|
652
|
+
ctx=ctx, provider=provider).collect()
|
|
653
|
+
if df.height == 0:
|
|
654
|
+
return None
|
|
655
|
+
return df
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
# ---------------------------------------------------------------------------
|
|
659
|
+
# apply_branch_cluster — single-pass entry for the apply_derived_g
|
|
660
|
+
# integration. Mutates ``flex_data`` in place.
|
|
661
|
+
# ---------------------------------------------------------------------------
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def apply_branch_cluster(
|
|
665
|
+
flex_data: object,
|
|
666
|
+
source: "InputSource",
|
|
667
|
+
workdir: Path,
|
|
668
|
+
active_solve: str | None = None,
|
|
669
|
+
*,
|
|
670
|
+
ctx: "object | None" = None,
|
|
671
|
+
provider: "object | None" = None,
|
|
672
|
+
) -> None:
|
|
673
|
+
"""Apply Cluster D Params to ``flex_data``.
|
|
674
|
+
|
|
675
|
+
Order:
|
|
676
|
+
1. ``period_branch_full`` (no dependencies; trivial CSV unroll).
|
|
677
|
+
2. ``period_in_use_set`` (depends on workdir or source).
|
|
678
|
+
3. ``dt_non_anticipativity`` (depends on realized + fix-storage CSVs).
|
|
679
|
+
4. ``pd_branch_weight`` (depends on period_branch_full + first_ts +
|
|
680
|
+
period_in_use_set + branch weights).
|
|
681
|
+
5. ``pdt_branch_weight`` (depends on dt + period_branch_full +
|
|
682
|
+
branch weights).
|
|
683
|
+
|
|
684
|
+
Δ.12b — assignment is now unconditional. Each helper returns
|
|
685
|
+
``None`` when the corresponding CSV is missing/empty (e.g.
|
|
686
|
+
single-solve fixtures with no branches); ``None`` is the explicit
|
|
687
|
+
"feature inactive" signal — same outcome the seed produces. No
|
|
688
|
+
silent fall-through to a CSV-loaded value.
|
|
689
|
+
|
|
690
|
+
R-O6 invariant: this helper does NOT touch ``invest_periods`` or
|
|
691
|
+
``v_invest`` — only the operational dispatch-side weights and
|
|
692
|
+
the non-anticipativity gate.
|
|
693
|
+
"""
|
|
694
|
+
dt = getattr(flex_data, "dt", None)
|
|
695
|
+
|
|
696
|
+
# 1-2. Set frames — None == "no branches / no realized periods" is
|
|
697
|
+
# the legitimate inactive-feature signal. Seed-preserving overlay:
|
|
698
|
+
# when this lazy override yields nothing (no provider + no workdir
|
|
699
|
+
# CSVs reachable, e.g. fixture-only ``load_flextool`` paths), keep
|
|
700
|
+
# the seed value set by ``_load_branch_artefacts`` — the override is
|
|
701
|
+
# the cascade-side producer, not a forced reset.
|
|
702
|
+
_new_pbf = period_branch_full_df(workdir, provider=provider)
|
|
703
|
+
if _new_pbf is not None or getattr(flex_data, "period_branch_full", None) is None:
|
|
704
|
+
flex_data.period_branch_full = _new_pbf
|
|
705
|
+
_new_piu = period_in_use_set_df(
|
|
706
|
+
workdir, source, active_solve, ctx=ctx, provider=provider)
|
|
707
|
+
if _new_piu is not None or getattr(flex_data, "period_in_use_set", None) is None:
|
|
708
|
+
flex_data.period_in_use_set = _new_piu
|
|
709
|
+
_new_dtn = dt_non_anticipativity_df(workdir, provider=provider)
|
|
710
|
+
if _new_dtn is not None or getattr(flex_data, "dt_non_anticipativity", None) is None:
|
|
711
|
+
flex_data.dt_non_anticipativity = _new_dtn
|
|
712
|
+
|
|
713
|
+
# 3-4. Branch-weight Params (lazy ports of the previous eager
|
|
714
|
+
# helpers in ``_derived_params.py``).
|
|
715
|
+
_new_pd_bw = pd_branch_weight_param(
|
|
716
|
+
workdir, source, active_solve, ctx=ctx, provider=provider)
|
|
717
|
+
if _new_pd_bw is not None or getattr(flex_data, "pd_branch_weight", None) is None:
|
|
718
|
+
flex_data.pd_branch_weight = _new_pd_bw
|
|
719
|
+
|
|
720
|
+
pdt_bw = pdt_branch_weight_param(workdir, source, active_solve, dt,
|
|
721
|
+
ctx=ctx, provider=provider)
|
|
722
|
+
if pdt_bw is not None and dt is not None and dt.height > 0:
|
|
723
|
+
# Match input.py's dense-dt semantics: when dt is supplied,
|
|
724
|
+
# build a (d, t)-dense Param via left-join + coalesce default
|
|
725
|
+
# of 1.0. This mirrors the previous CSV-cascade behaviour at
|
|
726
|
+
# input.py:2845-2870 (preserves exact frame shape).
|
|
727
|
+
# Defensive re-cast: cast d/t to canonical Enum so the left-join
|
|
728
|
+
# against ``pdt_bw.lazy`` (Enum d/t) composes cleanly even when
|
|
729
|
+
# ``dt`` arrives with Utf8 columns.
|
|
730
|
+
base = (dt.lazy()
|
|
731
|
+
.with_columns(value=pl.lit(1.0))
|
|
732
|
+
.select(alias_to_axis("d", "d"),
|
|
733
|
+
alias_to_axis("t", "t"),
|
|
734
|
+
"value"))
|
|
735
|
+
joined = (base
|
|
736
|
+
.join(pdt_bw.lazy,
|
|
737
|
+
on=["d", "t"], how="left", suffix="__r")
|
|
738
|
+
.with_columns(value=pl.coalesce(
|
|
739
|
+
pl.col("value__r"), pl.col("value")))
|
|
740
|
+
.select("d", "t", "value")
|
|
741
|
+
.collect())
|
|
742
|
+
pdt_bw = Param(("d", "t"), joined)
|
|
743
|
+
# Seed-preserving overlay (same rationale as the set frames above):
|
|
744
|
+
# only overwrite the seed when this cascade-side path produces a
|
|
745
|
+
# value. Fixture-only loads (no provider, no workdir scaffolding)
|
|
746
|
+
# rely on ``_load_branch_artefacts``' disk-fallback seed.
|
|
747
|
+
if pdt_bw is not None or getattr(flex_data, "pdt_branch_weight", None) is None:
|
|
748
|
+
flex_data.pdt_branch_weight = pdt_bw
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
__all__ = [
|
|
752
|
+
"period_branch_pairs_lf",
|
|
753
|
+
"solve_branch_weights_lf",
|
|
754
|
+
"first_timesteps_lf",
|
|
755
|
+
"period_in_use_set_lf",
|
|
756
|
+
"realized_dispatch_lf",
|
|
757
|
+
"fix_storage_timesteps_lf",
|
|
758
|
+
"steps_in_use_lf",
|
|
759
|
+
"pd_branch_weight_lf",
|
|
760
|
+
"pdt_branch_weight_lf",
|
|
761
|
+
"dt_non_anticipativity_lf",
|
|
762
|
+
"period_branch_full_lf",
|
|
763
|
+
"pd_branch_weight_param",
|
|
764
|
+
"pdt_branch_weight_param",
|
|
765
|
+
"dt_non_anticipativity_df",
|
|
766
|
+
"period_branch_full_df",
|
|
767
|
+
"period_in_use_set_df",
|
|
768
|
+
"apply_branch_cluster",
|
|
769
|
+
]
|