flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2398 @@
|
|
|
1
|
+
"""Heavy per-(d, t) emission writers.
|
|
2
|
+
|
|
3
|
+
Three ``write_pdt*`` helpers that emit per-(entity, period, time) CSVs
|
|
4
|
+
via mostly-procedural fallback cascades:
|
|
5
|
+
|
|
6
|
+
* :func:`write_pdtNodeInflow` (3-branch).
|
|
7
|
+
* :func:`write_pdtProfile` (5-branch).
|
|
8
|
+
* :func:`write_pdtConversion_rate_section_slope` (2 outputs — section + slope; conversion_rate emit pruned, dead).
|
|
9
|
+
|
|
10
|
+
The fallback cascades use simple dict-keyed lookups — optimal for the
|
|
11
|
+
per-row access pattern. See the module docstring on
|
|
12
|
+
:mod:`._pdt_lookup` for the broader rationale.
|
|
13
|
+
|
|
14
|
+
Branches 1 (stochastic fold-in) and 2 (parent-period fold-in) of
|
|
15
|
+
``write_pdtNodeInflow`` are mod's stochastic / parent-branch fold-ins
|
|
16
|
+
(Gap E in the migration tracker). No fixture in the repo's test data
|
|
17
|
+
carries non-empty ``pbt_node_inflow``, so these branches are inert in
|
|
18
|
+
parity tests — but we keep the structure for forward-compatibility.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import polars as pl
|
|
25
|
+
|
|
26
|
+
from flextool.engine_polars._emit_provider_io import (
|
|
27
|
+
_emit,
|
|
28
|
+
_provider_key,
|
|
29
|
+
)
|
|
30
|
+
from flextool.engine_polars._vectorize import (
|
|
31
|
+
build_entity_dt_grid,
|
|
32
|
+
build_entity_period_grid,
|
|
33
|
+
build_fold_frame,
|
|
34
|
+
coalesce_value,
|
|
35
|
+
collect_value_frame,
|
|
36
|
+
lift_dict_to_lookup,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _cell_str(value: "object | None") -> str:
|
|
41
|
+
"""Reproduce a ``csv.reader`` cell string for a native frame value.
|
|
42
|
+
|
|
43
|
+
``DataFrame.write_csv`` renders ``null`` as the empty string and every
|
|
44
|
+
other scalar as its textual form; ``csv.reader`` then reads those
|
|
45
|
+
strings back. Mirror that here so dict keys / structural string
|
|
46
|
+
columns stay byte-identical to the legacy CSV round-trip — ``None`` →
|
|
47
|
+
``""`` (skipped by the original truthiness guards) while a literal
|
|
48
|
+
``"0"`` is kept. ``provider.get`` returns DATA rows only (no header
|
|
49
|
+
row to skip); an empty / missing frame yields the same empty output
|
|
50
|
+
the legacy loop produced.
|
|
51
|
+
"""
|
|
52
|
+
return "" if value is None else str(value)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _utf8_frame(columns: dict[str, list[str]]) -> pl.DataFrame:
|
|
56
|
+
"""Build an all-Utf8 ``pl.DataFrame`` from column-name → string-list.
|
|
57
|
+
|
|
58
|
+
All columns (including ``value``) are stored as ``Utf8`` so the
|
|
59
|
+
legacy ``f"...,{repr(v)}\\n"`` byte-emission round-trips identically
|
|
60
|
+
through ``pl.DataFrame.write_csv``. See
|
|
61
|
+
:mod:`._emit_chain_params._ed_value_frame` for the rationale on
|
|
62
|
+
using ``repr(v)`` directly rather than coercing to ``float`` first.
|
|
63
|
+
"""
|
|
64
|
+
schema = {name: pl.Utf8 for name in columns}
|
|
65
|
+
return pl.DataFrame(columns, schema=schema)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
# Shared CSV readers (mirror legacy helpers byte-for-byte).
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _read_singles(path: Path,
|
|
74
|
+
*, provider: "object | None" = None) -> list[str]:
|
|
75
|
+
df = provider.get(_provider_key(path))
|
|
76
|
+
if df is None:
|
|
77
|
+
return []
|
|
78
|
+
out: list[str] = []
|
|
79
|
+
for row in df.iter_rows():
|
|
80
|
+
c0 = _cell_str(row[0]) if row else ""
|
|
81
|
+
if c0:
|
|
82
|
+
out.append(c0)
|
|
83
|
+
return out
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _read_pairs(path: Path,
|
|
87
|
+
*, provider: "object | None" = None) -> list[tuple[str, str]]:
|
|
88
|
+
df = provider.get(_provider_key(path))
|
|
89
|
+
if df is None:
|
|
90
|
+
return []
|
|
91
|
+
out: list[tuple[str, str]] = []
|
|
92
|
+
for row in df.iter_rows():
|
|
93
|
+
if len(row) < 2:
|
|
94
|
+
continue
|
|
95
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
96
|
+
if c0 and c1:
|
|
97
|
+
out.append((c0, c1))
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _read_pairs_to_dict(path: Path, key_col: int,
|
|
102
|
+
*, provider: "object | None" = None,
|
|
103
|
+
) -> dict[str, list[str]]:
|
|
104
|
+
"""Generic two-col CSV → ``key_col → list[other_col]``."""
|
|
105
|
+
out: dict[str, list[str]] = {}
|
|
106
|
+
df = provider.get(_provider_key(path))
|
|
107
|
+
if df is None:
|
|
108
|
+
return out
|
|
109
|
+
other_col = 1 - key_col
|
|
110
|
+
for row in df.iter_rows():
|
|
111
|
+
if len(row) < 2:
|
|
112
|
+
continue
|
|
113
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
114
|
+
if c0 and c1:
|
|
115
|
+
cells = (c0, c1)
|
|
116
|
+
out.setdefault(cells[key_col], []).append(cells[other_col])
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _read_stochastic_entities(group_entity_csv: Path,
|
|
121
|
+
group_stochastic_csv: Path,
|
|
122
|
+
*, provider: "object | None" = None,
|
|
123
|
+
) -> set[str]:
|
|
124
|
+
"""``stoch_entity = { e : exists g ∈ groupIncludeStochastics with (g, e) ∈ group__<entity> }``."""
|
|
125
|
+
stoch_groups = frozenset(
|
|
126
|
+
_read_singles(group_stochastic_csv, provider=provider)
|
|
127
|
+
)
|
|
128
|
+
out: set[str] = set()
|
|
129
|
+
df = provider.get(_provider_key(group_entity_csv))
|
|
130
|
+
if df is None:
|
|
131
|
+
return out
|
|
132
|
+
for row in df.iter_rows():
|
|
133
|
+
if len(row) < 2:
|
|
134
|
+
continue
|
|
135
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
136
|
+
if c0 in stoch_groups and c1:
|
|
137
|
+
out.add(c1)
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
# write_pdtNodeInflow — flextool.mod L1325 (3-branch).
|
|
143
|
+
# ---------------------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def derive_pdtNodeInflow(input_dir: Path, solve_data_dir: Path,
|
|
147
|
+
*, provider: "object | None" = None,
|
|
148
|
+
) -> pl.DataFrame:
|
|
149
|
+
"""Materialise the ``pdtNodeInflow`` frame.
|
|
150
|
+
|
|
151
|
+
Columns: ``node, period, time, value`` — all ``Utf8`` (value cells
|
|
152
|
+
are ``repr(v)`` so int/float distinction round-trips byte-identically
|
|
153
|
+
to the legacy ``fh.write(f"{n},{d},{t},{repr(v)}\\n")`` emission).
|
|
154
|
+
|
|
155
|
+
Branches:
|
|
156
|
+
1. Stochastic fold-in (``pbt_node_inflow`` over stochastic node).
|
|
157
|
+
2. Parent-period fold-in (``pbt_node_inflow`` over parent periods).
|
|
158
|
+
3. Deterministic additive sum of the 4 scaling methods:
|
|
159
|
+
* ``scale_to_annual_flow`` — pfa[n,d] * pti[n,t]
|
|
160
|
+
* ``scale_in_proportion`` — pfp[n,d] * pti[n,t]
|
|
161
|
+
* ``scale_to_annual_and_peak_flow`` — slope[n,d] * pti[n,t] - section[n,d]
|
|
162
|
+
* ``use_original`` — pti[n,t]
|
|
163
|
+
|
|
164
|
+
Domain: nodes whose method is anything BUT ``no_inflow``. Non-
|
|
165
|
+
balance-union nodes get 0 (mod L1280 guard).
|
|
166
|
+
"""
|
|
167
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
168
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
169
|
+
|
|
170
|
+
inflow_method_pairs = frozenset(
|
|
171
|
+
_read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
172
|
+
provider=provider)
|
|
173
|
+
)
|
|
174
|
+
n_balance = frozenset(
|
|
175
|
+
_read_singles(solve_data_dir / "nodeBalance.csv", provider=provider)
|
|
176
|
+
)
|
|
177
|
+
n_balance_period = frozenset(
|
|
178
|
+
_read_singles(solve_data_dir / "nodeBalancePeriod.csv",
|
|
179
|
+
provider=provider)
|
|
180
|
+
)
|
|
181
|
+
balance_union = n_balance | n_balance_period
|
|
182
|
+
|
|
183
|
+
stoch_node = _read_stochastic_entities(
|
|
184
|
+
input_dir / "group__node.csv",
|
|
185
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
186
|
+
provider=provider,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
ts_for_d = _read_pairs_to_dict(
|
|
190
|
+
solve_data_dir / "first_timesteps.csv", key_col=0,
|
|
191
|
+
provider=provider,
|
|
192
|
+
)
|
|
193
|
+
tb_for_d = _read_pairs_to_dict(
|
|
194
|
+
solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
|
|
195
|
+
provider=provider,
|
|
196
|
+
)
|
|
197
|
+
# period__branch.csv stores (db, d) — child key column is 1.
|
|
198
|
+
pe_for_d = _read_pairs_to_dict(
|
|
199
|
+
solve_data_dir / "period__branch.csv", key_col=1,
|
|
200
|
+
provider=provider,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
# pbt_node_inflow{(n, branch, ts, t) → value}
|
|
204
|
+
pbt_inflow: dict[tuple[str, str, str, str], float] = {}
|
|
205
|
+
pbt_path = input_dir / "pbt_node_inflow.csv"
|
|
206
|
+
pbt_df = provider.get(_provider_key(pbt_path))
|
|
207
|
+
if pbt_df is not None:
|
|
208
|
+
for row in pbt_df.iter_rows():
|
|
209
|
+
if len(row) < 5:
|
|
210
|
+
continue
|
|
211
|
+
c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
|
|
212
|
+
_cell_str(row[2]), _cell_str(row[3]))
|
|
213
|
+
if c0 and c1 and c2 and c3:
|
|
214
|
+
try:
|
|
215
|
+
pbt_inflow[(c0, c1, c2, c3)] = float(row[4])
|
|
216
|
+
except (ValueError, TypeError):
|
|
217
|
+
continue
|
|
218
|
+
|
|
219
|
+
# ptNode_inflow{(n, t) → value}
|
|
220
|
+
pt_inflow: dict[tuple[str, str], float] = {}
|
|
221
|
+
pti_path = solve_data_dir / "ptNode_inflow.csv"
|
|
222
|
+
pti_df = provider.get(_provider_key(pti_path))
|
|
223
|
+
if pti_df is not None:
|
|
224
|
+
for row in pti_df.iter_rows():
|
|
225
|
+
if len(row) < 3:
|
|
226
|
+
continue
|
|
227
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
228
|
+
if c0 and c1:
|
|
229
|
+
try:
|
|
230
|
+
pt_inflow[(c0, c1)] = float(row[2])
|
|
231
|
+
except (ValueError, TypeError):
|
|
232
|
+
continue
|
|
233
|
+
|
|
234
|
+
# pdNode lookup limited to (annual_flow, peak_inflow).
|
|
235
|
+
pdNode_af: dict[tuple[str, str], float] = {}
|
|
236
|
+
pdNode_pk: dict[tuple[str, str], float] = {}
|
|
237
|
+
pdn_path = solve_data_dir / "pdNode.csv"
|
|
238
|
+
pdn_df = provider.get(_provider_key(pdn_path))
|
|
239
|
+
if pdn_df is not None:
|
|
240
|
+
for row in pdn_df.iter_rows():
|
|
241
|
+
if len(row) < 4:
|
|
242
|
+
continue
|
|
243
|
+
c0 = _cell_str(row[0])
|
|
244
|
+
if c0:
|
|
245
|
+
try:
|
|
246
|
+
v = float(row[3])
|
|
247
|
+
except (ValueError, TypeError):
|
|
248
|
+
continue
|
|
249
|
+
c1, c2 = _cell_str(row[1]), _cell_str(row[2])
|
|
250
|
+
if c1 == "annual_flow":
|
|
251
|
+
pdNode_af[(c0, c2)] = v
|
|
252
|
+
elif c1 == "peak_inflow":
|
|
253
|
+
pdNode_pk[(c0, c2)] = v
|
|
254
|
+
|
|
255
|
+
def _read_2_keyed_value(path: Path) -> dict[tuple[str, str], float]:
|
|
256
|
+
out: dict[tuple[str, str], float] = {}
|
|
257
|
+
df = provider.get(_provider_key(path))
|
|
258
|
+
if df is None:
|
|
259
|
+
return out
|
|
260
|
+
for row in df.iter_rows():
|
|
261
|
+
if len(row) < 3:
|
|
262
|
+
continue
|
|
263
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
264
|
+
if c0 and c1:
|
|
265
|
+
try:
|
|
266
|
+
out[(c0, c1)] = float(row[2])
|
|
267
|
+
except (ValueError, TypeError):
|
|
268
|
+
continue
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
pfa = _read_2_keyed_value(
|
|
272
|
+
solve_data_dir / "period_flow_annual_multiplier.csv"
|
|
273
|
+
)
|
|
274
|
+
pfp = _read_2_keyed_value(
|
|
275
|
+
solve_data_dir / "period_flow_proportional_multiplier.csv"
|
|
276
|
+
)
|
|
277
|
+
nos_slope = _read_2_keyed_value(solve_data_dir / "new_old_slope.csv")
|
|
278
|
+
nos_section = _read_2_keyed_value(solve_data_dir / "new_old_section.csv")
|
|
279
|
+
|
|
280
|
+
eligible_nodes = [
|
|
281
|
+
n for n in nodes if (n, "no_inflow") not in inflow_method_pairs
|
|
282
|
+
]
|
|
283
|
+
|
|
284
|
+
nodes_col: list[str] = []
|
|
285
|
+
periods_col: list[str] = []
|
|
286
|
+
times_col: list[str] = []
|
|
287
|
+
values_col: list[str] = []
|
|
288
|
+
for n in eligible_nodes:
|
|
289
|
+
is_stoch = n in stoch_node
|
|
290
|
+
in_balance = n in balance_union
|
|
291
|
+
has_scale_annual = (n, "scale_to_annual_flow") in inflow_method_pairs
|
|
292
|
+
has_scale_proportion = (n, "scale_in_proportion") in inflow_method_pairs
|
|
293
|
+
has_scale_peak = (n, "scale_to_annual_and_peak_flow") in inflow_method_pairs
|
|
294
|
+
has_use_original = (n, "use_original") in inflow_method_pairs
|
|
295
|
+
for (d, t) in dt:
|
|
296
|
+
emit_v: float | None = None
|
|
297
|
+
# Branch 1: stochastic fold-in.
|
|
298
|
+
if is_stoch:
|
|
299
|
+
total = 0.0
|
|
300
|
+
hit = False
|
|
301
|
+
for tb in tb_for_d.get(d, ()):
|
|
302
|
+
for ts in ts_for_d.get(d, ()):
|
|
303
|
+
v = pbt_inflow.get((n, tb, ts, t))
|
|
304
|
+
if v is not None:
|
|
305
|
+
total += v
|
|
306
|
+
hit = True
|
|
307
|
+
if hit:
|
|
308
|
+
emit_v = total
|
|
309
|
+
# Branch 2: parent-period fold-in.
|
|
310
|
+
if emit_v is None:
|
|
311
|
+
pe_list = pe_for_d.get(d, ())
|
|
312
|
+
ts_list = ts_for_d.get(d, ())
|
|
313
|
+
if pe_list and ts_list:
|
|
314
|
+
total = 0.0
|
|
315
|
+
hit = False
|
|
316
|
+
for pe in pe_list:
|
|
317
|
+
for tb in tb_for_d.get(pe, ()):
|
|
318
|
+
for ts in ts_list:
|
|
319
|
+
v = pbt_inflow.get((n, tb, ts, t))
|
|
320
|
+
if v is not None:
|
|
321
|
+
total += v
|
|
322
|
+
hit = True
|
|
323
|
+
if hit:
|
|
324
|
+
emit_v = total
|
|
325
|
+
# Branch 3: deterministic additive sum.
|
|
326
|
+
if emit_v is None:
|
|
327
|
+
value = 0.0
|
|
328
|
+
if in_balance:
|
|
329
|
+
pti = pt_inflow.get((n, t), 0.0)
|
|
330
|
+
if has_scale_annual and pdNode_af.get((n, d), 0.0):
|
|
331
|
+
value += pfa.get((n, d), 0.0) * pti
|
|
332
|
+
if has_scale_proportion and pdNode_af.get((n, d), 0.0):
|
|
333
|
+
value += pfp.get((n, d), 0.0) * pti
|
|
334
|
+
if (has_scale_peak
|
|
335
|
+
and pdNode_af.get((n, d), 0.0)
|
|
336
|
+
and pdNode_pk.get((n, d), 0.0)):
|
|
337
|
+
value += nos_slope.get((n, d), 0.0) * pti \
|
|
338
|
+
- nos_section.get((n, d), 0.0)
|
|
339
|
+
if has_use_original:
|
|
340
|
+
value += pti
|
|
341
|
+
emit_v = value
|
|
342
|
+
nodes_col.append(n)
|
|
343
|
+
periods_col.append(d)
|
|
344
|
+
times_col.append(t)
|
|
345
|
+
values_col.append(repr(emit_v))
|
|
346
|
+
return _utf8_frame({
|
|
347
|
+
"node": nodes_col,
|
|
348
|
+
"period": periods_col,
|
|
349
|
+
"time": times_col,
|
|
350
|
+
"value": values_col,
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def derive_pdtNodeInflow_vectorized(input_dir: Path, solve_data_dir: Path,
|
|
355
|
+
*,
|
|
356
|
+
provider: "object | None" = None,
|
|
357
|
+
engine: str = "eager") -> pl.DataFrame:
|
|
358
|
+
"""Vectorized ``pdtNodeInflow`` derive — parity with the legacy.
|
|
359
|
+
|
|
360
|
+
Replaces the per-cell 3-branch scalar cascade in
|
|
361
|
+
:func:`derive_pdtNodeInflow` with vectorized polars, still per roll
|
|
362
|
+
over the roll's own window. Output is byte-identical to
|
|
363
|
+
:func:`derive_pdtNodeInflow`: columns ``node, period, time, value``
|
|
364
|
+
all ``Utf8``, entity-major row order, ``repr(v)`` value cells.
|
|
365
|
+
|
|
366
|
+
This family composes two already-proven patterns:
|
|
367
|
+
|
|
368
|
+
* the single-``node``-key grid + ``build_fold_frame`` fold skeleton
|
|
369
|
+
(branches 1 & 2 — stochastic + parent-period fold, identical to
|
|
370
|
+
:func:`derive_pdtProfile_vectorized`); and
|
|
371
|
+
* the gated additive SUM of branch 3 (the four inflow-scaling
|
|
372
|
+
methods), assembled membership-gated-left-join + ``fill_null(0.0)``
|
|
373
|
+
with a signed-zero normalization (identical mechanics to
|
|
374
|
+
:func:`_derive_varCost_pair_vectorized`).
|
|
375
|
+
|
|
376
|
+
The fold (branches 1-2) takes priority over the branch-3 deterministic
|
|
377
|
+
sum, which is always non-null (its 0.0 floor for non-balance / no-
|
|
378
|
+
method nodes), so ``coalesce(v_fold, v_b3)`` reproduces the legacy
|
|
379
|
+
``emit_v`` priority exactly.
|
|
380
|
+
|
|
381
|
+
The reader block is copied VERBATIM from :func:`derive_pdtNodeInflow`
|
|
382
|
+
so the deduped dicts / stoch set / float parses are byte-for-byte
|
|
383
|
+
identical, and the vectorized lookups are lifted from those dicts (not
|
|
384
|
+
re-read from the CSVs).
|
|
385
|
+
"""
|
|
386
|
+
# --- Step 1: reader block copied VERBATIM from derive_pdtNodeInflow ----
|
|
387
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
388
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
389
|
+
|
|
390
|
+
inflow_method_pairs = frozenset(
|
|
391
|
+
_read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
392
|
+
provider=provider)
|
|
393
|
+
)
|
|
394
|
+
n_balance = frozenset(
|
|
395
|
+
_read_singles(solve_data_dir / "nodeBalance.csv", provider=provider)
|
|
396
|
+
)
|
|
397
|
+
n_balance_period = frozenset(
|
|
398
|
+
_read_singles(solve_data_dir / "nodeBalancePeriod.csv",
|
|
399
|
+
provider=provider)
|
|
400
|
+
)
|
|
401
|
+
balance_union = n_balance | n_balance_period
|
|
402
|
+
|
|
403
|
+
stoch_node = _read_stochastic_entities(
|
|
404
|
+
input_dir / "group__node.csv",
|
|
405
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
406
|
+
provider=provider,
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
ts_for_d = _read_pairs_to_dict(
|
|
410
|
+
solve_data_dir / "first_timesteps.csv", key_col=0,
|
|
411
|
+
provider=provider,
|
|
412
|
+
)
|
|
413
|
+
tb_for_d = _read_pairs_to_dict(
|
|
414
|
+
solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
|
|
415
|
+
provider=provider,
|
|
416
|
+
)
|
|
417
|
+
# period__branch.csv stores (db, d) — child key column is 1.
|
|
418
|
+
pe_for_d = _read_pairs_to_dict(
|
|
419
|
+
solve_data_dir / "period__branch.csv", key_col=1,
|
|
420
|
+
provider=provider,
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
# pbt_node_inflow{(n, branch, ts, t) → value}
|
|
424
|
+
pbt_inflow: dict[tuple[str, str, str, str], float] = {}
|
|
425
|
+
pbt_path = input_dir / "pbt_node_inflow.csv"
|
|
426
|
+
pbt_df = provider.get(_provider_key(pbt_path))
|
|
427
|
+
if pbt_df is not None:
|
|
428
|
+
for row in pbt_df.iter_rows():
|
|
429
|
+
if len(row) < 5:
|
|
430
|
+
continue
|
|
431
|
+
c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
|
|
432
|
+
_cell_str(row[2]), _cell_str(row[3]))
|
|
433
|
+
if c0 and c1 and c2 and c3:
|
|
434
|
+
try:
|
|
435
|
+
pbt_inflow[(c0, c1, c2, c3)] = float(row[4])
|
|
436
|
+
except (ValueError, TypeError):
|
|
437
|
+
continue
|
|
438
|
+
|
|
439
|
+
# ptNode_inflow{(n, t) → value}
|
|
440
|
+
pt_inflow: dict[tuple[str, str], float] = {}
|
|
441
|
+
pti_path = solve_data_dir / "ptNode_inflow.csv"
|
|
442
|
+
pti_df = provider.get(_provider_key(pti_path))
|
|
443
|
+
if pti_df is not None:
|
|
444
|
+
for row in pti_df.iter_rows():
|
|
445
|
+
if len(row) < 3:
|
|
446
|
+
continue
|
|
447
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
448
|
+
if c0 and c1:
|
|
449
|
+
try:
|
|
450
|
+
pt_inflow[(c0, c1)] = float(row[2])
|
|
451
|
+
except (ValueError, TypeError):
|
|
452
|
+
continue
|
|
453
|
+
|
|
454
|
+
# pdNode lookup limited to (annual_flow, peak_inflow).
|
|
455
|
+
pdNode_af: dict[tuple[str, str], float] = {}
|
|
456
|
+
pdNode_pk: dict[tuple[str, str], float] = {}
|
|
457
|
+
pdn_path = solve_data_dir / "pdNode.csv"
|
|
458
|
+
pdn_df = provider.get(_provider_key(pdn_path))
|
|
459
|
+
if pdn_df is not None:
|
|
460
|
+
for row in pdn_df.iter_rows():
|
|
461
|
+
if len(row) < 4:
|
|
462
|
+
continue
|
|
463
|
+
c0 = _cell_str(row[0])
|
|
464
|
+
if c0:
|
|
465
|
+
try:
|
|
466
|
+
v = float(row[3])
|
|
467
|
+
except (ValueError, TypeError):
|
|
468
|
+
continue
|
|
469
|
+
c1, c2 = _cell_str(row[1]), _cell_str(row[2])
|
|
470
|
+
if c1 == "annual_flow":
|
|
471
|
+
pdNode_af[(c0, c2)] = v
|
|
472
|
+
elif c1 == "peak_inflow":
|
|
473
|
+
pdNode_pk[(c0, c2)] = v
|
|
474
|
+
|
|
475
|
+
def _read_2_keyed_value(path: Path) -> dict[tuple[str, str], float]:
|
|
476
|
+
out: dict[tuple[str, str], float] = {}
|
|
477
|
+
df = provider.get(_provider_key(path))
|
|
478
|
+
if df is None:
|
|
479
|
+
return out
|
|
480
|
+
for row in df.iter_rows():
|
|
481
|
+
if len(row) < 3:
|
|
482
|
+
continue
|
|
483
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
484
|
+
if c0 and c1:
|
|
485
|
+
try:
|
|
486
|
+
out[(c0, c1)] = float(row[2])
|
|
487
|
+
except (ValueError, TypeError):
|
|
488
|
+
continue
|
|
489
|
+
return out
|
|
490
|
+
|
|
491
|
+
pfa = _read_2_keyed_value(
|
|
492
|
+
solve_data_dir / "period_flow_annual_multiplier.csv"
|
|
493
|
+
)
|
|
494
|
+
pfp = _read_2_keyed_value(
|
|
495
|
+
solve_data_dir / "period_flow_proportional_multiplier.csv"
|
|
496
|
+
)
|
|
497
|
+
nos_slope = _read_2_keyed_value(solve_data_dir / "new_old_slope.csv")
|
|
498
|
+
nos_section = _read_2_keyed_value(solve_data_dir / "new_old_section.csv")
|
|
499
|
+
|
|
500
|
+
eligible_nodes = [
|
|
501
|
+
n for n in nodes if (n, "no_inflow") not in inflow_method_pairs
|
|
502
|
+
]
|
|
503
|
+
|
|
504
|
+
# --- Step 2: grid + fold (branches 1 & 2), pdtProfile-shaped ----------
|
|
505
|
+
key_cols = ["node"]
|
|
506
|
+
out_cols = [*key_cols, "period", "time"]
|
|
507
|
+
periods = [d for (d, _t) in dt]
|
|
508
|
+
|
|
509
|
+
grid = build_entity_dt_grid(
|
|
510
|
+
[(n,) for n in eligible_nodes], dt, key_cols=key_cols)
|
|
511
|
+
|
|
512
|
+
fold = build_fold_frame(
|
|
513
|
+
pbt=pbt_inflow,
|
|
514
|
+
pbt_key_cols=["node"],
|
|
515
|
+
out_key_cols=out_cols,
|
|
516
|
+
ts_for_d=ts_for_d,
|
|
517
|
+
tb_for_d=tb_for_d,
|
|
518
|
+
pe_for_d=pe_for_d,
|
|
519
|
+
stoch_entities=stoch_node,
|
|
520
|
+
stoch_filter_cols=["node"],
|
|
521
|
+
periods=periods,
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
# --- Step 3: branch 3 — deterministic gated additive sum --------------
|
|
525
|
+
# Lift the Float64 value lookups from the deduped dicts.
|
|
526
|
+
pti_df_lk = lift_dict_to_lookup(pt_inflow, ["node", "time"], "v_pti")
|
|
527
|
+
pfa_df = lift_dict_to_lookup(pfa, ["node", "period"], "v_pfa")
|
|
528
|
+
pfp_df = lift_dict_to_lookup(pfp, ["node", "period"], "v_pfp")
|
|
529
|
+
slope_df = lift_dict_to_lookup(nos_slope, ["node", "period"], "v_slope")
|
|
530
|
+
sect_df = lift_dict_to_lookup(nos_section, ["node", "period"], "v_sect")
|
|
531
|
+
af_df = lift_dict_to_lookup(pdNode_af, ["node", "period"], "v_af")
|
|
532
|
+
pk_df = lift_dict_to_lookup(pdNode_pk, ["node", "period"], "v_pk")
|
|
533
|
+
|
|
534
|
+
# Per-node Boolean membership frames with EXPLICIT schema (an empty set
|
|
535
|
+
# must NOT yield an all-Null frame). The four ``has_*`` gates are
|
|
536
|
+
# per-node memberships ``(n, method) ∈ inflow_method_pairs``.
|
|
537
|
+
def _bool_frame(names: "set[str] | list[str]", col: str) -> pl.DataFrame:
|
|
538
|
+
names = list(names)
|
|
539
|
+
return pl.DataFrame(
|
|
540
|
+
{"node": names, col: [True] * len(names)},
|
|
541
|
+
schema={"node": pl.Utf8, col: pl.Boolean},
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
in_bal_df = _bool_frame(balance_union, "__in_bal")
|
|
545
|
+
m_ann_df = _bool_frame(
|
|
546
|
+
{n for (n, m) in inflow_method_pairs
|
|
547
|
+
if m == "scale_to_annual_flow"}, "__m_ann")
|
|
548
|
+
m_prop_df = _bool_frame(
|
|
549
|
+
{n for (n, m) in inflow_method_pairs
|
|
550
|
+
if m == "scale_in_proportion"}, "__m_prop")
|
|
551
|
+
m_peak_df = _bool_frame(
|
|
552
|
+
{n for (n, m) in inflow_method_pairs
|
|
553
|
+
if m == "scale_to_annual_and_peak_flow"}, "__m_peak")
|
|
554
|
+
m_orig_df = _bool_frame(
|
|
555
|
+
{n for (n, m) in inflow_method_pairs
|
|
556
|
+
if m == "use_original"}, "__m_orig")
|
|
557
|
+
|
|
558
|
+
out = (
|
|
559
|
+
grid
|
|
560
|
+
.join(in_bal_df, on=["node"], how="left")
|
|
561
|
+
.join(m_ann_df, on=["node"], how="left")
|
|
562
|
+
.join(m_prop_df, on=["node"], how="left")
|
|
563
|
+
.join(m_peak_df, on=["node"], how="left")
|
|
564
|
+
.join(m_orig_df, on=["node"], how="left")
|
|
565
|
+
.join(pti_df_lk, on=["node", "time"], how="left")
|
|
566
|
+
.join(pfa_df, on=["node", "period"], how="left")
|
|
567
|
+
.join(pfp_df, on=["node", "period"], how="left")
|
|
568
|
+
.join(slope_df, on=["node", "period"], how="left")
|
|
569
|
+
.join(sect_df, on=["node", "period"], how="left")
|
|
570
|
+
.join(af_df, on=["node", "period"], how="left")
|
|
571
|
+
.join(pk_df, on=["node", "period"], how="left")
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
pti = pl.col("v_pti").fill_null(0.0)
|
|
575
|
+
# Gate on the VALUE being non-zero (NEVER ``is_not_null``): a real
|
|
576
|
+
# 0.0-valued af / pk key must gate the term OFF, matching the legacy
|
|
577
|
+
# ``if ... pdNode_af.get((n, d), 0.0):`` truthiness guard.
|
|
578
|
+
af_ok = pl.col("v_af").fill_null(0.0) != 0.0
|
|
579
|
+
pk_ok = pl.col("v_pk").fill_null(0.0) != 0.0
|
|
580
|
+
|
|
581
|
+
t_ann = (
|
|
582
|
+
pl.when(pl.col("__m_ann").fill_null(False) & af_ok)
|
|
583
|
+
.then(pl.col("v_pfa").fill_null(0.0) * pti)
|
|
584
|
+
.otherwise(0.0)
|
|
585
|
+
)
|
|
586
|
+
t_prop = (
|
|
587
|
+
pl.when(pl.col("__m_prop").fill_null(False) & af_ok)
|
|
588
|
+
.then(pl.col("v_pfp").fill_null(0.0) * pti)
|
|
589
|
+
.otherwise(0.0)
|
|
590
|
+
)
|
|
591
|
+
t_peak = (
|
|
592
|
+
pl.when(pl.col("__m_peak").fill_null(False) & af_ok & pk_ok)
|
|
593
|
+
.then(pl.col("v_slope").fill_null(0.0) * pti
|
|
594
|
+
- pl.col("v_sect").fill_null(0.0))
|
|
595
|
+
.otherwise(0.0)
|
|
596
|
+
)
|
|
597
|
+
t_orig = (
|
|
598
|
+
pl.when(pl.col("__m_orig").fill_null(False))
|
|
599
|
+
.then(pti)
|
|
600
|
+
.otherwise(0.0)
|
|
601
|
+
)
|
|
602
|
+
# Fixed add order (matches legacy :330-340).
|
|
603
|
+
branch3 = t_ann + t_prop + t_peak + t_orig
|
|
604
|
+
# Non-balance-union nodes get the 0.0 floor (legacy L328 guard).
|
|
605
|
+
v_b3 = (
|
|
606
|
+
pl.when(pl.col("__in_bal").fill_null(False))
|
|
607
|
+
.then(branch3)
|
|
608
|
+
.otherwise(0.0)
|
|
609
|
+
)
|
|
610
|
+
# Signed-zero normalization: the legacy seeds ``value = 0.0`` so any
|
|
611
|
+
# ``±0.0`` term collapses to ``+0.0`` (legacy never emits ``"-0.0"``);
|
|
612
|
+
# the peak term ``slope*pti - section`` can produce ``-0.0``. Wrap
|
|
613
|
+
# ``when(== 0.0).then(lit(0.0))`` (NOT ``lit(0.0) + sum`` — that form
|
|
614
|
+
# leaves ``-0.0`` across rows; proven broken in the varCost round).
|
|
615
|
+
v_b3 = (
|
|
616
|
+
pl.when(v_b3 == 0.0)
|
|
617
|
+
.then(pl.lit(0.0))
|
|
618
|
+
.otherwise(v_b3)
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
# --- Step 4: coalesce (fold-priority) + collect -----------------------
|
|
622
|
+
if fold is not None:
|
|
623
|
+
out = out.join(fold, on=out_cols, how="left")
|
|
624
|
+
else:
|
|
625
|
+
out = out.with_columns(
|
|
626
|
+
pl.lit(None, dtype=pl.Float64).alias("v_fold"))
|
|
627
|
+
|
|
628
|
+
out = out.with_columns(
|
|
629
|
+
coalesce_value([
|
|
630
|
+
pl.col("v_fold"), # branches 1-2 (stoch + parent fold)
|
|
631
|
+
v_b3, # branch 3 (always non-null = 0.0 floor)
|
|
632
|
+
])
|
|
633
|
+
)
|
|
634
|
+
return collect_value_frame(out, key_cols=out_cols, engine=engine)
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def emit_pdtNodeInflow(input_dir: Path, solve_data_dir: Path,
|
|
638
|
+
*, provider) -> None:
|
|
639
|
+
"""Emit ``pdtNodeInflow`` to the Provider."""
|
|
640
|
+
_emit(provider, "solve_data/pdtNodeInflow.csv",
|
|
641
|
+
derive_pdtNodeInflow_vectorized(
|
|
642
|
+
input_dir, solve_data_dir, provider=provider))
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
# ---------------------------------------------------------------------------
|
|
646
|
+
# write_pdtProfile — flextool.mod L1192 (5-branch fallback + stochastic UNION).
|
|
647
|
+
# ---------------------------------------------------------------------------
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def derive_pdtProfile(input_dir: Path, solve_data_dir: Path,
|
|
651
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
652
|
+
"""Materialise the ``pdtProfile`` frame.
|
|
653
|
+
|
|
654
|
+
Branches:
|
|
655
|
+
1. Stochastic fold-in (any of process / node / process_node refs
|
|
656
|
+
the profile under a stochastic group).
|
|
657
|
+
2. Parent-period fold-in.
|
|
658
|
+
3. ``pt_profile[p, t]``.
|
|
659
|
+
4. ``p_profile[p]``.
|
|
660
|
+
5. 0.
|
|
661
|
+
|
|
662
|
+
Domain: every profile in ``input/profile.csv`` × ``dt``.
|
|
663
|
+
"""
|
|
664
|
+
profiles = _read_singles(input_dir / "profile.csv", provider=provider)
|
|
665
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
666
|
+
|
|
667
|
+
# pbt / pt / p loaders.
|
|
668
|
+
pbt_profile: dict[tuple[str, str, str, str], float] = {}
|
|
669
|
+
pbt_path = input_dir / "pbt_profile.csv"
|
|
670
|
+
pbt_df = provider.get(_provider_key(pbt_path))
|
|
671
|
+
if pbt_df is not None:
|
|
672
|
+
for row in pbt_df.iter_rows():
|
|
673
|
+
if len(row) < 5:
|
|
674
|
+
continue
|
|
675
|
+
c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
|
|
676
|
+
_cell_str(row[2]), _cell_str(row[3]))
|
|
677
|
+
if c0 and c1 and c2 and c3:
|
|
678
|
+
try:
|
|
679
|
+
pbt_profile[(c0, c1, c2, c3)] = float(row[4])
|
|
680
|
+
except (ValueError, TypeError):
|
|
681
|
+
continue
|
|
682
|
+
pt_profile: dict[tuple[str, str], float] = {}
|
|
683
|
+
pt_path = solve_data_dir / "pt_profile.csv"
|
|
684
|
+
pt_df = provider.get(_provider_key(pt_path))
|
|
685
|
+
if pt_df is not None:
|
|
686
|
+
for row in pt_df.iter_rows():
|
|
687
|
+
if len(row) < 3:
|
|
688
|
+
continue
|
|
689
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
690
|
+
if c0 and c1:
|
|
691
|
+
try:
|
|
692
|
+
pt_profile[(c0, c1)] = float(row[2])
|
|
693
|
+
except (ValueError, TypeError):
|
|
694
|
+
continue
|
|
695
|
+
p_profile: dict[str, float] = {}
|
|
696
|
+
p_path = input_dir / "p_profile.csv"
|
|
697
|
+
p_df = provider.get(_provider_key(p_path))
|
|
698
|
+
if p_df is not None:
|
|
699
|
+
for row in p_df.iter_rows():
|
|
700
|
+
if len(row) < 2:
|
|
701
|
+
continue
|
|
702
|
+
c0 = _cell_str(row[0])
|
|
703
|
+
if c0:
|
|
704
|
+
try:
|
|
705
|
+
p_profile[c0] = float(row[1])
|
|
706
|
+
except (ValueError, TypeError):
|
|
707
|
+
continue
|
|
708
|
+
|
|
709
|
+
# Branch indices.
|
|
710
|
+
ts_for_d = _read_pairs_to_dict(
|
|
711
|
+
solve_data_dir / "first_timesteps.csv", key_col=0,
|
|
712
|
+
provider=provider,
|
|
713
|
+
)
|
|
714
|
+
tb_for_d = _read_pairs_to_dict(
|
|
715
|
+
solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
|
|
716
|
+
provider=provider,
|
|
717
|
+
)
|
|
718
|
+
pe_for_d = _read_pairs_to_dict(
|
|
719
|
+
solve_data_dir / "period__branch.csv", key_col=1,
|
|
720
|
+
provider=provider,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
# Stochastic profile UNION: any profile referenced via a stochastic
|
|
724
|
+
# process / node / process_node binding.
|
|
725
|
+
stoch_processes = _read_stochastic_entities(
|
|
726
|
+
input_dir / "group__process.csv",
|
|
727
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
728
|
+
provider=provider,
|
|
729
|
+
)
|
|
730
|
+
stoch_nodes = _read_stochastic_entities(
|
|
731
|
+
input_dir / "group__node.csv",
|
|
732
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
733
|
+
provider=provider,
|
|
734
|
+
)
|
|
735
|
+
stoch_profile: set[str] = set()
|
|
736
|
+
pp_path = input_dir / "process__profile__profile_method.csv"
|
|
737
|
+
pp_df = provider.get(_provider_key(pp_path))
|
|
738
|
+
if pp_df is not None:
|
|
739
|
+
for row in pp_df.iter_rows():
|
|
740
|
+
if len(row) < 2:
|
|
741
|
+
continue
|
|
742
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
743
|
+
if c0 in stoch_processes and c1:
|
|
744
|
+
stoch_profile.add(c1)
|
|
745
|
+
np_path = input_dir / "node__profile__profile_method.csv"
|
|
746
|
+
np_df = provider.get(_provider_key(np_path))
|
|
747
|
+
if np_df is not None:
|
|
748
|
+
for row in np_df.iter_rows():
|
|
749
|
+
if len(row) < 2:
|
|
750
|
+
continue
|
|
751
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
752
|
+
if c0 in stoch_nodes and c1:
|
|
753
|
+
stoch_profile.add(c1)
|
|
754
|
+
pnp_path = input_dir / "process__node__profile__profile_method.csv"
|
|
755
|
+
pnp_df = provider.get(_provider_key(pnp_path))
|
|
756
|
+
if pnp_df is not None:
|
|
757
|
+
for row in pnp_df.iter_rows():
|
|
758
|
+
if len(row) < 3:
|
|
759
|
+
continue
|
|
760
|
+
c0, c2 = _cell_str(row[0]), _cell_str(row[2])
|
|
761
|
+
if c0 in stoch_processes and c2:
|
|
762
|
+
stoch_profile.add(c2)
|
|
763
|
+
|
|
764
|
+
profiles_col: list[str] = []
|
|
765
|
+
periods_col: list[str] = []
|
|
766
|
+
times_col: list[str] = []
|
|
767
|
+
values_col: list[str] = []
|
|
768
|
+
for p in profiles:
|
|
769
|
+
is_stoch = p in stoch_profile
|
|
770
|
+
for (d, t) in dt:
|
|
771
|
+
cell: str | None = None
|
|
772
|
+
# Branch 1: stochastic fold-in.
|
|
773
|
+
if is_stoch:
|
|
774
|
+
total = 0.0
|
|
775
|
+
hit = False
|
|
776
|
+
for tb in tb_for_d.get(d, ()):
|
|
777
|
+
for ts in ts_for_d.get(d, ()):
|
|
778
|
+
v = pbt_profile.get((p, tb, ts, t))
|
|
779
|
+
if v is not None:
|
|
780
|
+
total += v
|
|
781
|
+
hit = True
|
|
782
|
+
if hit:
|
|
783
|
+
cell = repr(total)
|
|
784
|
+
# Branch 2: parent-period fold-in.
|
|
785
|
+
if cell is None:
|
|
786
|
+
pe_list = pe_for_d.get(d, ())
|
|
787
|
+
ts_list = ts_for_d.get(d, ())
|
|
788
|
+
if pe_list and ts_list:
|
|
789
|
+
total = 0.0
|
|
790
|
+
hit = False
|
|
791
|
+
for pe in pe_list:
|
|
792
|
+
for tb in tb_for_d.get(pe, ()):
|
|
793
|
+
for ts in ts_list:
|
|
794
|
+
v = pbt_profile.get((p, tb, ts, t))
|
|
795
|
+
if v is not None:
|
|
796
|
+
total += v
|
|
797
|
+
hit = True
|
|
798
|
+
if hit:
|
|
799
|
+
cell = repr(total)
|
|
800
|
+
# Branch 3: time axis.
|
|
801
|
+
if cell is None:
|
|
802
|
+
v = pt_profile.get((p, t))
|
|
803
|
+
if v is not None:
|
|
804
|
+
cell = repr(v)
|
|
805
|
+
# Branch 4: scalar.
|
|
806
|
+
if cell is None:
|
|
807
|
+
v = p_profile.get(p)
|
|
808
|
+
if v is not None:
|
|
809
|
+
cell = repr(v)
|
|
810
|
+
# Branch 5: 0.
|
|
811
|
+
if cell is None:
|
|
812
|
+
cell = "0.0"
|
|
813
|
+
profiles_col.append(p)
|
|
814
|
+
periods_col.append(d)
|
|
815
|
+
times_col.append(t)
|
|
816
|
+
values_col.append(cell)
|
|
817
|
+
return _utf8_frame({
|
|
818
|
+
"profile": profiles_col,
|
|
819
|
+
"period": periods_col,
|
|
820
|
+
"time": times_col,
|
|
821
|
+
"value": values_col,
|
|
822
|
+
})
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def derive_pdtProfile_vectorized(input_dir: Path, solve_data_dir: Path,
|
|
826
|
+
*,
|
|
827
|
+
provider: "object | None" = None,
|
|
828
|
+
engine: str = "eager") -> pl.DataFrame:
|
|
829
|
+
"""Vectorized ``pdtProfile`` derive — parity with the legacy.
|
|
830
|
+
|
|
831
|
+
Replaces the per-cell 5-branch scalar cascade in
|
|
832
|
+
:func:`derive_pdtProfile` with vectorized polars (left-joins +
|
|
833
|
+
``coalesce`` in cascade-priority order + the group-by-sum folds),
|
|
834
|
+
still per roll over the roll's own window. Output is byte-identical
|
|
835
|
+
to :func:`derive_pdtProfile`: columns ``profile, period, time,
|
|
836
|
+
value`` all ``Utf8``, entity-major row order, ``repr(v)`` value
|
|
837
|
+
cells.
|
|
838
|
+
|
|
839
|
+
Entity key = ``profile`` (single Utf8 col), NO param axis. Branches
|
|
840
|
+
(priority): (1) stochastic fold, (2) parent-period fold, (3)
|
|
841
|
+
``pt_profile[(profile, time)]``, (4) ``p_profile[profile]``, (5)
|
|
842
|
+
literal ``0.0``. The ``pbt_profile`` key is a 4-tuple
|
|
843
|
+
``(profile, tb, ts, t)`` (NO param), so the fold key cols are the
|
|
844
|
+
single ``["profile"]`` column.
|
|
845
|
+
|
|
846
|
+
The reader blocks (``pbt`` / ``pt`` / ``p`` loaders, the three fold-
|
|
847
|
+
index dicts, and the stochastic-profile UNION over process / node /
|
|
848
|
+
process_node bindings) are copied verbatim from
|
|
849
|
+
:func:`derive_pdtProfile` so the deduped dicts / stoch set match
|
|
850
|
+
byte-for-byte.
|
|
851
|
+
"""
|
|
852
|
+
profiles = _read_singles(input_dir / "profile.csv", provider=provider)
|
|
853
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
854
|
+
|
|
855
|
+
# pbt / pt / p loaders.
|
|
856
|
+
pbt_profile: dict[tuple[str, str, str, str], float] = {}
|
|
857
|
+
pbt_path = input_dir / "pbt_profile.csv"
|
|
858
|
+
pbt_df = provider.get(_provider_key(pbt_path))
|
|
859
|
+
if pbt_df is not None:
|
|
860
|
+
for row in pbt_df.iter_rows():
|
|
861
|
+
if len(row) < 5:
|
|
862
|
+
continue
|
|
863
|
+
c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
|
|
864
|
+
_cell_str(row[2]), _cell_str(row[3]))
|
|
865
|
+
if c0 and c1 and c2 and c3:
|
|
866
|
+
try:
|
|
867
|
+
pbt_profile[(c0, c1, c2, c3)] = float(row[4])
|
|
868
|
+
except (ValueError, TypeError):
|
|
869
|
+
continue
|
|
870
|
+
pt_profile: dict[tuple[str, str], float] = {}
|
|
871
|
+
pt_path = solve_data_dir / "pt_profile.csv"
|
|
872
|
+
pt_df = provider.get(_provider_key(pt_path))
|
|
873
|
+
if pt_df is not None:
|
|
874
|
+
for row in pt_df.iter_rows():
|
|
875
|
+
if len(row) < 3:
|
|
876
|
+
continue
|
|
877
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
878
|
+
if c0 and c1:
|
|
879
|
+
try:
|
|
880
|
+
pt_profile[(c0, c1)] = float(row[2])
|
|
881
|
+
except (ValueError, TypeError):
|
|
882
|
+
continue
|
|
883
|
+
p_profile: dict[str, float] = {}
|
|
884
|
+
p_path = input_dir / "p_profile.csv"
|
|
885
|
+
p_df = provider.get(_provider_key(p_path))
|
|
886
|
+
if p_df is not None:
|
|
887
|
+
for row in p_df.iter_rows():
|
|
888
|
+
if len(row) < 2:
|
|
889
|
+
continue
|
|
890
|
+
c0 = _cell_str(row[0])
|
|
891
|
+
if c0:
|
|
892
|
+
try:
|
|
893
|
+
p_profile[c0] = float(row[1])
|
|
894
|
+
except (ValueError, TypeError):
|
|
895
|
+
continue
|
|
896
|
+
|
|
897
|
+
# Branch indices.
|
|
898
|
+
ts_for_d = _read_pairs_to_dict(
|
|
899
|
+
solve_data_dir / "first_timesteps.csv", key_col=0,
|
|
900
|
+
provider=provider,
|
|
901
|
+
)
|
|
902
|
+
tb_for_d = _read_pairs_to_dict(
|
|
903
|
+
solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
|
|
904
|
+
provider=provider,
|
|
905
|
+
)
|
|
906
|
+
pe_for_d = _read_pairs_to_dict(
|
|
907
|
+
solve_data_dir / "period__branch.csv", key_col=1,
|
|
908
|
+
provider=provider,
|
|
909
|
+
)
|
|
910
|
+
|
|
911
|
+
# Stochastic profile UNION: any profile referenced via a stochastic
|
|
912
|
+
# process / node / process_node binding.
|
|
913
|
+
stoch_processes = _read_stochastic_entities(
|
|
914
|
+
input_dir / "group__process.csv",
|
|
915
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
916
|
+
provider=provider,
|
|
917
|
+
)
|
|
918
|
+
stoch_nodes = _read_stochastic_entities(
|
|
919
|
+
input_dir / "group__node.csv",
|
|
920
|
+
input_dir / "groupIncludeStochastics.csv",
|
|
921
|
+
provider=provider,
|
|
922
|
+
)
|
|
923
|
+
stoch_profile: set[str] = set()
|
|
924
|
+
pp_path = input_dir / "process__profile__profile_method.csv"
|
|
925
|
+
pp_df = provider.get(_provider_key(pp_path))
|
|
926
|
+
if pp_df is not None:
|
|
927
|
+
for row in pp_df.iter_rows():
|
|
928
|
+
if len(row) < 2:
|
|
929
|
+
continue
|
|
930
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
931
|
+
if c0 in stoch_processes and c1:
|
|
932
|
+
stoch_profile.add(c1)
|
|
933
|
+
np_path = input_dir / "node__profile__profile_method.csv"
|
|
934
|
+
np_df = provider.get(_provider_key(np_path))
|
|
935
|
+
if np_df is not None:
|
|
936
|
+
for row in np_df.iter_rows():
|
|
937
|
+
if len(row) < 2:
|
|
938
|
+
continue
|
|
939
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
940
|
+
if c0 in stoch_nodes and c1:
|
|
941
|
+
stoch_profile.add(c1)
|
|
942
|
+
pnp_path = input_dir / "process__node__profile__profile_method.csv"
|
|
943
|
+
pnp_df = provider.get(_provider_key(pnp_path))
|
|
944
|
+
if pnp_df is not None:
|
|
945
|
+
for row in pnp_df.iter_rows():
|
|
946
|
+
if len(row) < 3:
|
|
947
|
+
continue
|
|
948
|
+
c0, c2 = _cell_str(row[0]), _cell_str(row[2])
|
|
949
|
+
if c0 in stoch_processes and c2:
|
|
950
|
+
stoch_profile.add(c2)
|
|
951
|
+
|
|
952
|
+
# --- vectorized assembly ----------------------------------------------
|
|
953
|
+
key_cols = ["profile"]
|
|
954
|
+
out_cols = [*key_cols, "period", "time"]
|
|
955
|
+
periods = [d for (d, _t) in dt]
|
|
956
|
+
|
|
957
|
+
grid = build_entity_dt_grid(
|
|
958
|
+
[(p,) for p in profiles], dt, key_cols=key_cols)
|
|
959
|
+
|
|
960
|
+
pt_df_lk = lift_dict_to_lookup(pt_profile, ["profile", "time"], "v_pt")
|
|
961
|
+
p_df_lk = lift_dict_to_lookup(p_profile, ["profile"], "v_p")
|
|
962
|
+
|
|
963
|
+
fold = build_fold_frame(
|
|
964
|
+
pbt=pbt_profile,
|
|
965
|
+
pbt_key_cols=["profile"],
|
|
966
|
+
out_key_cols=out_cols,
|
|
967
|
+
ts_for_d=ts_for_d,
|
|
968
|
+
tb_for_d=tb_for_d,
|
|
969
|
+
pe_for_d=pe_for_d,
|
|
970
|
+
stoch_entities=stoch_profile,
|
|
971
|
+
stoch_filter_cols=["profile"],
|
|
972
|
+
periods=periods,
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
out = (
|
|
976
|
+
grid
|
|
977
|
+
.join(pt_df_lk, on=["profile", "time"], how="left")
|
|
978
|
+
.join(p_df_lk, on=["profile"], how="left")
|
|
979
|
+
)
|
|
980
|
+
if fold is not None:
|
|
981
|
+
out = out.join(fold, on=out_cols, how="left")
|
|
982
|
+
else:
|
|
983
|
+
out = out.with_columns(
|
|
984
|
+
pl.lit(None, dtype=pl.Float64).alias("v_fold"))
|
|
985
|
+
|
|
986
|
+
out = out.with_columns(
|
|
987
|
+
coalesce_value([
|
|
988
|
+
pl.col("v_fold"), # branches 1-2 (stoch + parent fold)
|
|
989
|
+
pl.col("v_pt"), # branch 3 (pt_profile)
|
|
990
|
+
pl.col("v_p"), # branch 4 (p_profile)
|
|
991
|
+
pl.lit(0.0), # branch 5 (literal 0.0)
|
|
992
|
+
])
|
|
993
|
+
)
|
|
994
|
+
return collect_value_frame(out, key_cols=out_cols, engine=engine)
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def emit_pdtProfile(input_dir: Path, solve_data_dir: Path,
|
|
998
|
+
*, provider) -> None:
|
|
999
|
+
"""Emit ``pdtProfile`` to the Provider."""
|
|
1000
|
+
_emit(provider, "solve_data/pdtProfile.csv",
|
|
1001
|
+
derive_pdtProfile_vectorized(
|
|
1002
|
+
input_dir, solve_data_dir, provider=provider))
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
# ---------------------------------------------------------------------------
|
|
1006
|
+
# write_pdtConversion_rate_section_slope — flextool.mod L1390-1400 (3 outputs).
|
|
1007
|
+
# ---------------------------------------------------------------------------
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def _derive_conversion_trio(
|
|
1011
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1012
|
+
*, provider: "object | None" = None,
|
|
1013
|
+
) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
|
|
1014
|
+
"""Build the three ``pdtConversion_rate`` / ``pdtProcess_section`` /
|
|
1015
|
+
``pdtProcess_slope`` frames in a single pass.
|
|
1016
|
+
|
|
1017
|
+
The slope formula reuses the ``conv_rate`` and ``section`` values
|
|
1018
|
+
computed for the first two frames; we share the intermediate dicts
|
|
1019
|
+
here so the work isn't duplicated across three independent
|
|
1020
|
+
``derive_*`` calls.
|
|
1021
|
+
"""
|
|
1022
|
+
processes = _read_singles(input_dir / "process.csv", provider=provider)
|
|
1023
|
+
process_minload = frozenset(
|
|
1024
|
+
_read_singles(solve_data_dir / "process_minload.csv",
|
|
1025
|
+
provider=provider)
|
|
1026
|
+
)
|
|
1027
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1028
|
+
|
|
1029
|
+
eff: dict[tuple[str, str, str], float] = {}
|
|
1030
|
+
min_load: dict[tuple[str, str, str], float] = {}
|
|
1031
|
+
eff_min: dict[tuple[str, str, str], float] = {}
|
|
1032
|
+
pdt_path = solve_data_dir / "pdtProcess.csv"
|
|
1033
|
+
pdt_df = provider.get(_provider_key(pdt_path))
|
|
1034
|
+
if pdt_df is not None:
|
|
1035
|
+
for row in pdt_df.iter_rows():
|
|
1036
|
+
if len(row) < 5:
|
|
1037
|
+
continue
|
|
1038
|
+
c0 = _cell_str(row[0])
|
|
1039
|
+
if not c0:
|
|
1040
|
+
continue
|
|
1041
|
+
try:
|
|
1042
|
+
v = float(row[4])
|
|
1043
|
+
except (ValueError, TypeError):
|
|
1044
|
+
continue
|
|
1045
|
+
c1, c2, c3 = _cell_str(row[1]), _cell_str(row[2]), _cell_str(row[3])
|
|
1046
|
+
key = (c0, c2, c3) # (process, period, time)
|
|
1047
|
+
if c1 == "efficiency":
|
|
1048
|
+
eff[key] = v
|
|
1049
|
+
elif c1 == "min_load":
|
|
1050
|
+
min_load[key] = v
|
|
1051
|
+
elif c1 == "efficiency_at_min_load":
|
|
1052
|
+
eff_min[key] = v
|
|
1053
|
+
|
|
1054
|
+
# pdtConversion_rate columns + intermediate conv_rate dict.
|
|
1055
|
+
conv_rate: dict[tuple[str, str, str], float] = {}
|
|
1056
|
+
cr_p: list[str] = []
|
|
1057
|
+
cr_d: list[str] = []
|
|
1058
|
+
cr_t: list[str] = []
|
|
1059
|
+
cr_v: list[str] = []
|
|
1060
|
+
for p in processes:
|
|
1061
|
+
for (d, t) in dt:
|
|
1062
|
+
e = eff.get((p, d, t), 0.0)
|
|
1063
|
+
v = round(1.0 / e, 6) if e else 0.0
|
|
1064
|
+
conv_rate[(p, d, t)] = v
|
|
1065
|
+
cr_p.append(p)
|
|
1066
|
+
cr_d.append(d)
|
|
1067
|
+
cr_t.append(t)
|
|
1068
|
+
cr_v.append(repr(v))
|
|
1069
|
+
conv_frame = _utf8_frame({
|
|
1070
|
+
"process": cr_p, "period": cr_d, "time": cr_t, "value": cr_v,
|
|
1071
|
+
})
|
|
1072
|
+
|
|
1073
|
+
# pdtProcess_section + intermediate section dict.
|
|
1074
|
+
section: dict[tuple[str, str, str], float] = {}
|
|
1075
|
+
sec_p: list[str] = []
|
|
1076
|
+
sec_d: list[str] = []
|
|
1077
|
+
sec_t: list[str] = []
|
|
1078
|
+
sec_v: list[str] = []
|
|
1079
|
+
for p in processes:
|
|
1080
|
+
if p not in process_minload:
|
|
1081
|
+
continue
|
|
1082
|
+
for (d, t) in dt:
|
|
1083
|
+
cr = conv_rate.get((p, d, t), 0.0)
|
|
1084
|
+
ml = min_load.get((p, d, t), 0.0)
|
|
1085
|
+
em = eff_min.get((p, d, t), 0.0)
|
|
1086
|
+
inv_em = (1.0 / em) if em else 0.0
|
|
1087
|
+
denom = 1.0 - ml
|
|
1088
|
+
rounded = round((cr - ml * inv_em) / denom, 6) if denom else 0.0
|
|
1089
|
+
v = cr - rounded
|
|
1090
|
+
section[(p, d, t)] = v
|
|
1091
|
+
sec_p.append(p)
|
|
1092
|
+
sec_d.append(d)
|
|
1093
|
+
sec_t.append(t)
|
|
1094
|
+
sec_v.append(repr(v))
|
|
1095
|
+
section_frame = _utf8_frame({
|
|
1096
|
+
"process": sec_p, "period": sec_d, "time": sec_t, "value": sec_v,
|
|
1097
|
+
})
|
|
1098
|
+
|
|
1099
|
+
# pdtProcess_slope.
|
|
1100
|
+
sl_p: list[str] = []
|
|
1101
|
+
sl_d: list[str] = []
|
|
1102
|
+
sl_t: list[str] = []
|
|
1103
|
+
sl_v: list[str] = []
|
|
1104
|
+
for p in processes:
|
|
1105
|
+
in_min = p in process_minload
|
|
1106
|
+
for (d, t) in dt:
|
|
1107
|
+
cr = conv_rate.get((p, d, t), 0.0)
|
|
1108
|
+
sec = section.get((p, d, t), 0.0) if in_min else 0.0
|
|
1109
|
+
v = cr - sec
|
|
1110
|
+
sl_p.append(p)
|
|
1111
|
+
sl_d.append(d)
|
|
1112
|
+
sl_t.append(t)
|
|
1113
|
+
sl_v.append(repr(v))
|
|
1114
|
+
slope_frame = _utf8_frame({
|
|
1115
|
+
"process": sl_p, "period": sl_d, "time": sl_t, "value": sl_v,
|
|
1116
|
+
})
|
|
1117
|
+
|
|
1118
|
+
return conv_frame, section_frame, slope_frame
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def emit_pdtConversion_rate_section_slope(
|
|
1122
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1123
|
+
*, provider,
|
|
1124
|
+
) -> None:
|
|
1125
|
+
"""Emit ``pdtConversion_rate_section_slope`` to the Provider."""
|
|
1126
|
+
_conv, sec, slope = _derive_conversion_trio(
|
|
1127
|
+
input_dir, solve_data_dir, provider=provider,
|
|
1128
|
+
)
|
|
1129
|
+
_emit(provider, "solve_data/pdtProcess_section.csv", sec)
|
|
1130
|
+
_emit(provider, "solve_data/pdtProcess_slope.csv", slope)
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
# ---------------------------------------------------------------------------
|
|
1134
|
+
# Group / commodity period-param fallbacks and the inflow
|
|
1135
|
+
# positive/negative split.
|
|
1136
|
+
#
|
|
1137
|
+
# Procedural shape with dict lookups in a nested loop — optimal for the
|
|
1138
|
+
# per-row access pattern. Output is byte-for-byte so the
|
|
1139
|
+
# parity tests can ``filecmp``.
|
|
1140
|
+
# ---------------------------------------------------------------------------
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
# flextool_base.dat L196-201 — group period param taxonomies.
|
|
1144
|
+
_GROUP_PERIOD_PARAM: frozenset[str] = frozenset((
|
|
1145
|
+
"capacity_margin", "co2_price", "co2_max_period", "co2_max_total",
|
|
1146
|
+
"inertia_limit", "invest_max_period", "invest_min_period",
|
|
1147
|
+
"max_cumulative_flow", "min_cumulative_flow", "non_synchronous_limit",
|
|
1148
|
+
"penalty_inertia", "penalty_non_synchronous",
|
|
1149
|
+
"max_instant_flow", "min_instant_flow", "penalty_capacity_margin",
|
|
1150
|
+
"retire_max_period", "retire_min_period",
|
|
1151
|
+
"cumulative_max_capacity", "cumulative_min_capacity",
|
|
1152
|
+
))
|
|
1153
|
+
_GROUP_TIME_PARAM: frozenset[str] = frozenset((
|
|
1154
|
+
"co2_price", "max_instant_flow", "min_instant_flow",
|
|
1155
|
+
))
|
|
1156
|
+
_GROUP_PARAM_DEFAULT_5000: frozenset[str] = frozenset((
|
|
1157
|
+
"penalty_inertia", "penalty_capacity_margin", "penalty_non_synchronous",
|
|
1158
|
+
))
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def _read_p_2(path: Path,
|
|
1162
|
+
*, provider: "object | None" = None,
|
|
1163
|
+
) -> dict[tuple[str, str], float]:
|
|
1164
|
+
"""Read a 3-col CSV ``(key1, key2, value)`` into a dict.
|
|
1165
|
+
|
|
1166
|
+
Mirrors legacy ``_read_p_2`` (entity_period_calc_params.py L1965).
|
|
1167
|
+
"""
|
|
1168
|
+
out: dict[tuple[str, str], float] = {}
|
|
1169
|
+
df = provider.get(_provider_key(path))
|
|
1170
|
+
if df is None:
|
|
1171
|
+
return out
|
|
1172
|
+
for row in df.iter_rows():
|
|
1173
|
+
if len(row) < 3:
|
|
1174
|
+
continue
|
|
1175
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
1176
|
+
if c0 and c1:
|
|
1177
|
+
try:
|
|
1178
|
+
out[(c0, c1)] = float(row[2])
|
|
1179
|
+
except (ValueError, TypeError):
|
|
1180
|
+
continue
|
|
1181
|
+
return out
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
def _read_pd_2(path: Path,
|
|
1185
|
+
*, provider: "object | None" = None,
|
|
1186
|
+
) -> dict[tuple[str, str, str], float]:
|
|
1187
|
+
"""Read a 4-col CSV ``(k1, k2, k3, value)`` into a dict."""
|
|
1188
|
+
out: dict[tuple[str, str, str], float] = {}
|
|
1189
|
+
df = provider.get(_provider_key(path))
|
|
1190
|
+
if df is None:
|
|
1191
|
+
return out
|
|
1192
|
+
for row in df.iter_rows():
|
|
1193
|
+
if len(row) < 4:
|
|
1194
|
+
continue
|
|
1195
|
+
c = [_cell_str(row[i]) for i in range(3)]
|
|
1196
|
+
if all(c):
|
|
1197
|
+
try:
|
|
1198
|
+
out[(c[0], c[1], c[2])] = float(row[3])
|
|
1199
|
+
except (ValueError, TypeError):
|
|
1200
|
+
continue
|
|
1201
|
+
return out
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _read_branches_for_d(period_branch_csv: Path,
|
|
1205
|
+
*, provider: "object | None" = None,
|
|
1206
|
+
) -> dict[str, list[str]]:
|
|
1207
|
+
"""``period__branch.csv`` is ``(branch_period, period)`` — index by
|
|
1208
|
+
the child period (column 1) and gather branch list."""
|
|
1209
|
+
out: dict[str, list[str]] = {}
|
|
1210
|
+
df = provider.get(_provider_key(period_branch_csv))
|
|
1211
|
+
if df is None:
|
|
1212
|
+
return out
|
|
1213
|
+
for row in df.iter_rows():
|
|
1214
|
+
if len(row) < 2:
|
|
1215
|
+
continue
|
|
1216
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
1217
|
+
if c0 and c1:
|
|
1218
|
+
out.setdefault(c1, []).append(c0)
|
|
1219
|
+
return out
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
# ---------------------------------------------------------------------------
|
|
1223
|
+
# write_pdGroup — flextool.mod L1115 (5-branch fallback).
|
|
1224
|
+
# ---------------------------------------------------------------------------
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
def derive_pdGroup(input_dir: Path, solve_data_dir: Path,
|
|
1228
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
1229
|
+
"""Materialise the ``pdGroup`` frame (5-branch fallback per (g, param, d)).
|
|
1230
|
+
|
|
1231
|
+
Branches:
|
|
1232
|
+
1. ``pd_group[g, param, d]`` — direct.
|
|
1233
|
+
2. ``sum_{db ∈ branches_for_d[d]} pd_group[g, param, db]`` — fold.
|
|
1234
|
+
3. ``p_group[g, param]`` — scalar fallback.
|
|
1235
|
+
4. ``5000`` when ``param`` is a 5000-default penalty.
|
|
1236
|
+
5. ``0``.
|
|
1237
|
+
"""
|
|
1238
|
+
pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
|
|
1239
|
+
p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
|
|
1240
|
+
branches_for_d = _read_branches_for_d(
|
|
1241
|
+
solve_data_dir / "period__branch.csv", provider=provider,
|
|
1242
|
+
)
|
|
1243
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
1244
|
+
period_in_use = _read_singles(
|
|
1245
|
+
solve_data_dir / "period_in_use_set.csv", provider=provider,
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
g_col: list[str] = []
|
|
1249
|
+
p_col: list[str] = []
|
|
1250
|
+
d_col: list[str] = []
|
|
1251
|
+
v_col: list[str] = []
|
|
1252
|
+
for g in groups:
|
|
1253
|
+
for param in _GROUP_PERIOD_PARAM:
|
|
1254
|
+
for d in period_in_use:
|
|
1255
|
+
if (g, param, d) in pd_g:
|
|
1256
|
+
v = pd_g[(g, param, d)]
|
|
1257
|
+
else:
|
|
1258
|
+
branched = [
|
|
1259
|
+
pd_g[(g, param, db)]
|
|
1260
|
+
for db in branches_for_d.get(d, ())
|
|
1261
|
+
if (g, param, db) in pd_g
|
|
1262
|
+
]
|
|
1263
|
+
if branched:
|
|
1264
|
+
v = sum(branched)
|
|
1265
|
+
elif (g, param) in p_g:
|
|
1266
|
+
v = p_g[(g, param)]
|
|
1267
|
+
elif param in _GROUP_PARAM_DEFAULT_5000:
|
|
1268
|
+
v = 5000.0
|
|
1269
|
+
else:
|
|
1270
|
+
v = 0.0
|
|
1271
|
+
g_col.append(g)
|
|
1272
|
+
p_col.append(param)
|
|
1273
|
+
d_col.append(d)
|
|
1274
|
+
v_col.append(repr(v))
|
|
1275
|
+
return _utf8_frame({
|
|
1276
|
+
"group": g_col, "param": p_col, "period": d_col, "value": v_col,
|
|
1277
|
+
})
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def derive_pdGroup_vectorized(input_dir: Path, solve_data_dir: Path,
|
|
1281
|
+
*,
|
|
1282
|
+
provider: "object | None" = None,
|
|
1283
|
+
engine: str = "eager") -> pl.DataFrame:
|
|
1284
|
+
"""Vectorized ``pdGroup`` derive — period-only, branch-sum, dedup-safe.
|
|
1285
|
+
|
|
1286
|
+
Replaces the per-cell cascade loop in :func:`derive_pdGroup` with
|
|
1287
|
+
vectorized polars (left-joins + a group-by-sum for the branch fold +
|
|
1288
|
+
``coalesce`` in cascade-priority order), still per roll over the
|
|
1289
|
+
roll's own window. Output columns ``group, param, period, value``
|
|
1290
|
+
all ``Utf8`` (NO time axis), entity-major row order, ``repr(v)``
|
|
1291
|
+
value cells.
|
|
1292
|
+
|
|
1293
|
+
Domain = ``group × _GROUP_PERIOD_PARAM`` (the live module frozenset,
|
|
1294
|
+
so iteration order matches legacy — S4) preserving order and
|
|
1295
|
+
duplicates; period axis = ``period_in_use`` (order + duplicates
|
|
1296
|
+
preserved).
|
|
1297
|
+
|
|
1298
|
+
Cascade per ``(g, param, d)`` (5-branch):
|
|
1299
|
+
|
|
1300
|
+
1. ``pd_group[g, param, d]`` — direct.
|
|
1301
|
+
2. ``sum_{db ∈ branches_for_d[d], (g,param,db)∈pd_group}
|
|
1302
|
+
pd_group[g, param, db]`` — fold,
|
|
1303
|
+
only when NON-empty.
|
|
1304
|
+
3. ``p_group[g, param]`` — scalar.
|
|
1305
|
+
4. ``5000`` when ``param ∈ _GROUP_PARAM_DEFAULT_5000``.
|
|
1306
|
+
5. ``0``.
|
|
1307
|
+
|
|
1308
|
+
The branch-sum (D3 critique fix) is computed on the **de-duplicated**
|
|
1309
|
+
``(group, param)`` set so a duplicated ``group`` entry in
|
|
1310
|
+
``group.csv`` does NOT double-count the fold; the dup-preserving final
|
|
1311
|
+
grid then left-joins the per-``(group,param,period)`` sum back,
|
|
1312
|
+
re-expanding to every output row.
|
|
1313
|
+
"""
|
|
1314
|
+
pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
|
|
1315
|
+
p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
|
|
1316
|
+
branches_for_d = _read_branches_for_d(
|
|
1317
|
+
solve_data_dir / "period__branch.csv", provider=provider,
|
|
1318
|
+
)
|
|
1319
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
1320
|
+
period_in_use = _read_singles(
|
|
1321
|
+
solve_data_dir / "period_in_use_set.csv", provider=provider,
|
|
1322
|
+
)
|
|
1323
|
+
|
|
1324
|
+
key_cols = ["group", "param"]
|
|
1325
|
+
out_cols = [*key_cols, "period"]
|
|
1326
|
+
|
|
1327
|
+
# Domain = group × _GROUP_PERIOD_PARAM, referencing the LIVE frozenset
|
|
1328
|
+
# object so iteration order matches the legacy loop (S4); preserve the
|
|
1329
|
+
# group list order + duplicates, never ``.unique()``.
|
|
1330
|
+
domain = [
|
|
1331
|
+
(g, param) for g in groups for param in list(_GROUP_PERIOD_PARAM)
|
|
1332
|
+
]
|
|
1333
|
+
|
|
1334
|
+
grid = build_entity_period_grid(
|
|
1335
|
+
domain, period_in_use, key_cols=key_cols,
|
|
1336
|
+
)
|
|
1337
|
+
|
|
1338
|
+
# Branch 1 — direct pd_group[(g, param, d)].
|
|
1339
|
+
pd_df = lift_dict_to_lookup(pd_g, ["group", "param", "period"], "v_pd")
|
|
1340
|
+
|
|
1341
|
+
# Branch 3 — scalar p_group[(g, param)].
|
|
1342
|
+
p_df = lift_dict_to_lookup(p_g, ["group", "param"], "v_p")
|
|
1343
|
+
|
|
1344
|
+
# Branch 4 — 5000 default for the penalty params.
|
|
1345
|
+
def5000_params = list(_GROUP_PARAM_DEFAULT_5000)
|
|
1346
|
+
def5000_df = pl.DataFrame(
|
|
1347
|
+
{
|
|
1348
|
+
"param": def5000_params,
|
|
1349
|
+
"v_5000": [5000.0] * len(def5000_params),
|
|
1350
|
+
},
|
|
1351
|
+
schema={"param": pl.Utf8, "v_5000": pl.Float64},
|
|
1352
|
+
)
|
|
1353
|
+
|
|
1354
|
+
# Branch 2 — branch-sum. Expand (period d → branch period db),
|
|
1355
|
+
# preserving duplicate (d, db) rows (D2: duplicates must double-count
|
|
1356
|
+
# to match the legacy ``sum(...)`` over the branches list).
|
|
1357
|
+
exp_period: list[str] = []
|
|
1358
|
+
exp_db: list[str] = []
|
|
1359
|
+
for d in period_in_use:
|
|
1360
|
+
for db in branches_for_d.get(d, ()):
|
|
1361
|
+
exp_period.append(d)
|
|
1362
|
+
exp_db.append(db)
|
|
1363
|
+
exp = pl.DataFrame(
|
|
1364
|
+
{"period": exp_period, "db": exp_db},
|
|
1365
|
+
schema={"period": pl.Utf8, "db": pl.Utf8},
|
|
1366
|
+
)
|
|
1367
|
+
|
|
1368
|
+
if exp.height > 0:
|
|
1369
|
+
pd_db = lift_dict_to_lookup(
|
|
1370
|
+
pd_g, ["group", "param", "db"], "v_pddb")
|
|
1371
|
+
# D3: de-dup the (group, param) set for the SUM ONLY so a
|
|
1372
|
+
# duplicated group does not double-count; the dup-preserving grid
|
|
1373
|
+
# re-expands the result back to every output row via the final
|
|
1374
|
+
# left-join.
|
|
1375
|
+
gp_unique = grid.select(["group", "param"]).unique()
|
|
1376
|
+
bsum = (
|
|
1377
|
+
gp_unique
|
|
1378
|
+
.join(exp, how="cross")
|
|
1379
|
+
# INNER join drops non-matching (g, param, db) → reproduces
|
|
1380
|
+
# both the ``if (g,param,db) in pd_g`` gate and the
|
|
1381
|
+
# ``if branched:`` non-empty gate.
|
|
1382
|
+
.join(pd_db, on=["group", "param", "db"], how="inner")
|
|
1383
|
+
.group_by(["group", "param", "period"])
|
|
1384
|
+
.agg(pl.col("v_pddb").sum().alias("v_branch"))
|
|
1385
|
+
)
|
|
1386
|
+
else:
|
|
1387
|
+
bsum = pl.DataFrame(
|
|
1388
|
+
{"group": [], "param": [], "period": [], "v_branch": []},
|
|
1389
|
+
schema={
|
|
1390
|
+
"group": pl.Utf8,
|
|
1391
|
+
"param": pl.Utf8,
|
|
1392
|
+
"period": pl.Utf8,
|
|
1393
|
+
"v_branch": pl.Float64,
|
|
1394
|
+
},
|
|
1395
|
+
)
|
|
1396
|
+
|
|
1397
|
+
out = (
|
|
1398
|
+
grid
|
|
1399
|
+
.join(pd_df, on=["group", "param", "period"], how="left")
|
|
1400
|
+
.join(bsum, on=["group", "param", "period"], how="left")
|
|
1401
|
+
.join(p_df, on=["group", "param"], how="left")
|
|
1402
|
+
.join(def5000_df, on=["param"], how="left")
|
|
1403
|
+
.with_columns(
|
|
1404
|
+
coalesce_value([
|
|
1405
|
+
pl.col("v_pd"), # branch 1 (direct)
|
|
1406
|
+
pl.col("v_branch"), # branch 2 (branch-sum, non-empty)
|
|
1407
|
+
pl.col("v_p"), # branch 3 (scalar)
|
|
1408
|
+
pl.col("v_5000"), # branch 4 (5000 default set)
|
|
1409
|
+
pl.lit(0.0), # branch 5 (default)
|
|
1410
|
+
])
|
|
1411
|
+
)
|
|
1412
|
+
)
|
|
1413
|
+
return collect_value_frame(
|
|
1414
|
+
out, key_cols=out_cols, sort_cols=["__eo", "__po"], engine=engine,
|
|
1415
|
+
)
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def emit_pdGroup(input_dir: Path, solve_data_dir: Path,
|
|
1419
|
+
*, provider) -> None:
|
|
1420
|
+
"""Emit ``pdGroup`` to the Provider."""
|
|
1421
|
+
_emit(provider, "solve_data/pdGroup.csv",
|
|
1422
|
+
derive_pdGroup_vectorized(
|
|
1423
|
+
input_dir, solve_data_dir, provider=provider))
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
# ---------------------------------------------------------------------------
|
|
1427
|
+
# write_pdtGroup — flextool.mod L1126 (4-branch fallback: pt → pd → p → 0).
|
|
1428
|
+
# ---------------------------------------------------------------------------
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def derive_pdtGroup(input_dir: Path, solve_data_dir: Path,
|
|
1432
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
1433
|
+
"""Materialise the ``pdtGroup`` frame.
|
|
1434
|
+
|
|
1435
|
+
Branches: ``pt_group[g, param, t]`` → ``pd_group[g, param, d]`` →
|
|
1436
|
+
``p_group[g, param]`` → 0.
|
|
1437
|
+
"""
|
|
1438
|
+
pt_g = _read_pd_2(input_dir / "pt_group.csv", provider=provider) # same (k1, k2, k3, v) shape
|
|
1439
|
+
pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
|
|
1440
|
+
p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
|
|
1441
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
1442
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1443
|
+
|
|
1444
|
+
g_col: list[str] = []
|
|
1445
|
+
p_col: list[str] = []
|
|
1446
|
+
d_col: list[str] = []
|
|
1447
|
+
t_col: list[str] = []
|
|
1448
|
+
v_col: list[str] = []
|
|
1449
|
+
for g in groups:
|
|
1450
|
+
for param in _GROUP_TIME_PARAM:
|
|
1451
|
+
for (d, t) in dt:
|
|
1452
|
+
if (g, param, t) in pt_g:
|
|
1453
|
+
v = pt_g[(g, param, t)]
|
|
1454
|
+
elif (g, param, d) in pd_g:
|
|
1455
|
+
v = pd_g[(g, param, d)]
|
|
1456
|
+
elif (g, param) in p_g:
|
|
1457
|
+
v = p_g[(g, param)]
|
|
1458
|
+
else:
|
|
1459
|
+
v = 0.0
|
|
1460
|
+
g_col.append(g)
|
|
1461
|
+
p_col.append(param)
|
|
1462
|
+
d_col.append(d)
|
|
1463
|
+
t_col.append(t)
|
|
1464
|
+
v_col.append(repr(v))
|
|
1465
|
+
return _utf8_frame({
|
|
1466
|
+
"group": g_col, "param": p_col, "period": d_col, "time": t_col,
|
|
1467
|
+
"value": v_col,
|
|
1468
|
+
})
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
def derive_pdtGroup_vectorized(input_dir: Path, solve_data_dir: Path,
|
|
1472
|
+
*,
|
|
1473
|
+
provider: "object | None" = None,
|
|
1474
|
+
engine: str = "eager") -> pl.DataFrame:
|
|
1475
|
+
"""Vectorized ``pdtGroup`` derive — byte-parity with the legacy.
|
|
1476
|
+
|
|
1477
|
+
Replaces the per-cell cascade loop in :func:`derive_pdtGroup` with
|
|
1478
|
+
vectorized polars (left-joins + ``coalesce`` in cascade-priority
|
|
1479
|
+
order), still per roll over the roll's own window. Output is
|
|
1480
|
+
byte-identical to :func:`derive_pdtGroup`: columns ``group, param,
|
|
1481
|
+
period, time, value`` all ``Utf8``, entity-major row order,
|
|
1482
|
+
``repr(v)`` value cells.
|
|
1483
|
+
|
|
1484
|
+
Domain = ``group × _GROUP_TIME_PARAM`` (the live module frozenset, so
|
|
1485
|
+
iteration order matches legacy — S4) preserving order and duplicates,
|
|
1486
|
+
crossed with ``dt`` from ``steps_in_use``.
|
|
1487
|
+
|
|
1488
|
+
Cascade (inline 4-branch, time-first):
|
|
1489
|
+
``pt_group`` → ``pd_group`` → ``p_group`` → ``0.0``.
|
|
1490
|
+
"""
|
|
1491
|
+
pt_g = _read_pd_2(input_dir / "pt_group.csv", provider=provider)
|
|
1492
|
+
pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
|
|
1493
|
+
p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
|
|
1494
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
1495
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1496
|
+
|
|
1497
|
+
key_cols = ["group", "param"]
|
|
1498
|
+
out_cols = [*key_cols, "period", "time"]
|
|
1499
|
+
|
|
1500
|
+
# Domain = group × _GROUP_TIME_PARAM, referencing the LIVE frozenset
|
|
1501
|
+
# object so iteration order matches the legacy loop (S4); preserve the
|
|
1502
|
+
# group list order + duplicates, never ``.unique()``.
|
|
1503
|
+
domain = [(g, param) for g in groups for param in list(_GROUP_TIME_PARAM)]
|
|
1504
|
+
|
|
1505
|
+
grid = build_entity_dt_grid(domain, dt, key_cols=key_cols)
|
|
1506
|
+
|
|
1507
|
+
pt_df = lift_dict_to_lookup(pt_g, ["group", "param", "time"], "v_pt")
|
|
1508
|
+
pd_df = lift_dict_to_lookup(pd_g, ["group", "param", "period"], "v_pd")
|
|
1509
|
+
p_df = lift_dict_to_lookup(p_g, ["group", "param"], "v_p")
|
|
1510
|
+
|
|
1511
|
+
out = (
|
|
1512
|
+
grid
|
|
1513
|
+
.join(pt_df, on=["group", "param", "time"], how="left")
|
|
1514
|
+
.join(pd_df, on=["group", "param", "period"], how="left")
|
|
1515
|
+
.join(p_df, on=["group", "param"], how="left")
|
|
1516
|
+
.with_columns(
|
|
1517
|
+
coalesce_value([
|
|
1518
|
+
pl.col("v_pt"), # branch 1 (time-first)
|
|
1519
|
+
pl.col("v_pd"), # branch 2 (period)
|
|
1520
|
+
pl.col("v_p"), # branch 3
|
|
1521
|
+
pl.lit(0.0), # branch 4 (default)
|
|
1522
|
+
])
|
|
1523
|
+
)
|
|
1524
|
+
)
|
|
1525
|
+
return collect_value_frame(out, key_cols=out_cols, engine=engine)
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
def emit_pdtGroup(input_dir: Path, solve_data_dir: Path,
|
|
1529
|
+
*, provider) -> None:
|
|
1530
|
+
"""Emit ``pdtGroup`` to the Provider."""
|
|
1531
|
+
_emit(provider, "solve_data/pdtGroup.csv",
|
|
1532
|
+
derive_pdtGroup_vectorized(
|
|
1533
|
+
input_dir, solve_data_dir, provider=provider))
|
|
1534
|
+
|
|
1535
|
+
|
|
1536
|
+
# ---------------------------------------------------------------------------
|
|
1537
|
+
# write_pdtCommodity — flextool.mod L1108 (3-branch: pt → pd → p → 0).
|
|
1538
|
+
# ---------------------------------------------------------------------------
|
|
1539
|
+
|
|
1540
|
+
# commodityTimeParam = {price} (flextool_base.dat L134)
|
|
1541
|
+
_COMMODITY_TIME_PARAM: tuple[str, ...] = ("price",)
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def derive_pdtCommodity(input_dir: Path, solve_data_dir: Path,
|
|
1545
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
1546
|
+
"""Materialise the ``pdtCommodity`` frame.
|
|
1547
|
+
|
|
1548
|
+
Domain: commodity × commodityTimeParam × dt.
|
|
1549
|
+
Branches: ``pt_commodity`` → ``pd_commodity`` → ``p_commodity`` → 0.
|
|
1550
|
+
"""
|
|
1551
|
+
pt = _read_pd_2(input_dir / "pt_commodity.csv", provider=provider)
|
|
1552
|
+
pd_ = _read_pd_2(input_dir / "pd_commodity.csv", provider=provider)
|
|
1553
|
+
p = _read_p_2(input_dir / "p_commodity.csv", provider=provider)
|
|
1554
|
+
commodities = _read_singles(input_dir / "commodity.csv", provider=provider)
|
|
1555
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1556
|
+
|
|
1557
|
+
c_col: list[str] = []
|
|
1558
|
+
p_col: list[str] = []
|
|
1559
|
+
d_col: list[str] = []
|
|
1560
|
+
t_col: list[str] = []
|
|
1561
|
+
v_col: list[str] = []
|
|
1562
|
+
for c in commodities:
|
|
1563
|
+
for param in _COMMODITY_TIME_PARAM:
|
|
1564
|
+
for (d, t) in dt:
|
|
1565
|
+
v = pt.get((c, param, t))
|
|
1566
|
+
if v is None:
|
|
1567
|
+
v = pd_.get((c, param, d))
|
|
1568
|
+
if v is None:
|
|
1569
|
+
v = p.get((c, param), 0.0)
|
|
1570
|
+
c_col.append(c)
|
|
1571
|
+
p_col.append(param)
|
|
1572
|
+
d_col.append(d)
|
|
1573
|
+
t_col.append(t)
|
|
1574
|
+
v_col.append(repr(v))
|
|
1575
|
+
return _utf8_frame({
|
|
1576
|
+
"commodity": c_col, "param": p_col, "period": d_col, "time": t_col,
|
|
1577
|
+
"value": v_col,
|
|
1578
|
+
})
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def derive_pdtCommodity_vectorized(input_dir: Path, solve_data_dir: Path,
|
|
1582
|
+
*,
|
|
1583
|
+
provider: "object | None" = None,
|
|
1584
|
+
engine: str = "eager") -> pl.DataFrame:
|
|
1585
|
+
"""Vectorized ``pdtCommodity`` derive — byte-parity with the legacy.
|
|
1586
|
+
|
|
1587
|
+
Replaces the per-cell cascade loop in :func:`derive_pdtCommodity`
|
|
1588
|
+
with vectorized polars (left-joins + ``coalesce`` in cascade-priority
|
|
1589
|
+
order), still per roll over the roll's own window. Output is
|
|
1590
|
+
byte-identical to :func:`derive_pdtCommodity`: columns ``commodity,
|
|
1591
|
+
param, period, time, value`` all ``Utf8``, entity-major row order,
|
|
1592
|
+
``repr(v)`` value cells.
|
|
1593
|
+
|
|
1594
|
+
Cascade (inline 3-branch, time-first):
|
|
1595
|
+
``pt_commodity`` → ``pd_commodity`` → ``p_commodity`` → ``0.0``.
|
|
1596
|
+
"""
|
|
1597
|
+
pt = _read_pd_2(input_dir / "pt_commodity.csv", provider=provider)
|
|
1598
|
+
pd_ = _read_pd_2(input_dir / "pd_commodity.csv", provider=provider)
|
|
1599
|
+
p = _read_p_2(input_dir / "p_commodity.csv", provider=provider)
|
|
1600
|
+
commodities = _read_singles(input_dir / "commodity.csv", provider=provider)
|
|
1601
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1602
|
+
|
|
1603
|
+
key_cols = ["commodity", "param"]
|
|
1604
|
+
out_cols = [*key_cols, "period", "time"]
|
|
1605
|
+
|
|
1606
|
+
# Domain = commodity × _COMMODITY_TIME_PARAM, preserving legacy
|
|
1607
|
+
# iteration order (commodity-major, param from the tuple) and never
|
|
1608
|
+
# ``.unique()``-d.
|
|
1609
|
+
domain = [(c, param) for c in commodities for param in _COMMODITY_TIME_PARAM]
|
|
1610
|
+
|
|
1611
|
+
grid = build_entity_dt_grid(domain, dt, key_cols=key_cols)
|
|
1612
|
+
|
|
1613
|
+
pt_df = lift_dict_to_lookup(
|
|
1614
|
+
pt, ["commodity", "param", "time"], "v_pt")
|
|
1615
|
+
pd_df = lift_dict_to_lookup(
|
|
1616
|
+
pd_, ["commodity", "param", "period"], "v_pd")
|
|
1617
|
+
p_df = lift_dict_to_lookup(
|
|
1618
|
+
p, ["commodity", "param"], "v_p")
|
|
1619
|
+
|
|
1620
|
+
out = (
|
|
1621
|
+
grid
|
|
1622
|
+
.join(pt_df, on=["commodity", "param", "time"], how="left")
|
|
1623
|
+
.join(pd_df, on=["commodity", "param", "period"], how="left")
|
|
1624
|
+
.join(p_df, on=["commodity", "param"], how="left")
|
|
1625
|
+
.with_columns(
|
|
1626
|
+
coalesce_value([
|
|
1627
|
+
pl.col("v_pt"), # branch 1 (time-first)
|
|
1628
|
+
pl.col("v_pd"), # branch 2 (period)
|
|
1629
|
+
pl.col("v_p"), # branch 3
|
|
1630
|
+
pl.lit(0.0), # branch 4 (default)
|
|
1631
|
+
])
|
|
1632
|
+
)
|
|
1633
|
+
)
|
|
1634
|
+
return collect_value_frame(out, key_cols=out_cols, engine=engine)
|
|
1635
|
+
|
|
1636
|
+
|
|
1637
|
+
def emit_pdtCommodity(input_dir: Path, solve_data_dir: Path,
|
|
1638
|
+
*, provider) -> None:
|
|
1639
|
+
"""Emit ``pdtCommodity`` to the Provider."""
|
|
1640
|
+
_emit(provider, "solve_data/pdtCommodity.csv",
|
|
1641
|
+
derive_pdtCommodity_vectorized(
|
|
1642
|
+
input_dir, solve_data_dir, provider=provider))
|
|
1643
|
+
|
|
1644
|
+
|
|
1645
|
+
# ---------------------------------------------------------------------------
|
|
1646
|
+
# write_p_positive_negative_inflow — flextool.mod L1672 / L1675.
|
|
1647
|
+
# ---------------------------------------------------------------------------
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def _derive_positive_negative_inflow(
|
|
1651
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1652
|
+
*, provider: "object | None" = None,
|
|
1653
|
+
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
1654
|
+
"""Build both ``p_positive_inflow`` and ``p_negative_inflow`` frames
|
|
1655
|
+
from a single read of ``pdtNodeInflow.csv``."""
|
|
1656
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
1657
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1658
|
+
inflow_method_pairs = frozenset(
|
|
1659
|
+
_read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
1660
|
+
provider=provider)
|
|
1661
|
+
)
|
|
1662
|
+
no_inflow_nodes = frozenset(
|
|
1663
|
+
n for n in nodes if (n, "no_inflow") in inflow_method_pairs
|
|
1664
|
+
)
|
|
1665
|
+
|
|
1666
|
+
pdt_inflow: dict[tuple[str, str, str], float] = {}
|
|
1667
|
+
pdtni_path = solve_data_dir / "pdtNodeInflow.csv"
|
|
1668
|
+
pdtni_df = provider.get(_provider_key(pdtni_path))
|
|
1669
|
+
if pdtni_df is not None:
|
|
1670
|
+
for row in pdtni_df.iter_rows():
|
|
1671
|
+
if len(row) < 4:
|
|
1672
|
+
continue
|
|
1673
|
+
c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
|
|
1674
|
+
if c0 and c1 and c2:
|
|
1675
|
+
try:
|
|
1676
|
+
pdt_inflow[(c0, c1, c2)] = float(row[3])
|
|
1677
|
+
except (ValueError, TypeError):
|
|
1678
|
+
continue
|
|
1679
|
+
|
|
1680
|
+
pos_n: list[str] = []
|
|
1681
|
+
pos_d: list[str] = []
|
|
1682
|
+
pos_t: list[str] = []
|
|
1683
|
+
pos_v: list[str] = []
|
|
1684
|
+
for n in nodes:
|
|
1685
|
+
if n in no_inflow_nodes:
|
|
1686
|
+
continue
|
|
1687
|
+
for (d, t) in dt:
|
|
1688
|
+
v = pdt_inflow.get((n, d, t), 0.0)
|
|
1689
|
+
pos_n.append(n)
|
|
1690
|
+
pos_d.append(d)
|
|
1691
|
+
pos_t.append(t)
|
|
1692
|
+
pos_v.append(repr(v if v >= 0 else 0.0))
|
|
1693
|
+
pos_frame = _utf8_frame({
|
|
1694
|
+
"node": pos_n, "period": pos_d, "time": pos_t, "value": pos_v,
|
|
1695
|
+
})
|
|
1696
|
+
|
|
1697
|
+
neg_n: list[str] = []
|
|
1698
|
+
neg_d: list[str] = []
|
|
1699
|
+
neg_t: list[str] = []
|
|
1700
|
+
neg_v: list[str] = []
|
|
1701
|
+
for n in nodes:
|
|
1702
|
+
for (d, t) in dt:
|
|
1703
|
+
if n in no_inflow_nodes:
|
|
1704
|
+
cell = "0.0"
|
|
1705
|
+
else:
|
|
1706
|
+
v = pdt_inflow.get((n, d, t), 0.0)
|
|
1707
|
+
cell = repr(v if v < 0 else 0.0)
|
|
1708
|
+
neg_n.append(n)
|
|
1709
|
+
neg_d.append(d)
|
|
1710
|
+
neg_t.append(t)
|
|
1711
|
+
neg_v.append(cell)
|
|
1712
|
+
neg_frame = _utf8_frame({
|
|
1713
|
+
"node": neg_n, "period": neg_d, "time": neg_t, "value": neg_v,
|
|
1714
|
+
})
|
|
1715
|
+
|
|
1716
|
+
return pos_frame, neg_frame
|
|
1717
|
+
|
|
1718
|
+
|
|
1719
|
+
def emit_p_positive_negative_inflow(
|
|
1720
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1721
|
+
*, provider,
|
|
1722
|
+
) -> None:
|
|
1723
|
+
"""Emit ``p_positive_negative_inflow`` to the Provider."""
|
|
1724
|
+
pos, neg = _derive_positive_negative_inflow(
|
|
1725
|
+
input_dir, solve_data_dir, provider=provider,
|
|
1726
|
+
)
|
|
1727
|
+
_emit(provider, "solve_data/p_positive_inflow.csv", pos)
|
|
1728
|
+
_emit(provider, "solve_data/p_negative_inflow.csv", neg)
|
|
1729
|
+
|
|
1730
|
+
|
|
1731
|
+
# ---------------------------------------------------------------------------
|
|
1732
|
+
# Phase 1 follow-up 5 — entity_period_calc_params: varCost + cap_reduction +
|
|
1733
|
+
# ed_period_params + pssdt_varCost filters.
|
|
1734
|
+
# ---------------------------------------------------------------------------
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def _read_triples(path: Path,
|
|
1738
|
+
*, provider: "object | None" = None,
|
|
1739
|
+
) -> list[tuple[str, str, str]]:
|
|
1740
|
+
df = provider.get(_provider_key(path))
|
|
1741
|
+
if df is None:
|
|
1742
|
+
return []
|
|
1743
|
+
out: list[tuple[str, str, str]] = []
|
|
1744
|
+
for row in df.iter_rows():
|
|
1745
|
+
if len(row) < 3:
|
|
1746
|
+
continue
|
|
1747
|
+
c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
|
|
1748
|
+
if c0 and c1 and c2:
|
|
1749
|
+
out.append((c0, c1, c2))
|
|
1750
|
+
return out
|
|
1751
|
+
|
|
1752
|
+
|
|
1753
|
+
def _read_pdt_at_param(path: Path, param_col: int, param_value: str,
|
|
1754
|
+
key_cols: tuple[int, ...],
|
|
1755
|
+
val_col: int,
|
|
1756
|
+
*, provider: "object | None" = None,
|
|
1757
|
+
) -> dict[tuple, float]:
|
|
1758
|
+
"""Read a long-format pdtX CSV, filter rows where col[param_col] ==
|
|
1759
|
+
param_value, return dict[tuple(_cell_str(row[c]) for c in key_cols)] =
|
|
1760
|
+
float(row[val_col]).
|
|
1761
|
+
"""
|
|
1762
|
+
out: dict[tuple, float] = {}
|
|
1763
|
+
df = provider.get(_provider_key(path))
|
|
1764
|
+
if df is None:
|
|
1765
|
+
return out
|
|
1766
|
+
for row in df.iter_rows():
|
|
1767
|
+
if (len(row) > max(param_col, val_col, *key_cols)
|
|
1768
|
+
and _cell_str(row[param_col]) == param_value):
|
|
1769
|
+
try:
|
|
1770
|
+
out[tuple(_cell_str(row[c]) for c in key_cols)] = float(row[val_col])
|
|
1771
|
+
except (ValueError, TypeError):
|
|
1772
|
+
continue
|
|
1773
|
+
return out
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
# ---- write_pdtProcess__source__sink__dt_varCost_pair (mod L1493, L1502) ----
|
|
1777
|
+
|
|
1778
|
+
def _derive_varCost_pair(
|
|
1779
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1780
|
+
*, provider: "object | None" = None,
|
|
1781
|
+
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
1782
|
+
"""Build both ``pdtProcess__source__sink__dt_varCost`` frames in one
|
|
1783
|
+
pass — the basic and ``_alwaysProcess`` variants share the same
|
|
1784
|
+
OOC dictionaries / proc_src / proc_snk lookups.
|
|
1785
|
+
"""
|
|
1786
|
+
pdt = _read_pdt_at_param(
|
|
1787
|
+
solve_data_dir / "pdtProcess.csv",
|
|
1788
|
+
param_col=1, param_value="other_operational_cost",
|
|
1789
|
+
key_cols=(0, 2, 3), val_col=4,
|
|
1790
|
+
provider=provider,
|
|
1791
|
+
) # (process, period, time) → value
|
|
1792
|
+
pdt_src = _read_pdt_at_param(
|
|
1793
|
+
solve_data_dir / "pdtProcess_source.csv",
|
|
1794
|
+
param_col=2, param_value="other_operational_cost",
|
|
1795
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
1796
|
+
provider=provider,
|
|
1797
|
+
) # (process, source, period, time) → value
|
|
1798
|
+
pdt_snk = _read_pdt_at_param(
|
|
1799
|
+
solve_data_dir / "pdtProcess_sink.csv",
|
|
1800
|
+
param_col=2, param_value="other_operational_cost",
|
|
1801
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
1802
|
+
provider=provider,
|
|
1803
|
+
) # (process, sink, period, time) → value
|
|
1804
|
+
proc_src = frozenset(
|
|
1805
|
+
_read_pairs(input_dir / "process__source.csv", provider=provider)
|
|
1806
|
+
)
|
|
1807
|
+
proc_snk = frozenset(
|
|
1808
|
+
_read_pairs(input_dir / "process__sink.csv", provider=provider)
|
|
1809
|
+
)
|
|
1810
|
+
pss = _read_triples(
|
|
1811
|
+
solve_data_dir / "process_source_sink.csv", provider=provider,
|
|
1812
|
+
)
|
|
1813
|
+
pss_always = _read_triples(
|
|
1814
|
+
solve_data_dir / "process_source_sink_alwaysProcess.csv",
|
|
1815
|
+
provider=provider,
|
|
1816
|
+
)
|
|
1817
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1818
|
+
|
|
1819
|
+
def _build(rows_iter: list[tuple[str, str, str]],
|
|
1820
|
+
always: bool) -> pl.DataFrame:
|
|
1821
|
+
p_col: list[str] = []
|
|
1822
|
+
s_col: list[str] = []
|
|
1823
|
+
k_col: list[str] = []
|
|
1824
|
+
d_col: list[str] = []
|
|
1825
|
+
t_col: list[str] = []
|
|
1826
|
+
v_col: list[str] = []
|
|
1827
|
+
for (p, src, snk) in rows_iter:
|
|
1828
|
+
for (d, t) in dt:
|
|
1829
|
+
v = 0.0
|
|
1830
|
+
if (p, src) in proc_src:
|
|
1831
|
+
v += pdt_src.get((p, src, d, t), 0.0)
|
|
1832
|
+
if (p, snk) in proc_snk:
|
|
1833
|
+
v += pdt_snk.get((p, snk, d, t), 0.0)
|
|
1834
|
+
if always:
|
|
1835
|
+
if (p, snk) in proc_snk or (p, snk) in proc_src:
|
|
1836
|
+
v += pdt.get((p, d, t), 0.0)
|
|
1837
|
+
else:
|
|
1838
|
+
v += pdt.get((p, d, t), 0.0)
|
|
1839
|
+
p_col.append(p)
|
|
1840
|
+
s_col.append(src)
|
|
1841
|
+
k_col.append(snk)
|
|
1842
|
+
d_col.append(d)
|
|
1843
|
+
t_col.append(t)
|
|
1844
|
+
v_col.append(repr(v))
|
|
1845
|
+
return _utf8_frame({
|
|
1846
|
+
"process": p_col, "source": s_col, "sink": k_col,
|
|
1847
|
+
"period": d_col, "time": t_col, "value": v_col,
|
|
1848
|
+
})
|
|
1849
|
+
|
|
1850
|
+
return _build(pss, always=False), _build(pss_always, always=True)
|
|
1851
|
+
|
|
1852
|
+
|
|
1853
|
+
def _derive_varCost_pair_vectorized(
|
|
1854
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1855
|
+
*, provider: "object | None" = None, engine: str = "eager",
|
|
1856
|
+
) -> tuple[pl.DataFrame, pl.DataFrame]:
|
|
1857
|
+
"""Vectorized twin of :func:`_derive_varCost_pair` (design §5 / S7).
|
|
1858
|
+
|
|
1859
|
+
Both the ``basic`` and ``_alwaysProcess`` variants are a
|
|
1860
|
+
membership-GATED SUM of three terms — NOT a coalesce cascade:
|
|
1861
|
+
|
|
1862
|
+
v = 0.0
|
|
1863
|
+
if (p, src) ∈ proc_src: v += pdt_src[(p, src, d, t)]
|
|
1864
|
+
if (p, snk) ∈ proc_snk: v += pdt_snk[(p, snk, d, t)]
|
|
1865
|
+
basic: v += pdt[(p, d, t)] # unconditional
|
|
1866
|
+
always: if (p, snk) ∈ proc_snk or (p, snk) ∈ proc_src:
|
|
1867
|
+
v += pdt[(p, d, t)]
|
|
1868
|
+
|
|
1869
|
+
Reproduced per cell as ``build_entity_dt_grid`` ⨯ membership/value
|
|
1870
|
+
left-joins ⨯ ``fill_null(0.0)`` ⨯ a fixed-order ``src + snk + pdt``
|
|
1871
|
+
add, then a SIGNED-ZERO normalization (critique D1): the legacy
|
|
1872
|
+
accumulator seeds ``v = 0.0`` so ``0.0 + (-0.0)`` already kills the
|
|
1873
|
+
sign on the first add → legacy never emits ``"-0.0"``; ``pl.lit(0.0)
|
|
1874
|
+
+ sum`` does NOT reproduce this (polars leaves ``-0.0`` as ``-0.0``
|
|
1875
|
+
across rows), so the value is wrapped
|
|
1876
|
+
``when(value_f == 0.0).then(0.0)`` to canonicalize ``±0.0 → +0.0``.
|
|
1877
|
+
|
|
1878
|
+
The two variants have DIFFERENT domains (``pss`` vs ``pss_always``)
|
|
1879
|
+
and a DIFFERENT pdt gate, so they are assembled separately and
|
|
1880
|
+
returned as a pair (mirroring the legacy shared producer).
|
|
1881
|
+
"""
|
|
1882
|
+
pdt = _read_pdt_at_param(
|
|
1883
|
+
solve_data_dir / "pdtProcess.csv",
|
|
1884
|
+
param_col=1, param_value="other_operational_cost",
|
|
1885
|
+
key_cols=(0, 2, 3), val_col=4,
|
|
1886
|
+
provider=provider,
|
|
1887
|
+
) # (process, period, time) → value
|
|
1888
|
+
pdt_src = _read_pdt_at_param(
|
|
1889
|
+
solve_data_dir / "pdtProcess_source.csv",
|
|
1890
|
+
param_col=2, param_value="other_operational_cost",
|
|
1891
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
1892
|
+
provider=provider,
|
|
1893
|
+
) # (process, source, period, time) → value
|
|
1894
|
+
pdt_snk = _read_pdt_at_param(
|
|
1895
|
+
solve_data_dir / "pdtProcess_sink.csv",
|
|
1896
|
+
param_col=2, param_value="other_operational_cost",
|
|
1897
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
1898
|
+
provider=provider,
|
|
1899
|
+
) # (process, sink, period, time) → value
|
|
1900
|
+
proc_src = frozenset(
|
|
1901
|
+
_read_pairs(input_dir / "process__source.csv", provider=provider)
|
|
1902
|
+
)
|
|
1903
|
+
proc_snk = frozenset(
|
|
1904
|
+
_read_pairs(input_dir / "process__sink.csv", provider=provider)
|
|
1905
|
+
)
|
|
1906
|
+
pss = _read_triples(
|
|
1907
|
+
solve_data_dir / "process_source_sink.csv", provider=provider,
|
|
1908
|
+
)
|
|
1909
|
+
pss_always = _read_triples(
|
|
1910
|
+
solve_data_dir / "process_source_sink_alwaysProcess.csv",
|
|
1911
|
+
provider=provider,
|
|
1912
|
+
)
|
|
1913
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
1914
|
+
|
|
1915
|
+
# Lift the three value dicts (last-wins-deduped) to lookup frames.
|
|
1916
|
+
src_df = lift_dict_to_lookup(
|
|
1917
|
+
pdt_src, ["process", "source", "period", "time"], "v_src")
|
|
1918
|
+
snk_df = lift_dict_to_lookup(
|
|
1919
|
+
pdt_snk, ["process", "sink", "period", "time"], "v_snk")
|
|
1920
|
+
pdt_df = lift_dict_to_lookup(
|
|
1921
|
+
pdt, ["process", "period", "time"], "v_pdt")
|
|
1922
|
+
|
|
1923
|
+
# Membership frames built from the frozensets with EXPLICIT schemas
|
|
1924
|
+
# (critique D7): an empty frozenset must NOT yield an all-Null frame.
|
|
1925
|
+
ps_df = pl.DataFrame(
|
|
1926
|
+
{
|
|
1927
|
+
"process": [a for (a, _b) in proc_src],
|
|
1928
|
+
"source": [b for (_a, b) in proc_src],
|
|
1929
|
+
"__in_src": [True] * len(proc_src),
|
|
1930
|
+
},
|
|
1931
|
+
schema={"process": pl.Utf8, "source": pl.Utf8,
|
|
1932
|
+
"__in_src": pl.Boolean},
|
|
1933
|
+
)
|
|
1934
|
+
pk_df = pl.DataFrame(
|
|
1935
|
+
{
|
|
1936
|
+
"process": [a for (a, _b) in proc_snk],
|
|
1937
|
+
"sink": [b for (_a, b) in proc_snk],
|
|
1938
|
+
"__in_snk": [True] * len(proc_snk),
|
|
1939
|
+
},
|
|
1940
|
+
schema={"process": pl.Utf8, "sink": pl.Utf8,
|
|
1941
|
+
"__in_snk": pl.Boolean},
|
|
1942
|
+
)
|
|
1943
|
+
# proc_src arcs renamed source→sink: expresses ``(p, snk) ∈ proc_src``
|
|
1944
|
+
# for the always-variant pdt gate (critique D2 — both gate terms key
|
|
1945
|
+
# on ``snk``, including against proc_src).
|
|
1946
|
+
ps_snk_df = pl.DataFrame(
|
|
1947
|
+
{
|
|
1948
|
+
"process": [a for (a, _b) in proc_src],
|
|
1949
|
+
"sink": [b for (_a, b) in proc_src],
|
|
1950
|
+
"__snk_in_src": [True] * len(proc_src),
|
|
1951
|
+
},
|
|
1952
|
+
schema={"process": pl.Utf8, "sink": pl.Utf8,
|
|
1953
|
+
"__snk_in_src": pl.Boolean},
|
|
1954
|
+
)
|
|
1955
|
+
|
|
1956
|
+
def _build_vec(domain: list[tuple[str, str, str]],
|
|
1957
|
+
always: bool) -> pl.DataFrame:
|
|
1958
|
+
grid = build_entity_dt_grid(
|
|
1959
|
+
domain, dt, key_cols=["process", "source", "sink"])
|
|
1960
|
+
out = (
|
|
1961
|
+
grid
|
|
1962
|
+
.join(ps_df, on=["process", "source"], how="left")
|
|
1963
|
+
.join(src_df, on=["process", "source", "period", "time"],
|
|
1964
|
+
how="left")
|
|
1965
|
+
.join(pk_df, on=["process", "sink"], how="left")
|
|
1966
|
+
.join(snk_df, on=["process", "sink", "period", "time"],
|
|
1967
|
+
how="left")
|
|
1968
|
+
.join(pdt_df, on=["process", "period", "time"], how="left")
|
|
1969
|
+
)
|
|
1970
|
+
t_src = (
|
|
1971
|
+
pl.when(pl.col("__in_src").fill_null(False))
|
|
1972
|
+
.then(pl.col("v_src").fill_null(0.0))
|
|
1973
|
+
.otherwise(0.0)
|
|
1974
|
+
)
|
|
1975
|
+
t_snk = (
|
|
1976
|
+
pl.when(pl.col("__in_snk").fill_null(False))
|
|
1977
|
+
.then(pl.col("v_snk").fill_null(0.0))
|
|
1978
|
+
.otherwise(0.0)
|
|
1979
|
+
)
|
|
1980
|
+
if always:
|
|
1981
|
+
out = out.join(ps_snk_df, on=["process", "sink"], how="left")
|
|
1982
|
+
t_pdt = (
|
|
1983
|
+
pl.when(
|
|
1984
|
+
pl.col("__in_snk").fill_null(False)
|
|
1985
|
+
| pl.col("__snk_in_src").fill_null(False)
|
|
1986
|
+
)
|
|
1987
|
+
.then(pl.col("v_pdt").fill_null(0.0))
|
|
1988
|
+
.otherwise(0.0)
|
|
1989
|
+
)
|
|
1990
|
+
else:
|
|
1991
|
+
t_pdt = pl.col("v_pdt").fill_null(0.0)
|
|
1992
|
+
# Fixed add order src + snk + pdt, then normalize signed zero so
|
|
1993
|
+
# ``±0.0`` renders ``"0.0"`` (legacy never emits ``"-0.0"``).
|
|
1994
|
+
value_raw = t_src + t_snk + t_pdt
|
|
1995
|
+
out = out.with_columns(
|
|
1996
|
+
pl.when(value_raw == 0.0)
|
|
1997
|
+
.then(pl.lit(0.0))
|
|
1998
|
+
.otherwise(value_raw)
|
|
1999
|
+
.alias("value_f")
|
|
2000
|
+
)
|
|
2001
|
+
return collect_value_frame(
|
|
2002
|
+
out,
|
|
2003
|
+
key_cols=["process", "source", "sink", "period", "time"],
|
|
2004
|
+
value_f_col="value_f", engine=engine,
|
|
2005
|
+
)
|
|
2006
|
+
|
|
2007
|
+
return _build_vec(pss, always=False), _build_vec(pss_always, always=True)
|
|
2008
|
+
|
|
2009
|
+
|
|
2010
|
+
def emit_pdtProcess__source__sink__dt_varCost_pair(
|
|
2011
|
+
input_dir: Path, solve_data_dir: Path,
|
|
2012
|
+
*, provider,
|
|
2013
|
+
) -> None:
|
|
2014
|
+
"""Emit ``pdtProcess__source__sink__dt_varCost_pair`` to the Provider."""
|
|
2015
|
+
basic, _always = _derive_varCost_pair_vectorized(
|
|
2016
|
+
input_dir, solve_data_dir, provider=provider,
|
|
2017
|
+
)
|
|
2018
|
+
_emit(provider, "solve_data/pdtProcess__source__sink__dt_varCost.csv",
|
|
2019
|
+
basic)
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
# ---- write_pssdt_varCost_filters (mod L1498-1501) -------------------------
|
|
2023
|
+
|
|
2024
|
+
def _filter_rows_to_frame(
|
|
2025
|
+
rows: list[tuple[str, str, str, str, str]],
|
|
2026
|
+
) -> pl.DataFrame:
|
|
2027
|
+
"""5-column key-only Utf8 frame for the pssdt_varCost filter outputs."""
|
|
2028
|
+
return _utf8_frame({
|
|
2029
|
+
"process": [r[0] for r in rows],
|
|
2030
|
+
"source": [r[1] for r in rows],
|
|
2031
|
+
"sink": [r[2] for r in rows],
|
|
2032
|
+
"period": [r[3] for r in rows],
|
|
2033
|
+
"time": [r[4] for r in rows],
|
|
2034
|
+
})
|
|
2035
|
+
|
|
2036
|
+
|
|
2037
|
+
def _derive_pssdt_varCost_filters(
|
|
2038
|
+
input_dir: Path, solve_data_dir: Path,
|
|
2039
|
+
*, provider: "object | None" = None,
|
|
2040
|
+
) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame, pl.DataFrame]:
|
|
2041
|
+
"""Build all four ``pssdt_varCost_*`` filter frames in one pass."""
|
|
2042
|
+
pdt = _read_pdt_at_param(
|
|
2043
|
+
solve_data_dir / "pdtProcess.csv",
|
|
2044
|
+
param_col=1, param_value="other_operational_cost",
|
|
2045
|
+
key_cols=(0, 2, 3), val_col=4,
|
|
2046
|
+
provider=provider,
|
|
2047
|
+
)
|
|
2048
|
+
pdt_src = _read_pdt_at_param(
|
|
2049
|
+
solve_data_dir / "pdtProcess_source.csv",
|
|
2050
|
+
param_col=2, param_value="other_operational_cost",
|
|
2051
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
2052
|
+
provider=provider,
|
|
2053
|
+
)
|
|
2054
|
+
pdt_snk = _read_pdt_at_param(
|
|
2055
|
+
solve_data_dir / "pdtProcess_sink.csv",
|
|
2056
|
+
param_col=2, param_value="other_operational_cost",
|
|
2057
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
2058
|
+
provider=provider,
|
|
2059
|
+
)
|
|
2060
|
+
varcost: dict[tuple[str, str, str, str, str], float] = {}
|
|
2061
|
+
vp = solve_data_dir / "pdtProcess__source__sink__dt_varCost.csv"
|
|
2062
|
+
vp_df = provider.get(_provider_key(vp))
|
|
2063
|
+
if vp_df is not None:
|
|
2064
|
+
for row in vp_df.iter_rows():
|
|
2065
|
+
if len(row) < 6:
|
|
2066
|
+
continue
|
|
2067
|
+
c = [_cell_str(row[i]) for i in range(5)]
|
|
2068
|
+
if all(c):
|
|
2069
|
+
try:
|
|
2070
|
+
varcost[(c[0], c[1], c[2], c[3], c[4])] = float(row[5])
|
|
2071
|
+
except (ValueError, TypeError):
|
|
2072
|
+
continue
|
|
2073
|
+
|
|
2074
|
+
proc_src = frozenset(
|
|
2075
|
+
_read_pairs(input_dir / "process__source.csv", provider=provider)
|
|
2076
|
+
)
|
|
2077
|
+
proc_snk = frozenset(
|
|
2078
|
+
_read_pairs(input_dir / "process__sink.csv", provider=provider)
|
|
2079
|
+
)
|
|
2080
|
+
pss_noEff = _read_triples(
|
|
2081
|
+
solve_data_dir / "process_source_sink_noEff.csv", provider=provider,
|
|
2082
|
+
)
|
|
2083
|
+
pss_eff = _read_triples(
|
|
2084
|
+
solve_data_dir / "process_source_sink_eff.csv", provider=provider,
|
|
2085
|
+
)
|
|
2086
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
2087
|
+
|
|
2088
|
+
no_eff: list[tuple[str, str, str, str, str]] = []
|
|
2089
|
+
for (p, src, snk) in pss_noEff:
|
|
2090
|
+
for (d, t) in dt:
|
|
2091
|
+
if varcost.get((p, src, snk, d, t), 0.0):
|
|
2092
|
+
no_eff.append((p, src, snk, d, t))
|
|
2093
|
+
|
|
2094
|
+
eff_src: list[tuple[str, str, str, str, str]] = []
|
|
2095
|
+
for (p, src, snk) in pss_eff:
|
|
2096
|
+
for (d, t) in dt:
|
|
2097
|
+
if (p, src) in proc_src and pdt_src.get((p, src, d, t), 0.0):
|
|
2098
|
+
eff_src.append((p, src, snk, d, t))
|
|
2099
|
+
|
|
2100
|
+
eff_snk: list[tuple[str, str, str, str, str]] = []
|
|
2101
|
+
for (p, src, snk) in pss_eff:
|
|
2102
|
+
for (d, t) in dt:
|
|
2103
|
+
if (p, snk) in proc_snk and pdt_snk.get((p, snk, d, t), 0.0):
|
|
2104
|
+
eff_snk.append((p, src, snk, d, t))
|
|
2105
|
+
|
|
2106
|
+
eff_conn: list[tuple[str, str, str, str, str]] = []
|
|
2107
|
+
for (p, src, snk) in pss_eff:
|
|
2108
|
+
for (d, t) in dt:
|
|
2109
|
+
if pdt.get((p, d, t), 0.0):
|
|
2110
|
+
eff_conn.append((p, src, snk, d, t))
|
|
2111
|
+
|
|
2112
|
+
return (
|
|
2113
|
+
_filter_rows_to_frame(no_eff),
|
|
2114
|
+
_filter_rows_to_frame(eff_src),
|
|
2115
|
+
_filter_rows_to_frame(eff_snk),
|
|
2116
|
+
_filter_rows_to_frame(eff_conn),
|
|
2117
|
+
)
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
def _derive_pssdt_varCost_filters_vectorized(
|
|
2121
|
+
input_dir: Path, solve_data_dir: Path,
|
|
2122
|
+
*, provider: "object | None" = None, engine: str = "eager",
|
|
2123
|
+
) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame, pl.DataFrame]:
|
|
2124
|
+
"""Vectorized twin of :func:`_derive_pssdt_varCost_filters`.
|
|
2125
|
+
|
|
2126
|
+
Each of the four outputs is a KEY-only coordinate predicate (no value
|
|
2127
|
+
column): for a ``(p, src, snk) × (d, t)`` grid keep the cell iff a
|
|
2128
|
+
membership-gated ``value ≠ 0`` predicate holds. The ``≠ 0`` test
|
|
2129
|
+
runs on the PARSED-FLOAT lifted value (NOT the rendered Utf8) so it
|
|
2130
|
+
matches the legacy truthiness exactly (``-0.0`` excluded, ``NaN``
|
|
2131
|
+
kept). Output: ``process, source, sink, period, time`` — all Utf8,
|
|
2132
|
+
entity-major order (``__eo, __to``), the order keys dropped.
|
|
2133
|
+
"""
|
|
2134
|
+
pdt = _read_pdt_at_param(
|
|
2135
|
+
solve_data_dir / "pdtProcess.csv",
|
|
2136
|
+
param_col=1, param_value="other_operational_cost",
|
|
2137
|
+
key_cols=(0, 2, 3), val_col=4,
|
|
2138
|
+
provider=provider,
|
|
2139
|
+
)
|
|
2140
|
+
pdt_src = _read_pdt_at_param(
|
|
2141
|
+
solve_data_dir / "pdtProcess_source.csv",
|
|
2142
|
+
param_col=2, param_value="other_operational_cost",
|
|
2143
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
2144
|
+
provider=provider,
|
|
2145
|
+
)
|
|
2146
|
+
pdt_snk = _read_pdt_at_param(
|
|
2147
|
+
solve_data_dir / "pdtProcess_sink.csv",
|
|
2148
|
+
param_col=2, param_value="other_operational_cost",
|
|
2149
|
+
key_cols=(0, 1, 3, 4), val_col=5,
|
|
2150
|
+
provider=provider,
|
|
2151
|
+
)
|
|
2152
|
+
varcost: dict[tuple[str, str, str, str, str], float] = {}
|
|
2153
|
+
vp = solve_data_dir / "pdtProcess__source__sink__dt_varCost.csv"
|
|
2154
|
+
vp_df = provider.get(_provider_key(vp))
|
|
2155
|
+
if vp_df is not None:
|
|
2156
|
+
for row in vp_df.iter_rows():
|
|
2157
|
+
if len(row) < 6:
|
|
2158
|
+
continue
|
|
2159
|
+
c = [_cell_str(row[i]) for i in range(5)]
|
|
2160
|
+
if all(c):
|
|
2161
|
+
try:
|
|
2162
|
+
varcost[(c[0], c[1], c[2], c[3], c[4])] = float(row[5])
|
|
2163
|
+
except (ValueError, TypeError):
|
|
2164
|
+
continue
|
|
2165
|
+
|
|
2166
|
+
proc_src = frozenset(
|
|
2167
|
+
_read_pairs(input_dir / "process__source.csv", provider=provider)
|
|
2168
|
+
)
|
|
2169
|
+
proc_snk = frozenset(
|
|
2170
|
+
_read_pairs(input_dir / "process__sink.csv", provider=provider)
|
|
2171
|
+
)
|
|
2172
|
+
pss_noEff = _read_triples(
|
|
2173
|
+
solve_data_dir / "process_source_sink_noEff.csv", provider=provider,
|
|
2174
|
+
)
|
|
2175
|
+
pss_eff = _read_triples(
|
|
2176
|
+
solve_data_dir / "process_source_sink_eff.csv", provider=provider,
|
|
2177
|
+
)
|
|
2178
|
+
dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
|
|
2179
|
+
|
|
2180
|
+
key5 = ["process", "source", "sink", "period", "time"]
|
|
2181
|
+
|
|
2182
|
+
# Lift predicate value sources.
|
|
2183
|
+
vc_df = lift_dict_to_lookup(
|
|
2184
|
+
varcost, ["process", "source", "sink", "period", "time"], "v_vc")
|
|
2185
|
+
src_df = lift_dict_to_lookup(
|
|
2186
|
+
pdt_src, ["process", "source", "period", "time"], "v_src")
|
|
2187
|
+
snk_df = lift_dict_to_lookup(
|
|
2188
|
+
pdt_snk, ["process", "sink", "period", "time"], "v_snk")
|
|
2189
|
+
pdt_df = lift_dict_to_lookup(
|
|
2190
|
+
pdt, ["process", "period", "time"], "v_pdt")
|
|
2191
|
+
|
|
2192
|
+
# Membership frames (explicit schemas — critique D7).
|
|
2193
|
+
ps_df = pl.DataFrame(
|
|
2194
|
+
{
|
|
2195
|
+
"process": [a for (a, _b) in proc_src],
|
|
2196
|
+
"source": [b for (_a, b) in proc_src],
|
|
2197
|
+
"__in_src": [True] * len(proc_src),
|
|
2198
|
+
},
|
|
2199
|
+
schema={"process": pl.Utf8, "source": pl.Utf8,
|
|
2200
|
+
"__in_src": pl.Boolean},
|
|
2201
|
+
)
|
|
2202
|
+
pk_df = pl.DataFrame(
|
|
2203
|
+
{
|
|
2204
|
+
"process": [a for (a, _b) in proc_snk],
|
|
2205
|
+
"sink": [b for (_a, b) in proc_snk],
|
|
2206
|
+
"__in_snk": [True] * len(proc_snk),
|
|
2207
|
+
},
|
|
2208
|
+
schema={"process": pl.Utf8, "sink": pl.Utf8,
|
|
2209
|
+
"__in_snk": pl.Boolean},
|
|
2210
|
+
)
|
|
2211
|
+
|
|
2212
|
+
def _finish(df: pl.DataFrame, predicate: pl.Expr) -> pl.DataFrame:
|
|
2213
|
+
out = (
|
|
2214
|
+
df.filter(predicate)
|
|
2215
|
+
.sort(["__eo", "__to"])
|
|
2216
|
+
.select(key5)
|
|
2217
|
+
)
|
|
2218
|
+
return out.with_columns(
|
|
2219
|
+
[pl.col(c).cast(pl.Utf8) for c in key5]
|
|
2220
|
+
)
|
|
2221
|
+
|
|
2222
|
+
# 1. noEff: varcost[(p, src, snk, d, t)] ≠ 0.
|
|
2223
|
+
g_no = build_entity_dt_grid(
|
|
2224
|
+
pss_noEff, dt, key_cols=["process", "source", "sink"])
|
|
2225
|
+
g_no = g_no.join(vc_df, on=key5, how="left")
|
|
2226
|
+
no_eff = _finish(g_no, pl.col("v_vc").fill_null(0.0) != 0.0)
|
|
2227
|
+
|
|
2228
|
+
# 2. eff_unit_source: (p, src) ∈ proc_src AND pdt_src ≠ 0.
|
|
2229
|
+
g_src = build_entity_dt_grid(
|
|
2230
|
+
pss_eff, dt, key_cols=["process", "source", "sink"])
|
|
2231
|
+
g_src = (
|
|
2232
|
+
g_src
|
|
2233
|
+
.join(ps_df, on=["process", "source"], how="left")
|
|
2234
|
+
.join(src_df, on=["process", "source", "period", "time"], how="left")
|
|
2235
|
+
)
|
|
2236
|
+
eff_src = _finish(
|
|
2237
|
+
g_src,
|
|
2238
|
+
pl.col("__in_src").fill_null(False)
|
|
2239
|
+
& (pl.col("v_src").fill_null(0.0) != 0.0),
|
|
2240
|
+
)
|
|
2241
|
+
|
|
2242
|
+
# 3. eff_unit_sink: (p, snk) ∈ proc_snk AND pdt_snk ≠ 0.
|
|
2243
|
+
g_snk = build_entity_dt_grid(
|
|
2244
|
+
pss_eff, dt, key_cols=["process", "source", "sink"])
|
|
2245
|
+
g_snk = (
|
|
2246
|
+
g_snk
|
|
2247
|
+
.join(pk_df, on=["process", "sink"], how="left")
|
|
2248
|
+
.join(snk_df, on=["process", "sink", "period", "time"], how="left")
|
|
2249
|
+
)
|
|
2250
|
+
eff_snk = _finish(
|
|
2251
|
+
g_snk,
|
|
2252
|
+
pl.col("__in_snk").fill_null(False)
|
|
2253
|
+
& (pl.col("v_snk").fill_null(0.0) != 0.0),
|
|
2254
|
+
)
|
|
2255
|
+
|
|
2256
|
+
# 4. eff_connection: pdt[(p, d, t)] ≠ 0 (NO membership gate).
|
|
2257
|
+
g_conn = build_entity_dt_grid(
|
|
2258
|
+
pss_eff, dt, key_cols=["process", "source", "sink"])
|
|
2259
|
+
g_conn = g_conn.join(pdt_df, on=["process", "period", "time"], how="left")
|
|
2260
|
+
eff_conn = _finish(g_conn, pl.col("v_pdt").fill_null(0.0) != 0.0)
|
|
2261
|
+
|
|
2262
|
+
return (no_eff, eff_src, eff_snk, eff_conn)
|
|
2263
|
+
|
|
2264
|
+
|
|
2265
|
+
def emit_pssdt_varCost_filters(
|
|
2266
|
+
input_dir: Path, solve_data_dir: Path,
|
|
2267
|
+
*, provider,
|
|
2268
|
+
) -> None:
|
|
2269
|
+
"""Emit ``pssdt_varCost_filters`` to the Provider."""
|
|
2270
|
+
no_eff, eff_src, eff_snk, eff_conn = (
|
|
2271
|
+
_derive_pssdt_varCost_filters_vectorized(
|
|
2272
|
+
input_dir, solve_data_dir, provider=provider,
|
|
2273
|
+
)
|
|
2274
|
+
)
|
|
2275
|
+
_emit(provider, "solve_data/pssdt_varCost_noEff.csv", no_eff)
|
|
2276
|
+
_emit(provider, "solve_data/pssdt_varCost_eff_unit_source.csv", eff_src)
|
|
2277
|
+
_emit(provider, "solve_data/pssdt_varCost_eff_unit_sink.csv", eff_snk)
|
|
2278
|
+
_emit(provider, "solve_data/pssdt_varCost_eff_connection.csv", eff_conn)
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
# ---- write_cap_reduction_params (mod L1637-1663) --------------------------
|
|
2282
|
+
|
|
2283
|
+
def _read_p_side_3(path: Path,
|
|
2284
|
+
*, provider: "object | None" = None,
|
|
2285
|
+
) -> dict[tuple[str, str, str], float]:
|
|
2286
|
+
"""Per-side 4-col CSV ``(k1, k2, k3, value)`` → ``(k1, k2, k3) → v``."""
|
|
2287
|
+
out: dict[tuple[str, str, str], float] = {}
|
|
2288
|
+
df = provider.get(_provider_key(path))
|
|
2289
|
+
if df is None:
|
|
2290
|
+
return out
|
|
2291
|
+
for row in df.iter_rows():
|
|
2292
|
+
if len(row) < 4:
|
|
2293
|
+
continue
|
|
2294
|
+
c = [_cell_str(row[i]) for i in range(3)]
|
|
2295
|
+
if all(c):
|
|
2296
|
+
try:
|
|
2297
|
+
out[(c[0], c[1], c[2])] = float(row[3])
|
|
2298
|
+
except (ValueError, TypeError):
|
|
2299
|
+
continue
|
|
2300
|
+
return out
|
|
2301
|
+
|
|
2302
|
+
|
|
2303
|
+
# ---- write_ed_period_params (mod L1252-1255 family, ed_*_period) ----------
|
|
2304
|
+
|
|
2305
|
+
def _read_ed_pairs(path: Path,
|
|
2306
|
+
*, provider: "object | None" = None,
|
|
2307
|
+
) -> list[tuple[str, str]]:
|
|
2308
|
+
df = provider.get(_provider_key(path))
|
|
2309
|
+
if df is None:
|
|
2310
|
+
return []
|
|
2311
|
+
out: list[tuple[str, str]] = []
|
|
2312
|
+
for row in df.iter_rows():
|
|
2313
|
+
if len(row) < 2:
|
|
2314
|
+
continue
|
|
2315
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
2316
|
+
if c0 and c1:
|
|
2317
|
+
out.append((c0, c1))
|
|
2318
|
+
return out
|
|
2319
|
+
|
|
2320
|
+
|
|
2321
|
+
def _ed_period_compute(
|
|
2322
|
+
src_pairs: list[tuple[str, str]],
|
|
2323
|
+
mod_param: str,
|
|
2324
|
+
pp, # PdLookup
|
|
2325
|
+
pn, # PdLookup
|
|
2326
|
+
process_set: frozenset[str],
|
|
2327
|
+
node_set: frozenset[str],
|
|
2328
|
+
) -> pl.DataFrame:
|
|
2329
|
+
"""One ed_period_param frame — 3-col Utf8 ``(entity, period, value)``."""
|
|
2330
|
+
e_col: list[str] = []
|
|
2331
|
+
d_col: list[str] = []
|
|
2332
|
+
v_col: list[str] = []
|
|
2333
|
+
for e, d in src_pairs:
|
|
2334
|
+
if e in process_set:
|
|
2335
|
+
v = pp.get(e, mod_param, d)
|
|
2336
|
+
elif e in node_set:
|
|
2337
|
+
v = pn.get(e, mod_param, d)
|
|
2338
|
+
else:
|
|
2339
|
+
v = 0.0
|
|
2340
|
+
e_col.append(e)
|
|
2341
|
+
d_col.append(d)
|
|
2342
|
+
v_col.append(repr(v))
|
|
2343
|
+
return _utf8_frame({"entity": e_col, "period": d_col, "value": v_col})
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
_ED_PERIOD_PARAM_SPECS: tuple[tuple[str, str, str], ...] = (
|
|
2347
|
+
# (basename, src-pair-tag ['invest'|'divest'], mod_param)
|
|
2348
|
+
("ed_invest_max_period.csv", "invest", "invest_max_period"),
|
|
2349
|
+
("ed_invest_min_period.csv", "invest", "invest_min_period"),
|
|
2350
|
+
("ed_divest_max_period.csv", "divest", "retire_max_period"),
|
|
2351
|
+
("ed_divest_min_period.csv", "divest", "retire_min_period"),
|
|
2352
|
+
("ed_cumulative_max_capacity.csv", "invest", "cumulative_max_capacity"),
|
|
2353
|
+
("ed_cumulative_min_capacity.csv", "invest", "cumulative_min_capacity"),
|
|
2354
|
+
)
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
def _ed_period_inputs(input_dir: Path, solve_data_dir: Path,
|
|
2358
|
+
*, provider: "object | None" = None):
|
|
2359
|
+
from flextool.engine_polars._pdt_lookup import PdLookup
|
|
2360
|
+
pp = PdLookup(
|
|
2361
|
+
pd_csv=input_dir / "pd_process.csv",
|
|
2362
|
+
p_csv=input_dir / "p_process.csv",
|
|
2363
|
+
period_branch_csv=solve_data_dir / "period__branch.csv",
|
|
2364
|
+
provider=provider,
|
|
2365
|
+
)
|
|
2366
|
+
pn = PdLookup(
|
|
2367
|
+
pd_csv=input_dir / "pd_node.csv",
|
|
2368
|
+
p_csv=input_dir / "p_node.csv",
|
|
2369
|
+
period_branch_csv=solve_data_dir / "period__branch.csv",
|
|
2370
|
+
provider=provider,
|
|
2371
|
+
)
|
|
2372
|
+
process_set = frozenset(
|
|
2373
|
+
_read_singles(input_dir / "process.csv", provider=provider)
|
|
2374
|
+
)
|
|
2375
|
+
node_set = frozenset(
|
|
2376
|
+
_read_singles(input_dir / "node.csv", provider=provider)
|
|
2377
|
+
)
|
|
2378
|
+
ed_invest_pairs = _read_ed_pairs(
|
|
2379
|
+
solve_data_dir / "ed_invest.csv", provider=provider,
|
|
2380
|
+
)
|
|
2381
|
+
ed_divest_pairs = _read_ed_pairs(
|
|
2382
|
+
solve_data_dir / "ed_divest.csv", provider=provider,
|
|
2383
|
+
)
|
|
2384
|
+
return pp, pn, process_set, node_set, ed_invest_pairs, ed_divest_pairs
|
|
2385
|
+
|
|
2386
|
+
|
|
2387
|
+
def emit_ed_period_params(
|
|
2388
|
+
input_dir: Path, solve_data_dir: Path,
|
|
2389
|
+
*, provider,
|
|
2390
|
+
) -> None:
|
|
2391
|
+
"""Emit ``ed_period_params`` to the Provider."""
|
|
2392
|
+
pp, pn, ps, ns, inv, div = _ed_period_inputs(
|
|
2393
|
+
input_dir, solve_data_dir, provider=provider,
|
|
2394
|
+
)
|
|
2395
|
+
pair_for = {"invest": inv, "divest": div}
|
|
2396
|
+
for fname, tag, mod_param in _ED_PERIOD_PARAM_SPECS:
|
|
2397
|
+
frame = _ed_period_compute(pair_for[tag], mod_param, pp, pn, ps, ns)
|
|
2398
|
+
_emit(provider, f"solve_data/{fname}", frame)
|