flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1420 @@
|
|
|
1
|
+
"""Node inflow scaling params — per-solve emitters.
|
|
2
|
+
|
|
3
|
+
Called per-solve from ``_emit_solve_time.run`` (batch 17).
|
|
4
|
+
|
|
5
|
+
Output CSVs (6 emitted — the 6 with a downstream consumer):
|
|
6
|
+
|
|
7
|
+
* ``ptNode_inflow.csv`` — (n, t) merged inflow
|
|
8
|
+
* ``_node_cap_inflow_fallback.csv`` — (n, d) abs(max)
|
|
9
|
+
* ``period_flow_annual_multiplier.csv`` — (n, d) cpsoy / psaf
|
|
10
|
+
* ``period_flow_proportional_multiplier.csv`` — (n, d) af / (abs(sum_t)/tdy)
|
|
11
|
+
* ``new_old_slope.csv`` — (n, d) npop * (1 + nom)
|
|
12
|
+
* ``new_old_section.csv`` — (n, d) peak * nom
|
|
13
|
+
|
|
14
|
+
These feed ``derive_pdtNodeInflow`` (5) and the lp-scaling emitter (1).
|
|
15
|
+
|
|
16
|
+
One further standalone diagnostic CSV is emitted alongside (NOT part of
|
|
17
|
+
the parity-gated 6, no LP consumer):
|
|
18
|
+
|
|
19
|
+
* ``inflow_scaling_diagnostics.csv`` — (node, period, inflow_method,
|
|
20
|
+
annual_flow, f, level_shift_pct), the Part-B annualisation-divergence
|
|
21
|
+
``f`` diagnostic. See ``_compute_inflow_scaling_diagnostics``.
|
|
22
|
+
|
|
23
|
+
The 9 internal middle parameters that used to be emitted as scratch CSVs
|
|
24
|
+
— ``orig_flow_sum``, ``period_share_of_annual_flow``, ``new_peak_sign``,
|
|
25
|
+
``old_peak_max``, ``old_peak_min``, ``old_peak_sign``,
|
|
26
|
+
``new_peak_divided_by_old_peak``, ``new_peak_inflow_sum``,
|
|
27
|
+
``new_old_multiplier`` — have NO external consumer (all internal middle
|
|
28
|
+
parameters); they are computed in-memory where still needed to feed the 6
|
|
29
|
+
consumed outputs, but no longer written. The legacy oracle
|
|
30
|
+
``_compute_inflow_scaling_frames`` still materialises all 15 as the
|
|
31
|
+
independent parity reference for the 6 consumed outputs.
|
|
32
|
+
|
|
33
|
+
Reuse note (Phase 2 sub-dispatch 4 brief)
|
|
34
|
+
-----------------------------------------
|
|
35
|
+
|
|
36
|
+
``flextool.engine_polars._inflow_scaling`` already implements every
|
|
37
|
+
per-(n, d) formula here (``_compute_period_share_of_annual_flow``,
|
|
38
|
+
``_compute_period_flow_annual_multiplier``,
|
|
39
|
+
``_compute_period_flow_proportional_multiplier``, ``_compute_peak_scaling``).
|
|
40
|
+
But that helper operates on :class:`InputSource` + per-solve aggregates
|
|
41
|
+
and folds all the intermediates into a single ``p_inflow`` Param —
|
|
42
|
+
the per-CSV intermediates are never materialised. Sharing it would
|
|
43
|
+
require either threading intermediate accessors through the helper
|
|
44
|
+
(cross-cutting) or running a different data path (legacy CSV reads
|
|
45
|
+
here, InputSource there). Instead this writer mirrors the legacy
|
|
46
|
+
preprocessing's CSV-in / CSV-out shape verbatim; the formulas match
|
|
47
|
+
``_inflow_scaling`` line-by-line so the two paths agree numerically.
|
|
48
|
+
|
|
49
|
+
Float values formatted with ``repr(float(v))`` for byte-identical
|
|
50
|
+
parity with the legacy emitter.
|
|
51
|
+
"""
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import logging
|
|
55
|
+
from pathlib import Path
|
|
56
|
+
|
|
57
|
+
import polars as pl
|
|
58
|
+
|
|
59
|
+
from flextool.engine_polars._emit_provider_io import (
|
|
60
|
+
_emit,
|
|
61
|
+
_provider_key,
|
|
62
|
+
)
|
|
63
|
+
from flextool.engine_polars._vectorize import _render_value_column
|
|
64
|
+
|
|
65
|
+
_logger = logging.getLogger(__name__)
|
|
66
|
+
|
|
67
|
+
# Scaling inflow methods whose annualisation depends on the per-(d, t)
|
|
68
|
+
# representative weight ``w = p_timestep_weight``. ``use_original`` /
|
|
69
|
+
# ``no_inflow`` do not annualise, so they carry no f diagnostic.
|
|
70
|
+
_SCALING_METHODS = (
|
|
71
|
+
"scale_to_annual_flow",
|
|
72
|
+
"scale_in_proportion",
|
|
73
|
+
"scale_to_annual_and_peak_flow",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ---------------------------------------------------------------------------
|
|
78
|
+
# Native-frame row helpers.
|
|
79
|
+
#
|
|
80
|
+
# These read the in-memory polars frame directly from the Provider via
|
|
81
|
+
# ``provider.get(_provider_key(path))`` instead of round-tripping through
|
|
82
|
+
# CSV text (the legacy ``_provider_open`` + manual ``line.split(",")``
|
|
83
|
+
# path, which had no quoted-field handling).
|
|
84
|
+
#
|
|
85
|
+
# Type-fidelity contract — reproduce *exactly* what the legacy
|
|
86
|
+
# ``line.rstrip("\n").split(",")`` over a ``DataFrame.write_csv``
|
|
87
|
+
# serialisation would have yielded:
|
|
88
|
+
#
|
|
89
|
+
# * Key columns (single-value entries and dict-key positions) were
|
|
90
|
+
# split strings. ``write_csv`` serialises ``null`` → ``""`` and any
|
|
91
|
+
# scalar (Enum / Int / Float / Utf8) → its string form. We coerce
|
|
92
|
+
# each key cell with :func:`_cell_str` (``None`` → ``""``, else
|
|
93
|
+
# ``str``) and apply the original truthiness guard to the *string*
|
|
94
|
+
# form so a null cell is skipped (matching the legacy ``if parts[i]``
|
|
95
|
+
# test) while a literal ``"0"`` is kept.
|
|
96
|
+
# * Value columns were re-coerced with ``float(...)``. We apply
|
|
97
|
+
# ``float(...)`` to the native cell — harmless on an already-float
|
|
98
|
+
# frame, necessary on an int frame, and identical to the legacy
|
|
99
|
+
# ``float(str_cell)`` on a stringified-value frame. A value that
|
|
100
|
+
# cannot be parsed as a float is skipped: the legacy ``except
|
|
101
|
+
# ValueError`` is widened to ``except (ValueError, TypeError)`` so a
|
|
102
|
+
# native ``None`` value cell (``float(None)`` raises ``TypeError``)
|
|
103
|
+
# is skipped exactly as the legacy ``float("")`` ``ValueError`` was.
|
|
104
|
+
#
|
|
105
|
+
# ``provider.get`` returns data rows only (no header), so there is no
|
|
106
|
+
# header row to skip; an empty / missing frame yields the same empty
|
|
107
|
+
# list / dict the legacy loop produced.
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _cell_str(value: "object | None") -> str:
|
|
112
|
+
"""Reproduce a split CSV cell string for a native frame value.
|
|
113
|
+
|
|
114
|
+
``DataFrame.write_csv`` renders ``null`` as the empty string and every
|
|
115
|
+
other scalar as its textual form; the legacy ``line.split(",")`` then
|
|
116
|
+
read those strings back. Mirror that here so dict keys / single
|
|
117
|
+
values stay byte-identical to the legacy CSV round-trip.
|
|
118
|
+
"""
|
|
119
|
+
return "" if value is None else str(value)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _read_singles(path: Path,
|
|
123
|
+
*, provider: "object | None" = None) -> list[str]:
|
|
124
|
+
"""First-column reader → list of non-empty first-column strings."""
|
|
125
|
+
df = provider.get(_provider_key(path))
|
|
126
|
+
if df is None:
|
|
127
|
+
return []
|
|
128
|
+
out: list[str] = []
|
|
129
|
+
for row in df.iter_rows():
|
|
130
|
+
if not row:
|
|
131
|
+
continue
|
|
132
|
+
c0 = _cell_str(row[0])
|
|
133
|
+
if c0:
|
|
134
|
+
out.append(c0)
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _read_pairs(path: Path,
|
|
139
|
+
*, provider: "object | None" = None) -> list[tuple[str, str]]:
|
|
140
|
+
"""First-two-column reader → list of (c0, c1) with both non-empty."""
|
|
141
|
+
df = provider.get(_provider_key(path))
|
|
142
|
+
if df is None:
|
|
143
|
+
return []
|
|
144
|
+
out: list[tuple[str, str]] = []
|
|
145
|
+
for row in df.iter_rows():
|
|
146
|
+
if len(row) < 2:
|
|
147
|
+
continue
|
|
148
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
149
|
+
if c0 and c1:
|
|
150
|
+
out.append((c0, c1))
|
|
151
|
+
return out
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _read_keyed2_float(path: Path,
|
|
155
|
+
*, provider: "object | None" = None,
|
|
156
|
+
) -> dict[tuple[str, str], float]:
|
|
157
|
+
"""Three-col frame (key1, key2, value) → {(k1, k2): float}.
|
|
158
|
+
|
|
159
|
+
Mirrors legacy ``_read_p`` / ``_read_pt_node_inflow``: malformed
|
|
160
|
+
or non-numeric rows silently skipped.
|
|
161
|
+
"""
|
|
162
|
+
out: dict[tuple[str, str], float] = {}
|
|
163
|
+
df = provider.get(_provider_key(path))
|
|
164
|
+
if df is None:
|
|
165
|
+
return out
|
|
166
|
+
for row in df.iter_rows():
|
|
167
|
+
if len(row) < 3:
|
|
168
|
+
continue
|
|
169
|
+
c0, c1 = _cell_str(row[0]), _cell_str(row[1])
|
|
170
|
+
if c0 and c1:
|
|
171
|
+
try:
|
|
172
|
+
out[(c0, c1)] = float(row[2])
|
|
173
|
+
except (ValueError, TypeError):
|
|
174
|
+
continue
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _read_keyed3_float(path: Path,
|
|
179
|
+
*, provider: "object | None" = None,
|
|
180
|
+
) -> dict[tuple[str, str, str], float]:
|
|
181
|
+
"""Four-col frame (k1, k2, k3, value) → {(k1, k2, k3): float}."""
|
|
182
|
+
out: dict[tuple[str, str, str], float] = {}
|
|
183
|
+
df = provider.get(_provider_key(path))
|
|
184
|
+
if df is None:
|
|
185
|
+
return out
|
|
186
|
+
for row in df.iter_rows():
|
|
187
|
+
if len(row) < 4:
|
|
188
|
+
continue
|
|
189
|
+
c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
|
|
190
|
+
if c0 and c1 and c2:
|
|
191
|
+
try:
|
|
192
|
+
out[(c0, c1, c2)] = float(row[3])
|
|
193
|
+
except (ValueError, TypeError):
|
|
194
|
+
continue
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _read_keyed_float(path: Path,
|
|
199
|
+
*, provider: "object | None" = None,
|
|
200
|
+
) -> dict[str, float]:
|
|
201
|
+
"""Two-col frame (key, value) → {key: float}."""
|
|
202
|
+
out: dict[str, float] = {}
|
|
203
|
+
df = provider.get(_provider_key(path))
|
|
204
|
+
if df is None:
|
|
205
|
+
return out
|
|
206
|
+
for row in df.iter_rows():
|
|
207
|
+
if len(row) < 2:
|
|
208
|
+
continue
|
|
209
|
+
c0 = _cell_str(row[0])
|
|
210
|
+
if c0:
|
|
211
|
+
try:
|
|
212
|
+
out[c0] = float(row[1])
|
|
213
|
+
except (ValueError, TypeError):
|
|
214
|
+
continue
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _rows_to_frame(
|
|
219
|
+
header: tuple[str, str, str],
|
|
220
|
+
rows: list[tuple[str, str, float]],
|
|
221
|
+
) -> pl.DataFrame:
|
|
222
|
+
"""Materialise (k1, k2, repr(value)) rows as an all-Utf8 frame.
|
|
223
|
+
|
|
224
|
+
Pre-stringifies values via ``repr(v)`` so the polars ``write_csv``
|
|
225
|
+
output is byte-identical to the legacy ``f"{a},{b},{repr(v)}\\n"``
|
|
226
|
+
text emitter. Note this uses ``repr(v)`` NOT ``repr(float(v))`` —
|
|
227
|
+
the legacy emitter assumes the caller already produced floats.
|
|
228
|
+
"""
|
|
229
|
+
return pl.DataFrame(
|
|
230
|
+
{
|
|
231
|
+
header[0]: [r[0] for r in rows],
|
|
232
|
+
header[1]: [r[1] for r in rows],
|
|
233
|
+
header[2]: [repr(r[2]) for r in rows],
|
|
234
|
+
},
|
|
235
|
+
schema={h: pl.Utf8 for h in header},
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# ---------------------------------------------------------------------------
|
|
240
|
+
# Family — write_node_inflow_scaling_params
|
|
241
|
+
# ---------------------------------------------------------------------------
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _compute_inflow_scaling_frames(
|
|
245
|
+
input_dir: Path, solve_data_dir: Path,
|
|
246
|
+
*, provider: "object | None" = None,
|
|
247
|
+
) -> dict[str, pl.DataFrame]:
|
|
248
|
+
"""Compute every inflow-scaling CSV in one pass, returning a dict
|
|
249
|
+
keyed by output basename.
|
|
250
|
+
|
|
251
|
+
Retained as the parity oracle for
|
|
252
|
+
:func:`_compute_inflow_scaling_frames_vectorized`
|
|
253
|
+
(``tests/engine_polars/test_vectorize_inflow_scaling_parity.py``); the
|
|
254
|
+
live emit path is the vectorized twin. The cross-CSV state (peak
|
|
255
|
+
family, npop / nom / etc) is heavy enough that splitting into
|
|
256
|
+
independent per-CSV passes would re-walk the time axis O(N) times — the
|
|
257
|
+
dict-of-frames pattern from the audit doc is the appropriate adapter
|
|
258
|
+
here.
|
|
259
|
+
"""
|
|
260
|
+
out: dict[str, pl.DataFrame] = {}
|
|
261
|
+
|
|
262
|
+
# ── Sources ────────────────────────────────────────────────────────
|
|
263
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
264
|
+
period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
|
|
265
|
+
provider=provider)
|
|
266
|
+
time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
|
|
267
|
+
p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
|
|
268
|
+
pt_node_inflow = _read_keyed2_float(
|
|
269
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
270
|
+
)
|
|
271
|
+
# The "(n, t) is explicitly set" predicate uses ONLY (n, t) presence
|
|
272
|
+
# in pt_node_inflow.csv — values are read separately above.
|
|
273
|
+
node_time_inflow = frozenset(_read_pairs(
|
|
274
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
275
|
+
))
|
|
276
|
+
|
|
277
|
+
inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
278
|
+
provider=provider)
|
|
279
|
+
methods_for_node: dict[str, set[str]] = {}
|
|
280
|
+
for n, m in inflow_method:
|
|
281
|
+
methods_for_node.setdefault(n, set()).add(m)
|
|
282
|
+
|
|
283
|
+
pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
|
|
284
|
+
cpsoy = _read_keyed_float(
|
|
285
|
+
solve_data_dir / "complete_period_share_of_year_calc.csv",
|
|
286
|
+
provider=provider,
|
|
287
|
+
)
|
|
288
|
+
p_tdy = _read_keyed_float(
|
|
289
|
+
solve_data_dir / "p_timeline_duration_in_years.csv",
|
|
290
|
+
provider=provider,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# Representative weight w[(d, t)] = p_timestep_weight[d, t] (the param
|
|
294
|
+
# the cost objective already uses). Sourced from the per-solve
|
|
295
|
+
# ``timestep_weight.csv`` (period, time, weight) emitted by
|
|
296
|
+
# ``emit_rp_data`` / ``emit_empty_rp_data`` BEFORE this batch runs; it
|
|
297
|
+
# is therefore in the Provider already. DEFAULT 1.0 for any (d, t) not
|
|
298
|
+
# present — when timeset_weights are absent/uniform the frame is empty
|
|
299
|
+
# (non-RP solve) or all-1.0, so w ≡ 1.0 and every weighted sum below
|
|
300
|
+
# reduces byte-for-byte to today's unweighted formula.
|
|
301
|
+
w_dt = _read_keyed2_float(
|
|
302
|
+
solve_data_dir / "timestep_weight.csv", provider=provider,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
# period__timeline (Python output): list of (period, timeline) pairs.
|
|
306
|
+
period_timeline = _read_pairs(solve_data_dir / "period__timeline_set.csv",
|
|
307
|
+
provider=provider)
|
|
308
|
+
timelines_for_d: dict[str, list[str]] = {}
|
|
309
|
+
for d, tl in period_timeline:
|
|
310
|
+
timelines_for_d.setdefault(d, []).append(tl)
|
|
311
|
+
|
|
312
|
+
# complete_time_in_use (Python output, _set suffix).
|
|
313
|
+
complete_time_in_use = _read_singles(
|
|
314
|
+
solve_data_dir / "complete_time_in_use_set.csv",
|
|
315
|
+
provider=provider,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# dt_complete from steps_complete_solve.csv — (period, step) pairs.
|
|
319
|
+
dt_complete_pairs = _read_pairs(
|
|
320
|
+
solve_data_dir / "steps_complete_solve.csv",
|
|
321
|
+
provider=provider,
|
|
322
|
+
)
|
|
323
|
+
dt_complete_for_d: dict[str, list[str]] = {}
|
|
324
|
+
for d, t in dt_complete_pairs:
|
|
325
|
+
dt_complete_for_d.setdefault(d, []).append(t)
|
|
326
|
+
|
|
327
|
+
# ── ptNode_inflow{n in node, t in time} ───────────────────────────
|
|
328
|
+
# Per legacy mod L1237: pass-through pt_node_inflow when (n, t) is
|
|
329
|
+
# in the explicit set, else the per-node scalar default from
|
|
330
|
+
# p_node[(n, "inflow")] (typically 0).
|
|
331
|
+
p_node_inflow_default = {
|
|
332
|
+
n: p_node.get((n, "inflow"), 0.0) for n in nodes
|
|
333
|
+
}
|
|
334
|
+
pti: dict[tuple[str, str], float] = {}
|
|
335
|
+
for n in nodes:
|
|
336
|
+
default = p_node_inflow_default[n]
|
|
337
|
+
for t in time_set:
|
|
338
|
+
if (n, t) in node_time_inflow:
|
|
339
|
+
pti[(n, t)] = pt_node_inflow.get((n, t), 0.0)
|
|
340
|
+
else:
|
|
341
|
+
pti[(n, t)] = default
|
|
342
|
+
|
|
343
|
+
rows_pt = [(n, t, pti[(n, t)]) for n in nodes for t in time_set]
|
|
344
|
+
out["ptNode_inflow.csv"] = _rows_to_frame(
|
|
345
|
+
("node", "time", "value"), rows_pt,
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
# ── _node_cap_inflow_fallback{n in node, d in period_in_use} ──────
|
|
349
|
+
# value = max_{t in time} abs(ptNode_inflow[n, t]); 0 if no time.
|
|
350
|
+
fallback_rows: list[tuple[str, str, float]] = []
|
|
351
|
+
if not time_set:
|
|
352
|
+
for n in nodes:
|
|
353
|
+
for d in period_in_use:
|
|
354
|
+
fallback_rows.append((n, d, 0.0))
|
|
355
|
+
else:
|
|
356
|
+
for n in nodes:
|
|
357
|
+
max_abs = max(abs(pti[(n, t)]) for t in time_set)
|
|
358
|
+
for d in period_in_use:
|
|
359
|
+
fallback_rows.append((n, d, max_abs))
|
|
360
|
+
out["_node_cap_inflow_fallback.csv"] = _rows_to_frame(
|
|
361
|
+
("node", "period", "value"), fallback_rows,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
# Helper: does node n have inflow method m?
|
|
365
|
+
def _has_method(n: str, m: str) -> bool:
|
|
366
|
+
return m in methods_for_node.get(n, ())
|
|
367
|
+
|
|
368
|
+
# Domain predicates -- legacy guards each writer with:
|
|
369
|
+
# methods AND pdNode[..annual_flow..] != 0 [AND peak_inflow != 0]
|
|
370
|
+
def _annual_eligible(n: str) -> bool:
|
|
371
|
+
return (_has_method(n, "scale_to_annual_flow")
|
|
372
|
+
or _has_method(n, "scale_to_annual_and_peak_flow"))
|
|
373
|
+
|
|
374
|
+
# ── orig_flow_sum ─────────────────────────────────────────────────
|
|
375
|
+
# value = sum_{t in complete_time_in_use} ptNode_inflow[n, t] · w[d, t]
|
|
376
|
+
# (A3 — annual-energy ingredient, weighted by the representative
|
|
377
|
+
# weight per period). Because w is period-specific the sum can
|
|
378
|
+
# no longer be cached period-independently; compute per (n, d).
|
|
379
|
+
# With w ≡ 1.0 this reduces byte-for-byte to the former
|
|
380
|
+
# period-independent t-sum.
|
|
381
|
+
# Domain: (n, d) where annual_eligible AND pdNode annual_flow != 0.
|
|
382
|
+
rows_orig: list[tuple[str, str, float]] = []
|
|
383
|
+
for n in nodes:
|
|
384
|
+
if not _annual_eligible(n):
|
|
385
|
+
continue
|
|
386
|
+
for d in period_in_use:
|
|
387
|
+
if pdNode.get((n, "annual_flow", d), 0.0) == 0.0:
|
|
388
|
+
continue
|
|
389
|
+
s = sum(
|
|
390
|
+
pti[(n, t)] * w_dt.get((d, t), 1.0)
|
|
391
|
+
for t in complete_time_in_use
|
|
392
|
+
)
|
|
393
|
+
rows_orig.append((n, d, s))
|
|
394
|
+
out["orig_flow_sum.csv"] = _rows_to_frame(
|
|
395
|
+
("node", "period", "value"), rows_orig,
|
|
396
|
+
)
|
|
397
|
+
orig_flow_sum = {(n, d): v for n, d, v in rows_orig}
|
|
398
|
+
|
|
399
|
+
# ── period_share_of_annual_flow ───────────────────────────────────
|
|
400
|
+
# value = abs(sum_{t in dt_complete[d]} ptNode_inflow[n, t])
|
|
401
|
+
# / pdNode[n, 'annual_flow', d].
|
|
402
|
+
rows_psaf: list[tuple[str, str, float]] = []
|
|
403
|
+
for n in nodes:
|
|
404
|
+
if not _annual_eligible(n):
|
|
405
|
+
continue
|
|
406
|
+
for d in period_in_use:
|
|
407
|
+
af = pdNode.get((n, "annual_flow", d), 0.0)
|
|
408
|
+
if af == 0.0:
|
|
409
|
+
continue
|
|
410
|
+
# A1 — weight each timestep's inflow by w[d, t] (representative
|
|
411
|
+
# weight); w ≡ 1.0 → today's unweighted sum byte-for-byte.
|
|
412
|
+
s = sum(pti[(n, t)] * w_dt.get((d, t), 1.0)
|
|
413
|
+
for t in dt_complete_for_d.get(d, ()))
|
|
414
|
+
rows_psaf.append((n, d, abs(s) / af))
|
|
415
|
+
out["period_share_of_annual_flow.csv"] = _rows_to_frame(
|
|
416
|
+
("node", "period", "value"), rows_psaf,
|
|
417
|
+
)
|
|
418
|
+
psaf = {(n, d): v for n, d, v in rows_psaf}
|
|
419
|
+
|
|
420
|
+
# ── period_flow_annual_multiplier ─────────────────────────────────
|
|
421
|
+
# value = complete_period_share_of_year[d] / period_share_of_annual_flow[n, d].
|
|
422
|
+
# Domain: (n, d) where scale_to_annual_flow AND pdNode annual_flow.
|
|
423
|
+
rows_pfam: list[tuple[str, str, float]] = []
|
|
424
|
+
for n in nodes:
|
|
425
|
+
if not _has_method(n, "scale_to_annual_flow"):
|
|
426
|
+
continue
|
|
427
|
+
for d in period_in_use:
|
|
428
|
+
if pdNode.get((n, "annual_flow", d), 0.0) == 0.0:
|
|
429
|
+
continue
|
|
430
|
+
denom = psaf.get((n, d), 0.0)
|
|
431
|
+
if denom == 0.0:
|
|
432
|
+
continue
|
|
433
|
+
rows_pfam.append((n, d, cpsoy.get(d, 0.0) / denom))
|
|
434
|
+
out["period_flow_annual_multiplier.csv"] = _rows_to_frame(
|
|
435
|
+
("node", "period", "value"), rows_pfam,
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
# ── period_flow_proportional_multiplier ───────────────────────────
|
|
439
|
+
# value = pdNode[n, 'annual_flow', d] /
|
|
440
|
+
# (abs(sum_{t in time} ptNode_inflow[n, t]) /
|
|
441
|
+
# sum_{tl in period__timeline[d]} p_timeline_duration_in_years[tl]).
|
|
442
|
+
rows_pfpm: list[tuple[str, str, float]] = []
|
|
443
|
+
# A2 — weight the inflow sum by w[d, t] (representative weight) per
|
|
444
|
+
# period. This makes the sum period-specific (was t-axis only); the
|
|
445
|
+
# iteration domain stays the full time_set, so with w ≡ 1.0 each term's
|
|
446
|
+
# factor is 1.0 and the sum is byte-identical to the former
|
|
447
|
+
# period-independent ``sum(pti[(n, t)] for t in time_set)``.
|
|
448
|
+
for n in nodes:
|
|
449
|
+
if not _has_method(n, "scale_in_proportion"):
|
|
450
|
+
continue
|
|
451
|
+
for d in period_in_use:
|
|
452
|
+
af = pdNode.get((n, "annual_flow", d), 0.0)
|
|
453
|
+
if af == 0.0:
|
|
454
|
+
continue
|
|
455
|
+
time_sum = sum(pti[(n, t)] * w_dt.get((d, t), 1.0)
|
|
456
|
+
for t in time_set)
|
|
457
|
+
tdy_sum = sum(p_tdy.get(tl, 0.0)
|
|
458
|
+
for tl in timelines_for_d.get(d, ()))
|
|
459
|
+
if tdy_sum == 0.0 or time_sum == 0.0:
|
|
460
|
+
continue
|
|
461
|
+
rows_pfpm.append((n, d, af / (abs(time_sum) / tdy_sum)))
|
|
462
|
+
out["period_flow_proportional_multiplier.csv"] = _rows_to_frame(
|
|
463
|
+
("node", "period", "value"), rows_pfpm,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
# ── Peak-flow family (annual_and_peak_flow) ────────────────────────
|
|
467
|
+
# Domain: (n, d) where scale_to_annual_and_peak_flow AND pdNode
|
|
468
|
+
# annual_flow != 0 AND pdNode peak_inflow != 0.
|
|
469
|
+
# Per-node availability of explicit (n, t) inflow rows — this drives
|
|
470
|
+
# the op_max / op_min "use scalar default" fallback path.
|
|
471
|
+
has_node_time_inflow: dict[str, bool] = {
|
|
472
|
+
n: any(nn == n for (nn, _t) in node_time_inflow) for n in nodes
|
|
473
|
+
}
|
|
474
|
+
# Per-node max/min across t (independent of d).
|
|
475
|
+
op_max_by_n: dict[str, float] = {}
|
|
476
|
+
op_min_by_n: dict[str, float] = {}
|
|
477
|
+
op_sign_by_n: dict[str, float] = {}
|
|
478
|
+
old_peak_by_n: dict[str, float] = {}
|
|
479
|
+
for n in nodes:
|
|
480
|
+
if has_node_time_inflow[n]:
|
|
481
|
+
inflow_vals = [pti[(n, t)] for t in time_set]
|
|
482
|
+
op_max = max(inflow_vals) if inflow_vals else 0.0
|
|
483
|
+
op_min = min(inflow_vals) if inflow_vals else 0.0
|
|
484
|
+
else:
|
|
485
|
+
scalar = p_node_inflow_default[n]
|
|
486
|
+
op_max = scalar
|
|
487
|
+
op_min = scalar
|
|
488
|
+
op_max_by_n[n] = op_max
|
|
489
|
+
op_min_by_n[n] = op_min
|
|
490
|
+
if has_node_time_inflow[n]:
|
|
491
|
+
op_sign = 1.0 if abs(op_max) >= abs(op_min) else -1.0
|
|
492
|
+
else:
|
|
493
|
+
op_sign = 1.0 if p_node_inflow_default[n] >= 0 else -1.0
|
|
494
|
+
op_sign_by_n[n] = op_sign
|
|
495
|
+
old_peak_by_n[n] = op_max if op_sign >= 0 else op_min
|
|
496
|
+
|
|
497
|
+
rows_nps: list[tuple[str, str, float]] = []
|
|
498
|
+
rows_opmax: list[tuple[str, str, float]] = []
|
|
499
|
+
rows_opmin: list[tuple[str, str, float]] = []
|
|
500
|
+
rows_ops: list[tuple[str, str, float]] = []
|
|
501
|
+
rows_npop: list[tuple[str, str, float]] = []
|
|
502
|
+
rows_npopinflow: list[tuple[str, str, float]] = []
|
|
503
|
+
|
|
504
|
+
def _peak_domain(n: str, d: str) -> bool:
|
|
505
|
+
return (
|
|
506
|
+
_has_method(n, "scale_to_annual_and_peak_flow")
|
|
507
|
+
and pdNode.get((n, "annual_flow", d), 0.0) != 0.0
|
|
508
|
+
and pdNode.get((n, "peak_inflow", d), 0.0) != 0.0
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
for n in nodes:
|
|
512
|
+
for d in period_in_use:
|
|
513
|
+
if not _peak_domain(n, d):
|
|
514
|
+
continue
|
|
515
|
+
peak = pdNode.get((n, "peak_inflow", d), 0.0)
|
|
516
|
+
rows_nps.append((n, d, 1.0 if peak >= 0 else -1.0))
|
|
517
|
+
rows_opmax.append((n, d, op_max_by_n[n]))
|
|
518
|
+
rows_opmin.append((n, d, op_min_by_n[n]))
|
|
519
|
+
rows_ops.append((n, d, op_sign_by_n[n]))
|
|
520
|
+
old_peak_val = old_peak_by_n[n]
|
|
521
|
+
if old_peak_val == 0.0:
|
|
522
|
+
# Legacy skips downstream rows when old_peak is 0 (avoids
|
|
523
|
+
# division by zero) — npop / npopinflow remain absent.
|
|
524
|
+
continue
|
|
525
|
+
npop = peak / old_peak_val
|
|
526
|
+
rows_npop.append((n, d, npop))
|
|
527
|
+
|
|
528
|
+
ofs = orig_flow_sum.get((n, d), 0.0)
|
|
529
|
+
cps = cpsoy.get(d, 0.0)
|
|
530
|
+
npopis = (npop * ofs / cps) if cps != 0.0 else 0.0
|
|
531
|
+
rows_npopinflow.append((n, d, npopis))
|
|
532
|
+
|
|
533
|
+
out["new_peak_sign.csv"] = _rows_to_frame(
|
|
534
|
+
("node", "period", "value"), rows_nps,
|
|
535
|
+
)
|
|
536
|
+
out["old_peak_max.csv"] = _rows_to_frame(
|
|
537
|
+
("node", "period", "value"), rows_opmax,
|
|
538
|
+
)
|
|
539
|
+
out["old_peak_min.csv"] = _rows_to_frame(
|
|
540
|
+
("node", "period", "value"), rows_opmin,
|
|
541
|
+
)
|
|
542
|
+
out["old_peak_sign.csv"] = _rows_to_frame(
|
|
543
|
+
("node", "period", "value"), rows_ops,
|
|
544
|
+
)
|
|
545
|
+
out["new_peak_divided_by_old_peak.csv"] = _rows_to_frame(
|
|
546
|
+
("node", "period", "value"), rows_npop,
|
|
547
|
+
)
|
|
548
|
+
|
|
549
|
+
# ── new_peak_inflow_sum, new_old_multiplier/slope/section ─────────
|
|
550
|
+
# Same domain as rows_nps; values derived from peak / npop / npopis.
|
|
551
|
+
npis_dict = {
|
|
552
|
+
(n, d): pdNode.get((n, "peak_inflow", d), 0.0) * 8760.0
|
|
553
|
+
for n, d, _ in rows_nps
|
|
554
|
+
}
|
|
555
|
+
rows_npis = [(n, d, npis_dict.get((n, d), 0.0)) for n, d, _ in rows_nps]
|
|
556
|
+
out["new_peak_inflow_sum.csv"] = _rows_to_frame(
|
|
557
|
+
("node", "period", "value"), rows_npis,
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
op_sign_dict = {(n, d): v for n, d, v in rows_ops}
|
|
561
|
+
npopinflow_dict = {(n, d): v for n, d, v in rows_npopinflow}
|
|
562
|
+
rows_nom: list[tuple[str, str, float]] = []
|
|
563
|
+
for n, d, _ in rows_nps:
|
|
564
|
+
npis = npis_dict.get((n, d), 0.0)
|
|
565
|
+
npopis = npopinflow_dict.get((n, d), 0.0)
|
|
566
|
+
os_sign = op_sign_dict.get((n, d), 0.0)
|
|
567
|
+
af = pdNode.get((n, "annual_flow", d), 0.0)
|
|
568
|
+
denom = npis - npopis
|
|
569
|
+
if denom == 0.0:
|
|
570
|
+
v = 0.0
|
|
571
|
+
else:
|
|
572
|
+
v = os_sign * (os_sign * npopis - af) / denom
|
|
573
|
+
rows_nom.append((n, d, v))
|
|
574
|
+
out["new_old_multiplier.csv"] = _rows_to_frame(
|
|
575
|
+
("node", "period", "value"), rows_nom,
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
nom_dict = {(n, d): v for n, d, v in rows_nom}
|
|
579
|
+
npop_dict = {(n, d): v for n, d, v in rows_npop}
|
|
580
|
+
rows_nos = [
|
|
581
|
+
(n, d, npop_dict.get((n, d), 0.0)
|
|
582
|
+
* (1.0 + nom_dict.get((n, d), 0.0)))
|
|
583
|
+
for n, d, _ in rows_nps
|
|
584
|
+
]
|
|
585
|
+
out["new_old_slope.csv"] = _rows_to_frame(
|
|
586
|
+
("node", "period", "value"), rows_nos,
|
|
587
|
+
)
|
|
588
|
+
|
|
589
|
+
rows_nosec = [
|
|
590
|
+
(n, d, pdNode.get((n, "peak_inflow", d), 0.0)
|
|
591
|
+
* nom_dict.get((n, d), 0.0))
|
|
592
|
+
for n, d, _ in rows_nps
|
|
593
|
+
]
|
|
594
|
+
out["new_old_section.csv"] = _rows_to_frame(
|
|
595
|
+
("node", "period", "value"), rows_nosec,
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
return out
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
# ---------------------------------------------------------------------------
|
|
602
|
+
# Vectorized twin of _compute_inflow_scaling_frames (vectorize-per-roll).
|
|
603
|
+
#
|
|
604
|
+
# Built as a FULL COPY of the legacy body, with each stage's COMPUTE
|
|
605
|
+
# replaced by vectorized polars. Only the 6 CONSUMED outputs are emitted;
|
|
606
|
+
# the 9 internal middle parameters are computed in-memory where still needed
|
|
607
|
+
# (orig_flow_sum / psaf dicts, inline fused912 npop/npis/nom) but never
|
|
608
|
+
# written. The legacy _compute_inflow_scaling_frames is KEPT as the parity
|
|
609
|
+
# oracle (it still materialises all 15; the test compares the 6).
|
|
610
|
+
#
|
|
611
|
+
# Tier policy: ptNode_inflow + _node_cap_inflow_fallback are sum-free
|
|
612
|
+
# (coalesce / max-abs) and MUST stay byte-identical (Tier A — read by the
|
|
613
|
+
# already-vectorized pdtNodeInflow and by lp-scaling). The sum-bearing
|
|
614
|
+
# consumed stages (period_flow_annual_multiplier,
|
|
615
|
+
# period_flow_proportional_multiplier, new_old_slope, new_old_section) are
|
|
616
|
+
# Tier B (last-ULP drift tolerated).
|
|
617
|
+
# ---------------------------------------------------------------------------
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _empty_value_frame(header: tuple[str, str, str]) -> pl.DataFrame:
|
|
621
|
+
"""An explicit all-Utf8 empty 3-col frame (key1, key2, value)."""
|
|
622
|
+
return pl.DataFrame(
|
|
623
|
+
{h: [] for h in header},
|
|
624
|
+
schema={h: pl.Utf8 for h in header},
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _ordered_value_frame(
|
|
629
|
+
df: pl.DataFrame,
|
|
630
|
+
header: tuple[str, str, str],
|
|
631
|
+
order_cols: list[str],
|
|
632
|
+
) -> pl.DataFrame:
|
|
633
|
+
"""Sort *df* by *order_cols*, render its ``value_f`` Float64 column via
|
|
634
|
+
``repr`` and project to the all-Utf8 ``(key1, key2, value)`` shape.
|
|
635
|
+
|
|
636
|
+
*df* must carry the two key columns named ``header[0]``/``header[1]``,
|
|
637
|
+
a Float64 ``value_f`` column, and the integer *order_cols*. An empty
|
|
638
|
+
*df* yields the explicit empty schema.
|
|
639
|
+
"""
|
|
640
|
+
if df.height == 0:
|
|
641
|
+
return _empty_value_frame(header)
|
|
642
|
+
df = df.sort(order_cols)
|
|
643
|
+
value = _render_value_column(df["value_f"])
|
|
644
|
+
return df.select([header[0], header[1]]).with_columns(
|
|
645
|
+
value.alias(header[2]),
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _compute_inflow_scaling_frames_vectorized(
|
|
650
|
+
input_dir: Path, solve_data_dir: Path,
|
|
651
|
+
*, provider: "object | None" = None,
|
|
652
|
+
) -> dict[str, pl.DataFrame]:
|
|
653
|
+
"""Vectorized twin of :func:`_compute_inflow_scaling_frames`.
|
|
654
|
+
|
|
655
|
+
Same reader block, same in-memory dicts, same stage order. Only the
|
|
656
|
+
6 CONSUMED outputs are assigned to ``out`` (``ptNode_inflow``,
|
|
657
|
+
``_node_cap_inflow_fallback``, ``period_flow_annual_multiplier``,
|
|
658
|
+
``period_flow_proportional_multiplier``, ``new_old_slope``,
|
|
659
|
+
``new_old_section``). The 9 internal middle parameters
|
|
660
|
+
(``orig_flow_sum``, ``period_share_of_annual_flow``, ``new_peak_sign``,
|
|
661
|
+
``old_peak_max``, ``old_peak_min``, ``old_peak_sign``,
|
|
662
|
+
``new_peak_divided_by_old_peak``, ``new_peak_inflow_sum``,
|
|
663
|
+
``new_old_multiplier``) have no external consumer and are no longer
|
|
664
|
+
emitted; the values still needed downstream live in-memory only
|
|
665
|
+
(``orig_flow_sum`` dict, ``psaf`` dict, and the inline fused912
|
|
666
|
+
``v_npop`` / ``v_npis`` / ``v_nom``). The legacy oracle
|
|
667
|
+
materialises all 15 and gates the 6 consumed.
|
|
668
|
+
"""
|
|
669
|
+
out: dict[str, pl.DataFrame] = {}
|
|
670
|
+
|
|
671
|
+
# ── Sources (copied verbatim from the legacy reader block) ─────────
|
|
672
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
673
|
+
period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
|
|
674
|
+
provider=provider)
|
|
675
|
+
time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
|
|
676
|
+
p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
|
|
677
|
+
pt_node_inflow = _read_keyed2_float(
|
|
678
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
679
|
+
)
|
|
680
|
+
node_time_inflow = frozenset(_read_pairs(
|
|
681
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
682
|
+
))
|
|
683
|
+
|
|
684
|
+
inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
685
|
+
provider=provider)
|
|
686
|
+
methods_for_node: dict[str, set[str]] = {}
|
|
687
|
+
for n, m in inflow_method:
|
|
688
|
+
methods_for_node.setdefault(n, set()).add(m)
|
|
689
|
+
|
|
690
|
+
pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
|
|
691
|
+
cpsoy = _read_keyed_float(
|
|
692
|
+
solve_data_dir / "complete_period_share_of_year_calc.csv",
|
|
693
|
+
provider=provider,
|
|
694
|
+
)
|
|
695
|
+
p_tdy = _read_keyed_float(
|
|
696
|
+
solve_data_dir / "p_timeline_duration_in_years.csv",
|
|
697
|
+
provider=provider,
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
# Representative weight w[(d, t)] = p_timestep_weight[d, t] (see the
|
|
701
|
+
# legacy reader block for the full rationale). DEFAULT 1.0 for any
|
|
702
|
+
# (d, t) absent → w ≡ 1.0 reduces every weighted sum below to today's.
|
|
703
|
+
w_dt = _read_keyed2_float(
|
|
704
|
+
solve_data_dir / "timestep_weight.csv", provider=provider,
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
period_timeline = _read_pairs(solve_data_dir / "period__timeline_set.csv",
|
|
708
|
+
provider=provider)
|
|
709
|
+
timelines_for_d: dict[str, list[str]] = {}
|
|
710
|
+
for d, tl in period_timeline:
|
|
711
|
+
timelines_for_d.setdefault(d, []).append(tl)
|
|
712
|
+
|
|
713
|
+
complete_time_in_use = _read_singles(
|
|
714
|
+
solve_data_dir / "complete_time_in_use_set.csv",
|
|
715
|
+
provider=provider,
|
|
716
|
+
)
|
|
717
|
+
|
|
718
|
+
dt_complete_pairs = _read_pairs(
|
|
719
|
+
solve_data_dir / "steps_complete_solve.csv",
|
|
720
|
+
provider=provider,
|
|
721
|
+
)
|
|
722
|
+
dt_complete_for_d: dict[str, list[str]] = {}
|
|
723
|
+
for d, t in dt_complete_pairs:
|
|
724
|
+
dt_complete_for_d.setdefault(d, []).append(t)
|
|
725
|
+
|
|
726
|
+
# ── Shared order frames (node list order, period order, time order) ─
|
|
727
|
+
node_eo = pl.DataFrame(
|
|
728
|
+
{"node": list(nodes), "__eo": list(range(len(nodes)))},
|
|
729
|
+
schema={"node": pl.Utf8, "__eo": pl.Int64},
|
|
730
|
+
)
|
|
731
|
+
period_po = pl.DataFrame(
|
|
732
|
+
{"period": list(period_in_use),
|
|
733
|
+
"__po": list(range(len(period_in_use)))},
|
|
734
|
+
schema={"period": pl.Utf8, "__po": pl.Int64},
|
|
735
|
+
)
|
|
736
|
+
time_to = pl.DataFrame(
|
|
737
|
+
{"time": list(time_set), "__to": list(range(len(time_set)))},
|
|
738
|
+
schema={"time": pl.Utf8, "__to": pl.Int64},
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
# ── Stage 1: ptNode_inflow{n in node, t in time} (Tier A) ──────────
|
|
742
|
+
# value = pt_node_inflow[(n, t)] if (n, t) in node_time_inflow
|
|
743
|
+
# else p_node[(n, "inflow")] (per-node scalar default).
|
|
744
|
+
p_node_inflow_default = {
|
|
745
|
+
n: p_node.get((n, "inflow"), 0.0) for n in nodes
|
|
746
|
+
}
|
|
747
|
+
nt_grid = node_eo.join(time_to, how="cross")
|
|
748
|
+
# pt_node_inflow lookup frame (value present when (n, t) explicit).
|
|
749
|
+
pti_lk = pl.DataFrame(
|
|
750
|
+
{"node": [k[0] for k in pt_node_inflow],
|
|
751
|
+
"time": [k[1] for k in pt_node_inflow],
|
|
752
|
+
"v_pti": list(pt_node_inflow.values())},
|
|
753
|
+
schema={"node": pl.Utf8, "time": pl.Utf8, "v_pti": pl.Float64},
|
|
754
|
+
)
|
|
755
|
+
# explicit-set membership frame (presence in node_time_inflow).
|
|
756
|
+
nti_lk = pl.DataFrame(
|
|
757
|
+
{"node": [k[0] for k in node_time_inflow],
|
|
758
|
+
"time": [k[1] for k in node_time_inflow],
|
|
759
|
+
"__nti": [True] * len(node_time_inflow)},
|
|
760
|
+
schema={"node": pl.Utf8, "time": pl.Utf8, "__nti": pl.Boolean},
|
|
761
|
+
)
|
|
762
|
+
# per-node scalar default frame.
|
|
763
|
+
dflt_lk = pl.DataFrame(
|
|
764
|
+
{"node": list(nodes),
|
|
765
|
+
"v_dflt": [p_node_inflow_default[n] for n in nodes]},
|
|
766
|
+
schema={"node": pl.Utf8, "v_dflt": pl.Float64},
|
|
767
|
+
)
|
|
768
|
+
pti_df = (
|
|
769
|
+
nt_grid
|
|
770
|
+
.join(nti_lk, on=["node", "time"], how="left")
|
|
771
|
+
.join(pti_lk, on=["node", "time"], how="left")
|
|
772
|
+
.join(dflt_lk, on="node", how="left")
|
|
773
|
+
.with_columns(
|
|
774
|
+
pl.when(pl.col("__nti").fill_null(False)) # noqa: FBT003
|
|
775
|
+
# explicit: pt_node_inflow.get((n,t), 0.0)
|
|
776
|
+
.then(pl.col("v_pti").fill_null(0.0))
|
|
777
|
+
# else: per-node scalar default
|
|
778
|
+
.otherwise(pl.col("v_dflt").fill_null(0.0))
|
|
779
|
+
.alias("value_f"),
|
|
780
|
+
)
|
|
781
|
+
)
|
|
782
|
+
out["ptNode_inflow.csv"] = _ordered_value_frame(
|
|
783
|
+
pti_df, ("node", "time", "value"), ["__eo", "__to"],
|
|
784
|
+
)
|
|
785
|
+
# NOTE: every downstream consumer of the per-(n, t) inflow series is now
|
|
786
|
+
# vectorized (it reads ``pti_df`` / ``pti_series`` directly), so the
|
|
787
|
+
# scalar ``pti`` dict is no longer reconstructed here.
|
|
788
|
+
|
|
789
|
+
# ── Stage 2: _node_cap_inflow_fallback{n, d} (Tier A) ──────────────
|
|
790
|
+
# value = max_t abs(pti[(n, t)]); 0.0 if no time.
|
|
791
|
+
if not time_set:
|
|
792
|
+
nd_grid = node_eo.join(period_po, how="cross").with_columns(
|
|
793
|
+
pl.lit(0.0, dtype=pl.Float64).alias("value_f"),
|
|
794
|
+
)
|
|
795
|
+
out["_node_cap_inflow_fallback.csv"] = _ordered_value_frame(
|
|
796
|
+
nd_grid, ("node", "period", "value"), ["__eo", "__po"],
|
|
797
|
+
)
|
|
798
|
+
else:
|
|
799
|
+
max_abs_df = (
|
|
800
|
+
pti_df.group_by("node")
|
|
801
|
+
.agg(pl.col("value_f").abs().max().alias("v_maxabs"))
|
|
802
|
+
)
|
|
803
|
+
nd_grid = (
|
|
804
|
+
node_eo.join(period_po, how="cross")
|
|
805
|
+
.join(max_abs_df, on="node", how="left")
|
|
806
|
+
.with_columns(
|
|
807
|
+
pl.col("v_maxabs").fill_null(0.0).alias("value_f"),
|
|
808
|
+
)
|
|
809
|
+
)
|
|
810
|
+
out["_node_cap_inflow_fallback.csv"] = _ordered_value_frame(
|
|
811
|
+
nd_grid, ("node", "period", "value"), ["__eo", "__po"],
|
|
812
|
+
)
|
|
813
|
+
|
|
814
|
+
# Helper: does node n have inflow method m? (legacy parity)
|
|
815
|
+
def _has_method(n: str, m: str) -> bool:
|
|
816
|
+
return m in methods_for_node.get(n, ())
|
|
817
|
+
|
|
818
|
+
def _annual_eligible(n: str) -> bool:
|
|
819
|
+
return (_has_method(n, "scale_to_annual_flow")
|
|
820
|
+
or _has_method(n, "scale_to_annual_and_peak_flow"))
|
|
821
|
+
|
|
822
|
+
# ── Shared C2 building blocks ──────────────────────────────────────
|
|
823
|
+
# FOUR DISTINCT method masks (Defect A — do NOT collapse onto one):
|
|
824
|
+
# annual_eligible → stages 3, 4
|
|
825
|
+
# scale_to_annual_flow → stage 5 (a peak-only node is EXCLUDED)
|
|
826
|
+
# scale_in_proportion → stage 6 (disjoint from the annual methods)
|
|
827
|
+
# scale_to_annual_and_peak_flow → stages 7, 8 (C3 peak family)
|
|
828
|
+
# Each as an ordered ``(node, __eo)`` frame so the entity-major
|
|
829
|
+
# emission order survives the joins.
|
|
830
|
+
_node_idx = {n: i for i, n in enumerate(nodes)}
|
|
831
|
+
|
|
832
|
+
def _ordered_node_frame(pred) -> pl.DataFrame:
|
|
833
|
+
sel = [n for n in nodes if pred(n)]
|
|
834
|
+
return pl.DataFrame(
|
|
835
|
+
{"node": sel, "__eo": [_node_idx[n] for n in sel]},
|
|
836
|
+
schema={"node": pl.Utf8, "__eo": pl.Int64},
|
|
837
|
+
)
|
|
838
|
+
|
|
839
|
+
annual_eligible_eo = _ordered_node_frame(_annual_eligible)
|
|
840
|
+
annual_flow_only_eo = _ordered_node_frame(
|
|
841
|
+
lambda n: _has_method(n, "scale_to_annual_flow"))
|
|
842
|
+
proportion_eo = _ordered_node_frame(
|
|
843
|
+
lambda n: _has_method(n, "scale_in_proportion"))
|
|
844
|
+
|
|
845
|
+
# pdNode annual_flow lookup keyed (node, period) → af.
|
|
846
|
+
af_keys = [(k[0], k[2]) for k in pdNode if k[1] == "annual_flow"]
|
|
847
|
+
af_vals = [v for k, v in pdNode.items() if k[1] == "annual_flow"]
|
|
848
|
+
af_lk = pl.DataFrame(
|
|
849
|
+
{"node": [k[0] for k in af_keys],
|
|
850
|
+
"period": [k[1] for k in af_keys],
|
|
851
|
+
"v_af": af_vals},
|
|
852
|
+
schema={"node": pl.Utf8, "period": pl.Utf8, "v_af": pl.Float64},
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
# Representative-weight lookup w[(period, time)] → weight; left-joined
|
|
856
|
+
# below with ``fill_null(1.0)`` so any (d, t) absent contributes the
|
|
857
|
+
# neutral factor 1.0 (matching the legacy ``w_dt.get((d, t), 1.0)``).
|
|
858
|
+
w_lk = pl.DataFrame(
|
|
859
|
+
{"period": [k[0] for k in w_dt],
|
|
860
|
+
"time": [k[1] for k in w_dt],
|
|
861
|
+
"v_w": list(w_dt.values())},
|
|
862
|
+
schema={"period": pl.Utf8, "time": pl.Utf8, "v_w": pl.Float64},
|
|
863
|
+
)
|
|
864
|
+
|
|
865
|
+
cti_lk = pl.DataFrame(
|
|
866
|
+
{"time": list(complete_time_in_use)},
|
|
867
|
+
schema={"time": pl.Utf8},
|
|
868
|
+
)
|
|
869
|
+
# pti as a (node, time, v_pti) frame restricted to the per-node series.
|
|
870
|
+
pti_series = pti_df.select(["node", "time", "value_f"]).rename(
|
|
871
|
+
{"value_f": "v_pti"})
|
|
872
|
+
# Per-(node, period) complete-timeline WEIGHTED sum (A3): the legacy
|
|
873
|
+
# ``sum(pti[(n,t)] · w[d,t] for t in complete_time_in_use)``. Build the
|
|
874
|
+
# dense (node, period, time) grid over the complete timeline, multiply
|
|
875
|
+
# pti by the period weight (fill 1.0), then group-sum per (node, period).
|
|
876
|
+
# With w ≡ 1.0 every weight factor is 1.0 → byte-identical to the former
|
|
877
|
+
# period-independent per-node sum.
|
|
878
|
+
complete_sum_df = (
|
|
879
|
+
annual_eligible_eo.select("node")
|
|
880
|
+
.join(cti_lk, how="cross")
|
|
881
|
+
.join(period_po.select("period"), how="cross")
|
|
882
|
+
.join(pti_series, on=["node", "time"], how="left")
|
|
883
|
+
.join(w_lk, on=["period", "time"], how="left")
|
|
884
|
+
.with_columns(
|
|
885
|
+
(pl.col("v_pti").fill_null(0.0)
|
|
886
|
+
* pl.col("v_w").fill_null(1.0)).alias("v_term"),
|
|
887
|
+
)
|
|
888
|
+
.group_by(["node", "period"])
|
|
889
|
+
.agg(pl.col("v_term").sum().alias("v_complete"))
|
|
890
|
+
)
|
|
891
|
+
|
|
892
|
+
# ── Stage 3: orig_flow_sum (in-memory only — feeds npopis) ─────────
|
|
893
|
+
# Domain: annual_eligible AND pdNode annual_flow != 0 (DROP on 0/miss).
|
|
894
|
+
# value = per-(node, period) complete-timeline WEIGHTED sum (A3). NOT
|
|
895
|
+
# emitted (no external consumer); only the dict below is used (by the
|
|
896
|
+
# fused912 npopis). Because the CSV is no longer written there is no
|
|
897
|
+
# byte-parity contract.
|
|
898
|
+
orig_df = (
|
|
899
|
+
annual_eligible_eo
|
|
900
|
+
.join(period_po, how="cross")
|
|
901
|
+
.join(af_lk, on=["node", "period"], how="left")
|
|
902
|
+
.filter(pl.col("v_af").fill_null(0.0) != 0.0)
|
|
903
|
+
.join(complete_sum_df, on=["node", "period"], how="left")
|
|
904
|
+
# A (node, period) with NO complete-timeline rows contributes 0:
|
|
905
|
+
# the legacy ``sum(())`` over an empty complete_time_in_use is 0.
|
|
906
|
+
.with_columns(pl.col("v_complete").fill_null(0.0).alias("value_f"))
|
|
907
|
+
)
|
|
908
|
+
# Reconstruct orig_flow_sum dict (float values) for downstream stages.
|
|
909
|
+
orig_flow_sum = {
|
|
910
|
+
(r[0], r[1]): r[2]
|
|
911
|
+
for r in orig_df.select(["node", "period", "value_f"]).iter_rows()
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
# ── Stage 4: period_share_of_annual_flow (Tier B) ──────────────────
|
|
915
|
+
# Domain: annual_eligible AND af != 0 (DROP on 0/miss).
|
|
916
|
+
# value = abs(sum_{t in dt_complete[d]} pti[(n,t)]) / af.
|
|
917
|
+
# Per-(node, period) dt_complete sum: lift the (period, time) pairs.
|
|
918
|
+
dtc_pairs = pl.DataFrame(
|
|
919
|
+
{"period": [d for d, _t in dt_complete_pairs],
|
|
920
|
+
"time": [t for _d, t in dt_complete_pairs]},
|
|
921
|
+
schema={"period": pl.Utf8, "time": pl.Utf8},
|
|
922
|
+
)
|
|
923
|
+
# node × (period, time) restricted to annual_eligible nodes only, then
|
|
924
|
+
# group-sum the WEIGHTED inflow series per (node, period) (A1): each
|
|
925
|
+
# term is pti[(n,t)] · w[d,t] (fill 1.0). With w ≡ 1.0 → today's sum.
|
|
926
|
+
dtc_sum_df = (
|
|
927
|
+
annual_eligible_eo.select("node")
|
|
928
|
+
.join(pti_series, on="node", how="inner")
|
|
929
|
+
.join(dtc_pairs, on="time", how="inner")
|
|
930
|
+
.join(w_lk, on=["period", "time"], how="left")
|
|
931
|
+
.with_columns(
|
|
932
|
+
(pl.col("v_pti") * pl.col("v_w").fill_null(1.0)).alias("v_term"),
|
|
933
|
+
)
|
|
934
|
+
.group_by(["node", "period"])
|
|
935
|
+
.agg(pl.col("v_term").sum().alias("v_dtc"))
|
|
936
|
+
)
|
|
937
|
+
psaf_df = (
|
|
938
|
+
annual_eligible_eo
|
|
939
|
+
.join(period_po, how="cross")
|
|
940
|
+
.join(af_lk, on=["node", "period"], how="left")
|
|
941
|
+
.filter(pl.col("v_af").fill_null(0.0) != 0.0)
|
|
942
|
+
.join(dtc_sum_df, on=["node", "period"], how="left")
|
|
943
|
+
# A (node, period) with no dt_complete rows: legacy sum(()) == 0.
|
|
944
|
+
.with_columns(
|
|
945
|
+
(pl.col("v_dtc").fill_null(0.0).abs() / pl.col("v_af"))
|
|
946
|
+
.alias("value_f"),
|
|
947
|
+
)
|
|
948
|
+
)
|
|
949
|
+
# NOT emitted (no external consumer); only the psaf dict below is used
|
|
950
|
+
# (by stage 5 = period_flow_annual_multiplier).
|
|
951
|
+
psaf = {
|
|
952
|
+
(r[0], r[1]): r[2]
|
|
953
|
+
for r in psaf_df.select(["node", "period", "value_f"]).iter_rows()
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
# ── Stage 6: period_flow_proportional_multiplier (Tier B) ──────────
|
|
957
|
+
# Domain: scale_in_proportion AND af != 0; DROP if tdy_sum==0 OR
|
|
958
|
+
# time_sum==0. value = af / (abs(time_sum) / tdy_sum).
|
|
959
|
+
# Per-(node, period) WEIGHTED time-axis sum over the WHOLE time_set (A2):
|
|
960
|
+
# each term is pti[(n,t)] · w[d,t] (fill 1.0). The iteration domain
|
|
961
|
+
# stays the full time_set (period-independent), so with w ≡ 1.0 the sum
|
|
962
|
+
# is byte-identical to the former per-node ``sum(pti) over time_set``.
|
|
963
|
+
time_sum_df = (
|
|
964
|
+
proportion_eo.select("node")
|
|
965
|
+
.join(pti_series, on="node", how="inner")
|
|
966
|
+
.join(period_po.select("period"), how="cross")
|
|
967
|
+
.join(w_lk, on=["period", "time"], how="left")
|
|
968
|
+
.with_columns(
|
|
969
|
+
(pl.col("v_pti") * pl.col("v_w").fill_null(1.0)).alias("v_term"),
|
|
970
|
+
)
|
|
971
|
+
.group_by(["node", "period"])
|
|
972
|
+
.agg(pl.col("v_term").sum().alias("v_timesum"))
|
|
973
|
+
)
|
|
974
|
+
# Per-period tdy sum: sum p_tdy over timelines_for_d[d].
|
|
975
|
+
pt_pairs = pl.DataFrame(
|
|
976
|
+
{"period": [d for d, _tl in period_timeline],
|
|
977
|
+
"timeline": [tl for _d, tl in period_timeline]},
|
|
978
|
+
schema={"period": pl.Utf8, "timeline": pl.Utf8},
|
|
979
|
+
)
|
|
980
|
+
tdy_lk = pl.DataFrame(
|
|
981
|
+
{"timeline": list(p_tdy.keys()), "v_tdy": list(p_tdy.values())},
|
|
982
|
+
schema={"timeline": pl.Utf8, "v_tdy": pl.Float64},
|
|
983
|
+
)
|
|
984
|
+
tdy_sum_df = (
|
|
985
|
+
pt_pairs
|
|
986
|
+
.join(tdy_lk, on="timeline", how="left")
|
|
987
|
+
.group_by("period")
|
|
988
|
+
.agg(pl.col("v_tdy").fill_null(0.0).sum().alias("v_tdysum"))
|
|
989
|
+
)
|
|
990
|
+
pfpm_df = (
|
|
991
|
+
proportion_eo
|
|
992
|
+
.join(period_po, how="cross")
|
|
993
|
+
.join(af_lk, on=["node", "period"], how="left")
|
|
994
|
+
.filter(pl.col("v_af").fill_null(0.0) != 0.0)
|
|
995
|
+
.join(time_sum_df, on=["node", "period"], how="left")
|
|
996
|
+
.join(tdy_sum_df, on="period", how="left")
|
|
997
|
+
# A (node, period) missing the time-sum join means no inflow rows ⇒
|
|
998
|
+
# legacy sum(()) == 0.0; a period missing tdy_sum ⇒ sum(()) == 0.0.
|
|
999
|
+
.with_columns(
|
|
1000
|
+
pl.col("v_timesum").fill_null(0.0).alias("v_timesum"),
|
|
1001
|
+
pl.col("v_tdysum").fill_null(0.0).alias("v_tdysum"),
|
|
1002
|
+
)
|
|
1003
|
+
.filter(
|
|
1004
|
+
(pl.col("v_tdysum") != 0.0) & (pl.col("v_timesum") != 0.0))
|
|
1005
|
+
.with_columns(
|
|
1006
|
+
(pl.col("v_af")
|
|
1007
|
+
/ (pl.col("v_timesum").abs() / pl.col("v_tdysum")))
|
|
1008
|
+
.alias("value_f"),
|
|
1009
|
+
)
|
|
1010
|
+
)
|
|
1011
|
+
out["period_flow_proportional_multiplier.csv"] = _ordered_value_frame(
|
|
1012
|
+
pfpm_df, ("node", "period", "value"), ["__eo", "__po"],
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
# ── Stage 5: period_flow_annual_multiplier (Tier B) ────────────────
|
|
1016
|
+
# Domain: scale_to_annual_flow ONLY (Defect A — a peak-only node is
|
|
1017
|
+
# annual_eligible but EXCLUDED here). THREE filters, TWO semantics
|
|
1018
|
+
# (Defect B):
|
|
1019
|
+
# (a) af==0 → DROP (inner-join af_lk + filter af!=0),
|
|
1020
|
+
# (b) psaf.get((n,d),0.0)==0 → DROP (inner-join psaf + filter !=0 →
|
|
1021
|
+
# drops BOTH a miss AND an exact-0),
|
|
1022
|
+
# (c) cpsoy.get(d,0.0) numerator → SURVIVE-with-0 (left-join cpsoy +
|
|
1023
|
+
# fill_null(0.0)).
|
|
1024
|
+
# value = cpsoy_filled / psaf.
|
|
1025
|
+
psaf_lk = pl.DataFrame(
|
|
1026
|
+
{"node": [k[0] for k in psaf],
|
|
1027
|
+
"period": [k[1] for k in psaf],
|
|
1028
|
+
"v_psaf": list(psaf.values())},
|
|
1029
|
+
schema={"node": pl.Utf8, "period": pl.Utf8, "v_psaf": pl.Float64},
|
|
1030
|
+
)
|
|
1031
|
+
cpsoy_lk = pl.DataFrame(
|
|
1032
|
+
{"period": list(cpsoy.keys()), "v_cpsoy": list(cpsoy.values())},
|
|
1033
|
+
schema={"period": pl.Utf8, "v_cpsoy": pl.Float64},
|
|
1034
|
+
)
|
|
1035
|
+
pfam_df = (
|
|
1036
|
+
annual_flow_only_eo
|
|
1037
|
+
.join(period_po, how="cross")
|
|
1038
|
+
# (a) af==0/miss → DROP.
|
|
1039
|
+
.join(af_lk, on=["node", "period"], how="left")
|
|
1040
|
+
.filter(pl.col("v_af").fill_null(0.0) != 0.0)
|
|
1041
|
+
# (b) psaf miss OR exact-0 → DROP.
|
|
1042
|
+
.join(psaf_lk, on=["node", "period"], how="left")
|
|
1043
|
+
.filter(pl.col("v_psaf").fill_null(0.0) != 0.0)
|
|
1044
|
+
# (c) cpsoy numerator → survive-with-0.
|
|
1045
|
+
.join(cpsoy_lk, on="period", how="left")
|
|
1046
|
+
.with_columns(
|
|
1047
|
+
(pl.col("v_cpsoy").fill_null(0.0) / pl.col("v_psaf"))
|
|
1048
|
+
.alias("value_f"),
|
|
1049
|
+
)
|
|
1050
|
+
)
|
|
1051
|
+
out["period_flow_annual_multiplier.csv"] = _ordered_value_frame(
|
|
1052
|
+
pfam_df, ("node", "period", "value"), ["__eo", "__po"],
|
|
1053
|
+
)
|
|
1054
|
+
|
|
1055
|
+
# ── Stage 7: per-node peak precompute (Tier A: max/min/sign) ───────
|
|
1056
|
+
# has_node_time_inflow[n] = node has ANY explicit (n, t) inflow row.
|
|
1057
|
+
peak_eo = _ordered_node_frame(
|
|
1058
|
+
lambda n: _has_method(n, "scale_to_annual_and_peak_flow"))
|
|
1059
|
+
hnti_nodes = {nn for (nn, _t) in node_time_inflow}
|
|
1060
|
+
# Per-node max/min over the WHOLE time series (dense grid). When
|
|
1061
|
+
# time_set is empty the group has no rows → null → fill 0.0 (matches
|
|
1062
|
+
# the legacy ``max(inflow_vals) if inflow_vals else 0.0``).
|
|
1063
|
+
minmax_df = (
|
|
1064
|
+
pti_df.group_by("node")
|
|
1065
|
+
.agg(
|
|
1066
|
+
pl.col("value_f").max().alias("v_tmax"),
|
|
1067
|
+
pl.col("value_f").min().alias("v_tmin"),
|
|
1068
|
+
)
|
|
1069
|
+
)
|
|
1070
|
+
# Per-node precompute over ALL nodes (the legacy loop iterates nodes).
|
|
1071
|
+
precomp = (
|
|
1072
|
+
node_eo
|
|
1073
|
+
.join(minmax_df, on="node", how="left")
|
|
1074
|
+
.with_columns(
|
|
1075
|
+
pl.col("node").is_in(list(hnti_nodes)).alias("__hnti"),
|
|
1076
|
+
pl.col("node").replace_strict(
|
|
1077
|
+
p_node_inflow_default, default=0.0,
|
|
1078
|
+
return_dtype=pl.Float64).alias("v_dflt"),
|
|
1079
|
+
)
|
|
1080
|
+
.with_columns(
|
|
1081
|
+
# op_max / op_min: time-series bound when has_node_time_inflow
|
|
1082
|
+
# (null → 0.0 for an empty time_set), else the scalar default.
|
|
1083
|
+
pl.when(pl.col("__hnti"))
|
|
1084
|
+
.then(pl.col("v_tmax").fill_null(0.0))
|
|
1085
|
+
.otherwise(pl.col("v_dflt")).alias("op_max"),
|
|
1086
|
+
pl.when(pl.col("__hnti"))
|
|
1087
|
+
.then(pl.col("v_tmin").fill_null(0.0))
|
|
1088
|
+
.otherwise(pl.col("v_dflt")).alias("op_min"),
|
|
1089
|
+
)
|
|
1090
|
+
.with_columns(
|
|
1091
|
+
# op_sign: has_node_time_inflow → 1 if |max|>=|min| else -1;
|
|
1092
|
+
# else 1 if default >= 0 else -1.
|
|
1093
|
+
pl.when(pl.col("__hnti"))
|
|
1094
|
+
.then(
|
|
1095
|
+
pl.when(pl.col("op_max").abs() >= pl.col("op_min").abs())
|
|
1096
|
+
.then(pl.lit(1.0)).otherwise(pl.lit(-1.0)))
|
|
1097
|
+
.otherwise(
|
|
1098
|
+
pl.when(pl.col("v_dflt") >= 0.0)
|
|
1099
|
+
.then(pl.lit(1.0)).otherwise(pl.lit(-1.0)))
|
|
1100
|
+
.alias("op_sign"),
|
|
1101
|
+
)
|
|
1102
|
+
.with_columns(
|
|
1103
|
+
# old_peak = op_max if op_sign >= 0 else op_min.
|
|
1104
|
+
pl.when(pl.col("op_sign") >= 0.0)
|
|
1105
|
+
.then(pl.col("op_max")).otherwise(pl.col("op_min"))
|
|
1106
|
+
.alias("old_peak"),
|
|
1107
|
+
)
|
|
1108
|
+
# op_max / op_min were only needed to DERIVE op_sign / old_peak
|
|
1109
|
+
# above; nothing downstream reads them, so drop them here.
|
|
1110
|
+
.select(["node", "op_sign", "old_peak"])
|
|
1111
|
+
)
|
|
1112
|
+
|
|
1113
|
+
# ── Stage 8: peak-domain family ────────────────────────────────────
|
|
1114
|
+
# peak_domain = scale_to_annual_and_peak_flow AND af!=0 AND peak!=0.
|
|
1115
|
+
peak_lk_keys = [(k[0], k[2]) for k in pdNode if k[1] == "peak_inflow"]
|
|
1116
|
+
peak_lk = pl.DataFrame(
|
|
1117
|
+
{"node": [k[0] for k in peak_lk_keys],
|
|
1118
|
+
"period": [k[1] for k in peak_lk_keys],
|
|
1119
|
+
"v_peak": [v for k, v in pdNode.items() if k[1] == "peak_inflow"]},
|
|
1120
|
+
schema={"node": pl.Utf8, "period": pl.Utf8, "v_peak": pl.Float64},
|
|
1121
|
+
)
|
|
1122
|
+
peak_domain_df = (
|
|
1123
|
+
peak_eo
|
|
1124
|
+
.join(period_po, how="cross")
|
|
1125
|
+
.join(af_lk, on=["node", "period"], how="left")
|
|
1126
|
+
.filter(pl.col("v_af").fill_null(0.0) != 0.0)
|
|
1127
|
+
.join(peak_lk, on=["node", "period"], how="left")
|
|
1128
|
+
.filter(pl.col("v_peak").fill_null(0.0) != 0.0)
|
|
1129
|
+
.join(precomp, on="node", how="left")
|
|
1130
|
+
.sort(["__eo", "__po"])
|
|
1131
|
+
)
|
|
1132
|
+
# The per-cell peak ingredients new_peak_sign / old_peak_max /
|
|
1133
|
+
# old_peak_min / old_peak_sign and new_peak_divided_by_old_peak (npop)
|
|
1134
|
+
# are NOT emitted (no external consumer). op_sign / old_peak are
|
|
1135
|
+
# carried on ``peak_domain_df`` from stage-7 ``precomp``;
|
|
1136
|
+
# npop is computed INLINE in the fused912 graph below (as ``v_npop``,
|
|
1137
|
+
# fill-0 on old_peak==0) feeding slope/section. No separate scratch
|
|
1138
|
+
# frame is built.
|
|
1139
|
+
# ── Stages 9-12 (Tier-B): ONE fused closed-form graph ──────────────
|
|
1140
|
+
# The whole sub-pipeline runs over the FULL peak-domain keyset
|
|
1141
|
+
# (peak_domain_df == new_peak_sign keyset). npop and npopis are
|
|
1142
|
+
# computed INLINE as ``when(old_peak == 0.0)`` expressions — an
|
|
1143
|
+
# old_peak==0 row is a FILL-to-0 (npop=0, npopis=0), NOT a row drop
|
|
1144
|
+
# (constraint 1): the row survives into slope/section/nom exactly as
|
|
1145
|
+
# the legacy left-join + fill_null(0.0) did, with the smaller
|
|
1146
|
+
# new_peak_divided_by_old_peak emit (npop_df, old_peak!=0 only) already
|
|
1147
|
+
# written above untouched (constraint 2). Collapsing ``npopis_df`` +
|
|
1148
|
+
# ``base912`` + the two npop/npopis re-joins into this single graph
|
|
1149
|
+
# removes the round-trip joins while preserving the exact float op
|
|
1150
|
+
# order / parenthesization (constraint 3) and the ``== 0.0`` guards
|
|
1151
|
+
# (constraint 4). op_sign / old_peak come from stage-7 ``precomp``,
|
|
1152
|
+
# carried on peak_domain_df, unchanged (constraint 5).
|
|
1153
|
+
ofs_lk = pl.DataFrame(
|
|
1154
|
+
{"node": [k[0] for k in orig_flow_sum],
|
|
1155
|
+
"period": [k[1] for k in orig_flow_sum],
|
|
1156
|
+
"v_ofs": list(orig_flow_sum.values())},
|
|
1157
|
+
schema={"node": pl.Utf8, "period": pl.Utf8, "v_ofs": pl.Float64},
|
|
1158
|
+
)
|
|
1159
|
+
fused912 = (
|
|
1160
|
+
peak_domain_df
|
|
1161
|
+
# Per-cell ingredients: orig_flow_sum (.get(k, 0.0)) + cpsoy
|
|
1162
|
+
# (.get(d, 0.0)) — left-join + fill_null(0.0) reproduces the dict
|
|
1163
|
+
# defaults the legacy stages read.
|
|
1164
|
+
.join(ofs_lk, on=["node", "period"], how="left")
|
|
1165
|
+
.join(cpsoy_lk, on="period", how="left")
|
|
1166
|
+
.with_columns(
|
|
1167
|
+
pl.col("v_ofs").fill_null(0.0).alias("v_ofs"),
|
|
1168
|
+
pl.col("v_cpsoy").fill_null(0.0).alias("v_cpsoy"),
|
|
1169
|
+
)
|
|
1170
|
+
.with_columns(
|
|
1171
|
+
# npis = peak * 8760.0 (constraint 3).
|
|
1172
|
+
(pl.col("v_peak") * 8760.0).alias("v_npis"),
|
|
1173
|
+
# npop = 0.0 if old_peak == 0.0 else peak / old_peak. FILL,
|
|
1174
|
+
# not drop (constraint 1); exact ``== 0.0`` guard (constraint 4).
|
|
1175
|
+
pl.when(pl.col("old_peak") == 0.0)
|
|
1176
|
+
.then(pl.lit(0.0))
|
|
1177
|
+
.otherwise(pl.col("v_peak") / pl.col("old_peak"))
|
|
1178
|
+
.alias("v_npop"),
|
|
1179
|
+
)
|
|
1180
|
+
.with_columns(
|
|
1181
|
+
# npopis = 0.0 if (cps == 0.0 or old_peak == 0.0) else
|
|
1182
|
+
# (npop * ofs) / cps. Left-to-right multiply-then-
|
|
1183
|
+
# divide (constraint 3). old_peak==0 ⇒ npop==0 above,
|
|
1184
|
+
# but the legacy code never reached the cps branch on
|
|
1185
|
+
# old_peak==0 (the row was skipped), defaulting npopis
|
|
1186
|
+
# to 0 — so guard old_peak==0 here too (constraint 1).
|
|
1187
|
+
pl.when((pl.col("v_cpsoy") != 0.0)
|
|
1188
|
+
& (pl.col("old_peak") != 0.0))
|
|
1189
|
+
.then(pl.col("v_npop") * pl.col("v_ofs") / pl.col("v_cpsoy"))
|
|
1190
|
+
.otherwise(pl.lit(0.0))
|
|
1191
|
+
.alias("v_npopis"),
|
|
1192
|
+
)
|
|
1193
|
+
.with_columns(
|
|
1194
|
+
# denom = npis - npopis; exact ``== 0.0`` guard (constraint 4).
|
|
1195
|
+
(pl.col("v_npis") - pl.col("v_npopis")).alias("v_denom"))
|
|
1196
|
+
.with_columns(
|
|
1197
|
+
# nom = 0.0 if denom == 0.0 else
|
|
1198
|
+
# op_sign * (op_sign * npopis - af) / denom. Inner parens
|
|
1199
|
+
# + multiply-before-divide preserved (constraint 3).
|
|
1200
|
+
pl.when(pl.col("v_denom") == 0.0)
|
|
1201
|
+
.then(pl.lit(0.0))
|
|
1202
|
+
.otherwise(
|
|
1203
|
+
pl.col("op_sign")
|
|
1204
|
+
* (pl.col("op_sign") * pl.col("v_npopis") - pl.col("v_af"))
|
|
1205
|
+
/ pl.col("v_denom"))
|
|
1206
|
+
.alias("v_nom"),
|
|
1207
|
+
)
|
|
1208
|
+
.with_columns(
|
|
1209
|
+
# slope = npop * (1.0 + nom); section = peak * nom (constraint 3).
|
|
1210
|
+
(pl.col("v_npop") * (1.0 + pl.col("v_nom"))).alias("v_slope"),
|
|
1211
|
+
(pl.col("v_peak") * pl.col("v_nom")).alias("v_section"),
|
|
1212
|
+
)
|
|
1213
|
+
)
|
|
1214
|
+
|
|
1215
|
+
# Stage 9 — new_peak_inflow_sum (npis = peak * 8760) and Stage 10 —
|
|
1216
|
+
# new_old_multiplier (nom) are NOT emitted (no external consumer); both
|
|
1217
|
+
# remain inline on ``fused912`` (v_npis / v_nom) feeding slope/section.
|
|
1218
|
+
# Stage 11 — new_old_slope = npop * (1 + nom).
|
|
1219
|
+
out["new_old_slope.csv"] = _ordered_value_frame(
|
|
1220
|
+
fused912.with_columns(pl.col("v_slope").alias("value_f")),
|
|
1221
|
+
("node", "period", "value"), ["__eo", "__po"],
|
|
1222
|
+
)
|
|
1223
|
+
# Stage 12 — new_old_section = peak * nom.
|
|
1224
|
+
out["new_old_section.csv"] = _ordered_value_frame(
|
|
1225
|
+
fused912.with_columns(pl.col("v_section").alias("value_f")),
|
|
1226
|
+
("node", "period", "value"), ["__eo", "__po"],
|
|
1227
|
+
)
|
|
1228
|
+
|
|
1229
|
+
return out
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
# ---------------------------------------------------------------------------
|
|
1233
|
+
# Part B — the ``f`` annualisation-divergence diagnostic.
|
|
1234
|
+
#
|
|
1235
|
+
# For every scaling node ``n`` and period ``d`` report the factor by which an
|
|
1236
|
+
# even-sample annualisation and the representative-weight annualisation
|
|
1237
|
+
# disagree:
|
|
1238
|
+
#
|
|
1239
|
+
# f[n, d] = ( Σ_{t in dt_complete[d]} I[n, t] )
|
|
1240
|
+
# / ( Σ_{t in dt_complete[d]} I[n, t]·w[d, t] )
|
|
1241
|
+
# = multiplier_new / multiplier_old
|
|
1242
|
+
#
|
|
1243
|
+
# ``f == 1.0`` exactly when the weights are uniform/absent (``w ≡ 1.0``).
|
|
1244
|
+
# This is a PURE diagnostic: it never enters the LP and is NOT added to the
|
|
1245
|
+
# scaling frames returned by ``_compute_inflow_scaling_frames*`` (the parity
|
|
1246
|
+
# gate asserts those return EXACTLY the 6 consumed keys). It is emitted as a
|
|
1247
|
+
# standalone ``solve_data/inflow_scaling_diagnostics.csv`` frame on the
|
|
1248
|
+
# Provider (surfaced by ``--csv-dump`` like the other solve_data artifacts),
|
|
1249
|
+
# and a WARNING is logged for any (n, d) whose timeset carries non-uniform
|
|
1250
|
+
# weights and whose ``|f - 1| > 0.01``.
|
|
1251
|
+
# ---------------------------------------------------------------------------
|
|
1252
|
+
|
|
1253
|
+
|
|
1254
|
+
def _compute_inflow_scaling_diagnostics(
|
|
1255
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1256
|
+
*, provider: "object | None" = None,
|
|
1257
|
+
) -> tuple[pl.DataFrame, list[tuple[str, str, float]]]:
|
|
1258
|
+
"""Compute the per-(node, period) ``f`` diagnostic.
|
|
1259
|
+
|
|
1260
|
+
Reads the same sources as the scaling compute (nodes, periods,
|
|
1261
|
+
inflow methods, the merged per-(n, t) inflow series, the
|
|
1262
|
+
representative weight ``w[(d, t)]`` and ``annual_flow``). Returns a
|
|
1263
|
+
two-tuple ``(frame, warnings)`` where:
|
|
1264
|
+
|
|
1265
|
+
* ``frame`` carries one row per scaling (node, period) with columns
|
|
1266
|
+
``node, period, inflow_method, annual_flow, f, level_shift_pct``
|
|
1267
|
+
(``level_shift_pct = 100·(f - 1)``). ``inflow_method`` is the set of
|
|
1268
|
+
scaling methods authored on the node joined by ``+`` (a node may
|
|
1269
|
+
legitimately carry more than one).
|
|
1270
|
+
* ``warnings`` is the subset of ``(node, period, f)`` for which the
|
|
1271
|
+
node's period carries NON-UNIFORM weights AND ``|f - 1| > 0.01`` —
|
|
1272
|
+
the rows the caller logs at WARNING level.
|
|
1273
|
+
|
|
1274
|
+
Guards mirror the scaling code: a (node, period) whose weighted
|
|
1275
|
+
denominator is 0 (an all-zero / cancelling profile under the weights)
|
|
1276
|
+
is reported with ``f = 1.0`` (no divergence to flag) rather than
|
|
1277
|
+
dividing by zero; the same when the unweighted numerator is 0.
|
|
1278
|
+
"""
|
|
1279
|
+
# ── Sources (subset of the scaling reader block) ───────────────────
|
|
1280
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
1281
|
+
period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
|
|
1282
|
+
provider=provider)
|
|
1283
|
+
time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
|
|
1284
|
+
p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
|
|
1285
|
+
pt_node_inflow = _read_keyed2_float(
|
|
1286
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
1287
|
+
)
|
|
1288
|
+
node_time_inflow = frozenset(_read_pairs(
|
|
1289
|
+
solve_data_dir / "pt_node_inflow.csv", provider=provider,
|
|
1290
|
+
))
|
|
1291
|
+
|
|
1292
|
+
inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
|
|
1293
|
+
provider=provider)
|
|
1294
|
+
methods_for_node: dict[str, set[str]] = {}
|
|
1295
|
+
for n, m in inflow_method:
|
|
1296
|
+
methods_for_node.setdefault(n, set()).add(m)
|
|
1297
|
+
|
|
1298
|
+
pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
|
|
1299
|
+
|
|
1300
|
+
# Representative weight w[(d, t)] = p_timestep_weight[d, t]; DEFAULT 1.0
|
|
1301
|
+
# for any (d, t) absent → f == 1.0 when weights are uniform/absent.
|
|
1302
|
+
w_dt = _read_keyed2_float(
|
|
1303
|
+
solve_data_dir / "timestep_weight.csv", provider=provider,
|
|
1304
|
+
)
|
|
1305
|
+
|
|
1306
|
+
# dt_complete from steps_complete_solve.csv — (period, step) pairs.
|
|
1307
|
+
dt_complete_pairs = _read_pairs(
|
|
1308
|
+
solve_data_dir / "steps_complete_solve.csv",
|
|
1309
|
+
provider=provider,
|
|
1310
|
+
)
|
|
1311
|
+
dt_complete_for_d: dict[str, list[str]] = {}
|
|
1312
|
+
for d, t in dt_complete_pairs:
|
|
1313
|
+
dt_complete_for_d.setdefault(d, []).append(t)
|
|
1314
|
+
|
|
1315
|
+
# Merged per-(n, t) inflow series (same rule as the scaling Stage 1):
|
|
1316
|
+
# explicit pt_node_inflow when (n, t) is set, else the per-node scalar
|
|
1317
|
+
# default p_node[(n, "inflow")].
|
|
1318
|
+
p_node_inflow_default = {
|
|
1319
|
+
n: p_node.get((n, "inflow"), 0.0) for n in nodes
|
|
1320
|
+
}
|
|
1321
|
+
pti: dict[tuple[str, str], float] = {}
|
|
1322
|
+
for n in nodes:
|
|
1323
|
+
default = p_node_inflow_default[n]
|
|
1324
|
+
for t in time_set:
|
|
1325
|
+
if (n, t) in node_time_inflow:
|
|
1326
|
+
pti[(n, t)] = pt_node_inflow.get((n, t), 0.0)
|
|
1327
|
+
else:
|
|
1328
|
+
pti[(n, t)] = default
|
|
1329
|
+
|
|
1330
|
+
rows: list[tuple[str, str, str, float, float, float]] = []
|
|
1331
|
+
warnings: list[tuple[str, str, float]] = []
|
|
1332
|
+
for n in nodes:
|
|
1333
|
+
scaling = [m for m in _SCALING_METHODS
|
|
1334
|
+
if m in methods_for_node.get(n, ())]
|
|
1335
|
+
if not scaling:
|
|
1336
|
+
continue
|
|
1337
|
+
method_label = "+".join(scaling)
|
|
1338
|
+
for d in period_in_use:
|
|
1339
|
+
af = pdNode.get((n, "annual_flow", d), 0.0)
|
|
1340
|
+
steps = dt_complete_for_d.get(d, ())
|
|
1341
|
+
# Unweighted numerator and weighted denominator over the
|
|
1342
|
+
# complete-period timesteps.
|
|
1343
|
+
num = sum(pti[(n, t)] for t in steps)
|
|
1344
|
+
den = sum(pti[(n, t)] * w_dt.get((d, t), 1.0) for t in steps)
|
|
1345
|
+
# Whether this period's timesteps carry a non-uniform weight
|
|
1346
|
+
# (a divergence is only meaningful — and only warned on — when
|
|
1347
|
+
# the representativeness actually varies across the sample).
|
|
1348
|
+
weights_here = [w_dt.get((d, t), 1.0) for t in steps]
|
|
1349
|
+
non_uniform = (
|
|
1350
|
+
len(weights_here) > 1
|
|
1351
|
+
and any(abs(wv - weights_here[0]) > 1e-12
|
|
1352
|
+
for wv in weights_here)
|
|
1353
|
+
)
|
|
1354
|
+
if den == 0.0 or num == 0.0:
|
|
1355
|
+
# Degenerate profile under the weights — no meaningful
|
|
1356
|
+
# divergence; report f = 1.0 (no shift) and never warn.
|
|
1357
|
+
f_val = 1.0
|
|
1358
|
+
else:
|
|
1359
|
+
f_val = num / den
|
|
1360
|
+
level_shift_pct = 100.0 * (f_val - 1.0)
|
|
1361
|
+
rows.append((n, d, method_label, af, f_val, level_shift_pct))
|
|
1362
|
+
if non_uniform and abs(f_val - 1.0) > 0.01:
|
|
1363
|
+
warnings.append((n, d, f_val))
|
|
1364
|
+
|
|
1365
|
+
frame = pl.DataFrame(
|
|
1366
|
+
{
|
|
1367
|
+
"node": [r[0] for r in rows],
|
|
1368
|
+
"period": [r[1] for r in rows],
|
|
1369
|
+
"inflow_method": [r[2] for r in rows],
|
|
1370
|
+
"annual_flow": [repr(r[3]) for r in rows],
|
|
1371
|
+
"f": [repr(r[4]) for r in rows],
|
|
1372
|
+
"level_shift_pct": [repr(r[5]) for r in rows],
|
|
1373
|
+
},
|
|
1374
|
+
schema={
|
|
1375
|
+
"node": pl.Utf8,
|
|
1376
|
+
"period": pl.Utf8,
|
|
1377
|
+
"inflow_method": pl.Utf8,
|
|
1378
|
+
"annual_flow": pl.Utf8,
|
|
1379
|
+
"f": pl.Utf8,
|
|
1380
|
+
"level_shift_pct": pl.Utf8,
|
|
1381
|
+
},
|
|
1382
|
+
)
|
|
1383
|
+
return frame, warnings
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
def emit_node_inflow_scaling_params(
|
|
1387
|
+
input_dir: Path, solve_data_dir: Path,
|
|
1388
|
+
*, provider,
|
|
1389
|
+
) -> None:
|
|
1390
|
+
"""Emit ``node_inflow_scaling_params`` to the Provider.
|
|
1391
|
+
|
|
1392
|
+
Emits the 6 consumed frames under ``solve_data/<basename>`` keys via
|
|
1393
|
+
:func:`_emit` (dual-key registration). Uses the vectorized compute
|
|
1394
|
+
(:func:`_compute_inflow_scaling_frames_vectorized`); the legacy
|
|
1395
|
+
:func:`_compute_inflow_scaling_frames` is retained as the parity
|
|
1396
|
+
oracle (``tests/engine_polars/test_vectorize_inflow_scaling_parity.py``).
|
|
1397
|
+
|
|
1398
|
+
Additionally emits the standalone ``inflow_scaling_diagnostics.csv``
|
|
1399
|
+
(the Part-B ``f`` divergence diagnostic) and logs a WARNING for any
|
|
1400
|
+
scaling node/period whose non-uniform ``timeset_weights`` imply an
|
|
1401
|
+
annualisation that differs from an even-sample one by ``|f - 1| > 1%``.
|
|
1402
|
+
The diagnostic is NOT part of the parity-gated scaling-frame dict.
|
|
1403
|
+
"""
|
|
1404
|
+
frames = _compute_inflow_scaling_frames_vectorized(
|
|
1405
|
+
input_dir, solve_data_dir, provider=provider)
|
|
1406
|
+
for basename, df in frames.items():
|
|
1407
|
+
_emit(provider, f"solve_data/{basename}", df)
|
|
1408
|
+
|
|
1409
|
+
diag, diag_warnings = _compute_inflow_scaling_diagnostics(
|
|
1410
|
+
input_dir, solve_data_dir, provider=provider)
|
|
1411
|
+
_emit(provider, "solve_data/inflow_scaling_diagnostics.csv", diag)
|
|
1412
|
+
for n, d, f_val in diag_warnings:
|
|
1413
|
+
pct = 100.0 * (f_val - 1.0)
|
|
1414
|
+
_logger.warning(
|
|
1415
|
+
"inflow scaling: node %s period %s: timeset_weights imply an "
|
|
1416
|
+
"annualisation that differs from an even-sample annualisation "
|
|
1417
|
+
"by f=%.5f (demand level %+.2f%%); check that timeset_weights "
|
|
1418
|
+
"match the representativeness of the selected timesteps.",
|
|
1419
|
+
n, d, f_val, pct,
|
|
1420
|
+
)
|