flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1420 @@
1
+ """Node inflow scaling params — per-solve emitters.
2
+
3
+ Called per-solve from ``_emit_solve_time.run`` (batch 17).
4
+
5
+ Output CSVs (6 emitted — the 6 with a downstream consumer):
6
+
7
+ * ``ptNode_inflow.csv`` — (n, t) merged inflow
8
+ * ``_node_cap_inflow_fallback.csv`` — (n, d) abs(max)
9
+ * ``period_flow_annual_multiplier.csv`` — (n, d) cpsoy / psaf
10
+ * ``period_flow_proportional_multiplier.csv`` — (n, d) af / (abs(sum_t)/tdy)
11
+ * ``new_old_slope.csv`` — (n, d) npop * (1 + nom)
12
+ * ``new_old_section.csv`` — (n, d) peak * nom
13
+
14
+ These feed ``derive_pdtNodeInflow`` (5) and the lp-scaling emitter (1).
15
+
16
+ One further standalone diagnostic CSV is emitted alongside (NOT part of
17
+ the parity-gated 6, no LP consumer):
18
+
19
+ * ``inflow_scaling_diagnostics.csv`` — (node, period, inflow_method,
20
+ annual_flow, f, level_shift_pct), the Part-B annualisation-divergence
21
+ ``f`` diagnostic. See ``_compute_inflow_scaling_diagnostics``.
22
+
23
+ The 9 internal middle parameters that used to be emitted as scratch CSVs
24
+ — ``orig_flow_sum``, ``period_share_of_annual_flow``, ``new_peak_sign``,
25
+ ``old_peak_max``, ``old_peak_min``, ``old_peak_sign``,
26
+ ``new_peak_divided_by_old_peak``, ``new_peak_inflow_sum``,
27
+ ``new_old_multiplier`` — have NO external consumer (all internal middle
28
+ parameters); they are computed in-memory where still needed to feed the 6
29
+ consumed outputs, but no longer written. The legacy oracle
30
+ ``_compute_inflow_scaling_frames`` still materialises all 15 as the
31
+ independent parity reference for the 6 consumed outputs.
32
+
33
+ Reuse note (Phase 2 sub-dispatch 4 brief)
34
+ -----------------------------------------
35
+
36
+ ``flextool.engine_polars._inflow_scaling`` already implements every
37
+ per-(n, d) formula here (``_compute_period_share_of_annual_flow``,
38
+ ``_compute_period_flow_annual_multiplier``,
39
+ ``_compute_period_flow_proportional_multiplier``, ``_compute_peak_scaling``).
40
+ But that helper operates on :class:`InputSource` + per-solve aggregates
41
+ and folds all the intermediates into a single ``p_inflow`` Param —
42
+ the per-CSV intermediates are never materialised. Sharing it would
43
+ require either threading intermediate accessors through the helper
44
+ (cross-cutting) or running a different data path (legacy CSV reads
45
+ here, InputSource there). Instead this writer mirrors the legacy
46
+ preprocessing's CSV-in / CSV-out shape verbatim; the formulas match
47
+ ``_inflow_scaling`` line-by-line so the two paths agree numerically.
48
+
49
+ Float values formatted with ``repr(float(v))`` for byte-identical
50
+ parity with the legacy emitter.
51
+ """
52
+ from __future__ import annotations
53
+
54
+ import logging
55
+ from pathlib import Path
56
+
57
+ import polars as pl
58
+
59
+ from flextool.engine_polars._emit_provider_io import (
60
+ _emit,
61
+ _provider_key,
62
+ )
63
+ from flextool.engine_polars._vectorize import _render_value_column
64
+
65
+ _logger = logging.getLogger(__name__)
66
+
67
+ # Scaling inflow methods whose annualisation depends on the per-(d, t)
68
+ # representative weight ``w = p_timestep_weight``. ``use_original`` /
69
+ # ``no_inflow`` do not annualise, so they carry no f diagnostic.
70
+ _SCALING_METHODS = (
71
+ "scale_to_annual_flow",
72
+ "scale_in_proportion",
73
+ "scale_to_annual_and_peak_flow",
74
+ )
75
+
76
+
77
+ # ---------------------------------------------------------------------------
78
+ # Native-frame row helpers.
79
+ #
80
+ # These read the in-memory polars frame directly from the Provider via
81
+ # ``provider.get(_provider_key(path))`` instead of round-tripping through
82
+ # CSV text (the legacy ``_provider_open`` + manual ``line.split(",")``
83
+ # path, which had no quoted-field handling).
84
+ #
85
+ # Type-fidelity contract — reproduce *exactly* what the legacy
86
+ # ``line.rstrip("\n").split(",")`` over a ``DataFrame.write_csv``
87
+ # serialisation would have yielded:
88
+ #
89
+ # * Key columns (single-value entries and dict-key positions) were
90
+ # split strings. ``write_csv`` serialises ``null`` → ``""`` and any
91
+ # scalar (Enum / Int / Float / Utf8) → its string form. We coerce
92
+ # each key cell with :func:`_cell_str` (``None`` → ``""``, else
93
+ # ``str``) and apply the original truthiness guard to the *string*
94
+ # form so a null cell is skipped (matching the legacy ``if parts[i]``
95
+ # test) while a literal ``"0"`` is kept.
96
+ # * Value columns were re-coerced with ``float(...)``. We apply
97
+ # ``float(...)`` to the native cell — harmless on an already-float
98
+ # frame, necessary on an int frame, and identical to the legacy
99
+ # ``float(str_cell)`` on a stringified-value frame. A value that
100
+ # cannot be parsed as a float is skipped: the legacy ``except
101
+ # ValueError`` is widened to ``except (ValueError, TypeError)`` so a
102
+ # native ``None`` value cell (``float(None)`` raises ``TypeError``)
103
+ # is skipped exactly as the legacy ``float("")`` ``ValueError`` was.
104
+ #
105
+ # ``provider.get`` returns data rows only (no header), so there is no
106
+ # header row to skip; an empty / missing frame yields the same empty
107
+ # list / dict the legacy loop produced.
108
+ # ---------------------------------------------------------------------------
109
+
110
+
111
+ def _cell_str(value: "object | None") -> str:
112
+ """Reproduce a split CSV cell string for a native frame value.
113
+
114
+ ``DataFrame.write_csv`` renders ``null`` as the empty string and every
115
+ other scalar as its textual form; the legacy ``line.split(",")`` then
116
+ read those strings back. Mirror that here so dict keys / single
117
+ values stay byte-identical to the legacy CSV round-trip.
118
+ """
119
+ return "" if value is None else str(value)
120
+
121
+
122
+ def _read_singles(path: Path,
123
+ *, provider: "object | None" = None) -> list[str]:
124
+ """First-column reader → list of non-empty first-column strings."""
125
+ df = provider.get(_provider_key(path))
126
+ if df is None:
127
+ return []
128
+ out: list[str] = []
129
+ for row in df.iter_rows():
130
+ if not row:
131
+ continue
132
+ c0 = _cell_str(row[0])
133
+ if c0:
134
+ out.append(c0)
135
+ return out
136
+
137
+
138
+ def _read_pairs(path: Path,
139
+ *, provider: "object | None" = None) -> list[tuple[str, str]]:
140
+ """First-two-column reader → list of (c0, c1) with both non-empty."""
141
+ df = provider.get(_provider_key(path))
142
+ if df is None:
143
+ return []
144
+ out: list[tuple[str, str]] = []
145
+ for row in df.iter_rows():
146
+ if len(row) < 2:
147
+ continue
148
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
149
+ if c0 and c1:
150
+ out.append((c0, c1))
151
+ return out
152
+
153
+
154
+ def _read_keyed2_float(path: Path,
155
+ *, provider: "object | None" = None,
156
+ ) -> dict[tuple[str, str], float]:
157
+ """Three-col frame (key1, key2, value) → {(k1, k2): float}.
158
+
159
+ Mirrors legacy ``_read_p`` / ``_read_pt_node_inflow``: malformed
160
+ or non-numeric rows silently skipped.
161
+ """
162
+ out: dict[tuple[str, str], float] = {}
163
+ df = provider.get(_provider_key(path))
164
+ if df is None:
165
+ return out
166
+ for row in df.iter_rows():
167
+ if len(row) < 3:
168
+ continue
169
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
170
+ if c0 and c1:
171
+ try:
172
+ out[(c0, c1)] = float(row[2])
173
+ except (ValueError, TypeError):
174
+ continue
175
+ return out
176
+
177
+
178
+ def _read_keyed3_float(path: Path,
179
+ *, provider: "object | None" = None,
180
+ ) -> dict[tuple[str, str, str], float]:
181
+ """Four-col frame (k1, k2, k3, value) → {(k1, k2, k3): float}."""
182
+ out: dict[tuple[str, str, str], float] = {}
183
+ df = provider.get(_provider_key(path))
184
+ if df is None:
185
+ return out
186
+ for row in df.iter_rows():
187
+ if len(row) < 4:
188
+ continue
189
+ c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
190
+ if c0 and c1 and c2:
191
+ try:
192
+ out[(c0, c1, c2)] = float(row[3])
193
+ except (ValueError, TypeError):
194
+ continue
195
+ return out
196
+
197
+
198
+ def _read_keyed_float(path: Path,
199
+ *, provider: "object | None" = None,
200
+ ) -> dict[str, float]:
201
+ """Two-col frame (key, value) → {key: float}."""
202
+ out: dict[str, float] = {}
203
+ df = provider.get(_provider_key(path))
204
+ if df is None:
205
+ return out
206
+ for row in df.iter_rows():
207
+ if len(row) < 2:
208
+ continue
209
+ c0 = _cell_str(row[0])
210
+ if c0:
211
+ try:
212
+ out[c0] = float(row[1])
213
+ except (ValueError, TypeError):
214
+ continue
215
+ return out
216
+
217
+
218
+ def _rows_to_frame(
219
+ header: tuple[str, str, str],
220
+ rows: list[tuple[str, str, float]],
221
+ ) -> pl.DataFrame:
222
+ """Materialise (k1, k2, repr(value)) rows as an all-Utf8 frame.
223
+
224
+ Pre-stringifies values via ``repr(v)`` so the polars ``write_csv``
225
+ output is byte-identical to the legacy ``f"{a},{b},{repr(v)}\\n"``
226
+ text emitter. Note this uses ``repr(v)`` NOT ``repr(float(v))`` —
227
+ the legacy emitter assumes the caller already produced floats.
228
+ """
229
+ return pl.DataFrame(
230
+ {
231
+ header[0]: [r[0] for r in rows],
232
+ header[1]: [r[1] for r in rows],
233
+ header[2]: [repr(r[2]) for r in rows],
234
+ },
235
+ schema={h: pl.Utf8 for h in header},
236
+ )
237
+
238
+
239
+ # ---------------------------------------------------------------------------
240
+ # Family — write_node_inflow_scaling_params
241
+ # ---------------------------------------------------------------------------
242
+
243
+
244
+ def _compute_inflow_scaling_frames(
245
+ input_dir: Path, solve_data_dir: Path,
246
+ *, provider: "object | None" = None,
247
+ ) -> dict[str, pl.DataFrame]:
248
+ """Compute every inflow-scaling CSV in one pass, returning a dict
249
+ keyed by output basename.
250
+
251
+ Retained as the parity oracle for
252
+ :func:`_compute_inflow_scaling_frames_vectorized`
253
+ (``tests/engine_polars/test_vectorize_inflow_scaling_parity.py``); the
254
+ live emit path is the vectorized twin. The cross-CSV state (peak
255
+ family, npop / nom / etc) is heavy enough that splitting into
256
+ independent per-CSV passes would re-walk the time axis O(N) times — the
257
+ dict-of-frames pattern from the audit doc is the appropriate adapter
258
+ here.
259
+ """
260
+ out: dict[str, pl.DataFrame] = {}
261
+
262
+ # ── Sources ────────────────────────────────────────────────────────
263
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
264
+ period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
265
+ provider=provider)
266
+ time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
267
+ p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
268
+ pt_node_inflow = _read_keyed2_float(
269
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
270
+ )
271
+ # The "(n, t) is explicitly set" predicate uses ONLY (n, t) presence
272
+ # in pt_node_inflow.csv — values are read separately above.
273
+ node_time_inflow = frozenset(_read_pairs(
274
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
275
+ ))
276
+
277
+ inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
278
+ provider=provider)
279
+ methods_for_node: dict[str, set[str]] = {}
280
+ for n, m in inflow_method:
281
+ methods_for_node.setdefault(n, set()).add(m)
282
+
283
+ pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
284
+ cpsoy = _read_keyed_float(
285
+ solve_data_dir / "complete_period_share_of_year_calc.csv",
286
+ provider=provider,
287
+ )
288
+ p_tdy = _read_keyed_float(
289
+ solve_data_dir / "p_timeline_duration_in_years.csv",
290
+ provider=provider,
291
+ )
292
+
293
+ # Representative weight w[(d, t)] = p_timestep_weight[d, t] (the param
294
+ # the cost objective already uses). Sourced from the per-solve
295
+ # ``timestep_weight.csv`` (period, time, weight) emitted by
296
+ # ``emit_rp_data`` / ``emit_empty_rp_data`` BEFORE this batch runs; it
297
+ # is therefore in the Provider already. DEFAULT 1.0 for any (d, t) not
298
+ # present — when timeset_weights are absent/uniform the frame is empty
299
+ # (non-RP solve) or all-1.0, so w ≡ 1.0 and every weighted sum below
300
+ # reduces byte-for-byte to today's unweighted formula.
301
+ w_dt = _read_keyed2_float(
302
+ solve_data_dir / "timestep_weight.csv", provider=provider,
303
+ )
304
+
305
+ # period__timeline (Python output): list of (period, timeline) pairs.
306
+ period_timeline = _read_pairs(solve_data_dir / "period__timeline_set.csv",
307
+ provider=provider)
308
+ timelines_for_d: dict[str, list[str]] = {}
309
+ for d, tl in period_timeline:
310
+ timelines_for_d.setdefault(d, []).append(tl)
311
+
312
+ # complete_time_in_use (Python output, _set suffix).
313
+ complete_time_in_use = _read_singles(
314
+ solve_data_dir / "complete_time_in_use_set.csv",
315
+ provider=provider,
316
+ )
317
+
318
+ # dt_complete from steps_complete_solve.csv — (period, step) pairs.
319
+ dt_complete_pairs = _read_pairs(
320
+ solve_data_dir / "steps_complete_solve.csv",
321
+ provider=provider,
322
+ )
323
+ dt_complete_for_d: dict[str, list[str]] = {}
324
+ for d, t in dt_complete_pairs:
325
+ dt_complete_for_d.setdefault(d, []).append(t)
326
+
327
+ # ── ptNode_inflow{n in node, t in time} ───────────────────────────
328
+ # Per legacy mod L1237: pass-through pt_node_inflow when (n, t) is
329
+ # in the explicit set, else the per-node scalar default from
330
+ # p_node[(n, "inflow")] (typically 0).
331
+ p_node_inflow_default = {
332
+ n: p_node.get((n, "inflow"), 0.0) for n in nodes
333
+ }
334
+ pti: dict[tuple[str, str], float] = {}
335
+ for n in nodes:
336
+ default = p_node_inflow_default[n]
337
+ for t in time_set:
338
+ if (n, t) in node_time_inflow:
339
+ pti[(n, t)] = pt_node_inflow.get((n, t), 0.0)
340
+ else:
341
+ pti[(n, t)] = default
342
+
343
+ rows_pt = [(n, t, pti[(n, t)]) for n in nodes for t in time_set]
344
+ out["ptNode_inflow.csv"] = _rows_to_frame(
345
+ ("node", "time", "value"), rows_pt,
346
+ )
347
+
348
+ # ── _node_cap_inflow_fallback{n in node, d in period_in_use} ──────
349
+ # value = max_{t in time} abs(ptNode_inflow[n, t]); 0 if no time.
350
+ fallback_rows: list[tuple[str, str, float]] = []
351
+ if not time_set:
352
+ for n in nodes:
353
+ for d in period_in_use:
354
+ fallback_rows.append((n, d, 0.0))
355
+ else:
356
+ for n in nodes:
357
+ max_abs = max(abs(pti[(n, t)]) for t in time_set)
358
+ for d in period_in_use:
359
+ fallback_rows.append((n, d, max_abs))
360
+ out["_node_cap_inflow_fallback.csv"] = _rows_to_frame(
361
+ ("node", "period", "value"), fallback_rows,
362
+ )
363
+
364
+ # Helper: does node n have inflow method m?
365
+ def _has_method(n: str, m: str) -> bool:
366
+ return m in methods_for_node.get(n, ())
367
+
368
+ # Domain predicates -- legacy guards each writer with:
369
+ # methods AND pdNode[..annual_flow..] != 0 [AND peak_inflow != 0]
370
+ def _annual_eligible(n: str) -> bool:
371
+ return (_has_method(n, "scale_to_annual_flow")
372
+ or _has_method(n, "scale_to_annual_and_peak_flow"))
373
+
374
+ # ── orig_flow_sum ─────────────────────────────────────────────────
375
+ # value = sum_{t in complete_time_in_use} ptNode_inflow[n, t] · w[d, t]
376
+ # (A3 — annual-energy ingredient, weighted by the representative
377
+ # weight per period). Because w is period-specific the sum can
378
+ # no longer be cached period-independently; compute per (n, d).
379
+ # With w ≡ 1.0 this reduces byte-for-byte to the former
380
+ # period-independent t-sum.
381
+ # Domain: (n, d) where annual_eligible AND pdNode annual_flow != 0.
382
+ rows_orig: list[tuple[str, str, float]] = []
383
+ for n in nodes:
384
+ if not _annual_eligible(n):
385
+ continue
386
+ for d in period_in_use:
387
+ if pdNode.get((n, "annual_flow", d), 0.0) == 0.0:
388
+ continue
389
+ s = sum(
390
+ pti[(n, t)] * w_dt.get((d, t), 1.0)
391
+ for t in complete_time_in_use
392
+ )
393
+ rows_orig.append((n, d, s))
394
+ out["orig_flow_sum.csv"] = _rows_to_frame(
395
+ ("node", "period", "value"), rows_orig,
396
+ )
397
+ orig_flow_sum = {(n, d): v for n, d, v in rows_orig}
398
+
399
+ # ── period_share_of_annual_flow ───────────────────────────────────
400
+ # value = abs(sum_{t in dt_complete[d]} ptNode_inflow[n, t])
401
+ # / pdNode[n, 'annual_flow', d].
402
+ rows_psaf: list[tuple[str, str, float]] = []
403
+ for n in nodes:
404
+ if not _annual_eligible(n):
405
+ continue
406
+ for d in period_in_use:
407
+ af = pdNode.get((n, "annual_flow", d), 0.0)
408
+ if af == 0.0:
409
+ continue
410
+ # A1 — weight each timestep's inflow by w[d, t] (representative
411
+ # weight); w ≡ 1.0 → today's unweighted sum byte-for-byte.
412
+ s = sum(pti[(n, t)] * w_dt.get((d, t), 1.0)
413
+ for t in dt_complete_for_d.get(d, ()))
414
+ rows_psaf.append((n, d, abs(s) / af))
415
+ out["period_share_of_annual_flow.csv"] = _rows_to_frame(
416
+ ("node", "period", "value"), rows_psaf,
417
+ )
418
+ psaf = {(n, d): v for n, d, v in rows_psaf}
419
+
420
+ # ── period_flow_annual_multiplier ─────────────────────────────────
421
+ # value = complete_period_share_of_year[d] / period_share_of_annual_flow[n, d].
422
+ # Domain: (n, d) where scale_to_annual_flow AND pdNode annual_flow.
423
+ rows_pfam: list[tuple[str, str, float]] = []
424
+ for n in nodes:
425
+ if not _has_method(n, "scale_to_annual_flow"):
426
+ continue
427
+ for d in period_in_use:
428
+ if pdNode.get((n, "annual_flow", d), 0.0) == 0.0:
429
+ continue
430
+ denom = psaf.get((n, d), 0.0)
431
+ if denom == 0.0:
432
+ continue
433
+ rows_pfam.append((n, d, cpsoy.get(d, 0.0) / denom))
434
+ out["period_flow_annual_multiplier.csv"] = _rows_to_frame(
435
+ ("node", "period", "value"), rows_pfam,
436
+ )
437
+
438
+ # ── period_flow_proportional_multiplier ───────────────────────────
439
+ # value = pdNode[n, 'annual_flow', d] /
440
+ # (abs(sum_{t in time} ptNode_inflow[n, t]) /
441
+ # sum_{tl in period__timeline[d]} p_timeline_duration_in_years[tl]).
442
+ rows_pfpm: list[tuple[str, str, float]] = []
443
+ # A2 — weight the inflow sum by w[d, t] (representative weight) per
444
+ # period. This makes the sum period-specific (was t-axis only); the
445
+ # iteration domain stays the full time_set, so with w ≡ 1.0 each term's
446
+ # factor is 1.0 and the sum is byte-identical to the former
447
+ # period-independent ``sum(pti[(n, t)] for t in time_set)``.
448
+ for n in nodes:
449
+ if not _has_method(n, "scale_in_proportion"):
450
+ continue
451
+ for d in period_in_use:
452
+ af = pdNode.get((n, "annual_flow", d), 0.0)
453
+ if af == 0.0:
454
+ continue
455
+ time_sum = sum(pti[(n, t)] * w_dt.get((d, t), 1.0)
456
+ for t in time_set)
457
+ tdy_sum = sum(p_tdy.get(tl, 0.0)
458
+ for tl in timelines_for_d.get(d, ()))
459
+ if tdy_sum == 0.0 or time_sum == 0.0:
460
+ continue
461
+ rows_pfpm.append((n, d, af / (abs(time_sum) / tdy_sum)))
462
+ out["period_flow_proportional_multiplier.csv"] = _rows_to_frame(
463
+ ("node", "period", "value"), rows_pfpm,
464
+ )
465
+
466
+ # ── Peak-flow family (annual_and_peak_flow) ────────────────────────
467
+ # Domain: (n, d) where scale_to_annual_and_peak_flow AND pdNode
468
+ # annual_flow != 0 AND pdNode peak_inflow != 0.
469
+ # Per-node availability of explicit (n, t) inflow rows — this drives
470
+ # the op_max / op_min "use scalar default" fallback path.
471
+ has_node_time_inflow: dict[str, bool] = {
472
+ n: any(nn == n for (nn, _t) in node_time_inflow) for n in nodes
473
+ }
474
+ # Per-node max/min across t (independent of d).
475
+ op_max_by_n: dict[str, float] = {}
476
+ op_min_by_n: dict[str, float] = {}
477
+ op_sign_by_n: dict[str, float] = {}
478
+ old_peak_by_n: dict[str, float] = {}
479
+ for n in nodes:
480
+ if has_node_time_inflow[n]:
481
+ inflow_vals = [pti[(n, t)] for t in time_set]
482
+ op_max = max(inflow_vals) if inflow_vals else 0.0
483
+ op_min = min(inflow_vals) if inflow_vals else 0.0
484
+ else:
485
+ scalar = p_node_inflow_default[n]
486
+ op_max = scalar
487
+ op_min = scalar
488
+ op_max_by_n[n] = op_max
489
+ op_min_by_n[n] = op_min
490
+ if has_node_time_inflow[n]:
491
+ op_sign = 1.0 if abs(op_max) >= abs(op_min) else -1.0
492
+ else:
493
+ op_sign = 1.0 if p_node_inflow_default[n] >= 0 else -1.0
494
+ op_sign_by_n[n] = op_sign
495
+ old_peak_by_n[n] = op_max if op_sign >= 0 else op_min
496
+
497
+ rows_nps: list[tuple[str, str, float]] = []
498
+ rows_opmax: list[tuple[str, str, float]] = []
499
+ rows_opmin: list[tuple[str, str, float]] = []
500
+ rows_ops: list[tuple[str, str, float]] = []
501
+ rows_npop: list[tuple[str, str, float]] = []
502
+ rows_npopinflow: list[tuple[str, str, float]] = []
503
+
504
+ def _peak_domain(n: str, d: str) -> bool:
505
+ return (
506
+ _has_method(n, "scale_to_annual_and_peak_flow")
507
+ and pdNode.get((n, "annual_flow", d), 0.0) != 0.0
508
+ and pdNode.get((n, "peak_inflow", d), 0.0) != 0.0
509
+ )
510
+
511
+ for n in nodes:
512
+ for d in period_in_use:
513
+ if not _peak_domain(n, d):
514
+ continue
515
+ peak = pdNode.get((n, "peak_inflow", d), 0.0)
516
+ rows_nps.append((n, d, 1.0 if peak >= 0 else -1.0))
517
+ rows_opmax.append((n, d, op_max_by_n[n]))
518
+ rows_opmin.append((n, d, op_min_by_n[n]))
519
+ rows_ops.append((n, d, op_sign_by_n[n]))
520
+ old_peak_val = old_peak_by_n[n]
521
+ if old_peak_val == 0.0:
522
+ # Legacy skips downstream rows when old_peak is 0 (avoids
523
+ # division by zero) — npop / npopinflow remain absent.
524
+ continue
525
+ npop = peak / old_peak_val
526
+ rows_npop.append((n, d, npop))
527
+
528
+ ofs = orig_flow_sum.get((n, d), 0.0)
529
+ cps = cpsoy.get(d, 0.0)
530
+ npopis = (npop * ofs / cps) if cps != 0.0 else 0.0
531
+ rows_npopinflow.append((n, d, npopis))
532
+
533
+ out["new_peak_sign.csv"] = _rows_to_frame(
534
+ ("node", "period", "value"), rows_nps,
535
+ )
536
+ out["old_peak_max.csv"] = _rows_to_frame(
537
+ ("node", "period", "value"), rows_opmax,
538
+ )
539
+ out["old_peak_min.csv"] = _rows_to_frame(
540
+ ("node", "period", "value"), rows_opmin,
541
+ )
542
+ out["old_peak_sign.csv"] = _rows_to_frame(
543
+ ("node", "period", "value"), rows_ops,
544
+ )
545
+ out["new_peak_divided_by_old_peak.csv"] = _rows_to_frame(
546
+ ("node", "period", "value"), rows_npop,
547
+ )
548
+
549
+ # ── new_peak_inflow_sum, new_old_multiplier/slope/section ─────────
550
+ # Same domain as rows_nps; values derived from peak / npop / npopis.
551
+ npis_dict = {
552
+ (n, d): pdNode.get((n, "peak_inflow", d), 0.0) * 8760.0
553
+ for n, d, _ in rows_nps
554
+ }
555
+ rows_npis = [(n, d, npis_dict.get((n, d), 0.0)) for n, d, _ in rows_nps]
556
+ out["new_peak_inflow_sum.csv"] = _rows_to_frame(
557
+ ("node", "period", "value"), rows_npis,
558
+ )
559
+
560
+ op_sign_dict = {(n, d): v for n, d, v in rows_ops}
561
+ npopinflow_dict = {(n, d): v for n, d, v in rows_npopinflow}
562
+ rows_nom: list[tuple[str, str, float]] = []
563
+ for n, d, _ in rows_nps:
564
+ npis = npis_dict.get((n, d), 0.0)
565
+ npopis = npopinflow_dict.get((n, d), 0.0)
566
+ os_sign = op_sign_dict.get((n, d), 0.0)
567
+ af = pdNode.get((n, "annual_flow", d), 0.0)
568
+ denom = npis - npopis
569
+ if denom == 0.0:
570
+ v = 0.0
571
+ else:
572
+ v = os_sign * (os_sign * npopis - af) / denom
573
+ rows_nom.append((n, d, v))
574
+ out["new_old_multiplier.csv"] = _rows_to_frame(
575
+ ("node", "period", "value"), rows_nom,
576
+ )
577
+
578
+ nom_dict = {(n, d): v for n, d, v in rows_nom}
579
+ npop_dict = {(n, d): v for n, d, v in rows_npop}
580
+ rows_nos = [
581
+ (n, d, npop_dict.get((n, d), 0.0)
582
+ * (1.0 + nom_dict.get((n, d), 0.0)))
583
+ for n, d, _ in rows_nps
584
+ ]
585
+ out["new_old_slope.csv"] = _rows_to_frame(
586
+ ("node", "period", "value"), rows_nos,
587
+ )
588
+
589
+ rows_nosec = [
590
+ (n, d, pdNode.get((n, "peak_inflow", d), 0.0)
591
+ * nom_dict.get((n, d), 0.0))
592
+ for n, d, _ in rows_nps
593
+ ]
594
+ out["new_old_section.csv"] = _rows_to_frame(
595
+ ("node", "period", "value"), rows_nosec,
596
+ )
597
+
598
+ return out
599
+
600
+
601
+ # ---------------------------------------------------------------------------
602
+ # Vectorized twin of _compute_inflow_scaling_frames (vectorize-per-roll).
603
+ #
604
+ # Built as a FULL COPY of the legacy body, with each stage's COMPUTE
605
+ # replaced by vectorized polars. Only the 6 CONSUMED outputs are emitted;
606
+ # the 9 internal middle parameters are computed in-memory where still needed
607
+ # (orig_flow_sum / psaf dicts, inline fused912 npop/npis/nom) but never
608
+ # written. The legacy _compute_inflow_scaling_frames is KEPT as the parity
609
+ # oracle (it still materialises all 15; the test compares the 6).
610
+ #
611
+ # Tier policy: ptNode_inflow + _node_cap_inflow_fallback are sum-free
612
+ # (coalesce / max-abs) and MUST stay byte-identical (Tier A — read by the
613
+ # already-vectorized pdtNodeInflow and by lp-scaling). The sum-bearing
614
+ # consumed stages (period_flow_annual_multiplier,
615
+ # period_flow_proportional_multiplier, new_old_slope, new_old_section) are
616
+ # Tier B (last-ULP drift tolerated).
617
+ # ---------------------------------------------------------------------------
618
+
619
+
620
+ def _empty_value_frame(header: tuple[str, str, str]) -> pl.DataFrame:
621
+ """An explicit all-Utf8 empty 3-col frame (key1, key2, value)."""
622
+ return pl.DataFrame(
623
+ {h: [] for h in header},
624
+ schema={h: pl.Utf8 for h in header},
625
+ )
626
+
627
+
628
+ def _ordered_value_frame(
629
+ df: pl.DataFrame,
630
+ header: tuple[str, str, str],
631
+ order_cols: list[str],
632
+ ) -> pl.DataFrame:
633
+ """Sort *df* by *order_cols*, render its ``value_f`` Float64 column via
634
+ ``repr`` and project to the all-Utf8 ``(key1, key2, value)`` shape.
635
+
636
+ *df* must carry the two key columns named ``header[0]``/``header[1]``,
637
+ a Float64 ``value_f`` column, and the integer *order_cols*. An empty
638
+ *df* yields the explicit empty schema.
639
+ """
640
+ if df.height == 0:
641
+ return _empty_value_frame(header)
642
+ df = df.sort(order_cols)
643
+ value = _render_value_column(df["value_f"])
644
+ return df.select([header[0], header[1]]).with_columns(
645
+ value.alias(header[2]),
646
+ )
647
+
648
+
649
+ def _compute_inflow_scaling_frames_vectorized(
650
+ input_dir: Path, solve_data_dir: Path,
651
+ *, provider: "object | None" = None,
652
+ ) -> dict[str, pl.DataFrame]:
653
+ """Vectorized twin of :func:`_compute_inflow_scaling_frames`.
654
+
655
+ Same reader block, same in-memory dicts, same stage order. Only the
656
+ 6 CONSUMED outputs are assigned to ``out`` (``ptNode_inflow``,
657
+ ``_node_cap_inflow_fallback``, ``period_flow_annual_multiplier``,
658
+ ``period_flow_proportional_multiplier``, ``new_old_slope``,
659
+ ``new_old_section``). The 9 internal middle parameters
660
+ (``orig_flow_sum``, ``period_share_of_annual_flow``, ``new_peak_sign``,
661
+ ``old_peak_max``, ``old_peak_min``, ``old_peak_sign``,
662
+ ``new_peak_divided_by_old_peak``, ``new_peak_inflow_sum``,
663
+ ``new_old_multiplier``) have no external consumer and are no longer
664
+ emitted; the values still needed downstream live in-memory only
665
+ (``orig_flow_sum`` dict, ``psaf`` dict, and the inline fused912
666
+ ``v_npop`` / ``v_npis`` / ``v_nom``). The legacy oracle
667
+ materialises all 15 and gates the 6 consumed.
668
+ """
669
+ out: dict[str, pl.DataFrame] = {}
670
+
671
+ # ── Sources (copied verbatim from the legacy reader block) ─────────
672
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
673
+ period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
674
+ provider=provider)
675
+ time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
676
+ p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
677
+ pt_node_inflow = _read_keyed2_float(
678
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
679
+ )
680
+ node_time_inflow = frozenset(_read_pairs(
681
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
682
+ ))
683
+
684
+ inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
685
+ provider=provider)
686
+ methods_for_node: dict[str, set[str]] = {}
687
+ for n, m in inflow_method:
688
+ methods_for_node.setdefault(n, set()).add(m)
689
+
690
+ pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
691
+ cpsoy = _read_keyed_float(
692
+ solve_data_dir / "complete_period_share_of_year_calc.csv",
693
+ provider=provider,
694
+ )
695
+ p_tdy = _read_keyed_float(
696
+ solve_data_dir / "p_timeline_duration_in_years.csv",
697
+ provider=provider,
698
+ )
699
+
700
+ # Representative weight w[(d, t)] = p_timestep_weight[d, t] (see the
701
+ # legacy reader block for the full rationale). DEFAULT 1.0 for any
702
+ # (d, t) absent → w ≡ 1.0 reduces every weighted sum below to today's.
703
+ w_dt = _read_keyed2_float(
704
+ solve_data_dir / "timestep_weight.csv", provider=provider,
705
+ )
706
+
707
+ period_timeline = _read_pairs(solve_data_dir / "period__timeline_set.csv",
708
+ provider=provider)
709
+ timelines_for_d: dict[str, list[str]] = {}
710
+ for d, tl in period_timeline:
711
+ timelines_for_d.setdefault(d, []).append(tl)
712
+
713
+ complete_time_in_use = _read_singles(
714
+ solve_data_dir / "complete_time_in_use_set.csv",
715
+ provider=provider,
716
+ )
717
+
718
+ dt_complete_pairs = _read_pairs(
719
+ solve_data_dir / "steps_complete_solve.csv",
720
+ provider=provider,
721
+ )
722
+ dt_complete_for_d: dict[str, list[str]] = {}
723
+ for d, t in dt_complete_pairs:
724
+ dt_complete_for_d.setdefault(d, []).append(t)
725
+
726
+ # ── Shared order frames (node list order, period order, time order) ─
727
+ node_eo = pl.DataFrame(
728
+ {"node": list(nodes), "__eo": list(range(len(nodes)))},
729
+ schema={"node": pl.Utf8, "__eo": pl.Int64},
730
+ )
731
+ period_po = pl.DataFrame(
732
+ {"period": list(period_in_use),
733
+ "__po": list(range(len(period_in_use)))},
734
+ schema={"period": pl.Utf8, "__po": pl.Int64},
735
+ )
736
+ time_to = pl.DataFrame(
737
+ {"time": list(time_set), "__to": list(range(len(time_set)))},
738
+ schema={"time": pl.Utf8, "__to": pl.Int64},
739
+ )
740
+
741
+ # ── Stage 1: ptNode_inflow{n in node, t in time} (Tier A) ──────────
742
+ # value = pt_node_inflow[(n, t)] if (n, t) in node_time_inflow
743
+ # else p_node[(n, "inflow")] (per-node scalar default).
744
+ p_node_inflow_default = {
745
+ n: p_node.get((n, "inflow"), 0.0) for n in nodes
746
+ }
747
+ nt_grid = node_eo.join(time_to, how="cross")
748
+ # pt_node_inflow lookup frame (value present when (n, t) explicit).
749
+ pti_lk = pl.DataFrame(
750
+ {"node": [k[0] for k in pt_node_inflow],
751
+ "time": [k[1] for k in pt_node_inflow],
752
+ "v_pti": list(pt_node_inflow.values())},
753
+ schema={"node": pl.Utf8, "time": pl.Utf8, "v_pti": pl.Float64},
754
+ )
755
+ # explicit-set membership frame (presence in node_time_inflow).
756
+ nti_lk = pl.DataFrame(
757
+ {"node": [k[0] for k in node_time_inflow],
758
+ "time": [k[1] for k in node_time_inflow],
759
+ "__nti": [True] * len(node_time_inflow)},
760
+ schema={"node": pl.Utf8, "time": pl.Utf8, "__nti": pl.Boolean},
761
+ )
762
+ # per-node scalar default frame.
763
+ dflt_lk = pl.DataFrame(
764
+ {"node": list(nodes),
765
+ "v_dflt": [p_node_inflow_default[n] for n in nodes]},
766
+ schema={"node": pl.Utf8, "v_dflt": pl.Float64},
767
+ )
768
+ pti_df = (
769
+ nt_grid
770
+ .join(nti_lk, on=["node", "time"], how="left")
771
+ .join(pti_lk, on=["node", "time"], how="left")
772
+ .join(dflt_lk, on="node", how="left")
773
+ .with_columns(
774
+ pl.when(pl.col("__nti").fill_null(False)) # noqa: FBT003
775
+ # explicit: pt_node_inflow.get((n,t), 0.0)
776
+ .then(pl.col("v_pti").fill_null(0.0))
777
+ # else: per-node scalar default
778
+ .otherwise(pl.col("v_dflt").fill_null(0.0))
779
+ .alias("value_f"),
780
+ )
781
+ )
782
+ out["ptNode_inflow.csv"] = _ordered_value_frame(
783
+ pti_df, ("node", "time", "value"), ["__eo", "__to"],
784
+ )
785
+ # NOTE: every downstream consumer of the per-(n, t) inflow series is now
786
+ # vectorized (it reads ``pti_df`` / ``pti_series`` directly), so the
787
+ # scalar ``pti`` dict is no longer reconstructed here.
788
+
789
+ # ── Stage 2: _node_cap_inflow_fallback{n, d} (Tier A) ──────────────
790
+ # value = max_t abs(pti[(n, t)]); 0.0 if no time.
791
+ if not time_set:
792
+ nd_grid = node_eo.join(period_po, how="cross").with_columns(
793
+ pl.lit(0.0, dtype=pl.Float64).alias("value_f"),
794
+ )
795
+ out["_node_cap_inflow_fallback.csv"] = _ordered_value_frame(
796
+ nd_grid, ("node", "period", "value"), ["__eo", "__po"],
797
+ )
798
+ else:
799
+ max_abs_df = (
800
+ pti_df.group_by("node")
801
+ .agg(pl.col("value_f").abs().max().alias("v_maxabs"))
802
+ )
803
+ nd_grid = (
804
+ node_eo.join(period_po, how="cross")
805
+ .join(max_abs_df, on="node", how="left")
806
+ .with_columns(
807
+ pl.col("v_maxabs").fill_null(0.0).alias("value_f"),
808
+ )
809
+ )
810
+ out["_node_cap_inflow_fallback.csv"] = _ordered_value_frame(
811
+ nd_grid, ("node", "period", "value"), ["__eo", "__po"],
812
+ )
813
+
814
+ # Helper: does node n have inflow method m? (legacy parity)
815
+ def _has_method(n: str, m: str) -> bool:
816
+ return m in methods_for_node.get(n, ())
817
+
818
+ def _annual_eligible(n: str) -> bool:
819
+ return (_has_method(n, "scale_to_annual_flow")
820
+ or _has_method(n, "scale_to_annual_and_peak_flow"))
821
+
822
+ # ── Shared C2 building blocks ──────────────────────────────────────
823
+ # FOUR DISTINCT method masks (Defect A — do NOT collapse onto one):
824
+ # annual_eligible → stages 3, 4
825
+ # scale_to_annual_flow → stage 5 (a peak-only node is EXCLUDED)
826
+ # scale_in_proportion → stage 6 (disjoint from the annual methods)
827
+ # scale_to_annual_and_peak_flow → stages 7, 8 (C3 peak family)
828
+ # Each as an ordered ``(node, __eo)`` frame so the entity-major
829
+ # emission order survives the joins.
830
+ _node_idx = {n: i for i, n in enumerate(nodes)}
831
+
832
+ def _ordered_node_frame(pred) -> pl.DataFrame:
833
+ sel = [n for n in nodes if pred(n)]
834
+ return pl.DataFrame(
835
+ {"node": sel, "__eo": [_node_idx[n] for n in sel]},
836
+ schema={"node": pl.Utf8, "__eo": pl.Int64},
837
+ )
838
+
839
+ annual_eligible_eo = _ordered_node_frame(_annual_eligible)
840
+ annual_flow_only_eo = _ordered_node_frame(
841
+ lambda n: _has_method(n, "scale_to_annual_flow"))
842
+ proportion_eo = _ordered_node_frame(
843
+ lambda n: _has_method(n, "scale_in_proportion"))
844
+
845
+ # pdNode annual_flow lookup keyed (node, period) → af.
846
+ af_keys = [(k[0], k[2]) for k in pdNode if k[1] == "annual_flow"]
847
+ af_vals = [v for k, v in pdNode.items() if k[1] == "annual_flow"]
848
+ af_lk = pl.DataFrame(
849
+ {"node": [k[0] for k in af_keys],
850
+ "period": [k[1] for k in af_keys],
851
+ "v_af": af_vals},
852
+ schema={"node": pl.Utf8, "period": pl.Utf8, "v_af": pl.Float64},
853
+ )
854
+
855
+ # Representative-weight lookup w[(period, time)] → weight; left-joined
856
+ # below with ``fill_null(1.0)`` so any (d, t) absent contributes the
857
+ # neutral factor 1.0 (matching the legacy ``w_dt.get((d, t), 1.0)``).
858
+ w_lk = pl.DataFrame(
859
+ {"period": [k[0] for k in w_dt],
860
+ "time": [k[1] for k in w_dt],
861
+ "v_w": list(w_dt.values())},
862
+ schema={"period": pl.Utf8, "time": pl.Utf8, "v_w": pl.Float64},
863
+ )
864
+
865
+ cti_lk = pl.DataFrame(
866
+ {"time": list(complete_time_in_use)},
867
+ schema={"time": pl.Utf8},
868
+ )
869
+ # pti as a (node, time, v_pti) frame restricted to the per-node series.
870
+ pti_series = pti_df.select(["node", "time", "value_f"]).rename(
871
+ {"value_f": "v_pti"})
872
+ # Per-(node, period) complete-timeline WEIGHTED sum (A3): the legacy
873
+ # ``sum(pti[(n,t)] · w[d,t] for t in complete_time_in_use)``. Build the
874
+ # dense (node, period, time) grid over the complete timeline, multiply
875
+ # pti by the period weight (fill 1.0), then group-sum per (node, period).
876
+ # With w ≡ 1.0 every weight factor is 1.0 → byte-identical to the former
877
+ # period-independent per-node sum.
878
+ complete_sum_df = (
879
+ annual_eligible_eo.select("node")
880
+ .join(cti_lk, how="cross")
881
+ .join(period_po.select("period"), how="cross")
882
+ .join(pti_series, on=["node", "time"], how="left")
883
+ .join(w_lk, on=["period", "time"], how="left")
884
+ .with_columns(
885
+ (pl.col("v_pti").fill_null(0.0)
886
+ * pl.col("v_w").fill_null(1.0)).alias("v_term"),
887
+ )
888
+ .group_by(["node", "period"])
889
+ .agg(pl.col("v_term").sum().alias("v_complete"))
890
+ )
891
+
892
+ # ── Stage 3: orig_flow_sum (in-memory only — feeds npopis) ─────────
893
+ # Domain: annual_eligible AND pdNode annual_flow != 0 (DROP on 0/miss).
894
+ # value = per-(node, period) complete-timeline WEIGHTED sum (A3). NOT
895
+ # emitted (no external consumer); only the dict below is used (by the
896
+ # fused912 npopis). Because the CSV is no longer written there is no
897
+ # byte-parity contract.
898
+ orig_df = (
899
+ annual_eligible_eo
900
+ .join(period_po, how="cross")
901
+ .join(af_lk, on=["node", "period"], how="left")
902
+ .filter(pl.col("v_af").fill_null(0.0) != 0.0)
903
+ .join(complete_sum_df, on=["node", "period"], how="left")
904
+ # A (node, period) with NO complete-timeline rows contributes 0:
905
+ # the legacy ``sum(())`` over an empty complete_time_in_use is 0.
906
+ .with_columns(pl.col("v_complete").fill_null(0.0).alias("value_f"))
907
+ )
908
+ # Reconstruct orig_flow_sum dict (float values) for downstream stages.
909
+ orig_flow_sum = {
910
+ (r[0], r[1]): r[2]
911
+ for r in orig_df.select(["node", "period", "value_f"]).iter_rows()
912
+ }
913
+
914
+ # ── Stage 4: period_share_of_annual_flow (Tier B) ──────────────────
915
+ # Domain: annual_eligible AND af != 0 (DROP on 0/miss).
916
+ # value = abs(sum_{t in dt_complete[d]} pti[(n,t)]) / af.
917
+ # Per-(node, period) dt_complete sum: lift the (period, time) pairs.
918
+ dtc_pairs = pl.DataFrame(
919
+ {"period": [d for d, _t in dt_complete_pairs],
920
+ "time": [t for _d, t in dt_complete_pairs]},
921
+ schema={"period": pl.Utf8, "time": pl.Utf8},
922
+ )
923
+ # node × (period, time) restricted to annual_eligible nodes only, then
924
+ # group-sum the WEIGHTED inflow series per (node, period) (A1): each
925
+ # term is pti[(n,t)] · w[d,t] (fill 1.0). With w ≡ 1.0 → today's sum.
926
+ dtc_sum_df = (
927
+ annual_eligible_eo.select("node")
928
+ .join(pti_series, on="node", how="inner")
929
+ .join(dtc_pairs, on="time", how="inner")
930
+ .join(w_lk, on=["period", "time"], how="left")
931
+ .with_columns(
932
+ (pl.col("v_pti") * pl.col("v_w").fill_null(1.0)).alias("v_term"),
933
+ )
934
+ .group_by(["node", "period"])
935
+ .agg(pl.col("v_term").sum().alias("v_dtc"))
936
+ )
937
+ psaf_df = (
938
+ annual_eligible_eo
939
+ .join(period_po, how="cross")
940
+ .join(af_lk, on=["node", "period"], how="left")
941
+ .filter(pl.col("v_af").fill_null(0.0) != 0.0)
942
+ .join(dtc_sum_df, on=["node", "period"], how="left")
943
+ # A (node, period) with no dt_complete rows: legacy sum(()) == 0.
944
+ .with_columns(
945
+ (pl.col("v_dtc").fill_null(0.0).abs() / pl.col("v_af"))
946
+ .alias("value_f"),
947
+ )
948
+ )
949
+ # NOT emitted (no external consumer); only the psaf dict below is used
950
+ # (by stage 5 = period_flow_annual_multiplier).
951
+ psaf = {
952
+ (r[0], r[1]): r[2]
953
+ for r in psaf_df.select(["node", "period", "value_f"]).iter_rows()
954
+ }
955
+
956
+ # ── Stage 6: period_flow_proportional_multiplier (Tier B) ──────────
957
+ # Domain: scale_in_proportion AND af != 0; DROP if tdy_sum==0 OR
958
+ # time_sum==0. value = af / (abs(time_sum) / tdy_sum).
959
+ # Per-(node, period) WEIGHTED time-axis sum over the WHOLE time_set (A2):
960
+ # each term is pti[(n,t)] · w[d,t] (fill 1.0). The iteration domain
961
+ # stays the full time_set (period-independent), so with w ≡ 1.0 the sum
962
+ # is byte-identical to the former per-node ``sum(pti) over time_set``.
963
+ time_sum_df = (
964
+ proportion_eo.select("node")
965
+ .join(pti_series, on="node", how="inner")
966
+ .join(period_po.select("period"), how="cross")
967
+ .join(w_lk, on=["period", "time"], how="left")
968
+ .with_columns(
969
+ (pl.col("v_pti") * pl.col("v_w").fill_null(1.0)).alias("v_term"),
970
+ )
971
+ .group_by(["node", "period"])
972
+ .agg(pl.col("v_term").sum().alias("v_timesum"))
973
+ )
974
+ # Per-period tdy sum: sum p_tdy over timelines_for_d[d].
975
+ pt_pairs = pl.DataFrame(
976
+ {"period": [d for d, _tl in period_timeline],
977
+ "timeline": [tl for _d, tl in period_timeline]},
978
+ schema={"period": pl.Utf8, "timeline": pl.Utf8},
979
+ )
980
+ tdy_lk = pl.DataFrame(
981
+ {"timeline": list(p_tdy.keys()), "v_tdy": list(p_tdy.values())},
982
+ schema={"timeline": pl.Utf8, "v_tdy": pl.Float64},
983
+ )
984
+ tdy_sum_df = (
985
+ pt_pairs
986
+ .join(tdy_lk, on="timeline", how="left")
987
+ .group_by("period")
988
+ .agg(pl.col("v_tdy").fill_null(0.0).sum().alias("v_tdysum"))
989
+ )
990
+ pfpm_df = (
991
+ proportion_eo
992
+ .join(period_po, how="cross")
993
+ .join(af_lk, on=["node", "period"], how="left")
994
+ .filter(pl.col("v_af").fill_null(0.0) != 0.0)
995
+ .join(time_sum_df, on=["node", "period"], how="left")
996
+ .join(tdy_sum_df, on="period", how="left")
997
+ # A (node, period) missing the time-sum join means no inflow rows ⇒
998
+ # legacy sum(()) == 0.0; a period missing tdy_sum ⇒ sum(()) == 0.0.
999
+ .with_columns(
1000
+ pl.col("v_timesum").fill_null(0.0).alias("v_timesum"),
1001
+ pl.col("v_tdysum").fill_null(0.0).alias("v_tdysum"),
1002
+ )
1003
+ .filter(
1004
+ (pl.col("v_tdysum") != 0.0) & (pl.col("v_timesum") != 0.0))
1005
+ .with_columns(
1006
+ (pl.col("v_af")
1007
+ / (pl.col("v_timesum").abs() / pl.col("v_tdysum")))
1008
+ .alias("value_f"),
1009
+ )
1010
+ )
1011
+ out["period_flow_proportional_multiplier.csv"] = _ordered_value_frame(
1012
+ pfpm_df, ("node", "period", "value"), ["__eo", "__po"],
1013
+ )
1014
+
1015
+ # ── Stage 5: period_flow_annual_multiplier (Tier B) ────────────────
1016
+ # Domain: scale_to_annual_flow ONLY (Defect A — a peak-only node is
1017
+ # annual_eligible but EXCLUDED here). THREE filters, TWO semantics
1018
+ # (Defect B):
1019
+ # (a) af==0 → DROP (inner-join af_lk + filter af!=0),
1020
+ # (b) psaf.get((n,d),0.0)==0 → DROP (inner-join psaf + filter !=0 →
1021
+ # drops BOTH a miss AND an exact-0),
1022
+ # (c) cpsoy.get(d,0.0) numerator → SURVIVE-with-0 (left-join cpsoy +
1023
+ # fill_null(0.0)).
1024
+ # value = cpsoy_filled / psaf.
1025
+ psaf_lk = pl.DataFrame(
1026
+ {"node": [k[0] for k in psaf],
1027
+ "period": [k[1] for k in psaf],
1028
+ "v_psaf": list(psaf.values())},
1029
+ schema={"node": pl.Utf8, "period": pl.Utf8, "v_psaf": pl.Float64},
1030
+ )
1031
+ cpsoy_lk = pl.DataFrame(
1032
+ {"period": list(cpsoy.keys()), "v_cpsoy": list(cpsoy.values())},
1033
+ schema={"period": pl.Utf8, "v_cpsoy": pl.Float64},
1034
+ )
1035
+ pfam_df = (
1036
+ annual_flow_only_eo
1037
+ .join(period_po, how="cross")
1038
+ # (a) af==0/miss → DROP.
1039
+ .join(af_lk, on=["node", "period"], how="left")
1040
+ .filter(pl.col("v_af").fill_null(0.0) != 0.0)
1041
+ # (b) psaf miss OR exact-0 → DROP.
1042
+ .join(psaf_lk, on=["node", "period"], how="left")
1043
+ .filter(pl.col("v_psaf").fill_null(0.0) != 0.0)
1044
+ # (c) cpsoy numerator → survive-with-0.
1045
+ .join(cpsoy_lk, on="period", how="left")
1046
+ .with_columns(
1047
+ (pl.col("v_cpsoy").fill_null(0.0) / pl.col("v_psaf"))
1048
+ .alias("value_f"),
1049
+ )
1050
+ )
1051
+ out["period_flow_annual_multiplier.csv"] = _ordered_value_frame(
1052
+ pfam_df, ("node", "period", "value"), ["__eo", "__po"],
1053
+ )
1054
+
1055
+ # ── Stage 7: per-node peak precompute (Tier A: max/min/sign) ───────
1056
+ # has_node_time_inflow[n] = node has ANY explicit (n, t) inflow row.
1057
+ peak_eo = _ordered_node_frame(
1058
+ lambda n: _has_method(n, "scale_to_annual_and_peak_flow"))
1059
+ hnti_nodes = {nn for (nn, _t) in node_time_inflow}
1060
+ # Per-node max/min over the WHOLE time series (dense grid). When
1061
+ # time_set is empty the group has no rows → null → fill 0.0 (matches
1062
+ # the legacy ``max(inflow_vals) if inflow_vals else 0.0``).
1063
+ minmax_df = (
1064
+ pti_df.group_by("node")
1065
+ .agg(
1066
+ pl.col("value_f").max().alias("v_tmax"),
1067
+ pl.col("value_f").min().alias("v_tmin"),
1068
+ )
1069
+ )
1070
+ # Per-node precompute over ALL nodes (the legacy loop iterates nodes).
1071
+ precomp = (
1072
+ node_eo
1073
+ .join(minmax_df, on="node", how="left")
1074
+ .with_columns(
1075
+ pl.col("node").is_in(list(hnti_nodes)).alias("__hnti"),
1076
+ pl.col("node").replace_strict(
1077
+ p_node_inflow_default, default=0.0,
1078
+ return_dtype=pl.Float64).alias("v_dflt"),
1079
+ )
1080
+ .with_columns(
1081
+ # op_max / op_min: time-series bound when has_node_time_inflow
1082
+ # (null → 0.0 for an empty time_set), else the scalar default.
1083
+ pl.when(pl.col("__hnti"))
1084
+ .then(pl.col("v_tmax").fill_null(0.0))
1085
+ .otherwise(pl.col("v_dflt")).alias("op_max"),
1086
+ pl.when(pl.col("__hnti"))
1087
+ .then(pl.col("v_tmin").fill_null(0.0))
1088
+ .otherwise(pl.col("v_dflt")).alias("op_min"),
1089
+ )
1090
+ .with_columns(
1091
+ # op_sign: has_node_time_inflow → 1 if |max|>=|min| else -1;
1092
+ # else 1 if default >= 0 else -1.
1093
+ pl.when(pl.col("__hnti"))
1094
+ .then(
1095
+ pl.when(pl.col("op_max").abs() >= pl.col("op_min").abs())
1096
+ .then(pl.lit(1.0)).otherwise(pl.lit(-1.0)))
1097
+ .otherwise(
1098
+ pl.when(pl.col("v_dflt") >= 0.0)
1099
+ .then(pl.lit(1.0)).otherwise(pl.lit(-1.0)))
1100
+ .alias("op_sign"),
1101
+ )
1102
+ .with_columns(
1103
+ # old_peak = op_max if op_sign >= 0 else op_min.
1104
+ pl.when(pl.col("op_sign") >= 0.0)
1105
+ .then(pl.col("op_max")).otherwise(pl.col("op_min"))
1106
+ .alias("old_peak"),
1107
+ )
1108
+ # op_max / op_min were only needed to DERIVE op_sign / old_peak
1109
+ # above; nothing downstream reads them, so drop them here.
1110
+ .select(["node", "op_sign", "old_peak"])
1111
+ )
1112
+
1113
+ # ── Stage 8: peak-domain family ────────────────────────────────────
1114
+ # peak_domain = scale_to_annual_and_peak_flow AND af!=0 AND peak!=0.
1115
+ peak_lk_keys = [(k[0], k[2]) for k in pdNode if k[1] == "peak_inflow"]
1116
+ peak_lk = pl.DataFrame(
1117
+ {"node": [k[0] for k in peak_lk_keys],
1118
+ "period": [k[1] for k in peak_lk_keys],
1119
+ "v_peak": [v for k, v in pdNode.items() if k[1] == "peak_inflow"]},
1120
+ schema={"node": pl.Utf8, "period": pl.Utf8, "v_peak": pl.Float64},
1121
+ )
1122
+ peak_domain_df = (
1123
+ peak_eo
1124
+ .join(period_po, how="cross")
1125
+ .join(af_lk, on=["node", "period"], how="left")
1126
+ .filter(pl.col("v_af").fill_null(0.0) != 0.0)
1127
+ .join(peak_lk, on=["node", "period"], how="left")
1128
+ .filter(pl.col("v_peak").fill_null(0.0) != 0.0)
1129
+ .join(precomp, on="node", how="left")
1130
+ .sort(["__eo", "__po"])
1131
+ )
1132
+ # The per-cell peak ingredients new_peak_sign / old_peak_max /
1133
+ # old_peak_min / old_peak_sign and new_peak_divided_by_old_peak (npop)
1134
+ # are NOT emitted (no external consumer). op_sign / old_peak are
1135
+ # carried on ``peak_domain_df`` from stage-7 ``precomp``;
1136
+ # npop is computed INLINE in the fused912 graph below (as ``v_npop``,
1137
+ # fill-0 on old_peak==0) feeding slope/section. No separate scratch
1138
+ # frame is built.
1139
+ # ── Stages 9-12 (Tier-B): ONE fused closed-form graph ──────────────
1140
+ # The whole sub-pipeline runs over the FULL peak-domain keyset
1141
+ # (peak_domain_df == new_peak_sign keyset). npop and npopis are
1142
+ # computed INLINE as ``when(old_peak == 0.0)`` expressions — an
1143
+ # old_peak==0 row is a FILL-to-0 (npop=0, npopis=0), NOT a row drop
1144
+ # (constraint 1): the row survives into slope/section/nom exactly as
1145
+ # the legacy left-join + fill_null(0.0) did, with the smaller
1146
+ # new_peak_divided_by_old_peak emit (npop_df, old_peak!=0 only) already
1147
+ # written above untouched (constraint 2). Collapsing ``npopis_df`` +
1148
+ # ``base912`` + the two npop/npopis re-joins into this single graph
1149
+ # removes the round-trip joins while preserving the exact float op
1150
+ # order / parenthesization (constraint 3) and the ``== 0.0`` guards
1151
+ # (constraint 4). op_sign / old_peak come from stage-7 ``precomp``,
1152
+ # carried on peak_domain_df, unchanged (constraint 5).
1153
+ ofs_lk = pl.DataFrame(
1154
+ {"node": [k[0] for k in orig_flow_sum],
1155
+ "period": [k[1] for k in orig_flow_sum],
1156
+ "v_ofs": list(orig_flow_sum.values())},
1157
+ schema={"node": pl.Utf8, "period": pl.Utf8, "v_ofs": pl.Float64},
1158
+ )
1159
+ fused912 = (
1160
+ peak_domain_df
1161
+ # Per-cell ingredients: orig_flow_sum (.get(k, 0.0)) + cpsoy
1162
+ # (.get(d, 0.0)) — left-join + fill_null(0.0) reproduces the dict
1163
+ # defaults the legacy stages read.
1164
+ .join(ofs_lk, on=["node", "period"], how="left")
1165
+ .join(cpsoy_lk, on="period", how="left")
1166
+ .with_columns(
1167
+ pl.col("v_ofs").fill_null(0.0).alias("v_ofs"),
1168
+ pl.col("v_cpsoy").fill_null(0.0).alias("v_cpsoy"),
1169
+ )
1170
+ .with_columns(
1171
+ # npis = peak * 8760.0 (constraint 3).
1172
+ (pl.col("v_peak") * 8760.0).alias("v_npis"),
1173
+ # npop = 0.0 if old_peak == 0.0 else peak / old_peak. FILL,
1174
+ # not drop (constraint 1); exact ``== 0.0`` guard (constraint 4).
1175
+ pl.when(pl.col("old_peak") == 0.0)
1176
+ .then(pl.lit(0.0))
1177
+ .otherwise(pl.col("v_peak") / pl.col("old_peak"))
1178
+ .alias("v_npop"),
1179
+ )
1180
+ .with_columns(
1181
+ # npopis = 0.0 if (cps == 0.0 or old_peak == 0.0) else
1182
+ # (npop * ofs) / cps. Left-to-right multiply-then-
1183
+ # divide (constraint 3). old_peak==0 ⇒ npop==0 above,
1184
+ # but the legacy code never reached the cps branch on
1185
+ # old_peak==0 (the row was skipped), defaulting npopis
1186
+ # to 0 — so guard old_peak==0 here too (constraint 1).
1187
+ pl.when((pl.col("v_cpsoy") != 0.0)
1188
+ & (pl.col("old_peak") != 0.0))
1189
+ .then(pl.col("v_npop") * pl.col("v_ofs") / pl.col("v_cpsoy"))
1190
+ .otherwise(pl.lit(0.0))
1191
+ .alias("v_npopis"),
1192
+ )
1193
+ .with_columns(
1194
+ # denom = npis - npopis; exact ``== 0.0`` guard (constraint 4).
1195
+ (pl.col("v_npis") - pl.col("v_npopis")).alias("v_denom"))
1196
+ .with_columns(
1197
+ # nom = 0.0 if denom == 0.0 else
1198
+ # op_sign * (op_sign * npopis - af) / denom. Inner parens
1199
+ # + multiply-before-divide preserved (constraint 3).
1200
+ pl.when(pl.col("v_denom") == 0.0)
1201
+ .then(pl.lit(0.0))
1202
+ .otherwise(
1203
+ pl.col("op_sign")
1204
+ * (pl.col("op_sign") * pl.col("v_npopis") - pl.col("v_af"))
1205
+ / pl.col("v_denom"))
1206
+ .alias("v_nom"),
1207
+ )
1208
+ .with_columns(
1209
+ # slope = npop * (1.0 + nom); section = peak * nom (constraint 3).
1210
+ (pl.col("v_npop") * (1.0 + pl.col("v_nom"))).alias("v_slope"),
1211
+ (pl.col("v_peak") * pl.col("v_nom")).alias("v_section"),
1212
+ )
1213
+ )
1214
+
1215
+ # Stage 9 — new_peak_inflow_sum (npis = peak * 8760) and Stage 10 —
1216
+ # new_old_multiplier (nom) are NOT emitted (no external consumer); both
1217
+ # remain inline on ``fused912`` (v_npis / v_nom) feeding slope/section.
1218
+ # Stage 11 — new_old_slope = npop * (1 + nom).
1219
+ out["new_old_slope.csv"] = _ordered_value_frame(
1220
+ fused912.with_columns(pl.col("v_slope").alias("value_f")),
1221
+ ("node", "period", "value"), ["__eo", "__po"],
1222
+ )
1223
+ # Stage 12 — new_old_section = peak * nom.
1224
+ out["new_old_section.csv"] = _ordered_value_frame(
1225
+ fused912.with_columns(pl.col("v_section").alias("value_f")),
1226
+ ("node", "period", "value"), ["__eo", "__po"],
1227
+ )
1228
+
1229
+ return out
1230
+
1231
+
1232
+ # ---------------------------------------------------------------------------
1233
+ # Part B — the ``f`` annualisation-divergence diagnostic.
1234
+ #
1235
+ # For every scaling node ``n`` and period ``d`` report the factor by which an
1236
+ # even-sample annualisation and the representative-weight annualisation
1237
+ # disagree:
1238
+ #
1239
+ # f[n, d] = ( Σ_{t in dt_complete[d]} I[n, t] )
1240
+ # / ( Σ_{t in dt_complete[d]} I[n, t]·w[d, t] )
1241
+ # = multiplier_new / multiplier_old
1242
+ #
1243
+ # ``f == 1.0`` exactly when the weights are uniform/absent (``w ≡ 1.0``).
1244
+ # This is a PURE diagnostic: it never enters the LP and is NOT added to the
1245
+ # scaling frames returned by ``_compute_inflow_scaling_frames*`` (the parity
1246
+ # gate asserts those return EXACTLY the 6 consumed keys). It is emitted as a
1247
+ # standalone ``solve_data/inflow_scaling_diagnostics.csv`` frame on the
1248
+ # Provider (surfaced by ``--csv-dump`` like the other solve_data artifacts),
1249
+ # and a WARNING is logged for any (n, d) whose timeset carries non-uniform
1250
+ # weights and whose ``|f - 1| > 0.01``.
1251
+ # ---------------------------------------------------------------------------
1252
+
1253
+
1254
+ def _compute_inflow_scaling_diagnostics(
1255
+ input_dir: Path, solve_data_dir: Path,
1256
+ *, provider: "object | None" = None,
1257
+ ) -> tuple[pl.DataFrame, list[tuple[str, str, float]]]:
1258
+ """Compute the per-(node, period) ``f`` diagnostic.
1259
+
1260
+ Reads the same sources as the scaling compute (nodes, periods,
1261
+ inflow methods, the merged per-(n, t) inflow series, the
1262
+ representative weight ``w[(d, t)]`` and ``annual_flow``). Returns a
1263
+ two-tuple ``(frame, warnings)`` where:
1264
+
1265
+ * ``frame`` carries one row per scaling (node, period) with columns
1266
+ ``node, period, inflow_method, annual_flow, f, level_shift_pct``
1267
+ (``level_shift_pct = 100·(f - 1)``). ``inflow_method`` is the set of
1268
+ scaling methods authored on the node joined by ``+`` (a node may
1269
+ legitimately carry more than one).
1270
+ * ``warnings`` is the subset of ``(node, period, f)`` for which the
1271
+ node's period carries NON-UNIFORM weights AND ``|f - 1| > 0.01`` —
1272
+ the rows the caller logs at WARNING level.
1273
+
1274
+ Guards mirror the scaling code: a (node, period) whose weighted
1275
+ denominator is 0 (an all-zero / cancelling profile under the weights)
1276
+ is reported with ``f = 1.0`` (no divergence to flag) rather than
1277
+ dividing by zero; the same when the unweighted numerator is 0.
1278
+ """
1279
+ # ── Sources (subset of the scaling reader block) ───────────────────
1280
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
1281
+ period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
1282
+ provider=provider)
1283
+ time_set = _read_singles(solve_data_dir / "time.csv", provider=provider)
1284
+ p_node = _read_keyed2_float(input_dir / "p_node.csv", provider=provider)
1285
+ pt_node_inflow = _read_keyed2_float(
1286
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
1287
+ )
1288
+ node_time_inflow = frozenset(_read_pairs(
1289
+ solve_data_dir / "pt_node_inflow.csv", provider=provider,
1290
+ ))
1291
+
1292
+ inflow_method = _read_pairs(solve_data_dir / "node__inflow_method.csv",
1293
+ provider=provider)
1294
+ methods_for_node: dict[str, set[str]] = {}
1295
+ for n, m in inflow_method:
1296
+ methods_for_node.setdefault(n, set()).add(m)
1297
+
1298
+ pdNode = _read_keyed3_float(solve_data_dir / "pdNode.csv", provider=provider)
1299
+
1300
+ # Representative weight w[(d, t)] = p_timestep_weight[d, t]; DEFAULT 1.0
1301
+ # for any (d, t) absent → f == 1.0 when weights are uniform/absent.
1302
+ w_dt = _read_keyed2_float(
1303
+ solve_data_dir / "timestep_weight.csv", provider=provider,
1304
+ )
1305
+
1306
+ # dt_complete from steps_complete_solve.csv — (period, step) pairs.
1307
+ dt_complete_pairs = _read_pairs(
1308
+ solve_data_dir / "steps_complete_solve.csv",
1309
+ provider=provider,
1310
+ )
1311
+ dt_complete_for_d: dict[str, list[str]] = {}
1312
+ for d, t in dt_complete_pairs:
1313
+ dt_complete_for_d.setdefault(d, []).append(t)
1314
+
1315
+ # Merged per-(n, t) inflow series (same rule as the scaling Stage 1):
1316
+ # explicit pt_node_inflow when (n, t) is set, else the per-node scalar
1317
+ # default p_node[(n, "inflow")].
1318
+ p_node_inflow_default = {
1319
+ n: p_node.get((n, "inflow"), 0.0) for n in nodes
1320
+ }
1321
+ pti: dict[tuple[str, str], float] = {}
1322
+ for n in nodes:
1323
+ default = p_node_inflow_default[n]
1324
+ for t in time_set:
1325
+ if (n, t) in node_time_inflow:
1326
+ pti[(n, t)] = pt_node_inflow.get((n, t), 0.0)
1327
+ else:
1328
+ pti[(n, t)] = default
1329
+
1330
+ rows: list[tuple[str, str, str, float, float, float]] = []
1331
+ warnings: list[tuple[str, str, float]] = []
1332
+ for n in nodes:
1333
+ scaling = [m for m in _SCALING_METHODS
1334
+ if m in methods_for_node.get(n, ())]
1335
+ if not scaling:
1336
+ continue
1337
+ method_label = "+".join(scaling)
1338
+ for d in period_in_use:
1339
+ af = pdNode.get((n, "annual_flow", d), 0.0)
1340
+ steps = dt_complete_for_d.get(d, ())
1341
+ # Unweighted numerator and weighted denominator over the
1342
+ # complete-period timesteps.
1343
+ num = sum(pti[(n, t)] for t in steps)
1344
+ den = sum(pti[(n, t)] * w_dt.get((d, t), 1.0) for t in steps)
1345
+ # Whether this period's timesteps carry a non-uniform weight
1346
+ # (a divergence is only meaningful — and only warned on — when
1347
+ # the representativeness actually varies across the sample).
1348
+ weights_here = [w_dt.get((d, t), 1.0) for t in steps]
1349
+ non_uniform = (
1350
+ len(weights_here) > 1
1351
+ and any(abs(wv - weights_here[0]) > 1e-12
1352
+ for wv in weights_here)
1353
+ )
1354
+ if den == 0.0 or num == 0.0:
1355
+ # Degenerate profile under the weights — no meaningful
1356
+ # divergence; report f = 1.0 (no shift) and never warn.
1357
+ f_val = 1.0
1358
+ else:
1359
+ f_val = num / den
1360
+ level_shift_pct = 100.0 * (f_val - 1.0)
1361
+ rows.append((n, d, method_label, af, f_val, level_shift_pct))
1362
+ if non_uniform and abs(f_val - 1.0) > 0.01:
1363
+ warnings.append((n, d, f_val))
1364
+
1365
+ frame = pl.DataFrame(
1366
+ {
1367
+ "node": [r[0] for r in rows],
1368
+ "period": [r[1] for r in rows],
1369
+ "inflow_method": [r[2] for r in rows],
1370
+ "annual_flow": [repr(r[3]) for r in rows],
1371
+ "f": [repr(r[4]) for r in rows],
1372
+ "level_shift_pct": [repr(r[5]) for r in rows],
1373
+ },
1374
+ schema={
1375
+ "node": pl.Utf8,
1376
+ "period": pl.Utf8,
1377
+ "inflow_method": pl.Utf8,
1378
+ "annual_flow": pl.Utf8,
1379
+ "f": pl.Utf8,
1380
+ "level_shift_pct": pl.Utf8,
1381
+ },
1382
+ )
1383
+ return frame, warnings
1384
+
1385
+
1386
+ def emit_node_inflow_scaling_params(
1387
+ input_dir: Path, solve_data_dir: Path,
1388
+ *, provider,
1389
+ ) -> None:
1390
+ """Emit ``node_inflow_scaling_params`` to the Provider.
1391
+
1392
+ Emits the 6 consumed frames under ``solve_data/<basename>`` keys via
1393
+ :func:`_emit` (dual-key registration). Uses the vectorized compute
1394
+ (:func:`_compute_inflow_scaling_frames_vectorized`); the legacy
1395
+ :func:`_compute_inflow_scaling_frames` is retained as the parity
1396
+ oracle (``tests/engine_polars/test_vectorize_inflow_scaling_parity.py``).
1397
+
1398
+ Additionally emits the standalone ``inflow_scaling_diagnostics.csv``
1399
+ (the Part-B ``f`` divergence diagnostic) and logs a WARNING for any
1400
+ scaling node/period whose non-uniform ``timeset_weights`` imply an
1401
+ annualisation that differs from an even-sample one by ``|f - 1| > 1%``.
1402
+ The diagnostic is NOT part of the parity-gated scaling-frame dict.
1403
+ """
1404
+ frames = _compute_inflow_scaling_frames_vectorized(
1405
+ input_dir, solve_data_dir, provider=provider)
1406
+ for basename, df in frames.items():
1407
+ _emit(provider, f"solve_data/{basename}", df)
1408
+
1409
+ diag, diag_warnings = _compute_inflow_scaling_diagnostics(
1410
+ input_dir, solve_data_dir, provider=provider)
1411
+ _emit(provider, "solve_data/inflow_scaling_diagnostics.csv", diag)
1412
+ for n, d, f_val in diag_warnings:
1413
+ pct = 100.0 * (f_val - 1.0)
1414
+ _logger.warning(
1415
+ "inflow scaling: node %s period %s: timeset_weights imply an "
1416
+ "annualisation that differs from an even-sample annualisation "
1417
+ "by f=%.5f (demand level %+.2f%%); check that timeset_weights "
1418
+ "match the representativeness of the selected timesteps.",
1419
+ n, d, f_val, pct,
1420
+ )