flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2398 @@
1
+ """Heavy per-(d, t) emission writers.
2
+
3
+ Three ``write_pdt*`` helpers that emit per-(entity, period, time) CSVs
4
+ via mostly-procedural fallback cascades:
5
+
6
+ * :func:`write_pdtNodeInflow` (3-branch).
7
+ * :func:`write_pdtProfile` (5-branch).
8
+ * :func:`write_pdtConversion_rate_section_slope` (2 outputs — section + slope; conversion_rate emit pruned, dead).
9
+
10
+ The fallback cascades use simple dict-keyed lookups — optimal for the
11
+ per-row access pattern. See the module docstring on
12
+ :mod:`._pdt_lookup` for the broader rationale.
13
+
14
+ Branches 1 (stochastic fold-in) and 2 (parent-period fold-in) of
15
+ ``write_pdtNodeInflow`` are mod's stochastic / parent-branch fold-ins
16
+ (Gap E in the migration tracker). No fixture in the repo's test data
17
+ carries non-empty ``pbt_node_inflow``, so these branches are inert in
18
+ parity tests — but we keep the structure for forward-compatibility.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ from pathlib import Path
23
+
24
+ import polars as pl
25
+
26
+ from flextool.engine_polars._emit_provider_io import (
27
+ _emit,
28
+ _provider_key,
29
+ )
30
+ from flextool.engine_polars._vectorize import (
31
+ build_entity_dt_grid,
32
+ build_entity_period_grid,
33
+ build_fold_frame,
34
+ coalesce_value,
35
+ collect_value_frame,
36
+ lift_dict_to_lookup,
37
+ )
38
+
39
+
40
+ def _cell_str(value: "object | None") -> str:
41
+ """Reproduce a ``csv.reader`` cell string for a native frame value.
42
+
43
+ ``DataFrame.write_csv`` renders ``null`` as the empty string and every
44
+ other scalar as its textual form; ``csv.reader`` then reads those
45
+ strings back. Mirror that here so dict keys / structural string
46
+ columns stay byte-identical to the legacy CSV round-trip — ``None`` →
47
+ ``""`` (skipped by the original truthiness guards) while a literal
48
+ ``"0"`` is kept. ``provider.get`` returns DATA rows only (no header
49
+ row to skip); an empty / missing frame yields the same empty output
50
+ the legacy loop produced.
51
+ """
52
+ return "" if value is None else str(value)
53
+
54
+
55
+ def _utf8_frame(columns: dict[str, list[str]]) -> pl.DataFrame:
56
+ """Build an all-Utf8 ``pl.DataFrame`` from column-name → string-list.
57
+
58
+ All columns (including ``value``) are stored as ``Utf8`` so the
59
+ legacy ``f"...,{repr(v)}\\n"`` byte-emission round-trips identically
60
+ through ``pl.DataFrame.write_csv``. See
61
+ :mod:`._emit_chain_params._ed_value_frame` for the rationale on
62
+ using ``repr(v)`` directly rather than coercing to ``float`` first.
63
+ """
64
+ schema = {name: pl.Utf8 for name in columns}
65
+ return pl.DataFrame(columns, schema=schema)
66
+
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # Shared CSV readers (mirror legacy helpers byte-for-byte).
70
+ # ---------------------------------------------------------------------------
71
+
72
+
73
+ def _read_singles(path: Path,
74
+ *, provider: "object | None" = None) -> list[str]:
75
+ df = provider.get(_provider_key(path))
76
+ if df is None:
77
+ return []
78
+ out: list[str] = []
79
+ for row in df.iter_rows():
80
+ c0 = _cell_str(row[0]) if row else ""
81
+ if c0:
82
+ out.append(c0)
83
+ return out
84
+
85
+
86
+ def _read_pairs(path: Path,
87
+ *, provider: "object | None" = None) -> list[tuple[str, str]]:
88
+ df = provider.get(_provider_key(path))
89
+ if df is None:
90
+ return []
91
+ out: list[tuple[str, str]] = []
92
+ for row in df.iter_rows():
93
+ if len(row) < 2:
94
+ continue
95
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
96
+ if c0 and c1:
97
+ out.append((c0, c1))
98
+ return out
99
+
100
+
101
+ def _read_pairs_to_dict(path: Path, key_col: int,
102
+ *, provider: "object | None" = None,
103
+ ) -> dict[str, list[str]]:
104
+ """Generic two-col CSV → ``key_col → list[other_col]``."""
105
+ out: dict[str, list[str]] = {}
106
+ df = provider.get(_provider_key(path))
107
+ if df is None:
108
+ return out
109
+ other_col = 1 - key_col
110
+ for row in df.iter_rows():
111
+ if len(row) < 2:
112
+ continue
113
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
114
+ if c0 and c1:
115
+ cells = (c0, c1)
116
+ out.setdefault(cells[key_col], []).append(cells[other_col])
117
+ return out
118
+
119
+
120
+ def _read_stochastic_entities(group_entity_csv: Path,
121
+ group_stochastic_csv: Path,
122
+ *, provider: "object | None" = None,
123
+ ) -> set[str]:
124
+ """``stoch_entity = { e : exists g ∈ groupIncludeStochastics with (g, e) ∈ group__<entity> }``."""
125
+ stoch_groups = frozenset(
126
+ _read_singles(group_stochastic_csv, provider=provider)
127
+ )
128
+ out: set[str] = set()
129
+ df = provider.get(_provider_key(group_entity_csv))
130
+ if df is None:
131
+ return out
132
+ for row in df.iter_rows():
133
+ if len(row) < 2:
134
+ continue
135
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
136
+ if c0 in stoch_groups and c1:
137
+ out.add(c1)
138
+ return out
139
+
140
+
141
+ # ---------------------------------------------------------------------------
142
+ # write_pdtNodeInflow — flextool.mod L1325 (3-branch).
143
+ # ---------------------------------------------------------------------------
144
+
145
+
146
+ def derive_pdtNodeInflow(input_dir: Path, solve_data_dir: Path,
147
+ *, provider: "object | None" = None,
148
+ ) -> pl.DataFrame:
149
+ """Materialise the ``pdtNodeInflow`` frame.
150
+
151
+ Columns: ``node, period, time, value`` — all ``Utf8`` (value cells
152
+ are ``repr(v)`` so int/float distinction round-trips byte-identically
153
+ to the legacy ``fh.write(f"{n},{d},{t},{repr(v)}\\n")`` emission).
154
+
155
+ Branches:
156
+ 1. Stochastic fold-in (``pbt_node_inflow`` over stochastic node).
157
+ 2. Parent-period fold-in (``pbt_node_inflow`` over parent periods).
158
+ 3. Deterministic additive sum of the 4 scaling methods:
159
+ * ``scale_to_annual_flow`` — pfa[n,d] * pti[n,t]
160
+ * ``scale_in_proportion`` — pfp[n,d] * pti[n,t]
161
+ * ``scale_to_annual_and_peak_flow`` — slope[n,d] * pti[n,t] - section[n,d]
162
+ * ``use_original`` — pti[n,t]
163
+
164
+ Domain: nodes whose method is anything BUT ``no_inflow``. Non-
165
+ balance-union nodes get 0 (mod L1280 guard).
166
+ """
167
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
168
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
169
+
170
+ inflow_method_pairs = frozenset(
171
+ _read_pairs(solve_data_dir / "node__inflow_method.csv",
172
+ provider=provider)
173
+ )
174
+ n_balance = frozenset(
175
+ _read_singles(solve_data_dir / "nodeBalance.csv", provider=provider)
176
+ )
177
+ n_balance_period = frozenset(
178
+ _read_singles(solve_data_dir / "nodeBalancePeriod.csv",
179
+ provider=provider)
180
+ )
181
+ balance_union = n_balance | n_balance_period
182
+
183
+ stoch_node = _read_stochastic_entities(
184
+ input_dir / "group__node.csv",
185
+ input_dir / "groupIncludeStochastics.csv",
186
+ provider=provider,
187
+ )
188
+
189
+ ts_for_d = _read_pairs_to_dict(
190
+ solve_data_dir / "first_timesteps.csv", key_col=0,
191
+ provider=provider,
192
+ )
193
+ tb_for_d = _read_pairs_to_dict(
194
+ solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
195
+ provider=provider,
196
+ )
197
+ # period__branch.csv stores (db, d) — child key column is 1.
198
+ pe_for_d = _read_pairs_to_dict(
199
+ solve_data_dir / "period__branch.csv", key_col=1,
200
+ provider=provider,
201
+ )
202
+
203
+ # pbt_node_inflow{(n, branch, ts, t) → value}
204
+ pbt_inflow: dict[tuple[str, str, str, str], float] = {}
205
+ pbt_path = input_dir / "pbt_node_inflow.csv"
206
+ pbt_df = provider.get(_provider_key(pbt_path))
207
+ if pbt_df is not None:
208
+ for row in pbt_df.iter_rows():
209
+ if len(row) < 5:
210
+ continue
211
+ c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
212
+ _cell_str(row[2]), _cell_str(row[3]))
213
+ if c0 and c1 and c2 and c3:
214
+ try:
215
+ pbt_inflow[(c0, c1, c2, c3)] = float(row[4])
216
+ except (ValueError, TypeError):
217
+ continue
218
+
219
+ # ptNode_inflow{(n, t) → value}
220
+ pt_inflow: dict[tuple[str, str], float] = {}
221
+ pti_path = solve_data_dir / "ptNode_inflow.csv"
222
+ pti_df = provider.get(_provider_key(pti_path))
223
+ if pti_df is not None:
224
+ for row in pti_df.iter_rows():
225
+ if len(row) < 3:
226
+ continue
227
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
228
+ if c0 and c1:
229
+ try:
230
+ pt_inflow[(c0, c1)] = float(row[2])
231
+ except (ValueError, TypeError):
232
+ continue
233
+
234
+ # pdNode lookup limited to (annual_flow, peak_inflow).
235
+ pdNode_af: dict[tuple[str, str], float] = {}
236
+ pdNode_pk: dict[tuple[str, str], float] = {}
237
+ pdn_path = solve_data_dir / "pdNode.csv"
238
+ pdn_df = provider.get(_provider_key(pdn_path))
239
+ if pdn_df is not None:
240
+ for row in pdn_df.iter_rows():
241
+ if len(row) < 4:
242
+ continue
243
+ c0 = _cell_str(row[0])
244
+ if c0:
245
+ try:
246
+ v = float(row[3])
247
+ except (ValueError, TypeError):
248
+ continue
249
+ c1, c2 = _cell_str(row[1]), _cell_str(row[2])
250
+ if c1 == "annual_flow":
251
+ pdNode_af[(c0, c2)] = v
252
+ elif c1 == "peak_inflow":
253
+ pdNode_pk[(c0, c2)] = v
254
+
255
+ def _read_2_keyed_value(path: Path) -> dict[tuple[str, str], float]:
256
+ out: dict[tuple[str, str], float] = {}
257
+ df = provider.get(_provider_key(path))
258
+ if df is None:
259
+ return out
260
+ for row in df.iter_rows():
261
+ if len(row) < 3:
262
+ continue
263
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
264
+ if c0 and c1:
265
+ try:
266
+ out[(c0, c1)] = float(row[2])
267
+ except (ValueError, TypeError):
268
+ continue
269
+ return out
270
+
271
+ pfa = _read_2_keyed_value(
272
+ solve_data_dir / "period_flow_annual_multiplier.csv"
273
+ )
274
+ pfp = _read_2_keyed_value(
275
+ solve_data_dir / "period_flow_proportional_multiplier.csv"
276
+ )
277
+ nos_slope = _read_2_keyed_value(solve_data_dir / "new_old_slope.csv")
278
+ nos_section = _read_2_keyed_value(solve_data_dir / "new_old_section.csv")
279
+
280
+ eligible_nodes = [
281
+ n for n in nodes if (n, "no_inflow") not in inflow_method_pairs
282
+ ]
283
+
284
+ nodes_col: list[str] = []
285
+ periods_col: list[str] = []
286
+ times_col: list[str] = []
287
+ values_col: list[str] = []
288
+ for n in eligible_nodes:
289
+ is_stoch = n in stoch_node
290
+ in_balance = n in balance_union
291
+ has_scale_annual = (n, "scale_to_annual_flow") in inflow_method_pairs
292
+ has_scale_proportion = (n, "scale_in_proportion") in inflow_method_pairs
293
+ has_scale_peak = (n, "scale_to_annual_and_peak_flow") in inflow_method_pairs
294
+ has_use_original = (n, "use_original") in inflow_method_pairs
295
+ for (d, t) in dt:
296
+ emit_v: float | None = None
297
+ # Branch 1: stochastic fold-in.
298
+ if is_stoch:
299
+ total = 0.0
300
+ hit = False
301
+ for tb in tb_for_d.get(d, ()):
302
+ for ts in ts_for_d.get(d, ()):
303
+ v = pbt_inflow.get((n, tb, ts, t))
304
+ if v is not None:
305
+ total += v
306
+ hit = True
307
+ if hit:
308
+ emit_v = total
309
+ # Branch 2: parent-period fold-in.
310
+ if emit_v is None:
311
+ pe_list = pe_for_d.get(d, ())
312
+ ts_list = ts_for_d.get(d, ())
313
+ if pe_list and ts_list:
314
+ total = 0.0
315
+ hit = False
316
+ for pe in pe_list:
317
+ for tb in tb_for_d.get(pe, ()):
318
+ for ts in ts_list:
319
+ v = pbt_inflow.get((n, tb, ts, t))
320
+ if v is not None:
321
+ total += v
322
+ hit = True
323
+ if hit:
324
+ emit_v = total
325
+ # Branch 3: deterministic additive sum.
326
+ if emit_v is None:
327
+ value = 0.0
328
+ if in_balance:
329
+ pti = pt_inflow.get((n, t), 0.0)
330
+ if has_scale_annual and pdNode_af.get((n, d), 0.0):
331
+ value += pfa.get((n, d), 0.0) * pti
332
+ if has_scale_proportion and pdNode_af.get((n, d), 0.0):
333
+ value += pfp.get((n, d), 0.0) * pti
334
+ if (has_scale_peak
335
+ and pdNode_af.get((n, d), 0.0)
336
+ and pdNode_pk.get((n, d), 0.0)):
337
+ value += nos_slope.get((n, d), 0.0) * pti \
338
+ - nos_section.get((n, d), 0.0)
339
+ if has_use_original:
340
+ value += pti
341
+ emit_v = value
342
+ nodes_col.append(n)
343
+ periods_col.append(d)
344
+ times_col.append(t)
345
+ values_col.append(repr(emit_v))
346
+ return _utf8_frame({
347
+ "node": nodes_col,
348
+ "period": periods_col,
349
+ "time": times_col,
350
+ "value": values_col,
351
+ })
352
+
353
+
354
+ def derive_pdtNodeInflow_vectorized(input_dir: Path, solve_data_dir: Path,
355
+ *,
356
+ provider: "object | None" = None,
357
+ engine: str = "eager") -> pl.DataFrame:
358
+ """Vectorized ``pdtNodeInflow`` derive — parity with the legacy.
359
+
360
+ Replaces the per-cell 3-branch scalar cascade in
361
+ :func:`derive_pdtNodeInflow` with vectorized polars, still per roll
362
+ over the roll's own window. Output is byte-identical to
363
+ :func:`derive_pdtNodeInflow`: columns ``node, period, time, value``
364
+ all ``Utf8``, entity-major row order, ``repr(v)`` value cells.
365
+
366
+ This family composes two already-proven patterns:
367
+
368
+ * the single-``node``-key grid + ``build_fold_frame`` fold skeleton
369
+ (branches 1 & 2 — stochastic + parent-period fold, identical to
370
+ :func:`derive_pdtProfile_vectorized`); and
371
+ * the gated additive SUM of branch 3 (the four inflow-scaling
372
+ methods), assembled membership-gated-left-join + ``fill_null(0.0)``
373
+ with a signed-zero normalization (identical mechanics to
374
+ :func:`_derive_varCost_pair_vectorized`).
375
+
376
+ The fold (branches 1-2) takes priority over the branch-3 deterministic
377
+ sum, which is always non-null (its 0.0 floor for non-balance / no-
378
+ method nodes), so ``coalesce(v_fold, v_b3)`` reproduces the legacy
379
+ ``emit_v`` priority exactly.
380
+
381
+ The reader block is copied VERBATIM from :func:`derive_pdtNodeInflow`
382
+ so the deduped dicts / stoch set / float parses are byte-for-byte
383
+ identical, and the vectorized lookups are lifted from those dicts (not
384
+ re-read from the CSVs).
385
+ """
386
+ # --- Step 1: reader block copied VERBATIM from derive_pdtNodeInflow ----
387
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
388
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
389
+
390
+ inflow_method_pairs = frozenset(
391
+ _read_pairs(solve_data_dir / "node__inflow_method.csv",
392
+ provider=provider)
393
+ )
394
+ n_balance = frozenset(
395
+ _read_singles(solve_data_dir / "nodeBalance.csv", provider=provider)
396
+ )
397
+ n_balance_period = frozenset(
398
+ _read_singles(solve_data_dir / "nodeBalancePeriod.csv",
399
+ provider=provider)
400
+ )
401
+ balance_union = n_balance | n_balance_period
402
+
403
+ stoch_node = _read_stochastic_entities(
404
+ input_dir / "group__node.csv",
405
+ input_dir / "groupIncludeStochastics.csv",
406
+ provider=provider,
407
+ )
408
+
409
+ ts_for_d = _read_pairs_to_dict(
410
+ solve_data_dir / "first_timesteps.csv", key_col=0,
411
+ provider=provider,
412
+ )
413
+ tb_for_d = _read_pairs_to_dict(
414
+ solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
415
+ provider=provider,
416
+ )
417
+ # period__branch.csv stores (db, d) — child key column is 1.
418
+ pe_for_d = _read_pairs_to_dict(
419
+ solve_data_dir / "period__branch.csv", key_col=1,
420
+ provider=provider,
421
+ )
422
+
423
+ # pbt_node_inflow{(n, branch, ts, t) → value}
424
+ pbt_inflow: dict[tuple[str, str, str, str], float] = {}
425
+ pbt_path = input_dir / "pbt_node_inflow.csv"
426
+ pbt_df = provider.get(_provider_key(pbt_path))
427
+ if pbt_df is not None:
428
+ for row in pbt_df.iter_rows():
429
+ if len(row) < 5:
430
+ continue
431
+ c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
432
+ _cell_str(row[2]), _cell_str(row[3]))
433
+ if c0 and c1 and c2 and c3:
434
+ try:
435
+ pbt_inflow[(c0, c1, c2, c3)] = float(row[4])
436
+ except (ValueError, TypeError):
437
+ continue
438
+
439
+ # ptNode_inflow{(n, t) → value}
440
+ pt_inflow: dict[tuple[str, str], float] = {}
441
+ pti_path = solve_data_dir / "ptNode_inflow.csv"
442
+ pti_df = provider.get(_provider_key(pti_path))
443
+ if pti_df is not None:
444
+ for row in pti_df.iter_rows():
445
+ if len(row) < 3:
446
+ continue
447
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
448
+ if c0 and c1:
449
+ try:
450
+ pt_inflow[(c0, c1)] = float(row[2])
451
+ except (ValueError, TypeError):
452
+ continue
453
+
454
+ # pdNode lookup limited to (annual_flow, peak_inflow).
455
+ pdNode_af: dict[tuple[str, str], float] = {}
456
+ pdNode_pk: dict[tuple[str, str], float] = {}
457
+ pdn_path = solve_data_dir / "pdNode.csv"
458
+ pdn_df = provider.get(_provider_key(pdn_path))
459
+ if pdn_df is not None:
460
+ for row in pdn_df.iter_rows():
461
+ if len(row) < 4:
462
+ continue
463
+ c0 = _cell_str(row[0])
464
+ if c0:
465
+ try:
466
+ v = float(row[3])
467
+ except (ValueError, TypeError):
468
+ continue
469
+ c1, c2 = _cell_str(row[1]), _cell_str(row[2])
470
+ if c1 == "annual_flow":
471
+ pdNode_af[(c0, c2)] = v
472
+ elif c1 == "peak_inflow":
473
+ pdNode_pk[(c0, c2)] = v
474
+
475
+ def _read_2_keyed_value(path: Path) -> dict[tuple[str, str], float]:
476
+ out: dict[tuple[str, str], float] = {}
477
+ df = provider.get(_provider_key(path))
478
+ if df is None:
479
+ return out
480
+ for row in df.iter_rows():
481
+ if len(row) < 3:
482
+ continue
483
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
484
+ if c0 and c1:
485
+ try:
486
+ out[(c0, c1)] = float(row[2])
487
+ except (ValueError, TypeError):
488
+ continue
489
+ return out
490
+
491
+ pfa = _read_2_keyed_value(
492
+ solve_data_dir / "period_flow_annual_multiplier.csv"
493
+ )
494
+ pfp = _read_2_keyed_value(
495
+ solve_data_dir / "period_flow_proportional_multiplier.csv"
496
+ )
497
+ nos_slope = _read_2_keyed_value(solve_data_dir / "new_old_slope.csv")
498
+ nos_section = _read_2_keyed_value(solve_data_dir / "new_old_section.csv")
499
+
500
+ eligible_nodes = [
501
+ n for n in nodes if (n, "no_inflow") not in inflow_method_pairs
502
+ ]
503
+
504
+ # --- Step 2: grid + fold (branches 1 & 2), pdtProfile-shaped ----------
505
+ key_cols = ["node"]
506
+ out_cols = [*key_cols, "period", "time"]
507
+ periods = [d for (d, _t) in dt]
508
+
509
+ grid = build_entity_dt_grid(
510
+ [(n,) for n in eligible_nodes], dt, key_cols=key_cols)
511
+
512
+ fold = build_fold_frame(
513
+ pbt=pbt_inflow,
514
+ pbt_key_cols=["node"],
515
+ out_key_cols=out_cols,
516
+ ts_for_d=ts_for_d,
517
+ tb_for_d=tb_for_d,
518
+ pe_for_d=pe_for_d,
519
+ stoch_entities=stoch_node,
520
+ stoch_filter_cols=["node"],
521
+ periods=periods,
522
+ )
523
+
524
+ # --- Step 3: branch 3 — deterministic gated additive sum --------------
525
+ # Lift the Float64 value lookups from the deduped dicts.
526
+ pti_df_lk = lift_dict_to_lookup(pt_inflow, ["node", "time"], "v_pti")
527
+ pfa_df = lift_dict_to_lookup(pfa, ["node", "period"], "v_pfa")
528
+ pfp_df = lift_dict_to_lookup(pfp, ["node", "period"], "v_pfp")
529
+ slope_df = lift_dict_to_lookup(nos_slope, ["node", "period"], "v_slope")
530
+ sect_df = lift_dict_to_lookup(nos_section, ["node", "period"], "v_sect")
531
+ af_df = lift_dict_to_lookup(pdNode_af, ["node", "period"], "v_af")
532
+ pk_df = lift_dict_to_lookup(pdNode_pk, ["node", "period"], "v_pk")
533
+
534
+ # Per-node Boolean membership frames with EXPLICIT schema (an empty set
535
+ # must NOT yield an all-Null frame). The four ``has_*`` gates are
536
+ # per-node memberships ``(n, method) ∈ inflow_method_pairs``.
537
+ def _bool_frame(names: "set[str] | list[str]", col: str) -> pl.DataFrame:
538
+ names = list(names)
539
+ return pl.DataFrame(
540
+ {"node": names, col: [True] * len(names)},
541
+ schema={"node": pl.Utf8, col: pl.Boolean},
542
+ )
543
+
544
+ in_bal_df = _bool_frame(balance_union, "__in_bal")
545
+ m_ann_df = _bool_frame(
546
+ {n for (n, m) in inflow_method_pairs
547
+ if m == "scale_to_annual_flow"}, "__m_ann")
548
+ m_prop_df = _bool_frame(
549
+ {n for (n, m) in inflow_method_pairs
550
+ if m == "scale_in_proportion"}, "__m_prop")
551
+ m_peak_df = _bool_frame(
552
+ {n for (n, m) in inflow_method_pairs
553
+ if m == "scale_to_annual_and_peak_flow"}, "__m_peak")
554
+ m_orig_df = _bool_frame(
555
+ {n for (n, m) in inflow_method_pairs
556
+ if m == "use_original"}, "__m_orig")
557
+
558
+ out = (
559
+ grid
560
+ .join(in_bal_df, on=["node"], how="left")
561
+ .join(m_ann_df, on=["node"], how="left")
562
+ .join(m_prop_df, on=["node"], how="left")
563
+ .join(m_peak_df, on=["node"], how="left")
564
+ .join(m_orig_df, on=["node"], how="left")
565
+ .join(pti_df_lk, on=["node", "time"], how="left")
566
+ .join(pfa_df, on=["node", "period"], how="left")
567
+ .join(pfp_df, on=["node", "period"], how="left")
568
+ .join(slope_df, on=["node", "period"], how="left")
569
+ .join(sect_df, on=["node", "period"], how="left")
570
+ .join(af_df, on=["node", "period"], how="left")
571
+ .join(pk_df, on=["node", "period"], how="left")
572
+ )
573
+
574
+ pti = pl.col("v_pti").fill_null(0.0)
575
+ # Gate on the VALUE being non-zero (NEVER ``is_not_null``): a real
576
+ # 0.0-valued af / pk key must gate the term OFF, matching the legacy
577
+ # ``if ... pdNode_af.get((n, d), 0.0):`` truthiness guard.
578
+ af_ok = pl.col("v_af").fill_null(0.0) != 0.0
579
+ pk_ok = pl.col("v_pk").fill_null(0.0) != 0.0
580
+
581
+ t_ann = (
582
+ pl.when(pl.col("__m_ann").fill_null(False) & af_ok)
583
+ .then(pl.col("v_pfa").fill_null(0.0) * pti)
584
+ .otherwise(0.0)
585
+ )
586
+ t_prop = (
587
+ pl.when(pl.col("__m_prop").fill_null(False) & af_ok)
588
+ .then(pl.col("v_pfp").fill_null(0.0) * pti)
589
+ .otherwise(0.0)
590
+ )
591
+ t_peak = (
592
+ pl.when(pl.col("__m_peak").fill_null(False) & af_ok & pk_ok)
593
+ .then(pl.col("v_slope").fill_null(0.0) * pti
594
+ - pl.col("v_sect").fill_null(0.0))
595
+ .otherwise(0.0)
596
+ )
597
+ t_orig = (
598
+ pl.when(pl.col("__m_orig").fill_null(False))
599
+ .then(pti)
600
+ .otherwise(0.0)
601
+ )
602
+ # Fixed add order (matches legacy :330-340).
603
+ branch3 = t_ann + t_prop + t_peak + t_orig
604
+ # Non-balance-union nodes get the 0.0 floor (legacy L328 guard).
605
+ v_b3 = (
606
+ pl.when(pl.col("__in_bal").fill_null(False))
607
+ .then(branch3)
608
+ .otherwise(0.0)
609
+ )
610
+ # Signed-zero normalization: the legacy seeds ``value = 0.0`` so any
611
+ # ``±0.0`` term collapses to ``+0.0`` (legacy never emits ``"-0.0"``);
612
+ # the peak term ``slope*pti - section`` can produce ``-0.0``. Wrap
613
+ # ``when(== 0.0).then(lit(0.0))`` (NOT ``lit(0.0) + sum`` — that form
614
+ # leaves ``-0.0`` across rows; proven broken in the varCost round).
615
+ v_b3 = (
616
+ pl.when(v_b3 == 0.0)
617
+ .then(pl.lit(0.0))
618
+ .otherwise(v_b3)
619
+ )
620
+
621
+ # --- Step 4: coalesce (fold-priority) + collect -----------------------
622
+ if fold is not None:
623
+ out = out.join(fold, on=out_cols, how="left")
624
+ else:
625
+ out = out.with_columns(
626
+ pl.lit(None, dtype=pl.Float64).alias("v_fold"))
627
+
628
+ out = out.with_columns(
629
+ coalesce_value([
630
+ pl.col("v_fold"), # branches 1-2 (stoch + parent fold)
631
+ v_b3, # branch 3 (always non-null = 0.0 floor)
632
+ ])
633
+ )
634
+ return collect_value_frame(out, key_cols=out_cols, engine=engine)
635
+
636
+
637
+ def emit_pdtNodeInflow(input_dir: Path, solve_data_dir: Path,
638
+ *, provider) -> None:
639
+ """Emit ``pdtNodeInflow`` to the Provider."""
640
+ _emit(provider, "solve_data/pdtNodeInflow.csv",
641
+ derive_pdtNodeInflow_vectorized(
642
+ input_dir, solve_data_dir, provider=provider))
643
+
644
+
645
+ # ---------------------------------------------------------------------------
646
+ # write_pdtProfile — flextool.mod L1192 (5-branch fallback + stochastic UNION).
647
+ # ---------------------------------------------------------------------------
648
+
649
+
650
+ def derive_pdtProfile(input_dir: Path, solve_data_dir: Path,
651
+ *, provider: "object | None" = None) -> pl.DataFrame:
652
+ """Materialise the ``pdtProfile`` frame.
653
+
654
+ Branches:
655
+ 1. Stochastic fold-in (any of process / node / process_node refs
656
+ the profile under a stochastic group).
657
+ 2. Parent-period fold-in.
658
+ 3. ``pt_profile[p, t]``.
659
+ 4. ``p_profile[p]``.
660
+ 5. 0.
661
+
662
+ Domain: every profile in ``input/profile.csv`` × ``dt``.
663
+ """
664
+ profiles = _read_singles(input_dir / "profile.csv", provider=provider)
665
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
666
+
667
+ # pbt / pt / p loaders.
668
+ pbt_profile: dict[tuple[str, str, str, str], float] = {}
669
+ pbt_path = input_dir / "pbt_profile.csv"
670
+ pbt_df = provider.get(_provider_key(pbt_path))
671
+ if pbt_df is not None:
672
+ for row in pbt_df.iter_rows():
673
+ if len(row) < 5:
674
+ continue
675
+ c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
676
+ _cell_str(row[2]), _cell_str(row[3]))
677
+ if c0 and c1 and c2 and c3:
678
+ try:
679
+ pbt_profile[(c0, c1, c2, c3)] = float(row[4])
680
+ except (ValueError, TypeError):
681
+ continue
682
+ pt_profile: dict[tuple[str, str], float] = {}
683
+ pt_path = solve_data_dir / "pt_profile.csv"
684
+ pt_df = provider.get(_provider_key(pt_path))
685
+ if pt_df is not None:
686
+ for row in pt_df.iter_rows():
687
+ if len(row) < 3:
688
+ continue
689
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
690
+ if c0 and c1:
691
+ try:
692
+ pt_profile[(c0, c1)] = float(row[2])
693
+ except (ValueError, TypeError):
694
+ continue
695
+ p_profile: dict[str, float] = {}
696
+ p_path = input_dir / "p_profile.csv"
697
+ p_df = provider.get(_provider_key(p_path))
698
+ if p_df is not None:
699
+ for row in p_df.iter_rows():
700
+ if len(row) < 2:
701
+ continue
702
+ c0 = _cell_str(row[0])
703
+ if c0:
704
+ try:
705
+ p_profile[c0] = float(row[1])
706
+ except (ValueError, TypeError):
707
+ continue
708
+
709
+ # Branch indices.
710
+ ts_for_d = _read_pairs_to_dict(
711
+ solve_data_dir / "first_timesteps.csv", key_col=0,
712
+ provider=provider,
713
+ )
714
+ tb_for_d = _read_pairs_to_dict(
715
+ solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
716
+ provider=provider,
717
+ )
718
+ pe_for_d = _read_pairs_to_dict(
719
+ solve_data_dir / "period__branch.csv", key_col=1,
720
+ provider=provider,
721
+ )
722
+
723
+ # Stochastic profile UNION: any profile referenced via a stochastic
724
+ # process / node / process_node binding.
725
+ stoch_processes = _read_stochastic_entities(
726
+ input_dir / "group__process.csv",
727
+ input_dir / "groupIncludeStochastics.csv",
728
+ provider=provider,
729
+ )
730
+ stoch_nodes = _read_stochastic_entities(
731
+ input_dir / "group__node.csv",
732
+ input_dir / "groupIncludeStochastics.csv",
733
+ provider=provider,
734
+ )
735
+ stoch_profile: set[str] = set()
736
+ pp_path = input_dir / "process__profile__profile_method.csv"
737
+ pp_df = provider.get(_provider_key(pp_path))
738
+ if pp_df is not None:
739
+ for row in pp_df.iter_rows():
740
+ if len(row) < 2:
741
+ continue
742
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
743
+ if c0 in stoch_processes and c1:
744
+ stoch_profile.add(c1)
745
+ np_path = input_dir / "node__profile__profile_method.csv"
746
+ np_df = provider.get(_provider_key(np_path))
747
+ if np_df is not None:
748
+ for row in np_df.iter_rows():
749
+ if len(row) < 2:
750
+ continue
751
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
752
+ if c0 in stoch_nodes and c1:
753
+ stoch_profile.add(c1)
754
+ pnp_path = input_dir / "process__node__profile__profile_method.csv"
755
+ pnp_df = provider.get(_provider_key(pnp_path))
756
+ if pnp_df is not None:
757
+ for row in pnp_df.iter_rows():
758
+ if len(row) < 3:
759
+ continue
760
+ c0, c2 = _cell_str(row[0]), _cell_str(row[2])
761
+ if c0 in stoch_processes and c2:
762
+ stoch_profile.add(c2)
763
+
764
+ profiles_col: list[str] = []
765
+ periods_col: list[str] = []
766
+ times_col: list[str] = []
767
+ values_col: list[str] = []
768
+ for p in profiles:
769
+ is_stoch = p in stoch_profile
770
+ for (d, t) in dt:
771
+ cell: str | None = None
772
+ # Branch 1: stochastic fold-in.
773
+ if is_stoch:
774
+ total = 0.0
775
+ hit = False
776
+ for tb in tb_for_d.get(d, ()):
777
+ for ts in ts_for_d.get(d, ()):
778
+ v = pbt_profile.get((p, tb, ts, t))
779
+ if v is not None:
780
+ total += v
781
+ hit = True
782
+ if hit:
783
+ cell = repr(total)
784
+ # Branch 2: parent-period fold-in.
785
+ if cell is None:
786
+ pe_list = pe_for_d.get(d, ())
787
+ ts_list = ts_for_d.get(d, ())
788
+ if pe_list and ts_list:
789
+ total = 0.0
790
+ hit = False
791
+ for pe in pe_list:
792
+ for tb in tb_for_d.get(pe, ()):
793
+ for ts in ts_list:
794
+ v = pbt_profile.get((p, tb, ts, t))
795
+ if v is not None:
796
+ total += v
797
+ hit = True
798
+ if hit:
799
+ cell = repr(total)
800
+ # Branch 3: time axis.
801
+ if cell is None:
802
+ v = pt_profile.get((p, t))
803
+ if v is not None:
804
+ cell = repr(v)
805
+ # Branch 4: scalar.
806
+ if cell is None:
807
+ v = p_profile.get(p)
808
+ if v is not None:
809
+ cell = repr(v)
810
+ # Branch 5: 0.
811
+ if cell is None:
812
+ cell = "0.0"
813
+ profiles_col.append(p)
814
+ periods_col.append(d)
815
+ times_col.append(t)
816
+ values_col.append(cell)
817
+ return _utf8_frame({
818
+ "profile": profiles_col,
819
+ "period": periods_col,
820
+ "time": times_col,
821
+ "value": values_col,
822
+ })
823
+
824
+
825
+ def derive_pdtProfile_vectorized(input_dir: Path, solve_data_dir: Path,
826
+ *,
827
+ provider: "object | None" = None,
828
+ engine: str = "eager") -> pl.DataFrame:
829
+ """Vectorized ``pdtProfile`` derive — parity with the legacy.
830
+
831
+ Replaces the per-cell 5-branch scalar cascade in
832
+ :func:`derive_pdtProfile` with vectorized polars (left-joins +
833
+ ``coalesce`` in cascade-priority order + the group-by-sum folds),
834
+ still per roll over the roll's own window. Output is byte-identical
835
+ to :func:`derive_pdtProfile`: columns ``profile, period, time,
836
+ value`` all ``Utf8``, entity-major row order, ``repr(v)`` value
837
+ cells.
838
+
839
+ Entity key = ``profile`` (single Utf8 col), NO param axis. Branches
840
+ (priority): (1) stochastic fold, (2) parent-period fold, (3)
841
+ ``pt_profile[(profile, time)]``, (4) ``p_profile[profile]``, (5)
842
+ literal ``0.0``. The ``pbt_profile`` key is a 4-tuple
843
+ ``(profile, tb, ts, t)`` (NO param), so the fold key cols are the
844
+ single ``["profile"]`` column.
845
+
846
+ The reader blocks (``pbt`` / ``pt`` / ``p`` loaders, the three fold-
847
+ index dicts, and the stochastic-profile UNION over process / node /
848
+ process_node bindings) are copied verbatim from
849
+ :func:`derive_pdtProfile` so the deduped dicts / stoch set match
850
+ byte-for-byte.
851
+ """
852
+ profiles = _read_singles(input_dir / "profile.csv", provider=provider)
853
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
854
+
855
+ # pbt / pt / p loaders.
856
+ pbt_profile: dict[tuple[str, str, str, str], float] = {}
857
+ pbt_path = input_dir / "pbt_profile.csv"
858
+ pbt_df = provider.get(_provider_key(pbt_path))
859
+ if pbt_df is not None:
860
+ for row in pbt_df.iter_rows():
861
+ if len(row) < 5:
862
+ continue
863
+ c0, c1, c2, c3 = (_cell_str(row[0]), _cell_str(row[1]),
864
+ _cell_str(row[2]), _cell_str(row[3]))
865
+ if c0 and c1 and c2 and c3:
866
+ try:
867
+ pbt_profile[(c0, c1, c2, c3)] = float(row[4])
868
+ except (ValueError, TypeError):
869
+ continue
870
+ pt_profile: dict[tuple[str, str], float] = {}
871
+ pt_path = solve_data_dir / "pt_profile.csv"
872
+ pt_df = provider.get(_provider_key(pt_path))
873
+ if pt_df is not None:
874
+ for row in pt_df.iter_rows():
875
+ if len(row) < 3:
876
+ continue
877
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
878
+ if c0 and c1:
879
+ try:
880
+ pt_profile[(c0, c1)] = float(row[2])
881
+ except (ValueError, TypeError):
882
+ continue
883
+ p_profile: dict[str, float] = {}
884
+ p_path = input_dir / "p_profile.csv"
885
+ p_df = provider.get(_provider_key(p_path))
886
+ if p_df is not None:
887
+ for row in p_df.iter_rows():
888
+ if len(row) < 2:
889
+ continue
890
+ c0 = _cell_str(row[0])
891
+ if c0:
892
+ try:
893
+ p_profile[c0] = float(row[1])
894
+ except (ValueError, TypeError):
895
+ continue
896
+
897
+ # Branch indices.
898
+ ts_for_d = _read_pairs_to_dict(
899
+ solve_data_dir / "first_timesteps.csv", key_col=0,
900
+ provider=provider,
901
+ )
902
+ tb_for_d = _read_pairs_to_dict(
903
+ solve_data_dir / "solve_branch__time_branch.csv", key_col=0,
904
+ provider=provider,
905
+ )
906
+ pe_for_d = _read_pairs_to_dict(
907
+ solve_data_dir / "period__branch.csv", key_col=1,
908
+ provider=provider,
909
+ )
910
+
911
+ # Stochastic profile UNION: any profile referenced via a stochastic
912
+ # process / node / process_node binding.
913
+ stoch_processes = _read_stochastic_entities(
914
+ input_dir / "group__process.csv",
915
+ input_dir / "groupIncludeStochastics.csv",
916
+ provider=provider,
917
+ )
918
+ stoch_nodes = _read_stochastic_entities(
919
+ input_dir / "group__node.csv",
920
+ input_dir / "groupIncludeStochastics.csv",
921
+ provider=provider,
922
+ )
923
+ stoch_profile: set[str] = set()
924
+ pp_path = input_dir / "process__profile__profile_method.csv"
925
+ pp_df = provider.get(_provider_key(pp_path))
926
+ if pp_df is not None:
927
+ for row in pp_df.iter_rows():
928
+ if len(row) < 2:
929
+ continue
930
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
931
+ if c0 in stoch_processes and c1:
932
+ stoch_profile.add(c1)
933
+ np_path = input_dir / "node__profile__profile_method.csv"
934
+ np_df = provider.get(_provider_key(np_path))
935
+ if np_df is not None:
936
+ for row in np_df.iter_rows():
937
+ if len(row) < 2:
938
+ continue
939
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
940
+ if c0 in stoch_nodes and c1:
941
+ stoch_profile.add(c1)
942
+ pnp_path = input_dir / "process__node__profile__profile_method.csv"
943
+ pnp_df = provider.get(_provider_key(pnp_path))
944
+ if pnp_df is not None:
945
+ for row in pnp_df.iter_rows():
946
+ if len(row) < 3:
947
+ continue
948
+ c0, c2 = _cell_str(row[0]), _cell_str(row[2])
949
+ if c0 in stoch_processes and c2:
950
+ stoch_profile.add(c2)
951
+
952
+ # --- vectorized assembly ----------------------------------------------
953
+ key_cols = ["profile"]
954
+ out_cols = [*key_cols, "period", "time"]
955
+ periods = [d for (d, _t) in dt]
956
+
957
+ grid = build_entity_dt_grid(
958
+ [(p,) for p in profiles], dt, key_cols=key_cols)
959
+
960
+ pt_df_lk = lift_dict_to_lookup(pt_profile, ["profile", "time"], "v_pt")
961
+ p_df_lk = lift_dict_to_lookup(p_profile, ["profile"], "v_p")
962
+
963
+ fold = build_fold_frame(
964
+ pbt=pbt_profile,
965
+ pbt_key_cols=["profile"],
966
+ out_key_cols=out_cols,
967
+ ts_for_d=ts_for_d,
968
+ tb_for_d=tb_for_d,
969
+ pe_for_d=pe_for_d,
970
+ stoch_entities=stoch_profile,
971
+ stoch_filter_cols=["profile"],
972
+ periods=periods,
973
+ )
974
+
975
+ out = (
976
+ grid
977
+ .join(pt_df_lk, on=["profile", "time"], how="left")
978
+ .join(p_df_lk, on=["profile"], how="left")
979
+ )
980
+ if fold is not None:
981
+ out = out.join(fold, on=out_cols, how="left")
982
+ else:
983
+ out = out.with_columns(
984
+ pl.lit(None, dtype=pl.Float64).alias("v_fold"))
985
+
986
+ out = out.with_columns(
987
+ coalesce_value([
988
+ pl.col("v_fold"), # branches 1-2 (stoch + parent fold)
989
+ pl.col("v_pt"), # branch 3 (pt_profile)
990
+ pl.col("v_p"), # branch 4 (p_profile)
991
+ pl.lit(0.0), # branch 5 (literal 0.0)
992
+ ])
993
+ )
994
+ return collect_value_frame(out, key_cols=out_cols, engine=engine)
995
+
996
+
997
+ def emit_pdtProfile(input_dir: Path, solve_data_dir: Path,
998
+ *, provider) -> None:
999
+ """Emit ``pdtProfile`` to the Provider."""
1000
+ _emit(provider, "solve_data/pdtProfile.csv",
1001
+ derive_pdtProfile_vectorized(
1002
+ input_dir, solve_data_dir, provider=provider))
1003
+
1004
+
1005
+ # ---------------------------------------------------------------------------
1006
+ # write_pdtConversion_rate_section_slope — flextool.mod L1390-1400 (3 outputs).
1007
+ # ---------------------------------------------------------------------------
1008
+
1009
+
1010
+ def _derive_conversion_trio(
1011
+ input_dir: Path, solve_data_dir: Path,
1012
+ *, provider: "object | None" = None,
1013
+ ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame]:
1014
+ """Build the three ``pdtConversion_rate`` / ``pdtProcess_section`` /
1015
+ ``pdtProcess_slope`` frames in a single pass.
1016
+
1017
+ The slope formula reuses the ``conv_rate`` and ``section`` values
1018
+ computed for the first two frames; we share the intermediate dicts
1019
+ here so the work isn't duplicated across three independent
1020
+ ``derive_*`` calls.
1021
+ """
1022
+ processes = _read_singles(input_dir / "process.csv", provider=provider)
1023
+ process_minload = frozenset(
1024
+ _read_singles(solve_data_dir / "process_minload.csv",
1025
+ provider=provider)
1026
+ )
1027
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1028
+
1029
+ eff: dict[tuple[str, str, str], float] = {}
1030
+ min_load: dict[tuple[str, str, str], float] = {}
1031
+ eff_min: dict[tuple[str, str, str], float] = {}
1032
+ pdt_path = solve_data_dir / "pdtProcess.csv"
1033
+ pdt_df = provider.get(_provider_key(pdt_path))
1034
+ if pdt_df is not None:
1035
+ for row in pdt_df.iter_rows():
1036
+ if len(row) < 5:
1037
+ continue
1038
+ c0 = _cell_str(row[0])
1039
+ if not c0:
1040
+ continue
1041
+ try:
1042
+ v = float(row[4])
1043
+ except (ValueError, TypeError):
1044
+ continue
1045
+ c1, c2, c3 = _cell_str(row[1]), _cell_str(row[2]), _cell_str(row[3])
1046
+ key = (c0, c2, c3) # (process, period, time)
1047
+ if c1 == "efficiency":
1048
+ eff[key] = v
1049
+ elif c1 == "min_load":
1050
+ min_load[key] = v
1051
+ elif c1 == "efficiency_at_min_load":
1052
+ eff_min[key] = v
1053
+
1054
+ # pdtConversion_rate columns + intermediate conv_rate dict.
1055
+ conv_rate: dict[tuple[str, str, str], float] = {}
1056
+ cr_p: list[str] = []
1057
+ cr_d: list[str] = []
1058
+ cr_t: list[str] = []
1059
+ cr_v: list[str] = []
1060
+ for p in processes:
1061
+ for (d, t) in dt:
1062
+ e = eff.get((p, d, t), 0.0)
1063
+ v = round(1.0 / e, 6) if e else 0.0
1064
+ conv_rate[(p, d, t)] = v
1065
+ cr_p.append(p)
1066
+ cr_d.append(d)
1067
+ cr_t.append(t)
1068
+ cr_v.append(repr(v))
1069
+ conv_frame = _utf8_frame({
1070
+ "process": cr_p, "period": cr_d, "time": cr_t, "value": cr_v,
1071
+ })
1072
+
1073
+ # pdtProcess_section + intermediate section dict.
1074
+ section: dict[tuple[str, str, str], float] = {}
1075
+ sec_p: list[str] = []
1076
+ sec_d: list[str] = []
1077
+ sec_t: list[str] = []
1078
+ sec_v: list[str] = []
1079
+ for p in processes:
1080
+ if p not in process_minload:
1081
+ continue
1082
+ for (d, t) in dt:
1083
+ cr = conv_rate.get((p, d, t), 0.0)
1084
+ ml = min_load.get((p, d, t), 0.0)
1085
+ em = eff_min.get((p, d, t), 0.0)
1086
+ inv_em = (1.0 / em) if em else 0.0
1087
+ denom = 1.0 - ml
1088
+ rounded = round((cr - ml * inv_em) / denom, 6) if denom else 0.0
1089
+ v = cr - rounded
1090
+ section[(p, d, t)] = v
1091
+ sec_p.append(p)
1092
+ sec_d.append(d)
1093
+ sec_t.append(t)
1094
+ sec_v.append(repr(v))
1095
+ section_frame = _utf8_frame({
1096
+ "process": sec_p, "period": sec_d, "time": sec_t, "value": sec_v,
1097
+ })
1098
+
1099
+ # pdtProcess_slope.
1100
+ sl_p: list[str] = []
1101
+ sl_d: list[str] = []
1102
+ sl_t: list[str] = []
1103
+ sl_v: list[str] = []
1104
+ for p in processes:
1105
+ in_min = p in process_minload
1106
+ for (d, t) in dt:
1107
+ cr = conv_rate.get((p, d, t), 0.0)
1108
+ sec = section.get((p, d, t), 0.0) if in_min else 0.0
1109
+ v = cr - sec
1110
+ sl_p.append(p)
1111
+ sl_d.append(d)
1112
+ sl_t.append(t)
1113
+ sl_v.append(repr(v))
1114
+ slope_frame = _utf8_frame({
1115
+ "process": sl_p, "period": sl_d, "time": sl_t, "value": sl_v,
1116
+ })
1117
+
1118
+ return conv_frame, section_frame, slope_frame
1119
+
1120
+
1121
+ def emit_pdtConversion_rate_section_slope(
1122
+ input_dir: Path, solve_data_dir: Path,
1123
+ *, provider,
1124
+ ) -> None:
1125
+ """Emit ``pdtConversion_rate_section_slope`` to the Provider."""
1126
+ _conv, sec, slope = _derive_conversion_trio(
1127
+ input_dir, solve_data_dir, provider=provider,
1128
+ )
1129
+ _emit(provider, "solve_data/pdtProcess_section.csv", sec)
1130
+ _emit(provider, "solve_data/pdtProcess_slope.csv", slope)
1131
+
1132
+
1133
+ # ---------------------------------------------------------------------------
1134
+ # Group / commodity period-param fallbacks and the inflow
1135
+ # positive/negative split.
1136
+ #
1137
+ # Procedural shape with dict lookups in a nested loop — optimal for the
1138
+ # per-row access pattern. Output is byte-for-byte so the
1139
+ # parity tests can ``filecmp``.
1140
+ # ---------------------------------------------------------------------------
1141
+
1142
+
1143
+ # flextool_base.dat L196-201 — group period param taxonomies.
1144
+ _GROUP_PERIOD_PARAM: frozenset[str] = frozenset((
1145
+ "capacity_margin", "co2_price", "co2_max_period", "co2_max_total",
1146
+ "inertia_limit", "invest_max_period", "invest_min_period",
1147
+ "max_cumulative_flow", "min_cumulative_flow", "non_synchronous_limit",
1148
+ "penalty_inertia", "penalty_non_synchronous",
1149
+ "max_instant_flow", "min_instant_flow", "penalty_capacity_margin",
1150
+ "retire_max_period", "retire_min_period",
1151
+ "cumulative_max_capacity", "cumulative_min_capacity",
1152
+ ))
1153
+ _GROUP_TIME_PARAM: frozenset[str] = frozenset((
1154
+ "co2_price", "max_instant_flow", "min_instant_flow",
1155
+ ))
1156
+ _GROUP_PARAM_DEFAULT_5000: frozenset[str] = frozenset((
1157
+ "penalty_inertia", "penalty_capacity_margin", "penalty_non_synchronous",
1158
+ ))
1159
+
1160
+
1161
+ def _read_p_2(path: Path,
1162
+ *, provider: "object | None" = None,
1163
+ ) -> dict[tuple[str, str], float]:
1164
+ """Read a 3-col CSV ``(key1, key2, value)`` into a dict.
1165
+
1166
+ Mirrors legacy ``_read_p_2`` (entity_period_calc_params.py L1965).
1167
+ """
1168
+ out: dict[tuple[str, str], float] = {}
1169
+ df = provider.get(_provider_key(path))
1170
+ if df is None:
1171
+ return out
1172
+ for row in df.iter_rows():
1173
+ if len(row) < 3:
1174
+ continue
1175
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
1176
+ if c0 and c1:
1177
+ try:
1178
+ out[(c0, c1)] = float(row[2])
1179
+ except (ValueError, TypeError):
1180
+ continue
1181
+ return out
1182
+
1183
+
1184
+ def _read_pd_2(path: Path,
1185
+ *, provider: "object | None" = None,
1186
+ ) -> dict[tuple[str, str, str], float]:
1187
+ """Read a 4-col CSV ``(k1, k2, k3, value)`` into a dict."""
1188
+ out: dict[tuple[str, str, str], float] = {}
1189
+ df = provider.get(_provider_key(path))
1190
+ if df is None:
1191
+ return out
1192
+ for row in df.iter_rows():
1193
+ if len(row) < 4:
1194
+ continue
1195
+ c = [_cell_str(row[i]) for i in range(3)]
1196
+ if all(c):
1197
+ try:
1198
+ out[(c[0], c[1], c[2])] = float(row[3])
1199
+ except (ValueError, TypeError):
1200
+ continue
1201
+ return out
1202
+
1203
+
1204
+ def _read_branches_for_d(period_branch_csv: Path,
1205
+ *, provider: "object | None" = None,
1206
+ ) -> dict[str, list[str]]:
1207
+ """``period__branch.csv`` is ``(branch_period, period)`` — index by
1208
+ the child period (column 1) and gather branch list."""
1209
+ out: dict[str, list[str]] = {}
1210
+ df = provider.get(_provider_key(period_branch_csv))
1211
+ if df is None:
1212
+ return out
1213
+ for row in df.iter_rows():
1214
+ if len(row) < 2:
1215
+ continue
1216
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
1217
+ if c0 and c1:
1218
+ out.setdefault(c1, []).append(c0)
1219
+ return out
1220
+
1221
+
1222
+ # ---------------------------------------------------------------------------
1223
+ # write_pdGroup — flextool.mod L1115 (5-branch fallback).
1224
+ # ---------------------------------------------------------------------------
1225
+
1226
+
1227
+ def derive_pdGroup(input_dir: Path, solve_data_dir: Path,
1228
+ *, provider: "object | None" = None) -> pl.DataFrame:
1229
+ """Materialise the ``pdGroup`` frame (5-branch fallback per (g, param, d)).
1230
+
1231
+ Branches:
1232
+ 1. ``pd_group[g, param, d]`` — direct.
1233
+ 2. ``sum_{db ∈ branches_for_d[d]} pd_group[g, param, db]`` — fold.
1234
+ 3. ``p_group[g, param]`` — scalar fallback.
1235
+ 4. ``5000`` when ``param`` is a 5000-default penalty.
1236
+ 5. ``0``.
1237
+ """
1238
+ pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
1239
+ p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
1240
+ branches_for_d = _read_branches_for_d(
1241
+ solve_data_dir / "period__branch.csv", provider=provider,
1242
+ )
1243
+ groups = _read_singles(input_dir / "group.csv", provider=provider)
1244
+ period_in_use = _read_singles(
1245
+ solve_data_dir / "period_in_use_set.csv", provider=provider,
1246
+ )
1247
+
1248
+ g_col: list[str] = []
1249
+ p_col: list[str] = []
1250
+ d_col: list[str] = []
1251
+ v_col: list[str] = []
1252
+ for g in groups:
1253
+ for param in _GROUP_PERIOD_PARAM:
1254
+ for d in period_in_use:
1255
+ if (g, param, d) in pd_g:
1256
+ v = pd_g[(g, param, d)]
1257
+ else:
1258
+ branched = [
1259
+ pd_g[(g, param, db)]
1260
+ for db in branches_for_d.get(d, ())
1261
+ if (g, param, db) in pd_g
1262
+ ]
1263
+ if branched:
1264
+ v = sum(branched)
1265
+ elif (g, param) in p_g:
1266
+ v = p_g[(g, param)]
1267
+ elif param in _GROUP_PARAM_DEFAULT_5000:
1268
+ v = 5000.0
1269
+ else:
1270
+ v = 0.0
1271
+ g_col.append(g)
1272
+ p_col.append(param)
1273
+ d_col.append(d)
1274
+ v_col.append(repr(v))
1275
+ return _utf8_frame({
1276
+ "group": g_col, "param": p_col, "period": d_col, "value": v_col,
1277
+ })
1278
+
1279
+
1280
+ def derive_pdGroup_vectorized(input_dir: Path, solve_data_dir: Path,
1281
+ *,
1282
+ provider: "object | None" = None,
1283
+ engine: str = "eager") -> pl.DataFrame:
1284
+ """Vectorized ``pdGroup`` derive — period-only, branch-sum, dedup-safe.
1285
+
1286
+ Replaces the per-cell cascade loop in :func:`derive_pdGroup` with
1287
+ vectorized polars (left-joins + a group-by-sum for the branch fold +
1288
+ ``coalesce`` in cascade-priority order), still per roll over the
1289
+ roll's own window. Output columns ``group, param, period, value``
1290
+ all ``Utf8`` (NO time axis), entity-major row order, ``repr(v)``
1291
+ value cells.
1292
+
1293
+ Domain = ``group × _GROUP_PERIOD_PARAM`` (the live module frozenset,
1294
+ so iteration order matches legacy — S4) preserving order and
1295
+ duplicates; period axis = ``period_in_use`` (order + duplicates
1296
+ preserved).
1297
+
1298
+ Cascade per ``(g, param, d)`` (5-branch):
1299
+
1300
+ 1. ``pd_group[g, param, d]`` — direct.
1301
+ 2. ``sum_{db ∈ branches_for_d[d], (g,param,db)∈pd_group}
1302
+ pd_group[g, param, db]`` — fold,
1303
+ only when NON-empty.
1304
+ 3. ``p_group[g, param]`` — scalar.
1305
+ 4. ``5000`` when ``param ∈ _GROUP_PARAM_DEFAULT_5000``.
1306
+ 5. ``0``.
1307
+
1308
+ The branch-sum (D3 critique fix) is computed on the **de-duplicated**
1309
+ ``(group, param)`` set so a duplicated ``group`` entry in
1310
+ ``group.csv`` does NOT double-count the fold; the dup-preserving final
1311
+ grid then left-joins the per-``(group,param,period)`` sum back,
1312
+ re-expanding to every output row.
1313
+ """
1314
+ pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
1315
+ p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
1316
+ branches_for_d = _read_branches_for_d(
1317
+ solve_data_dir / "period__branch.csv", provider=provider,
1318
+ )
1319
+ groups = _read_singles(input_dir / "group.csv", provider=provider)
1320
+ period_in_use = _read_singles(
1321
+ solve_data_dir / "period_in_use_set.csv", provider=provider,
1322
+ )
1323
+
1324
+ key_cols = ["group", "param"]
1325
+ out_cols = [*key_cols, "period"]
1326
+
1327
+ # Domain = group × _GROUP_PERIOD_PARAM, referencing the LIVE frozenset
1328
+ # object so iteration order matches the legacy loop (S4); preserve the
1329
+ # group list order + duplicates, never ``.unique()``.
1330
+ domain = [
1331
+ (g, param) for g in groups for param in list(_GROUP_PERIOD_PARAM)
1332
+ ]
1333
+
1334
+ grid = build_entity_period_grid(
1335
+ domain, period_in_use, key_cols=key_cols,
1336
+ )
1337
+
1338
+ # Branch 1 — direct pd_group[(g, param, d)].
1339
+ pd_df = lift_dict_to_lookup(pd_g, ["group", "param", "period"], "v_pd")
1340
+
1341
+ # Branch 3 — scalar p_group[(g, param)].
1342
+ p_df = lift_dict_to_lookup(p_g, ["group", "param"], "v_p")
1343
+
1344
+ # Branch 4 — 5000 default for the penalty params.
1345
+ def5000_params = list(_GROUP_PARAM_DEFAULT_5000)
1346
+ def5000_df = pl.DataFrame(
1347
+ {
1348
+ "param": def5000_params,
1349
+ "v_5000": [5000.0] * len(def5000_params),
1350
+ },
1351
+ schema={"param": pl.Utf8, "v_5000": pl.Float64},
1352
+ )
1353
+
1354
+ # Branch 2 — branch-sum. Expand (period d → branch period db),
1355
+ # preserving duplicate (d, db) rows (D2: duplicates must double-count
1356
+ # to match the legacy ``sum(...)`` over the branches list).
1357
+ exp_period: list[str] = []
1358
+ exp_db: list[str] = []
1359
+ for d in period_in_use:
1360
+ for db in branches_for_d.get(d, ()):
1361
+ exp_period.append(d)
1362
+ exp_db.append(db)
1363
+ exp = pl.DataFrame(
1364
+ {"period": exp_period, "db": exp_db},
1365
+ schema={"period": pl.Utf8, "db": pl.Utf8},
1366
+ )
1367
+
1368
+ if exp.height > 0:
1369
+ pd_db = lift_dict_to_lookup(
1370
+ pd_g, ["group", "param", "db"], "v_pddb")
1371
+ # D3: de-dup the (group, param) set for the SUM ONLY so a
1372
+ # duplicated group does not double-count; the dup-preserving grid
1373
+ # re-expands the result back to every output row via the final
1374
+ # left-join.
1375
+ gp_unique = grid.select(["group", "param"]).unique()
1376
+ bsum = (
1377
+ gp_unique
1378
+ .join(exp, how="cross")
1379
+ # INNER join drops non-matching (g, param, db) → reproduces
1380
+ # both the ``if (g,param,db) in pd_g`` gate and the
1381
+ # ``if branched:`` non-empty gate.
1382
+ .join(pd_db, on=["group", "param", "db"], how="inner")
1383
+ .group_by(["group", "param", "period"])
1384
+ .agg(pl.col("v_pddb").sum().alias("v_branch"))
1385
+ )
1386
+ else:
1387
+ bsum = pl.DataFrame(
1388
+ {"group": [], "param": [], "period": [], "v_branch": []},
1389
+ schema={
1390
+ "group": pl.Utf8,
1391
+ "param": pl.Utf8,
1392
+ "period": pl.Utf8,
1393
+ "v_branch": pl.Float64,
1394
+ },
1395
+ )
1396
+
1397
+ out = (
1398
+ grid
1399
+ .join(pd_df, on=["group", "param", "period"], how="left")
1400
+ .join(bsum, on=["group", "param", "period"], how="left")
1401
+ .join(p_df, on=["group", "param"], how="left")
1402
+ .join(def5000_df, on=["param"], how="left")
1403
+ .with_columns(
1404
+ coalesce_value([
1405
+ pl.col("v_pd"), # branch 1 (direct)
1406
+ pl.col("v_branch"), # branch 2 (branch-sum, non-empty)
1407
+ pl.col("v_p"), # branch 3 (scalar)
1408
+ pl.col("v_5000"), # branch 4 (5000 default set)
1409
+ pl.lit(0.0), # branch 5 (default)
1410
+ ])
1411
+ )
1412
+ )
1413
+ return collect_value_frame(
1414
+ out, key_cols=out_cols, sort_cols=["__eo", "__po"], engine=engine,
1415
+ )
1416
+
1417
+
1418
+ def emit_pdGroup(input_dir: Path, solve_data_dir: Path,
1419
+ *, provider) -> None:
1420
+ """Emit ``pdGroup`` to the Provider."""
1421
+ _emit(provider, "solve_data/pdGroup.csv",
1422
+ derive_pdGroup_vectorized(
1423
+ input_dir, solve_data_dir, provider=provider))
1424
+
1425
+
1426
+ # ---------------------------------------------------------------------------
1427
+ # write_pdtGroup — flextool.mod L1126 (4-branch fallback: pt → pd → p → 0).
1428
+ # ---------------------------------------------------------------------------
1429
+
1430
+
1431
+ def derive_pdtGroup(input_dir: Path, solve_data_dir: Path,
1432
+ *, provider: "object | None" = None) -> pl.DataFrame:
1433
+ """Materialise the ``pdtGroup`` frame.
1434
+
1435
+ Branches: ``pt_group[g, param, t]`` → ``pd_group[g, param, d]`` →
1436
+ ``p_group[g, param]`` → 0.
1437
+ """
1438
+ pt_g = _read_pd_2(input_dir / "pt_group.csv", provider=provider) # same (k1, k2, k3, v) shape
1439
+ pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
1440
+ p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
1441
+ groups = _read_singles(input_dir / "group.csv", provider=provider)
1442
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1443
+
1444
+ g_col: list[str] = []
1445
+ p_col: list[str] = []
1446
+ d_col: list[str] = []
1447
+ t_col: list[str] = []
1448
+ v_col: list[str] = []
1449
+ for g in groups:
1450
+ for param in _GROUP_TIME_PARAM:
1451
+ for (d, t) in dt:
1452
+ if (g, param, t) in pt_g:
1453
+ v = pt_g[(g, param, t)]
1454
+ elif (g, param, d) in pd_g:
1455
+ v = pd_g[(g, param, d)]
1456
+ elif (g, param) in p_g:
1457
+ v = p_g[(g, param)]
1458
+ else:
1459
+ v = 0.0
1460
+ g_col.append(g)
1461
+ p_col.append(param)
1462
+ d_col.append(d)
1463
+ t_col.append(t)
1464
+ v_col.append(repr(v))
1465
+ return _utf8_frame({
1466
+ "group": g_col, "param": p_col, "period": d_col, "time": t_col,
1467
+ "value": v_col,
1468
+ })
1469
+
1470
+
1471
+ def derive_pdtGroup_vectorized(input_dir: Path, solve_data_dir: Path,
1472
+ *,
1473
+ provider: "object | None" = None,
1474
+ engine: str = "eager") -> pl.DataFrame:
1475
+ """Vectorized ``pdtGroup`` derive — byte-parity with the legacy.
1476
+
1477
+ Replaces the per-cell cascade loop in :func:`derive_pdtGroup` with
1478
+ vectorized polars (left-joins + ``coalesce`` in cascade-priority
1479
+ order), still per roll over the roll's own window. Output is
1480
+ byte-identical to :func:`derive_pdtGroup`: columns ``group, param,
1481
+ period, time, value`` all ``Utf8``, entity-major row order,
1482
+ ``repr(v)`` value cells.
1483
+
1484
+ Domain = ``group × _GROUP_TIME_PARAM`` (the live module frozenset, so
1485
+ iteration order matches legacy — S4) preserving order and duplicates,
1486
+ crossed with ``dt`` from ``steps_in_use``.
1487
+
1488
+ Cascade (inline 4-branch, time-first):
1489
+ ``pt_group`` → ``pd_group`` → ``p_group`` → ``0.0``.
1490
+ """
1491
+ pt_g = _read_pd_2(input_dir / "pt_group.csv", provider=provider)
1492
+ pd_g = _read_pd_2(input_dir / "pd_group.csv", provider=provider)
1493
+ p_g = _read_p_2(input_dir / "p_group.csv", provider=provider)
1494
+ groups = _read_singles(input_dir / "group.csv", provider=provider)
1495
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1496
+
1497
+ key_cols = ["group", "param"]
1498
+ out_cols = [*key_cols, "period", "time"]
1499
+
1500
+ # Domain = group × _GROUP_TIME_PARAM, referencing the LIVE frozenset
1501
+ # object so iteration order matches the legacy loop (S4); preserve the
1502
+ # group list order + duplicates, never ``.unique()``.
1503
+ domain = [(g, param) for g in groups for param in list(_GROUP_TIME_PARAM)]
1504
+
1505
+ grid = build_entity_dt_grid(domain, dt, key_cols=key_cols)
1506
+
1507
+ pt_df = lift_dict_to_lookup(pt_g, ["group", "param", "time"], "v_pt")
1508
+ pd_df = lift_dict_to_lookup(pd_g, ["group", "param", "period"], "v_pd")
1509
+ p_df = lift_dict_to_lookup(p_g, ["group", "param"], "v_p")
1510
+
1511
+ out = (
1512
+ grid
1513
+ .join(pt_df, on=["group", "param", "time"], how="left")
1514
+ .join(pd_df, on=["group", "param", "period"], how="left")
1515
+ .join(p_df, on=["group", "param"], how="left")
1516
+ .with_columns(
1517
+ coalesce_value([
1518
+ pl.col("v_pt"), # branch 1 (time-first)
1519
+ pl.col("v_pd"), # branch 2 (period)
1520
+ pl.col("v_p"), # branch 3
1521
+ pl.lit(0.0), # branch 4 (default)
1522
+ ])
1523
+ )
1524
+ )
1525
+ return collect_value_frame(out, key_cols=out_cols, engine=engine)
1526
+
1527
+
1528
+ def emit_pdtGroup(input_dir: Path, solve_data_dir: Path,
1529
+ *, provider) -> None:
1530
+ """Emit ``pdtGroup`` to the Provider."""
1531
+ _emit(provider, "solve_data/pdtGroup.csv",
1532
+ derive_pdtGroup_vectorized(
1533
+ input_dir, solve_data_dir, provider=provider))
1534
+
1535
+
1536
+ # ---------------------------------------------------------------------------
1537
+ # write_pdtCommodity — flextool.mod L1108 (3-branch: pt → pd → p → 0).
1538
+ # ---------------------------------------------------------------------------
1539
+
1540
+ # commodityTimeParam = {price} (flextool_base.dat L134)
1541
+ _COMMODITY_TIME_PARAM: tuple[str, ...] = ("price",)
1542
+
1543
+
1544
+ def derive_pdtCommodity(input_dir: Path, solve_data_dir: Path,
1545
+ *, provider: "object | None" = None) -> pl.DataFrame:
1546
+ """Materialise the ``pdtCommodity`` frame.
1547
+
1548
+ Domain: commodity × commodityTimeParam × dt.
1549
+ Branches: ``pt_commodity`` → ``pd_commodity`` → ``p_commodity`` → 0.
1550
+ """
1551
+ pt = _read_pd_2(input_dir / "pt_commodity.csv", provider=provider)
1552
+ pd_ = _read_pd_2(input_dir / "pd_commodity.csv", provider=provider)
1553
+ p = _read_p_2(input_dir / "p_commodity.csv", provider=provider)
1554
+ commodities = _read_singles(input_dir / "commodity.csv", provider=provider)
1555
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1556
+
1557
+ c_col: list[str] = []
1558
+ p_col: list[str] = []
1559
+ d_col: list[str] = []
1560
+ t_col: list[str] = []
1561
+ v_col: list[str] = []
1562
+ for c in commodities:
1563
+ for param in _COMMODITY_TIME_PARAM:
1564
+ for (d, t) in dt:
1565
+ v = pt.get((c, param, t))
1566
+ if v is None:
1567
+ v = pd_.get((c, param, d))
1568
+ if v is None:
1569
+ v = p.get((c, param), 0.0)
1570
+ c_col.append(c)
1571
+ p_col.append(param)
1572
+ d_col.append(d)
1573
+ t_col.append(t)
1574
+ v_col.append(repr(v))
1575
+ return _utf8_frame({
1576
+ "commodity": c_col, "param": p_col, "period": d_col, "time": t_col,
1577
+ "value": v_col,
1578
+ })
1579
+
1580
+
1581
+ def derive_pdtCommodity_vectorized(input_dir: Path, solve_data_dir: Path,
1582
+ *,
1583
+ provider: "object | None" = None,
1584
+ engine: str = "eager") -> pl.DataFrame:
1585
+ """Vectorized ``pdtCommodity`` derive — byte-parity with the legacy.
1586
+
1587
+ Replaces the per-cell cascade loop in :func:`derive_pdtCommodity`
1588
+ with vectorized polars (left-joins + ``coalesce`` in cascade-priority
1589
+ order), still per roll over the roll's own window. Output is
1590
+ byte-identical to :func:`derive_pdtCommodity`: columns ``commodity,
1591
+ param, period, time, value`` all ``Utf8``, entity-major row order,
1592
+ ``repr(v)`` value cells.
1593
+
1594
+ Cascade (inline 3-branch, time-first):
1595
+ ``pt_commodity`` → ``pd_commodity`` → ``p_commodity`` → ``0.0``.
1596
+ """
1597
+ pt = _read_pd_2(input_dir / "pt_commodity.csv", provider=provider)
1598
+ pd_ = _read_pd_2(input_dir / "pd_commodity.csv", provider=provider)
1599
+ p = _read_p_2(input_dir / "p_commodity.csv", provider=provider)
1600
+ commodities = _read_singles(input_dir / "commodity.csv", provider=provider)
1601
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1602
+
1603
+ key_cols = ["commodity", "param"]
1604
+ out_cols = [*key_cols, "period", "time"]
1605
+
1606
+ # Domain = commodity × _COMMODITY_TIME_PARAM, preserving legacy
1607
+ # iteration order (commodity-major, param from the tuple) and never
1608
+ # ``.unique()``-d.
1609
+ domain = [(c, param) for c in commodities for param in _COMMODITY_TIME_PARAM]
1610
+
1611
+ grid = build_entity_dt_grid(domain, dt, key_cols=key_cols)
1612
+
1613
+ pt_df = lift_dict_to_lookup(
1614
+ pt, ["commodity", "param", "time"], "v_pt")
1615
+ pd_df = lift_dict_to_lookup(
1616
+ pd_, ["commodity", "param", "period"], "v_pd")
1617
+ p_df = lift_dict_to_lookup(
1618
+ p, ["commodity", "param"], "v_p")
1619
+
1620
+ out = (
1621
+ grid
1622
+ .join(pt_df, on=["commodity", "param", "time"], how="left")
1623
+ .join(pd_df, on=["commodity", "param", "period"], how="left")
1624
+ .join(p_df, on=["commodity", "param"], how="left")
1625
+ .with_columns(
1626
+ coalesce_value([
1627
+ pl.col("v_pt"), # branch 1 (time-first)
1628
+ pl.col("v_pd"), # branch 2 (period)
1629
+ pl.col("v_p"), # branch 3
1630
+ pl.lit(0.0), # branch 4 (default)
1631
+ ])
1632
+ )
1633
+ )
1634
+ return collect_value_frame(out, key_cols=out_cols, engine=engine)
1635
+
1636
+
1637
+ def emit_pdtCommodity(input_dir: Path, solve_data_dir: Path,
1638
+ *, provider) -> None:
1639
+ """Emit ``pdtCommodity`` to the Provider."""
1640
+ _emit(provider, "solve_data/pdtCommodity.csv",
1641
+ derive_pdtCommodity_vectorized(
1642
+ input_dir, solve_data_dir, provider=provider))
1643
+
1644
+
1645
+ # ---------------------------------------------------------------------------
1646
+ # write_p_positive_negative_inflow — flextool.mod L1672 / L1675.
1647
+ # ---------------------------------------------------------------------------
1648
+
1649
+
1650
+ def _derive_positive_negative_inflow(
1651
+ input_dir: Path, solve_data_dir: Path,
1652
+ *, provider: "object | None" = None,
1653
+ ) -> tuple[pl.DataFrame, pl.DataFrame]:
1654
+ """Build both ``p_positive_inflow`` and ``p_negative_inflow`` frames
1655
+ from a single read of ``pdtNodeInflow.csv``."""
1656
+ nodes = _read_singles(input_dir / "node.csv", provider=provider)
1657
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1658
+ inflow_method_pairs = frozenset(
1659
+ _read_pairs(solve_data_dir / "node__inflow_method.csv",
1660
+ provider=provider)
1661
+ )
1662
+ no_inflow_nodes = frozenset(
1663
+ n for n in nodes if (n, "no_inflow") in inflow_method_pairs
1664
+ )
1665
+
1666
+ pdt_inflow: dict[tuple[str, str, str], float] = {}
1667
+ pdtni_path = solve_data_dir / "pdtNodeInflow.csv"
1668
+ pdtni_df = provider.get(_provider_key(pdtni_path))
1669
+ if pdtni_df is not None:
1670
+ for row in pdtni_df.iter_rows():
1671
+ if len(row) < 4:
1672
+ continue
1673
+ c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
1674
+ if c0 and c1 and c2:
1675
+ try:
1676
+ pdt_inflow[(c0, c1, c2)] = float(row[3])
1677
+ except (ValueError, TypeError):
1678
+ continue
1679
+
1680
+ pos_n: list[str] = []
1681
+ pos_d: list[str] = []
1682
+ pos_t: list[str] = []
1683
+ pos_v: list[str] = []
1684
+ for n in nodes:
1685
+ if n in no_inflow_nodes:
1686
+ continue
1687
+ for (d, t) in dt:
1688
+ v = pdt_inflow.get((n, d, t), 0.0)
1689
+ pos_n.append(n)
1690
+ pos_d.append(d)
1691
+ pos_t.append(t)
1692
+ pos_v.append(repr(v if v >= 0 else 0.0))
1693
+ pos_frame = _utf8_frame({
1694
+ "node": pos_n, "period": pos_d, "time": pos_t, "value": pos_v,
1695
+ })
1696
+
1697
+ neg_n: list[str] = []
1698
+ neg_d: list[str] = []
1699
+ neg_t: list[str] = []
1700
+ neg_v: list[str] = []
1701
+ for n in nodes:
1702
+ for (d, t) in dt:
1703
+ if n in no_inflow_nodes:
1704
+ cell = "0.0"
1705
+ else:
1706
+ v = pdt_inflow.get((n, d, t), 0.0)
1707
+ cell = repr(v if v < 0 else 0.0)
1708
+ neg_n.append(n)
1709
+ neg_d.append(d)
1710
+ neg_t.append(t)
1711
+ neg_v.append(cell)
1712
+ neg_frame = _utf8_frame({
1713
+ "node": neg_n, "period": neg_d, "time": neg_t, "value": neg_v,
1714
+ })
1715
+
1716
+ return pos_frame, neg_frame
1717
+
1718
+
1719
+ def emit_p_positive_negative_inflow(
1720
+ input_dir: Path, solve_data_dir: Path,
1721
+ *, provider,
1722
+ ) -> None:
1723
+ """Emit ``p_positive_negative_inflow`` to the Provider."""
1724
+ pos, neg = _derive_positive_negative_inflow(
1725
+ input_dir, solve_data_dir, provider=provider,
1726
+ )
1727
+ _emit(provider, "solve_data/p_positive_inflow.csv", pos)
1728
+ _emit(provider, "solve_data/p_negative_inflow.csv", neg)
1729
+
1730
+
1731
+ # ---------------------------------------------------------------------------
1732
+ # Phase 1 follow-up 5 — entity_period_calc_params: varCost + cap_reduction +
1733
+ # ed_period_params + pssdt_varCost filters.
1734
+ # ---------------------------------------------------------------------------
1735
+
1736
+
1737
+ def _read_triples(path: Path,
1738
+ *, provider: "object | None" = None,
1739
+ ) -> list[tuple[str, str, str]]:
1740
+ df = provider.get(_provider_key(path))
1741
+ if df is None:
1742
+ return []
1743
+ out: list[tuple[str, str, str]] = []
1744
+ for row in df.iter_rows():
1745
+ if len(row) < 3:
1746
+ continue
1747
+ c0, c1, c2 = _cell_str(row[0]), _cell_str(row[1]), _cell_str(row[2])
1748
+ if c0 and c1 and c2:
1749
+ out.append((c0, c1, c2))
1750
+ return out
1751
+
1752
+
1753
+ def _read_pdt_at_param(path: Path, param_col: int, param_value: str,
1754
+ key_cols: tuple[int, ...],
1755
+ val_col: int,
1756
+ *, provider: "object | None" = None,
1757
+ ) -> dict[tuple, float]:
1758
+ """Read a long-format pdtX CSV, filter rows where col[param_col] ==
1759
+ param_value, return dict[tuple(_cell_str(row[c]) for c in key_cols)] =
1760
+ float(row[val_col]).
1761
+ """
1762
+ out: dict[tuple, float] = {}
1763
+ df = provider.get(_provider_key(path))
1764
+ if df is None:
1765
+ return out
1766
+ for row in df.iter_rows():
1767
+ if (len(row) > max(param_col, val_col, *key_cols)
1768
+ and _cell_str(row[param_col]) == param_value):
1769
+ try:
1770
+ out[tuple(_cell_str(row[c]) for c in key_cols)] = float(row[val_col])
1771
+ except (ValueError, TypeError):
1772
+ continue
1773
+ return out
1774
+
1775
+
1776
+ # ---- write_pdtProcess__source__sink__dt_varCost_pair (mod L1493, L1502) ----
1777
+
1778
+ def _derive_varCost_pair(
1779
+ input_dir: Path, solve_data_dir: Path,
1780
+ *, provider: "object | None" = None,
1781
+ ) -> tuple[pl.DataFrame, pl.DataFrame]:
1782
+ """Build both ``pdtProcess__source__sink__dt_varCost`` frames in one
1783
+ pass — the basic and ``_alwaysProcess`` variants share the same
1784
+ OOC dictionaries / proc_src / proc_snk lookups.
1785
+ """
1786
+ pdt = _read_pdt_at_param(
1787
+ solve_data_dir / "pdtProcess.csv",
1788
+ param_col=1, param_value="other_operational_cost",
1789
+ key_cols=(0, 2, 3), val_col=4,
1790
+ provider=provider,
1791
+ ) # (process, period, time) → value
1792
+ pdt_src = _read_pdt_at_param(
1793
+ solve_data_dir / "pdtProcess_source.csv",
1794
+ param_col=2, param_value="other_operational_cost",
1795
+ key_cols=(0, 1, 3, 4), val_col=5,
1796
+ provider=provider,
1797
+ ) # (process, source, period, time) → value
1798
+ pdt_snk = _read_pdt_at_param(
1799
+ solve_data_dir / "pdtProcess_sink.csv",
1800
+ param_col=2, param_value="other_operational_cost",
1801
+ key_cols=(0, 1, 3, 4), val_col=5,
1802
+ provider=provider,
1803
+ ) # (process, sink, period, time) → value
1804
+ proc_src = frozenset(
1805
+ _read_pairs(input_dir / "process__source.csv", provider=provider)
1806
+ )
1807
+ proc_snk = frozenset(
1808
+ _read_pairs(input_dir / "process__sink.csv", provider=provider)
1809
+ )
1810
+ pss = _read_triples(
1811
+ solve_data_dir / "process_source_sink.csv", provider=provider,
1812
+ )
1813
+ pss_always = _read_triples(
1814
+ solve_data_dir / "process_source_sink_alwaysProcess.csv",
1815
+ provider=provider,
1816
+ )
1817
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1818
+
1819
+ def _build(rows_iter: list[tuple[str, str, str]],
1820
+ always: bool) -> pl.DataFrame:
1821
+ p_col: list[str] = []
1822
+ s_col: list[str] = []
1823
+ k_col: list[str] = []
1824
+ d_col: list[str] = []
1825
+ t_col: list[str] = []
1826
+ v_col: list[str] = []
1827
+ for (p, src, snk) in rows_iter:
1828
+ for (d, t) in dt:
1829
+ v = 0.0
1830
+ if (p, src) in proc_src:
1831
+ v += pdt_src.get((p, src, d, t), 0.0)
1832
+ if (p, snk) in proc_snk:
1833
+ v += pdt_snk.get((p, snk, d, t), 0.0)
1834
+ if always:
1835
+ if (p, snk) in proc_snk or (p, snk) in proc_src:
1836
+ v += pdt.get((p, d, t), 0.0)
1837
+ else:
1838
+ v += pdt.get((p, d, t), 0.0)
1839
+ p_col.append(p)
1840
+ s_col.append(src)
1841
+ k_col.append(snk)
1842
+ d_col.append(d)
1843
+ t_col.append(t)
1844
+ v_col.append(repr(v))
1845
+ return _utf8_frame({
1846
+ "process": p_col, "source": s_col, "sink": k_col,
1847
+ "period": d_col, "time": t_col, "value": v_col,
1848
+ })
1849
+
1850
+ return _build(pss, always=False), _build(pss_always, always=True)
1851
+
1852
+
1853
+ def _derive_varCost_pair_vectorized(
1854
+ input_dir: Path, solve_data_dir: Path,
1855
+ *, provider: "object | None" = None, engine: str = "eager",
1856
+ ) -> tuple[pl.DataFrame, pl.DataFrame]:
1857
+ """Vectorized twin of :func:`_derive_varCost_pair` (design §5 / S7).
1858
+
1859
+ Both the ``basic`` and ``_alwaysProcess`` variants are a
1860
+ membership-GATED SUM of three terms — NOT a coalesce cascade:
1861
+
1862
+ v = 0.0
1863
+ if (p, src) ∈ proc_src: v += pdt_src[(p, src, d, t)]
1864
+ if (p, snk) ∈ proc_snk: v += pdt_snk[(p, snk, d, t)]
1865
+ basic: v += pdt[(p, d, t)] # unconditional
1866
+ always: if (p, snk) ∈ proc_snk or (p, snk) ∈ proc_src:
1867
+ v += pdt[(p, d, t)]
1868
+
1869
+ Reproduced per cell as ``build_entity_dt_grid`` ⨯ membership/value
1870
+ left-joins ⨯ ``fill_null(0.0)`` ⨯ a fixed-order ``src + snk + pdt``
1871
+ add, then a SIGNED-ZERO normalization (critique D1): the legacy
1872
+ accumulator seeds ``v = 0.0`` so ``0.0 + (-0.0)`` already kills the
1873
+ sign on the first add → legacy never emits ``"-0.0"``; ``pl.lit(0.0)
1874
+ + sum`` does NOT reproduce this (polars leaves ``-0.0`` as ``-0.0``
1875
+ across rows), so the value is wrapped
1876
+ ``when(value_f == 0.0).then(0.0)`` to canonicalize ``±0.0 → +0.0``.
1877
+
1878
+ The two variants have DIFFERENT domains (``pss`` vs ``pss_always``)
1879
+ and a DIFFERENT pdt gate, so they are assembled separately and
1880
+ returned as a pair (mirroring the legacy shared producer).
1881
+ """
1882
+ pdt = _read_pdt_at_param(
1883
+ solve_data_dir / "pdtProcess.csv",
1884
+ param_col=1, param_value="other_operational_cost",
1885
+ key_cols=(0, 2, 3), val_col=4,
1886
+ provider=provider,
1887
+ ) # (process, period, time) → value
1888
+ pdt_src = _read_pdt_at_param(
1889
+ solve_data_dir / "pdtProcess_source.csv",
1890
+ param_col=2, param_value="other_operational_cost",
1891
+ key_cols=(0, 1, 3, 4), val_col=5,
1892
+ provider=provider,
1893
+ ) # (process, source, period, time) → value
1894
+ pdt_snk = _read_pdt_at_param(
1895
+ solve_data_dir / "pdtProcess_sink.csv",
1896
+ param_col=2, param_value="other_operational_cost",
1897
+ key_cols=(0, 1, 3, 4), val_col=5,
1898
+ provider=provider,
1899
+ ) # (process, sink, period, time) → value
1900
+ proc_src = frozenset(
1901
+ _read_pairs(input_dir / "process__source.csv", provider=provider)
1902
+ )
1903
+ proc_snk = frozenset(
1904
+ _read_pairs(input_dir / "process__sink.csv", provider=provider)
1905
+ )
1906
+ pss = _read_triples(
1907
+ solve_data_dir / "process_source_sink.csv", provider=provider,
1908
+ )
1909
+ pss_always = _read_triples(
1910
+ solve_data_dir / "process_source_sink_alwaysProcess.csv",
1911
+ provider=provider,
1912
+ )
1913
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
1914
+
1915
+ # Lift the three value dicts (last-wins-deduped) to lookup frames.
1916
+ src_df = lift_dict_to_lookup(
1917
+ pdt_src, ["process", "source", "period", "time"], "v_src")
1918
+ snk_df = lift_dict_to_lookup(
1919
+ pdt_snk, ["process", "sink", "period", "time"], "v_snk")
1920
+ pdt_df = lift_dict_to_lookup(
1921
+ pdt, ["process", "period", "time"], "v_pdt")
1922
+
1923
+ # Membership frames built from the frozensets with EXPLICIT schemas
1924
+ # (critique D7): an empty frozenset must NOT yield an all-Null frame.
1925
+ ps_df = pl.DataFrame(
1926
+ {
1927
+ "process": [a for (a, _b) in proc_src],
1928
+ "source": [b for (_a, b) in proc_src],
1929
+ "__in_src": [True] * len(proc_src),
1930
+ },
1931
+ schema={"process": pl.Utf8, "source": pl.Utf8,
1932
+ "__in_src": pl.Boolean},
1933
+ )
1934
+ pk_df = pl.DataFrame(
1935
+ {
1936
+ "process": [a for (a, _b) in proc_snk],
1937
+ "sink": [b for (_a, b) in proc_snk],
1938
+ "__in_snk": [True] * len(proc_snk),
1939
+ },
1940
+ schema={"process": pl.Utf8, "sink": pl.Utf8,
1941
+ "__in_snk": pl.Boolean},
1942
+ )
1943
+ # proc_src arcs renamed source→sink: expresses ``(p, snk) ∈ proc_src``
1944
+ # for the always-variant pdt gate (critique D2 — both gate terms key
1945
+ # on ``snk``, including against proc_src).
1946
+ ps_snk_df = pl.DataFrame(
1947
+ {
1948
+ "process": [a for (a, _b) in proc_src],
1949
+ "sink": [b for (_a, b) in proc_src],
1950
+ "__snk_in_src": [True] * len(proc_src),
1951
+ },
1952
+ schema={"process": pl.Utf8, "sink": pl.Utf8,
1953
+ "__snk_in_src": pl.Boolean},
1954
+ )
1955
+
1956
+ def _build_vec(domain: list[tuple[str, str, str]],
1957
+ always: bool) -> pl.DataFrame:
1958
+ grid = build_entity_dt_grid(
1959
+ domain, dt, key_cols=["process", "source", "sink"])
1960
+ out = (
1961
+ grid
1962
+ .join(ps_df, on=["process", "source"], how="left")
1963
+ .join(src_df, on=["process", "source", "period", "time"],
1964
+ how="left")
1965
+ .join(pk_df, on=["process", "sink"], how="left")
1966
+ .join(snk_df, on=["process", "sink", "period", "time"],
1967
+ how="left")
1968
+ .join(pdt_df, on=["process", "period", "time"], how="left")
1969
+ )
1970
+ t_src = (
1971
+ pl.when(pl.col("__in_src").fill_null(False))
1972
+ .then(pl.col("v_src").fill_null(0.0))
1973
+ .otherwise(0.0)
1974
+ )
1975
+ t_snk = (
1976
+ pl.when(pl.col("__in_snk").fill_null(False))
1977
+ .then(pl.col("v_snk").fill_null(0.0))
1978
+ .otherwise(0.0)
1979
+ )
1980
+ if always:
1981
+ out = out.join(ps_snk_df, on=["process", "sink"], how="left")
1982
+ t_pdt = (
1983
+ pl.when(
1984
+ pl.col("__in_snk").fill_null(False)
1985
+ | pl.col("__snk_in_src").fill_null(False)
1986
+ )
1987
+ .then(pl.col("v_pdt").fill_null(0.0))
1988
+ .otherwise(0.0)
1989
+ )
1990
+ else:
1991
+ t_pdt = pl.col("v_pdt").fill_null(0.0)
1992
+ # Fixed add order src + snk + pdt, then normalize signed zero so
1993
+ # ``±0.0`` renders ``"0.0"`` (legacy never emits ``"-0.0"``).
1994
+ value_raw = t_src + t_snk + t_pdt
1995
+ out = out.with_columns(
1996
+ pl.when(value_raw == 0.0)
1997
+ .then(pl.lit(0.0))
1998
+ .otherwise(value_raw)
1999
+ .alias("value_f")
2000
+ )
2001
+ return collect_value_frame(
2002
+ out,
2003
+ key_cols=["process", "source", "sink", "period", "time"],
2004
+ value_f_col="value_f", engine=engine,
2005
+ )
2006
+
2007
+ return _build_vec(pss, always=False), _build_vec(pss_always, always=True)
2008
+
2009
+
2010
+ def emit_pdtProcess__source__sink__dt_varCost_pair(
2011
+ input_dir: Path, solve_data_dir: Path,
2012
+ *, provider,
2013
+ ) -> None:
2014
+ """Emit ``pdtProcess__source__sink__dt_varCost_pair`` to the Provider."""
2015
+ basic, _always = _derive_varCost_pair_vectorized(
2016
+ input_dir, solve_data_dir, provider=provider,
2017
+ )
2018
+ _emit(provider, "solve_data/pdtProcess__source__sink__dt_varCost.csv",
2019
+ basic)
2020
+
2021
+
2022
+ # ---- write_pssdt_varCost_filters (mod L1498-1501) -------------------------
2023
+
2024
+ def _filter_rows_to_frame(
2025
+ rows: list[tuple[str, str, str, str, str]],
2026
+ ) -> pl.DataFrame:
2027
+ """5-column key-only Utf8 frame for the pssdt_varCost filter outputs."""
2028
+ return _utf8_frame({
2029
+ "process": [r[0] for r in rows],
2030
+ "source": [r[1] for r in rows],
2031
+ "sink": [r[2] for r in rows],
2032
+ "period": [r[3] for r in rows],
2033
+ "time": [r[4] for r in rows],
2034
+ })
2035
+
2036
+
2037
+ def _derive_pssdt_varCost_filters(
2038
+ input_dir: Path, solve_data_dir: Path,
2039
+ *, provider: "object | None" = None,
2040
+ ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame, pl.DataFrame]:
2041
+ """Build all four ``pssdt_varCost_*`` filter frames in one pass."""
2042
+ pdt = _read_pdt_at_param(
2043
+ solve_data_dir / "pdtProcess.csv",
2044
+ param_col=1, param_value="other_operational_cost",
2045
+ key_cols=(0, 2, 3), val_col=4,
2046
+ provider=provider,
2047
+ )
2048
+ pdt_src = _read_pdt_at_param(
2049
+ solve_data_dir / "pdtProcess_source.csv",
2050
+ param_col=2, param_value="other_operational_cost",
2051
+ key_cols=(0, 1, 3, 4), val_col=5,
2052
+ provider=provider,
2053
+ )
2054
+ pdt_snk = _read_pdt_at_param(
2055
+ solve_data_dir / "pdtProcess_sink.csv",
2056
+ param_col=2, param_value="other_operational_cost",
2057
+ key_cols=(0, 1, 3, 4), val_col=5,
2058
+ provider=provider,
2059
+ )
2060
+ varcost: dict[tuple[str, str, str, str, str], float] = {}
2061
+ vp = solve_data_dir / "pdtProcess__source__sink__dt_varCost.csv"
2062
+ vp_df = provider.get(_provider_key(vp))
2063
+ if vp_df is not None:
2064
+ for row in vp_df.iter_rows():
2065
+ if len(row) < 6:
2066
+ continue
2067
+ c = [_cell_str(row[i]) for i in range(5)]
2068
+ if all(c):
2069
+ try:
2070
+ varcost[(c[0], c[1], c[2], c[3], c[4])] = float(row[5])
2071
+ except (ValueError, TypeError):
2072
+ continue
2073
+
2074
+ proc_src = frozenset(
2075
+ _read_pairs(input_dir / "process__source.csv", provider=provider)
2076
+ )
2077
+ proc_snk = frozenset(
2078
+ _read_pairs(input_dir / "process__sink.csv", provider=provider)
2079
+ )
2080
+ pss_noEff = _read_triples(
2081
+ solve_data_dir / "process_source_sink_noEff.csv", provider=provider,
2082
+ )
2083
+ pss_eff = _read_triples(
2084
+ solve_data_dir / "process_source_sink_eff.csv", provider=provider,
2085
+ )
2086
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
2087
+
2088
+ no_eff: list[tuple[str, str, str, str, str]] = []
2089
+ for (p, src, snk) in pss_noEff:
2090
+ for (d, t) in dt:
2091
+ if varcost.get((p, src, snk, d, t), 0.0):
2092
+ no_eff.append((p, src, snk, d, t))
2093
+
2094
+ eff_src: list[tuple[str, str, str, str, str]] = []
2095
+ for (p, src, snk) in pss_eff:
2096
+ for (d, t) in dt:
2097
+ if (p, src) in proc_src and pdt_src.get((p, src, d, t), 0.0):
2098
+ eff_src.append((p, src, snk, d, t))
2099
+
2100
+ eff_snk: list[tuple[str, str, str, str, str]] = []
2101
+ for (p, src, snk) in pss_eff:
2102
+ for (d, t) in dt:
2103
+ if (p, snk) in proc_snk and pdt_snk.get((p, snk, d, t), 0.0):
2104
+ eff_snk.append((p, src, snk, d, t))
2105
+
2106
+ eff_conn: list[tuple[str, str, str, str, str]] = []
2107
+ for (p, src, snk) in pss_eff:
2108
+ for (d, t) in dt:
2109
+ if pdt.get((p, d, t), 0.0):
2110
+ eff_conn.append((p, src, snk, d, t))
2111
+
2112
+ return (
2113
+ _filter_rows_to_frame(no_eff),
2114
+ _filter_rows_to_frame(eff_src),
2115
+ _filter_rows_to_frame(eff_snk),
2116
+ _filter_rows_to_frame(eff_conn),
2117
+ )
2118
+
2119
+
2120
+ def _derive_pssdt_varCost_filters_vectorized(
2121
+ input_dir: Path, solve_data_dir: Path,
2122
+ *, provider: "object | None" = None, engine: str = "eager",
2123
+ ) -> tuple[pl.DataFrame, pl.DataFrame, pl.DataFrame, pl.DataFrame]:
2124
+ """Vectorized twin of :func:`_derive_pssdt_varCost_filters`.
2125
+
2126
+ Each of the four outputs is a KEY-only coordinate predicate (no value
2127
+ column): for a ``(p, src, snk) × (d, t)`` grid keep the cell iff a
2128
+ membership-gated ``value ≠ 0`` predicate holds. The ``≠ 0`` test
2129
+ runs on the PARSED-FLOAT lifted value (NOT the rendered Utf8) so it
2130
+ matches the legacy truthiness exactly (``-0.0`` excluded, ``NaN``
2131
+ kept). Output: ``process, source, sink, period, time`` — all Utf8,
2132
+ entity-major order (``__eo, __to``), the order keys dropped.
2133
+ """
2134
+ pdt = _read_pdt_at_param(
2135
+ solve_data_dir / "pdtProcess.csv",
2136
+ param_col=1, param_value="other_operational_cost",
2137
+ key_cols=(0, 2, 3), val_col=4,
2138
+ provider=provider,
2139
+ )
2140
+ pdt_src = _read_pdt_at_param(
2141
+ solve_data_dir / "pdtProcess_source.csv",
2142
+ param_col=2, param_value="other_operational_cost",
2143
+ key_cols=(0, 1, 3, 4), val_col=5,
2144
+ provider=provider,
2145
+ )
2146
+ pdt_snk = _read_pdt_at_param(
2147
+ solve_data_dir / "pdtProcess_sink.csv",
2148
+ param_col=2, param_value="other_operational_cost",
2149
+ key_cols=(0, 1, 3, 4), val_col=5,
2150
+ provider=provider,
2151
+ )
2152
+ varcost: dict[tuple[str, str, str, str, str], float] = {}
2153
+ vp = solve_data_dir / "pdtProcess__source__sink__dt_varCost.csv"
2154
+ vp_df = provider.get(_provider_key(vp))
2155
+ if vp_df is not None:
2156
+ for row in vp_df.iter_rows():
2157
+ if len(row) < 6:
2158
+ continue
2159
+ c = [_cell_str(row[i]) for i in range(5)]
2160
+ if all(c):
2161
+ try:
2162
+ varcost[(c[0], c[1], c[2], c[3], c[4])] = float(row[5])
2163
+ except (ValueError, TypeError):
2164
+ continue
2165
+
2166
+ proc_src = frozenset(
2167
+ _read_pairs(input_dir / "process__source.csv", provider=provider)
2168
+ )
2169
+ proc_snk = frozenset(
2170
+ _read_pairs(input_dir / "process__sink.csv", provider=provider)
2171
+ )
2172
+ pss_noEff = _read_triples(
2173
+ solve_data_dir / "process_source_sink_noEff.csv", provider=provider,
2174
+ )
2175
+ pss_eff = _read_triples(
2176
+ solve_data_dir / "process_source_sink_eff.csv", provider=provider,
2177
+ )
2178
+ dt = _read_pairs(solve_data_dir / "steps_in_use.csv", provider=provider)
2179
+
2180
+ key5 = ["process", "source", "sink", "period", "time"]
2181
+
2182
+ # Lift predicate value sources.
2183
+ vc_df = lift_dict_to_lookup(
2184
+ varcost, ["process", "source", "sink", "period", "time"], "v_vc")
2185
+ src_df = lift_dict_to_lookup(
2186
+ pdt_src, ["process", "source", "period", "time"], "v_src")
2187
+ snk_df = lift_dict_to_lookup(
2188
+ pdt_snk, ["process", "sink", "period", "time"], "v_snk")
2189
+ pdt_df = lift_dict_to_lookup(
2190
+ pdt, ["process", "period", "time"], "v_pdt")
2191
+
2192
+ # Membership frames (explicit schemas — critique D7).
2193
+ ps_df = pl.DataFrame(
2194
+ {
2195
+ "process": [a for (a, _b) in proc_src],
2196
+ "source": [b for (_a, b) in proc_src],
2197
+ "__in_src": [True] * len(proc_src),
2198
+ },
2199
+ schema={"process": pl.Utf8, "source": pl.Utf8,
2200
+ "__in_src": pl.Boolean},
2201
+ )
2202
+ pk_df = pl.DataFrame(
2203
+ {
2204
+ "process": [a for (a, _b) in proc_snk],
2205
+ "sink": [b for (_a, b) in proc_snk],
2206
+ "__in_snk": [True] * len(proc_snk),
2207
+ },
2208
+ schema={"process": pl.Utf8, "sink": pl.Utf8,
2209
+ "__in_snk": pl.Boolean},
2210
+ )
2211
+
2212
+ def _finish(df: pl.DataFrame, predicate: pl.Expr) -> pl.DataFrame:
2213
+ out = (
2214
+ df.filter(predicate)
2215
+ .sort(["__eo", "__to"])
2216
+ .select(key5)
2217
+ )
2218
+ return out.with_columns(
2219
+ [pl.col(c).cast(pl.Utf8) for c in key5]
2220
+ )
2221
+
2222
+ # 1. noEff: varcost[(p, src, snk, d, t)] ≠ 0.
2223
+ g_no = build_entity_dt_grid(
2224
+ pss_noEff, dt, key_cols=["process", "source", "sink"])
2225
+ g_no = g_no.join(vc_df, on=key5, how="left")
2226
+ no_eff = _finish(g_no, pl.col("v_vc").fill_null(0.0) != 0.0)
2227
+
2228
+ # 2. eff_unit_source: (p, src) ∈ proc_src AND pdt_src ≠ 0.
2229
+ g_src = build_entity_dt_grid(
2230
+ pss_eff, dt, key_cols=["process", "source", "sink"])
2231
+ g_src = (
2232
+ g_src
2233
+ .join(ps_df, on=["process", "source"], how="left")
2234
+ .join(src_df, on=["process", "source", "period", "time"], how="left")
2235
+ )
2236
+ eff_src = _finish(
2237
+ g_src,
2238
+ pl.col("__in_src").fill_null(False)
2239
+ & (pl.col("v_src").fill_null(0.0) != 0.0),
2240
+ )
2241
+
2242
+ # 3. eff_unit_sink: (p, snk) ∈ proc_snk AND pdt_snk ≠ 0.
2243
+ g_snk = build_entity_dt_grid(
2244
+ pss_eff, dt, key_cols=["process", "source", "sink"])
2245
+ g_snk = (
2246
+ g_snk
2247
+ .join(pk_df, on=["process", "sink"], how="left")
2248
+ .join(snk_df, on=["process", "sink", "period", "time"], how="left")
2249
+ )
2250
+ eff_snk = _finish(
2251
+ g_snk,
2252
+ pl.col("__in_snk").fill_null(False)
2253
+ & (pl.col("v_snk").fill_null(0.0) != 0.0),
2254
+ )
2255
+
2256
+ # 4. eff_connection: pdt[(p, d, t)] ≠ 0 (NO membership gate).
2257
+ g_conn = build_entity_dt_grid(
2258
+ pss_eff, dt, key_cols=["process", "source", "sink"])
2259
+ g_conn = g_conn.join(pdt_df, on=["process", "period", "time"], how="left")
2260
+ eff_conn = _finish(g_conn, pl.col("v_pdt").fill_null(0.0) != 0.0)
2261
+
2262
+ return (no_eff, eff_src, eff_snk, eff_conn)
2263
+
2264
+
2265
+ def emit_pssdt_varCost_filters(
2266
+ input_dir: Path, solve_data_dir: Path,
2267
+ *, provider,
2268
+ ) -> None:
2269
+ """Emit ``pssdt_varCost_filters`` to the Provider."""
2270
+ no_eff, eff_src, eff_snk, eff_conn = (
2271
+ _derive_pssdt_varCost_filters_vectorized(
2272
+ input_dir, solve_data_dir, provider=provider,
2273
+ )
2274
+ )
2275
+ _emit(provider, "solve_data/pssdt_varCost_noEff.csv", no_eff)
2276
+ _emit(provider, "solve_data/pssdt_varCost_eff_unit_source.csv", eff_src)
2277
+ _emit(provider, "solve_data/pssdt_varCost_eff_unit_sink.csv", eff_snk)
2278
+ _emit(provider, "solve_data/pssdt_varCost_eff_connection.csv", eff_conn)
2279
+
2280
+
2281
+ # ---- write_cap_reduction_params (mod L1637-1663) --------------------------
2282
+
2283
+ def _read_p_side_3(path: Path,
2284
+ *, provider: "object | None" = None,
2285
+ ) -> dict[tuple[str, str, str], float]:
2286
+ """Per-side 4-col CSV ``(k1, k2, k3, value)`` → ``(k1, k2, k3) → v``."""
2287
+ out: dict[tuple[str, str, str], float] = {}
2288
+ df = provider.get(_provider_key(path))
2289
+ if df is None:
2290
+ return out
2291
+ for row in df.iter_rows():
2292
+ if len(row) < 4:
2293
+ continue
2294
+ c = [_cell_str(row[i]) for i in range(3)]
2295
+ if all(c):
2296
+ try:
2297
+ out[(c[0], c[1], c[2])] = float(row[3])
2298
+ except (ValueError, TypeError):
2299
+ continue
2300
+ return out
2301
+
2302
+
2303
+ # ---- write_ed_period_params (mod L1252-1255 family, ed_*_period) ----------
2304
+
2305
+ def _read_ed_pairs(path: Path,
2306
+ *, provider: "object | None" = None,
2307
+ ) -> list[tuple[str, str]]:
2308
+ df = provider.get(_provider_key(path))
2309
+ if df is None:
2310
+ return []
2311
+ out: list[tuple[str, str]] = []
2312
+ for row in df.iter_rows():
2313
+ if len(row) < 2:
2314
+ continue
2315
+ c0, c1 = _cell_str(row[0]), _cell_str(row[1])
2316
+ if c0 and c1:
2317
+ out.append((c0, c1))
2318
+ return out
2319
+
2320
+
2321
+ def _ed_period_compute(
2322
+ src_pairs: list[tuple[str, str]],
2323
+ mod_param: str,
2324
+ pp, # PdLookup
2325
+ pn, # PdLookup
2326
+ process_set: frozenset[str],
2327
+ node_set: frozenset[str],
2328
+ ) -> pl.DataFrame:
2329
+ """One ed_period_param frame — 3-col Utf8 ``(entity, period, value)``."""
2330
+ e_col: list[str] = []
2331
+ d_col: list[str] = []
2332
+ v_col: list[str] = []
2333
+ for e, d in src_pairs:
2334
+ if e in process_set:
2335
+ v = pp.get(e, mod_param, d)
2336
+ elif e in node_set:
2337
+ v = pn.get(e, mod_param, d)
2338
+ else:
2339
+ v = 0.0
2340
+ e_col.append(e)
2341
+ d_col.append(d)
2342
+ v_col.append(repr(v))
2343
+ return _utf8_frame({"entity": e_col, "period": d_col, "value": v_col})
2344
+
2345
+
2346
+ _ED_PERIOD_PARAM_SPECS: tuple[tuple[str, str, str], ...] = (
2347
+ # (basename, src-pair-tag ['invest'|'divest'], mod_param)
2348
+ ("ed_invest_max_period.csv", "invest", "invest_max_period"),
2349
+ ("ed_invest_min_period.csv", "invest", "invest_min_period"),
2350
+ ("ed_divest_max_period.csv", "divest", "retire_max_period"),
2351
+ ("ed_divest_min_period.csv", "divest", "retire_min_period"),
2352
+ ("ed_cumulative_max_capacity.csv", "invest", "cumulative_max_capacity"),
2353
+ ("ed_cumulative_min_capacity.csv", "invest", "cumulative_min_capacity"),
2354
+ )
2355
+
2356
+
2357
+ def _ed_period_inputs(input_dir: Path, solve_data_dir: Path,
2358
+ *, provider: "object | None" = None):
2359
+ from flextool.engine_polars._pdt_lookup import PdLookup
2360
+ pp = PdLookup(
2361
+ pd_csv=input_dir / "pd_process.csv",
2362
+ p_csv=input_dir / "p_process.csv",
2363
+ period_branch_csv=solve_data_dir / "period__branch.csv",
2364
+ provider=provider,
2365
+ )
2366
+ pn = PdLookup(
2367
+ pd_csv=input_dir / "pd_node.csv",
2368
+ p_csv=input_dir / "p_node.csv",
2369
+ period_branch_csv=solve_data_dir / "period__branch.csv",
2370
+ provider=provider,
2371
+ )
2372
+ process_set = frozenset(
2373
+ _read_singles(input_dir / "process.csv", provider=provider)
2374
+ )
2375
+ node_set = frozenset(
2376
+ _read_singles(input_dir / "node.csv", provider=provider)
2377
+ )
2378
+ ed_invest_pairs = _read_ed_pairs(
2379
+ solve_data_dir / "ed_invest.csv", provider=provider,
2380
+ )
2381
+ ed_divest_pairs = _read_ed_pairs(
2382
+ solve_data_dir / "ed_divest.csv", provider=provider,
2383
+ )
2384
+ return pp, pn, process_set, node_set, ed_invest_pairs, ed_divest_pairs
2385
+
2386
+
2387
+ def emit_ed_period_params(
2388
+ input_dir: Path, solve_data_dir: Path,
2389
+ *, provider,
2390
+ ) -> None:
2391
+ """Emit ``ed_period_params`` to the Provider."""
2392
+ pp, pn, ps, ns, inv, div = _ed_period_inputs(
2393
+ input_dir, solve_data_dir, provider=provider,
2394
+ )
2395
+ pair_for = {"invest": inv, "divest": div}
2396
+ for fname, tag, mod_param in _ED_PERIOD_PARAM_SPECS:
2397
+ frame = _ed_period_compute(pair_for[tag], mod_param, pp, pn, ps, ns)
2398
+ _emit(provider, f"solve_data/{fname}", frame)