flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2056 @@
1
+ """Γ.2 Projection Param helpers.
2
+
3
+ Each function in this module takes an :class:`InputSource` and returns
4
+ the corresponding polar_high frame (or ``None`` if the projection is empty
5
+ on the supplied scenario). A Projection is defined (per the
6
+ ``audit/db_direct_param_map.md`` spec §1) as a set-algebra layer over
7
+ ``source.entities(...)`` and ``source.parameter(...)`` results — a
8
+ filter, a distinct projection, or a structural join. No per-row
9
+ algorithms, no per-period state cascades.
10
+
11
+ Lazy-evaluation pattern
12
+ -----------------------
13
+ ``InputSource.parameter()`` and ``.entities()`` already collect at the
14
+ source boundary. Helpers compose those results via :class:`pl.LazyFrame`
15
+ chains and ``.collect()`` once at the boundary — so polars can fuse
16
+ projections / filter pushdowns across joins.
17
+
18
+ The helpers are intentionally tiny (5–25 LOC each). Common patterns:
19
+
20
+ 1. **Filter on a method discriminator.** Pull the entity-level method
21
+ parameter, filter to a specific value, project the entity column.
22
+ Example: ``process_minload`` is the units with non-zero ``min_load``.
23
+
24
+ 2. **Distinct projection.** Pull a Map / 1d_map parameter, take
25
+ ``.unique()`` over the entity dimension(s). Example:
26
+ ``commodity__tier_ann`` is the distinct ``(commodity, tier)`` pairs
27
+ where the commodity carries an annual ladder.
28
+
29
+ 3. **Structural join.** Inner-join two membership sets. Example:
30
+ ``flow_from_commodity_eff`` is ``pss_eff ⋈ commodity__node`` on
31
+ ``source = node``.
32
+
33
+ 4. **Union.** Concatenate two membership sets with column alignment.
34
+ Example: ``group_entity`` unions ``group__node``, ``group__unit``,
35
+ ``group__connection`` into one ``(group, entity)`` frame.
36
+
37
+ Block-aware filters (``flow_to_n``, ``flow_from_n``, etc.) are NOT
38
+ implemented here — they depend on multi-resolution preprocessed
39
+ auxiliary tables (``process_side_block``, ``entity_block``,
40
+ ``overlap_set``) that are themselves Derived (Γ.3) artefacts.
41
+
42
+ See ``audit/db_direct_param_map.md`` §1, §5, §7.2 for the contract.
43
+ """
44
+ from __future__ import annotations
45
+
46
+ from typing import TYPE_CHECKING
47
+
48
+ import polars as pl
49
+
50
+ from flextool.engine_polars._axis_enums import (
51
+ alias_to_axis,
52
+ cast_dim,
53
+ cast_frame_axes,
54
+ get_global_axis_enums,
55
+ rename_to_axis,
56
+ schema_dtype,
57
+ )
58
+
59
+
60
+ def _to_e(expr: pl.Expr) -> pl.Expr:
61
+ """Cast *expr* to the entity-union (``e``) axis enum if active.
62
+
63
+ Used at ``pl.col("p").alias("source"/"sink")`` sites where a
64
+ process-typed column is reused as an entity-union slot. Without
65
+ the cast, ``vertical_relaxed`` concat downstream raises
66
+ ``SchemaError: failed to determine supertype of enum and enum``.
67
+ """
68
+ return cast_dim(expr, get_global_axis_enums(), "e")
69
+
70
+ if TYPE_CHECKING:
71
+ from flextool.engine_polars._input_source import InputSource
72
+
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Helpers for the common shape of "rename source to polar_high column names"
76
+ # ---------------------------------------------------------------------------
77
+
78
+ def _empty(schema: dict[str, type[pl.DataType]]) -> pl.DataFrame:
79
+ return pl.DataFrame(schema=schema)
80
+
81
+
82
+ def _try_param(source: "InputSource", entity_class: str,
83
+ parameter_name: str) -> pl.DataFrame | None:
84
+ """Return ``source.parameter(...)`` or ``None`` if unknown / empty."""
85
+ try:
86
+ df = source.parameter(entity_class, parameter_name)
87
+ except KeyError:
88
+ return None
89
+ if df.height == 0:
90
+ return None
91
+ return df
92
+
93
+
94
+ def _try_entities(source: "InputSource", entity_class: str) -> pl.DataFrame | None:
95
+ try:
96
+ df = source.entities(entity_class)
97
+ except KeyError:
98
+ return None
99
+ if df.height == 0:
100
+ return None
101
+ return df
102
+
103
+
104
+ # ---------------------------------------------------------------------------
105
+ # §1.2 — nodeBalance (P): node_type ∈ {balance}
106
+ # ---------------------------------------------------------------------------
107
+
108
+ def nodeBalance(source: "InputSource") -> pl.DataFrame:
109
+ """Nodes with ``node_type ∈ {'balance', 'storage'}``.
110
+
111
+ Schema: ``[n]``. flextool's `node_type_sets.py:69` defines
112
+ `nodeBalance` as the nodes whose type is **either** balance or
113
+ storage — both participate in the LHS of `nodeBalance_eq`.
114
+ """
115
+ df = _try_param(source, "node", "node_type")
116
+ if df is None:
117
+ return _empty({"n": pl.Utf8})
118
+ return (df.lazy()
119
+ .filter(pl.col("value").is_in(["balance", "storage"]))
120
+ .pipe(rename_to_axis, {"name": "n"})
121
+ .select("n")
122
+ .sort("n")
123
+ .collect())
124
+
125
+
126
+ def nodeBalancePeriod(source: "InputSource") -> pl.DataFrame:
127
+ """Nodes with ``node_type == 'balance_within_period'``.
128
+
129
+ Schema: ``[n]``. Mirrors :func:`nodeBalance` exactly except for the
130
+ filter literal — these nodes enforce their energy balance once per
131
+ period (legacy ``balance_within_period``) and also carry slack vars
132
+ ``vq_state_up/down``, so they participate in the same slack-scaling /
133
+ penalty params as ``nodeBalance``.
134
+ """
135
+ df = _try_param(source, "node", "node_type")
136
+ if df is None:
137
+ return _empty({"n": pl.Utf8})
138
+ return (df.lazy()
139
+ .filter(pl.col("value").is_in(["balance_within_period"]))
140
+ .pipe(rename_to_axis, {"name": "n"})
141
+ .select("n")
142
+ .sort("n")
143
+ .collect())
144
+
145
+
146
+ # ---------------------------------------------------------------------------
147
+ # §1.3 — Process topology Projections
148
+ # ---------------------------------------------------------------------------
149
+
150
+ def process_source_sink(source: "InputSource") -> pl.DataFrame:
151
+ """Union of ``unit__inputNode``, ``unit__outputNode``, and
152
+ ``connection__node__node`` into the canonical ``(p, source, sink)``
153
+ arc table.
154
+
155
+ For unit input arcs: ``source = node``, ``sink = unit``.
156
+ For unit output arcs: ``source = unit``, ``sink = node``.
157
+ For connection arcs: a connection's ``node_1 → node_2``, and the
158
+ reverse ``node_2 → node_1`` if the connection is bidirectional —
159
+ here we materialise both directions as the .mod's
160
+ ``process_source_sink`` includes them when the connection is
161
+ bidirectional. flextool's preprocessor decides directionality
162
+ via ``connection.is_DC`` / transfer-method semantics; for the
163
+ Projection-only port we mirror flextool's CSV: process_source_sink
164
+ is the **union** of the input/output unit arcs and **all**
165
+ (connection, node_1, node_2) and (connection, node_2, node_1)
166
+ pairs.
167
+
168
+ Schema: ``[p, source, sink]``.
169
+
170
+ This produces the **expanded** form (one arc per input + one arc per
171
+ output) where indirect / no-collapse units appear as ``(p, source, p)``
172
+ and ``(p, p, sink)``. Use :func:`process_source_sink_collapsed` to
173
+ obtain flextool's preprocessing-side **collapsed** shape (direct-
174
+ method units flattened to ``(p, source, sink)``) — that's the form
175
+ written to ``solve_data/process_source_sink.csv``.
176
+ """
177
+ parts: list[pl.LazyFrame] = []
178
+
179
+ uin = _try_entities(source, "unit__inputNode")
180
+ if uin is not None:
181
+ parts.append(uin.lazy().select(
182
+ alias_to_axis("unit", "p"),
183
+ alias_to_axis("node", "source"),
184
+ alias_to_axis("unit", "sink"),
185
+ ))
186
+ uout = _try_entities(source, "unit__outputNode")
187
+ if uout is not None:
188
+ parts.append(uout.lazy().select(
189
+ alias_to_axis("unit", "p"),
190
+ alias_to_axis("unit", "source"),
191
+ alias_to_axis("node", "sink"),
192
+ ))
193
+ cnn = _try_entities(source, "connection__node__node")
194
+ if cnn is not None:
195
+ parts.append(cnn.lazy().select(
196
+ alias_to_axis("connection", "p"),
197
+ alias_to_axis("node_1", "source"),
198
+ alias_to_axis("node_2", "sink"),
199
+ ))
200
+
201
+ if not parts:
202
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
203
+ return pl.concat(parts).unique().sort("p", "source", "sink").collect()
204
+
205
+
206
+ # ---------------------------------------------------------------------------
207
+ # Δ.17b Gap B — flextool preprocessing-style collapsed pss
208
+ # ---------------------------------------------------------------------------
209
+ #
210
+ # flextool's preprocessing produces ``solve_data/process_source_sink.csv``
211
+ # as the union of 10 sub-sets (see preprocessing/process_arc_unions.py
212
+ # and process_method_sets.py). The shape per process depends on the
213
+ # internal method:
214
+ #
215
+ # * DIRECT methods (method_1way_1var_*, method_2way_1var_*,
216
+ # method_2way_2var_*): cross-product of (p, source) × (p, sink) →
217
+ # ``(p, source, sink)``. 2-way variants also emit reverse arcs
218
+ # ``(p, sink, source)``.
219
+ # * INDIRECT methods (method_*_nvar_*): ``(p, source, p)`` and
220
+ # ``(p, p, sink)`` — the unit name appears as the intermediate
221
+ # "node" in two arcs. 2way_nvar also emits ``(p, p, source)``.
222
+ # * 1way_1var with no source: ``(p, p, sink)`` (process_process_toSink_noConversion).
223
+ # * 1way_1var with no sink: ``(p, source, p)`` (process_source_toProcess_noConversion).
224
+ # * Connections: always ``(c, node_1, node_2)`` (already 1-arc form).
225
+ #
226
+ # Internal method comes from the classifier (input_writer.METHODS_MAPPING).
227
+ # We reuse the classifier from ``_derived_params.py``.
228
+
229
+
230
+ _METHOD_2WAY_NVAR_LOCAL = frozenset(("method_2way_nvar_off",))
231
+ _METHOD_2WAY_2VAR_LOCAL = frozenset((
232
+ "method_2way_2var_off", "method_2way_2var_exclude",
233
+ "method_2way_2var_MIP_exclude",
234
+ ))
235
+ _METHOD_1WAY_1VAR_LOCAL = frozenset((
236
+ "method_1way_1var_off", "method_1way_1var_LP", "method_1way_1var_MIP",
237
+ ))
238
+ _METHOD_DIRECT_LOCAL = frozenset((
239
+ "method_1way_1var_off", "method_1way_1var_LP", "method_1way_1var_MIP",
240
+ "method_2way_1var_off", "method_2way_2var_off",
241
+ "method_2way_2var_exclude", "method_2way_2var_MIP_exclude",
242
+ ))
243
+ _METHOD_INDIRECT_LOCAL = frozenset((
244
+ "method_1way_nvar_off", "method_1way_nvar_LP", "method_1way_nvar_MIP",
245
+ "method_2way_nvar_off",
246
+ ))
247
+
248
+
249
+ def process_source_sink_collapsed(source: "InputSource",
250
+ classified: pl.DataFrame | None = None
251
+ ) -> pl.DataFrame:
252
+ """flextool-preprocessing-shaped ``process_source_sink`` — same as
253
+ :func:`process_source_sink_canonical` minus the ``method`` column.
254
+ Schema: ``[p, source, sink]``.
255
+
256
+ Δ.17b Gap B closure. Prior to this, ``input.py:_load_process_topology``
257
+ had to read the three preprocessed ``process_source_sink*.csv`` files
258
+ from disk because the canonical helper produced the *expanded*
259
+ ``process_source_sink`` shape (``unit__inputNode`` ∪ ``unit__outputNode``
260
+ — 2 arcs per direct+indirect unit) — not flextool's collapsed shape.
261
+ This helper produces the same 10-way union flextool's
262
+ ``write_process_arc_unions`` builds.
263
+ """
264
+ del classified # canonical computes its own
265
+ canonical = process_source_sink_canonical(source)
266
+ if canonical.height == 0:
267
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
268
+ return (canonical.lazy()
269
+ .select("p", "source", "sink")
270
+ .unique()
271
+ .sort("p", "source", "sink")
272
+ .collect())
273
+
274
+
275
+ # Conversion methods that are "efficiency-based" (eff partition). The
276
+ # .mod calls these the ``method_X_LP`` family; flextool's preprocessor
277
+ # also includes ``min_load_efficiency`` and ``constant_efficiency`` for
278
+ # the Projection. See preprocessing/process_method_sets.py.
279
+ _EFF_CONVERSION_METHODS = ("min_load_efficiency", "constant_efficiency",
280
+ "method_X_LP")
281
+
282
+
283
+ def process_source_sink_canonical(source: "InputSource",
284
+ pss: pl.DataFrame | None = None) -> pl.DataFrame:
285
+ """Method-tagged canonical frame for the ``process_source_sink``
286
+ family. Built once and projected to ``_eff`` / ``_noEff`` by the
287
+ consumers below — replaces the prior pattern of computing each
288
+ partition from scratch.
289
+
290
+ Schema: ``[p, source, sink, method]`` where ``method ∈ {'eff', 'noEff'}``.
291
+
292
+ Δ.17b Gap B: produces flextool's preprocessing-side **collapsed
293
+ shape** (DIRECT methods flattened to ``(p, source, sink)``; INDIRECT
294
+ methods kept as 2-arc form ``(p, source, p)``+``(p, p, sink)``).
295
+ Partition keyed by internal METHOD (DIRECT → eff, else → noEff),
296
+ not by ``conversion_method``. This matches flextool's
297
+ ``write_process_arc_unions``:
298
+
299
+ * ``_eff`` = ``process_source_toSink ∪ process_sink_toSource``
300
+ (DIRECT-method arcs).
301
+ * ``_noEff`` = 8-way union of indirect / noConversion / profile
302
+ sub-sets.
303
+
304
+ The legacy ``pss`` parameter is accepted for backward compatibility
305
+ but ignored when the classifier is available — the classifier dictates
306
+ both the shape (collapsed vs expanded) and the eff/noEff partition.
307
+ """
308
+ del pss # legacy parameter
309
+
310
+ # Late import to avoid a circular reference.
311
+ from flextool.engine_polars._derived_params import _classify_process_method
312
+ classified = _classify_process_method(source)
313
+
314
+ # Connections are always noEff and 1-arc.
315
+ cnn = _try_entities(source, "connection__node__node")
316
+ parts: list[pl.LazyFrame] = []
317
+ if cnn is not None and cnn.height > 0:
318
+ # Connections classify into the eff partition only when they
319
+ # carry a transfer_method that the .mod treats as efficiency-
320
+ # bearing (i.e. DIRECT-family internal methods). Use the
321
+ # classifier output rather than re-deriving here.
322
+ cls_conns = (classified.lazy()
323
+ .filter(pl.col("klass") == "connection")
324
+ .select("p", "method"))
325
+ cnn_lf = cnn.lazy().pipe(
326
+ rename_to_axis,
327
+ {"connection": "p", "node_1": "source", "node_2": "sink"},
328
+ ).select("p", "source", "sink")
329
+ # Join with classifier to get internal method per arc.
330
+ cnn_with_method = cnn_lf.join(cls_conns, on="p", how="left")
331
+ # Forward direction: tag eff/noEff by method.
332
+ cnn_eff = (cnn_with_method
333
+ .filter(pl.col("method").is_in(list(_METHOD_DIRECT_LOCAL)))
334
+ .select("p", "source", "sink",
335
+ pl.lit("eff").alias("method")))
336
+ cnn_noEff = (cnn_with_method
337
+ .filter(~pl.col("method").is_in(list(_METHOD_DIRECT_LOCAL))
338
+ | pl.col("method").is_null())
339
+ .select("p", "source", "sink",
340
+ pl.lit("noEff").alias("method")))
341
+ parts.append(cnn_eff)
342
+ parts.append(cnn_noEff)
343
+ # Reverse direction for 2way_2var connections.
344
+ rev_eff = (cnn_with_method
345
+ .filter(pl.col("method").is_in(list(_METHOD_2WAY_2VAR_LOCAL)))
346
+ .select("p",
347
+ alias_to_axis("sink", "source"),
348
+ alias_to_axis("source", "sink"),
349
+ pl.lit("eff").alias("method")))
350
+ parts.append(rev_eff)
351
+ # 2way_nvar connections — emits indirect 2-arc form. In practice
352
+ # connections almost never have 2way_nvar; guard via classifier.
353
+ rev_noEff = (cnn_with_method
354
+ .filter(pl.col("method").is_in(list(_METHOD_2WAY_NVAR_LOCAL)))
355
+ .select("p",
356
+ alias_to_axis("sink", "source"),
357
+ alias_to_axis("source", "sink"),
358
+ pl.lit("noEff").alias("method")))
359
+ parts.append(rev_noEff)
360
+
361
+ # Per-unit dispatch keyed by classified method. Build empty-but-typed
362
+ # source/sink LazyFrames when the entity-class is missing, so the
363
+ # noConversion branches still see "no source rows" / "no sink rows"
364
+ # rather than skipping entirely. Phase 4: dim columns must carry
365
+ # their canonical Enum dtype so joins against Enum-keyed frames
366
+ # don't SchemaError on the empty branch.
367
+ _enums_empty = get_global_axis_enums()
368
+
369
+ def _empty_lf(*cols):
370
+ return pl.DataFrame(
371
+ {c: [] for c in cols},
372
+ schema={c: schema_dtype(_enums_empty, c) for c in cols},
373
+ ).lazy()
374
+ if classified is not None and classified.height > 0:
375
+ cls_units = (classified.lazy()
376
+ .filter(pl.col("klass") == "unit")
377
+ .select("p", "method"))
378
+ uin = _try_entities(source, "unit__inputNode")
379
+ sources_lf = (
380
+ uin.lazy().pipe(rename_to_axis,
381
+ {"unit": "p", "node": "source"})
382
+ .select("p", "source")
383
+ ) if uin is not None else _empty_lf("p", "source")
384
+ uout = _try_entities(source, "unit__outputNode")
385
+ sinks_lf = (
386
+ uout.lazy().pipe(rename_to_axis,
387
+ {"unit": "p", "node": "sink"})
388
+ .select("p", "sink")
389
+ ) if uout is not None else _empty_lf("p", "sink")
390
+
391
+ # ── DIRECT methods: cross-join → ``_eff`` partition ──
392
+ direct_p = (cls_units
393
+ .filter(pl.col("method").is_in(list(_METHOD_DIRECT_LOCAL)))
394
+ .select("p"))
395
+ direct_arcs = (sources_lf
396
+ .join(direct_p, on="p", how="inner")
397
+ .join(sinks_lf, on="p", how="inner")
398
+ .select("p", "source", "sink",
399
+ pl.lit("eff").alias("method")))
400
+ parts.append(direct_arcs)
401
+
402
+ # 2way_2var: reverse arcs → also ``_eff``.
403
+ two_way_p = (cls_units
404
+ .filter(pl.col("method").is_in(list(_METHOD_2WAY_2VAR_LOCAL)))
405
+ .select("p"))
406
+ rev_arcs = (sources_lf
407
+ .join(two_way_p, on="p", how="inner")
408
+ .join(sinks_lf, on="p", how="inner")
409
+ .select(pl.col("p"),
410
+ alias_to_axis("sink", "source"),
411
+ alias_to_axis("source", "sink"),
412
+ pl.lit("eff").alias("method")))
413
+ parts.append(rev_arcs)
414
+
415
+ # ── INDIRECT methods: 2-arc form → ``_noEff`` partition ──
416
+ indirect_p = (cls_units
417
+ .filter(pl.col("method").is_in(list(_METHOD_INDIRECT_LOCAL)))
418
+ .select("p"))
419
+ indirect_inputs = (sources_lf
420
+ .join(indirect_p, on="p", how="inner")
421
+ .select("p", "source", alias_to_axis(pl.col("p"), "sink"),
422
+ pl.lit("noEff").alias("method")))
423
+ parts.append(indirect_inputs)
424
+
425
+ # 2way_nvar: process_process_toSource — (p, p, source).
426
+ two_way_nvar_p = (cls_units
427
+ .filter(pl.col("method").is_in(list(_METHOD_2WAY_NVAR_LOCAL)))
428
+ .select("p"))
429
+ indirect_inputs_rev = (sources_lf
430
+ .join(two_way_nvar_p, on="p", how="inner")
431
+ .select("p", alias_to_axis(pl.col("p"), "source"),
432
+ alias_to_axis("source", "sink"),
433
+ pl.lit("noEff").alias("method")))
434
+ parts.append(indirect_inputs_rev)
435
+
436
+ indirect_outputs = (sinks_lf
437
+ .join(indirect_p, on="p", how="inner")
438
+ .select("p", alias_to_axis(pl.col("p"), "source"), "sink",
439
+ pl.lit("noEff").alias("method")))
440
+ parts.append(indirect_outputs)
441
+
442
+ # ── 1way_1var DIRECT, missing one side: noConversion ``_noEff`` ──
443
+ one_way_1var_p = (cls_units
444
+ .filter(pl.col("method").is_in(list(_METHOD_1WAY_1VAR_LOCAL)))
445
+ .select("p"))
446
+ sources_exist = sources_lf.select("p").unique()
447
+ sinks_exist = sinks_lf.select("p").unique()
448
+ no_sink_p = one_way_1var_p.join(sinks_exist, on="p", how="anti")
449
+ no_source_p = one_way_1var_p.join(sources_exist, on="p", how="anti")
450
+ no_sink_arcs = (sources_lf
451
+ .join(no_sink_p, on="p", how="inner")
452
+ .select("p", "source", alias_to_axis(pl.col("p"), "sink"),
453
+ pl.lit("noEff").alias("method")))
454
+ parts.append(no_sink_arcs)
455
+ no_source_arcs = (sinks_lf
456
+ .join(no_source_p, on="p", how="inner")
457
+ .select("p", alias_to_axis(pl.col("p"), "source"), "sink",
458
+ pl.lit("noEff").alias("method")))
459
+ parts.append(no_source_arcs)
460
+
461
+ # Phase 4.6: when activation is on, the empty fallback must use the
462
+ # canonical Enum dtypes so downstream joins against Enum-typed
463
+ # frames don't SchemaError on the empty branch.
464
+ _enums = get_global_axis_enums()
465
+ _empty_schema = {
466
+ "p": schema_dtype(_enums, "p"),
467
+ "source": schema_dtype(_enums, "source"),
468
+ "sink": schema_dtype(_enums, "sink"),
469
+ "method": pl.Utf8,
470
+ }
471
+ if not parts:
472
+ return _empty(_empty_schema)
473
+
474
+ # Final dedup: arcs may appear in multiple partitions due to method
475
+ # categories; flextool's union takes a set-union (dedup). Tie-break:
476
+ # if same (p, source, sink) appears as both eff and noEff (shouldn't
477
+ # in practice but defensive), prefer eff.
478
+ out = (pl.concat(parts, how="vertical_relaxed")
479
+ .unique(subset=["p", "source", "sink", "method"])
480
+ .sort("p", "source", "sink", "method")
481
+ .collect())
482
+ if out.height == 0:
483
+ return _empty(_empty_schema)
484
+ # Cast dim columns to canonical axis enums. Many partitions emit
485
+ # ``source``/``sink`` via ``pl.col("p").alias(...)`` (cross-axis from
486
+ # the ``p`` enum into the ``source``/``sink`` slot whose contract
487
+ # axis is ``e``). The concat above is ``vertical_relaxed`` so
488
+ # types fall to a common supertype (string when mixing enum
489
+ # vocabularies); the cast here re-establishes canonical Enum
490
+ # dtypes once at the boundary.
491
+ enums = get_global_axis_enums()
492
+ if enums is not None:
493
+ out = cast_frame_axes(out, enums)
494
+ return out
495
+
496
+
497
+ def process_source_sink_eff(source: "InputSource",
498
+ pss: pl.DataFrame | None = None) -> pl.DataFrame:
499
+ """``pss`` filtered to processes with an efficiency-based
500
+ ``conversion_method`` (units only; connections never participate
501
+ in eff).
502
+
503
+ Schema: ``[p, source, sink]``.
504
+
505
+ Δ.3: thin filter over :func:`process_source_sink_canonical`.
506
+ """
507
+ canonical = process_source_sink_canonical(source, pss)
508
+ if canonical.height == 0:
509
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
510
+ return (canonical.lazy()
511
+ .filter(pl.col("method") == "eff")
512
+ .select("p", "source", "sink")
513
+ .sort("p", "source", "sink")
514
+ .collect())
515
+
516
+
517
+ def process_source_sink_noEff(source: "InputSource",
518
+ pss: pl.DataFrame | None = None,
519
+ pss_eff: pl.DataFrame | None = None) -> pl.DataFrame:
520
+ """Complement of ``process_source_sink_eff`` within ``pss`` — the
521
+ arcs whose process is NOT efficiency-based (connections + units
522
+ with ``conversion_method ∉ eff family``).
523
+
524
+ Schema: ``[p, source, sink]``.
525
+
526
+ Δ.3: thin filter over :func:`process_source_sink_canonical`. The
527
+ legacy ``pss_eff`` parameter is accepted for backward compatibility
528
+ but no longer used — the canonical frame already encodes the
529
+ partition via the ``method`` column.
530
+ """
531
+ del pss_eff # legacy parameter; see docstring
532
+ canonical = process_source_sink_canonical(source, pss)
533
+ if canonical.height == 0:
534
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
535
+ return (canonical.lazy()
536
+ .filter(pl.col("method") == "noEff")
537
+ .select("p", "source", "sink")
538
+ .sort("p", "source", "sink")
539
+ .collect())
540
+
541
+
542
+ # ---------------------------------------------------------------------------
543
+ # process_source_sink_ramp_limit_* / _ramp_cost helpers
544
+ # ---------------------------------------------------------------------------
545
+ #
546
+ # Five sets:
547
+ #
548
+ # * ``process_source_sink_ramp_limit_source_up`` — (p, src, sink) where
549
+ # (p, src, m) ∈ process__node__ramp_method, m ∈ RAMP_LIMIT_METHOD
550
+ # AND p_process_source[p, src, 'ramp_speed_up'] > 0.
551
+ # * Symmetric for sink/up, source/down, sink/down.
552
+ # * ``process_source_sink_ramp_cost`` — OR of source/sink-side membership
553
+ # in RAMP_COST_METHOD (no speed-gate).
554
+ #
555
+ # In Spine, the per-arc ramp_method + ramp_speed_* parameters live on
556
+ # ``unit__inputNode`` (source side) and ``unit__outputNode`` (sink side).
557
+ # Connections never carry ramp methods — flextool's preprocessing reads
558
+ # only ``input/process__node__ramp_method.csv`` which is the union of
559
+ # ``unit__inputNode.ramp_method`` ∪ ``unit__outputNode.ramp_method``.
560
+ #
561
+ # User-supplied MathProg snippets (Δ.17c dispatch, Gap D):
562
+ #
563
+ # set process_source_sink_ramp_limit_source_up :=
564
+ # {(p, source, sink) in process_source_sink :
565
+ # ( sum{(p, source, m) in process_node_ramp_method
566
+ # : m in ramp_limit_method} 1
567
+ # && p_process_source[p, source, 'ramp_speed_up'] > 0
568
+ # )};
569
+ #
570
+ # set process_source_sink_ramp_cost :=
571
+ # {(p, source, sink) in process_source_sink :
572
+ # sum{(p, source, m) in process_node_ramp_method
573
+ # : m in ramp_cost_method} 1
574
+ # || sum{(p, sink, m) in process_node_ramp_method
575
+ # : m in ramp_cost_method} 1};
576
+
577
+
578
+ # Mirror of :mod:`flextool.input_derivation._method_constants`.
579
+ # Kept locally to avoid an import dependency.
580
+ _RAMP_LIMIT_METHOD: frozenset[str] = frozenset(("ramp_limit", "both"))
581
+ _RAMP_COST_METHOD: frozenset[str] = frozenset(("ramp_cost", "both"))
582
+
583
+
584
+ def _ramp_pairs(source: "InputSource",
585
+ relationship_class: str,
586
+ method_set: "frozenset[str]") -> "pl.LazyFrame":
587
+ """Return ``[unit, node]`` pairs whose ``ramp_method`` value is in
588
+ *method_set*, drawn from *relationship_class* ∈
589
+ ``{unit__inputNode, unit__outputNode}``.
590
+
591
+ Empty / missing parameter → empty LazyFrame with schema ``[unit, node]``.
592
+ """
593
+ df = _try_param(source, relationship_class, "ramp_method")
594
+ if df is None:
595
+ return pl.DataFrame(
596
+ schema={"unit": pl.Utf8, "node": pl.Utf8}).lazy()
597
+ return (df.lazy()
598
+ .filter(pl.col("value").is_in(list(method_set)))
599
+ .select("unit", "node"))
600
+
601
+
602
+ def _ramp_speed_positive_pairs(source: "InputSource",
603
+ relationship_class: str,
604
+ parameter_name: str) -> "pl.LazyFrame":
605
+ """Return ``[unit, node]`` pairs whose ``parameter_name`` value
606
+ (``ramp_speed_up`` / ``ramp_speed_down``) is strictly > 0.
607
+ """
608
+ df = _try_param(source, relationship_class, parameter_name)
609
+ if df is None:
610
+ return pl.DataFrame(
611
+ schema={"unit": pl.Utf8, "node": pl.Utf8}).lazy()
612
+ return (df.lazy()
613
+ .filter(pl.col("value").is_not_null() & (pl.col("value") > 0.0))
614
+ .select("unit", "node"))
615
+
616
+
617
+ def _ramp_limit_set(source: "InputSource",
618
+ side: str,
619
+ direction: str) -> pl.DataFrame:
620
+ """Build ``process_source_sink_ramp_limit_<side>_<direction>``.
621
+
622
+ *side* — ``"source"`` or ``"sink"`` (the column the relationship-side
623
+ constraint applies to).
624
+ *direction* — ``"up"`` or ``"down"`` (which speed parameter to gate).
625
+ """
626
+ if side == "source":
627
+ rel_class = "unit__inputNode"
628
+ ramp_pairs = _ramp_pairs(source, rel_class, _RAMP_LIMIT_METHOD) \
629
+ .pipe(rename_to_axis, {"unit": "p", "node": "source"})
630
+ speed_pairs = _ramp_speed_positive_pairs(
631
+ source, rel_class, f"ramp_speed_{direction}") \
632
+ .pipe(rename_to_axis, {"unit": "p", "node": "source"})
633
+ gated = ramp_pairs.join(speed_pairs, on=["p", "source"],
634
+ how="inner")
635
+ canonical = process_source_sink_canonical(source).lazy()
636
+ return (canonical
637
+ .select("p", "source", "sink")
638
+ .unique()
639
+ .join(gated, on=["p", "source"], how="inner")
640
+ .select("p", "source", "sink")
641
+ .unique()
642
+ .sort("p", "source", "sink")
643
+ .collect())
644
+ if side == "sink":
645
+ rel_class = "unit__outputNode"
646
+ ramp_pairs = _ramp_pairs(source, rel_class, _RAMP_LIMIT_METHOD) \
647
+ .pipe(rename_to_axis, {"unit": "p", "node": "sink"})
648
+ speed_pairs = _ramp_speed_positive_pairs(
649
+ source, rel_class, f"ramp_speed_{direction}") \
650
+ .pipe(rename_to_axis, {"unit": "p", "node": "sink"})
651
+ gated = ramp_pairs.join(speed_pairs, on=["p", "sink"],
652
+ how="inner")
653
+ canonical = process_source_sink_canonical(source).lazy()
654
+ return (canonical
655
+ .select("p", "source", "sink")
656
+ .unique()
657
+ .join(gated, on=["p", "sink"], how="inner")
658
+ .select("p", "source", "sink")
659
+ .unique()
660
+ .sort("p", "source", "sink")
661
+ .collect())
662
+ raise ValueError(f"_ramp_limit_set: unknown side {side!r}")
663
+
664
+
665
+ def process_source_sink_ramp_limit_source_up(
666
+ source: "InputSource",
667
+ ) -> pl.DataFrame:
668
+ """Δ.17c Gap D — see module docstring above for the MathProg shape."""
669
+ return _ramp_limit_set(source, "source", "up")
670
+
671
+
672
+ def process_source_sink_ramp_limit_source_down(
673
+ source: "InputSource",
674
+ ) -> pl.DataFrame:
675
+ return _ramp_limit_set(source, "source", "down")
676
+
677
+
678
+ def process_source_sink_ramp_limit_sink_up(
679
+ source: "InputSource",
680
+ ) -> pl.DataFrame:
681
+ return _ramp_limit_set(source, "sink", "up")
682
+
683
+
684
+ def process_source_sink_ramp_limit_sink_down(
685
+ source: "InputSource",
686
+ ) -> pl.DataFrame:
687
+ return _ramp_limit_set(source, "sink", "down")
688
+
689
+
690
+ def process_source_sink_ramp_cost(source: "InputSource") -> pl.DataFrame:
691
+ """``{(p, src, sink) ∈ pss : ramp_cost on source-side OR sink-side}``.
692
+
693
+ Mirrors mod L1115-1119 + the user's Δ.17c MathProg snippet (no
694
+ speed-gate; pure method membership).
695
+ """
696
+ src_cost = (_ramp_pairs(source, "unit__inputNode", _RAMP_COST_METHOD)
697
+ .pipe(rename_to_axis, {"unit": "p", "node": "source"}))
698
+ sink_cost = (_ramp_pairs(source, "unit__outputNode", _RAMP_COST_METHOD)
699
+ .pipe(rename_to_axis, {"unit": "p", "node": "sink"}))
700
+ canonical = process_source_sink_canonical(source).lazy().select(
701
+ "p", "source", "sink").unique()
702
+ via_source = canonical.join(src_cost, on=["p", "source"], how="inner")
703
+ via_sink = canonical.join(sink_cost, on=["p", "sink"], how="inner")
704
+ return (pl.concat([via_source, via_sink], how="vertical_relaxed")
705
+ .select("p", "source", "sink")
706
+ .unique()
707
+ .sort("p", "source", "sink")
708
+ .collect())
709
+
710
+
711
+ _METHOD_2WAY_1VAR_OFF_LOCAL = frozenset(("method_2way_1var_off",))
712
+
713
+
714
+ def process_source_sink_2way_1var(source: "InputSource") -> pl.DataFrame:
715
+ """``{(p, source, sink) ∈ pss : method == method_2way_1var_off}``.
716
+
717
+ The single-signed-flow 2-way method (the .mod's ``v_flow ∈ [-cap,
718
+ +cap]``). The engine emits ``v_flow ≥ 0`` on the FORWARD arc only, so
719
+ these arcs need a non-negative reverse auxiliary ``v_flow_back`` to
720
+ carry sink→source flow. This set is the union of DC-power-flow and
721
+ non-DC ``method_2way_1var_off`` arcs (the classifier doesn't gate on
722
+ the DC feature); ``model.py`` declares the shared ``v_flow_back`` over
723
+ it and ``_dc_power_flow`` consumes the same Var on the DC subset.
724
+
725
+ Schema: ``[p, source, sink]`` in the canonical forward orientation
726
+ (matching ``process_source_sink``).
727
+ """
728
+ from flextool.engine_polars._derived_params import _classify_process_method
729
+
730
+ classified = _classify_process_method(source)
731
+ if classified is None or classified.height == 0:
732
+ return pl.DataFrame(schema={
733
+ "p": schema_dtype(get_global_axis_enums(), "p"),
734
+ "source": schema_dtype(get_global_axis_enums(), "source"),
735
+ "sink": schema_dtype(get_global_axis_enums(), "sink"),
736
+ })
737
+ twovar_p = (classified.lazy()
738
+ .filter(pl.col("method").is_in(list(_METHOD_2WAY_1VAR_OFF_LOCAL)))
739
+ .select("p")
740
+ .unique())
741
+ canonical = process_source_sink_canonical(source).lazy().select(
742
+ "p", "source", "sink").unique()
743
+ return (canonical
744
+ .join(twovar_p, on="p", how="inner")
745
+ .select("p", "source", "sink")
746
+ .unique()
747
+ .sort("p", "source", "sink")
748
+ .collect())
749
+
750
+
751
+ def flow_from_commodity_eff(source: "InputSource",
752
+ pss_eff: pl.DataFrame | None = None) -> pl.DataFrame:
753
+ """``pss_eff ⋈ commodity__node`` on ``source = node`` →
754
+ ``(p, source, sink, c)``. Empty if either side is empty.
755
+ """
756
+ if pss_eff is None:
757
+ pss_eff = process_source_sink_eff(source)
758
+ if pss_eff.height == 0:
759
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
760
+ "sink": pl.Utf8, "c": pl.Utf8})
761
+ cn = _try_entities(source, "commodity__node")
762
+ if cn is None:
763
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
764
+ "sink": pl.Utf8, "c": pl.Utf8})
765
+ return (pss_eff.lazy()
766
+ .join(cn.lazy(), left_on="source", right_on="node", how="inner")
767
+ .pipe(rename_to_axis, {"commodity": "c"})
768
+ .select("p", "source", "sink", "c")
769
+ .sort("p", "source", "sink", "c")
770
+ .collect())
771
+
772
+
773
+ def flow_from_commodity_noEff(source: "InputSource",
774
+ pss_noEff: pl.DataFrame | None = None) -> pl.DataFrame:
775
+ """``pss_noEff ⋈ commodity__node`` on ``source = node``."""
776
+ if pss_noEff is None:
777
+ pss_noEff = process_source_sink_noEff(source)
778
+ if pss_noEff.height == 0:
779
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
780
+ "sink": pl.Utf8, "c": pl.Utf8})
781
+ cn = _try_entities(source, "commodity__node")
782
+ if cn is None:
783
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
784
+ "sink": pl.Utf8, "c": pl.Utf8})
785
+ return (pss_noEff.lazy()
786
+ .join(cn.lazy(), left_on="source", right_on="node", how="inner")
787
+ .pipe(rename_to_axis, {"commodity": "c"})
788
+ .select("p", "source", "sink", "c")
789
+ .sort("p", "source", "sink", "c")
790
+ .collect())
791
+
792
+
793
+ def flow_to_commodity(source: "InputSource",
794
+ pss: pl.DataFrame | None = None) -> pl.DataFrame:
795
+ """``pss ⋈ commodity__node`` on ``sink = node`` →
796
+ ``(p, source, sink, c)``. Sink-side flow into a priced
797
+ commodity node.
798
+ """
799
+ if pss is None:
800
+ pss = process_source_sink(source)
801
+ if pss.height == 0:
802
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
803
+ "sink": pl.Utf8, "c": pl.Utf8})
804
+ cn = _try_entities(source, "commodity__node")
805
+ if cn is None:
806
+ return _empty({"p": pl.Utf8, "source": pl.Utf8,
807
+ "sink": pl.Utf8, "c": pl.Utf8})
808
+ return (pss.lazy()
809
+ .join(cn.lazy(), left_on="sink", right_on="node", how="inner")
810
+ .pipe(rename_to_axis, {"commodity": "c"})
811
+ .select("p", "source", "sink", "c")
812
+ .sort("p", "source", "sink", "c")
813
+ .collect())
814
+
815
+
816
+ # ---------------------------------------------------------------------------
817
+ # §1.4 — CO2 group set Projections
818
+ # ---------------------------------------------------------------------------
819
+
820
+ def group_co2_max_period(source: "InputSource") -> pl.DataFrame:
821
+ """Distinct groups with a non-empty ``co2_max_period`` Map.
822
+
823
+ Schema: ``[g]``.
824
+ """
825
+ df = _try_param(source, "group", "co2_max_period")
826
+ if df is None:
827
+ return _empty({"g": pl.Utf8})
828
+ return (df.lazy()
829
+ .select(alias_to_axis("name", "g"))
830
+ .unique()
831
+ .sort("g")
832
+ .collect())
833
+
834
+
835
+ # ---------------------------------------------------------------------------
836
+ # §1.5 — Indirect-conversion (CHP) Projections
837
+ # ---------------------------------------------------------------------------
838
+
839
+ _INDIRECT_CONVERSION_METHODS = ("method_indirect", "method_indirect_LP")
840
+
841
+
842
+ def process_indirect(source: "InputSource") -> pl.DataFrame:
843
+ """Units with ``conversion_method`` in the indirect family.
844
+
845
+ Schema: ``[p]``.
846
+ """
847
+ df = _try_param(source, "unit", "conversion_method")
848
+ if df is None:
849
+ return _empty({"p": pl.Utf8})
850
+ return (df.lazy()
851
+ .filter(pl.col("value").is_in(_INDIRECT_CONVERSION_METHODS))
852
+ .select(alias_to_axis("name", "p"))
853
+ .sort("p")
854
+ .collect())
855
+
856
+
857
+ def process_input_flows(source: "InputSource",
858
+ pss: pl.DataFrame | None = None,
859
+ process_indirect_set: pl.DataFrame | None = None) -> pl.DataFrame:
860
+ """Indirect-process input arcs: ``pss`` filtered to ``sink == p``
861
+ AND ``p ∈ process_indirect``. In the indirect family the unit
862
+ name appears as both ``p`` and ``sink`` (one row per (p, source)).
863
+
864
+ Schema: ``[p, source, sink]``.
865
+ """
866
+ if pss is None:
867
+ pss = process_source_sink(source)
868
+ if process_indirect_set is None:
869
+ process_indirect_set = process_indirect(source)
870
+ if pss.height == 0 or process_indirect_set.height == 0:
871
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
872
+ return (pss.lazy()
873
+ # Cross-axis compare: sink (e) vs p (p). Per contract
874
+ # p ⊂ e; up-cast p to e so the compare runs in Enum.
875
+ .filter(pl.col("sink") == cast_dim(pl.col("p"), None, "e"))
876
+ .join(process_indirect_set.lazy(), on="p", how="inner")
877
+ .sort("p", "source", "sink")
878
+ .collect())
879
+
880
+
881
+ def process_output_flows(source: "InputSource",
882
+ pss: pl.DataFrame | None = None,
883
+ process_indirect_set: pl.DataFrame | None = None) -> pl.DataFrame:
884
+ """Indirect-process output arcs: ``pss`` filtered to ``source == p``
885
+ AND ``p ∈ process_indirect``.
886
+
887
+ Schema: ``[p, source, sink]``.
888
+ """
889
+ if pss is None:
890
+ pss = process_source_sink(source)
891
+ if process_indirect_set is None:
892
+ process_indirect_set = process_indirect(source)
893
+ if pss.height == 0 or process_indirect_set.height == 0:
894
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
895
+ return (pss.lazy()
896
+ # Cross-axis compare: source (e) vs p (p). Per contract
897
+ # p ⊂ e; up-cast p to e so the compare runs in Enum.
898
+ .filter(pl.col("source") == cast_dim(pl.col("p"), None, "e"))
899
+ .join(process_indirect_set.lazy(), on="p", how="inner")
900
+ .sort("p", "source", "sink")
901
+ .collect())
902
+
903
+
904
+ def process_indirect_dt(process_indirect_set: pl.DataFrame,
905
+ dt: pl.DataFrame) -> pl.DataFrame:
906
+ """``process_indirect × dt`` cross-product. Schema: ``[p, d, t]``.
907
+ """
908
+ if process_indirect_set is None or process_indirect_set.height == 0:
909
+ return _empty({"p": pl.Utf8, "d": pl.Utf8, "t": pl.Utf8})
910
+ if dt is None or dt.height == 0:
911
+ return _empty({"p": pl.Utf8, "d": pl.Utf8, "t": pl.Utf8})
912
+ # Defensive re-cast: ensure d/t are canonical Enum on the cross-join
913
+ # output (process_indirect_set carries Enum p).
914
+ return (process_indirect_set.lazy()
915
+ .join(dt.lazy()
916
+ .with_columns(alias_to_axis(pl.col("d"), "d"),
917
+ alias_to_axis(pl.col("t"), "t")),
918
+ how="cross")
919
+ .sort("p", "d", "t")
920
+ .collect())
921
+
922
+
923
+ # ---------------------------------------------------------------------------
924
+ # §1.6 — Constraint sense partitions (cdt_eq / cdt_le / cdt_ge)
925
+ # ---------------------------------------------------------------------------
926
+
927
+ def cdt_filter(source: "InputSource", sense: str,
928
+ dt: pl.DataFrame) -> pl.DataFrame | None:
929
+ """Cross-product of constraints with the named ``sense`` and
930
+ ``dt``. Returns ``None`` when the partition is empty (so the caller
931
+ can skip emitting the constraint group).
932
+
933
+ Senses: ``equal`` / ``less_than`` / ``greater_than``.
934
+ """
935
+ s = _try_param(source, "constraint", "sense")
936
+ if s is None or dt is None or dt.height == 0:
937
+ return None
938
+ rows = (s.lazy()
939
+ .filter(pl.col("value") == sense)
940
+ .select(alias_to_axis("name", "cn"))
941
+ .collect())
942
+ if rows.height == 0:
943
+ return None
944
+ # Defensive re-cast: ensure d/t are canonical Enum on the cross-join
945
+ # output (rows carries Enum cn via alias_to_axis above).
946
+ return (rows.lazy()
947
+ .join(dt.lazy()
948
+ .with_columns(alias_to_axis(pl.col("d"), "d"),
949
+ alias_to_axis(pl.col("t"), "t")),
950
+ how="cross")
951
+ .sort("cn", "d", "t")
952
+ .collect())
953
+
954
+
955
+ # ---------------------------------------------------------------------------
956
+ # §1.7 — Profile filter Projections
957
+ # ---------------------------------------------------------------------------
958
+
959
+ def _profile_filter(source: "InputSource",
960
+ profile_class: str,
961
+ dim_renames: dict[str, str],
962
+ out_cols: tuple[str, ...],
963
+ method: str) -> pl.DataFrame:
964
+ """Filter a profile-method relationship by ``method`` value.
965
+
966
+ ``dim_renames`` maps Spine column names → polar_high column names;
967
+ ``out_cols`` is the projection.
968
+ """
969
+ df = _try_param(source, profile_class, "profile_method")
970
+ if df is None:
971
+ empty_schema = {c: pl.Utf8 for c in out_cols}
972
+ return _empty(empty_schema)
973
+ return (df.lazy()
974
+ .filter(pl.col("value") == method)
975
+ .pipe(rename_to_axis, dim_renames)
976
+ .select(*out_cols)
977
+ .sort(*out_cols)
978
+ .collect())
979
+
980
+
981
+ def process_profile_upper(source: "InputSource") -> pl.DataFrame:
982
+ """Unit / connection profile arcs with ``profile_method=upper_limit``.
983
+
984
+ Schema: ``[p, source, sink, profile]``. Each row is keyed by
985
+ ``(p, source, sink, profile)``. Built as the union of the
986
+ ``unit__node__profile`` (for unit-side arcs) and
987
+ ``connection__node__profile`` (for connection-side arcs)
988
+ relationships filtered by method.
989
+
990
+ For ``unit__node__profile`` the node is either the source (input
991
+ arc) or the sink (output arc); the .mod resolves this from
992
+ ``unit__inputNode`` / ``unit__outputNode``. We replicate that by
993
+ cross-joining with ``pss_eff`` / ``pss`` on ``(p, node)``.
994
+ """
995
+ return _profile_method_arc(source, "upper_limit")
996
+
997
+
998
+ def process_profile_lower(source: "InputSource") -> pl.DataFrame:
999
+ return _profile_method_arc(source, "lower_limit")
1000
+
1001
+
1002
+ def process_profile_fixed(source: "InputSource") -> pl.DataFrame:
1003
+ return _profile_method_arc(source, "fixed")
1004
+
1005
+
1006
+ def _profile_method_arc(source: "InputSource", method: str) -> pl.DataFrame:
1007
+ """Internal: join unit/connection profile-method rows back to
1008
+ process_source_sink to reconstruct the (p, source, sink, f) arc
1009
+ tuples. Column ``f`` here is flextool's name for the profile.
1010
+
1011
+ For each ``(unit, node, profile)`` row with the requested method,
1012
+ emit two rows when the (unit, node) is a unit__inputNode (sink=unit,
1013
+ source=node) and one row per unit__outputNode (sink=node,
1014
+ source=unit). Connection profile-method rows are mapped via
1015
+ connection__node__node similarly.
1016
+ """
1017
+ out_schema = {"p": pl.Utf8, "source": pl.Utf8,
1018
+ "sink": pl.Utf8, "f": pl.Utf8}
1019
+
1020
+ parts: list[pl.LazyFrame] = []
1021
+
1022
+ # Unit profiles: unit__node__profile filtered by method, then map
1023
+ # node to the (unit, node) arc (input or output side).
1024
+ unp = _try_param(source, "unit__node__profile", "profile_method")
1025
+ if unp is not None:
1026
+ unp_lz = (unp.lazy()
1027
+ .filter(pl.col("value") == method)
1028
+ .select("unit", "node",
1029
+ alias_to_axis("profile", "f")))
1030
+ # Input side (node=source, sink=unit).
1031
+ uin = _try_entities(source, "unit__inputNode")
1032
+ if uin is not None:
1033
+ parts.append(
1034
+ unp_lz.join(uin.lazy(), on=["unit", "node"], how="inner")
1035
+ .select(
1036
+ alias_to_axis("unit", "p"),
1037
+ alias_to_axis("node", "source"),
1038
+ alias_to_axis("unit", "sink"),
1039
+ "f",
1040
+ ))
1041
+ # Output side (sink=node). The ``source`` slot is the unit's REAL
1042
+ # input-node, NOT the unit name: a DIRECT (constant_efficiency)
1043
+ # unit with an input node carries its output flow on the arc
1044
+ # ``v_flow[unit, input_node, output_node]`` (the preprocessing
1045
+ # ``process_source_sink`` collapses the (input × output) cross-
1046
+ # product to a single ``(p, source, sink)`` arc). Aliasing
1047
+ # ``source = unit`` here keyed the profile_flow constraint on a
1048
+ # non-existent ``(unit, unit, node)`` arc → empty LHS → infeasible
1049
+ # ``0 == rhs`` row. We recover the real source by joining the
1050
+ # ``(unit, output_node)`` rows back to the collapsed arc table on
1051
+ # ``(p, sink)``. For a source-less generator
1052
+ # (``conversion_method = none``, no input node) the collapsed arc
1053
+ # IS ``(p, p, sink)`` so the recovered ``source`` equals the unit
1054
+ # name — this branch is then a byte-exact no-op for VRE units.
1055
+ uout = _try_entities(source, "unit__outputNode")
1056
+ if uout is not None:
1057
+ collapsed = process_source_sink_collapsed(source)
1058
+ out_pf = (unp_lz
1059
+ .join(uout.lazy(), on=["unit", "node"], how="inner")
1060
+ .select(alias_to_axis("unit", "p"),
1061
+ alias_to_axis("node", "sink"),
1062
+ "f"))
1063
+ parts.append(
1064
+ out_pf.join(collapsed.lazy(), on=["p", "sink"], how="inner")
1065
+ .select("p", "source", "sink", "f"))
1066
+
1067
+ # Connection profile rows (rare in current fixtures but covered for
1068
+ # completeness). connection__node__profile: (connection, node, profile).
1069
+ cnp = _try_param(source, "connection__node__profile", "profile_method")
1070
+ if cnp is not None:
1071
+ cnp_lz = (cnp.lazy()
1072
+ .filter(pl.col("value") == method)
1073
+ .select("connection", "node",
1074
+ alias_to_axis("profile", "f")))
1075
+ cnn = _try_entities(source, "connection__node__node")
1076
+ if cnn is not None:
1077
+ cnn_lz = cnn.lazy()
1078
+ parts.append(
1079
+ cnp_lz.join(cnn_lz, left_on=["connection", "node"],
1080
+ right_on=["connection", "node_1"], how="inner")
1081
+ .select(
1082
+ alias_to_axis("connection", "p"),
1083
+ alias_to_axis("node", "source"),
1084
+ alias_to_axis("node_2", "sink"),
1085
+ "f",
1086
+ ))
1087
+ parts.append(
1088
+ cnp_lz.join(cnn_lz, left_on=["connection", "node"],
1089
+ right_on=["connection", "node_2"], how="inner")
1090
+ .select(
1091
+ alias_to_axis("connection", "p"),
1092
+ alias_to_axis("node_1", "source"),
1093
+ alias_to_axis("node", "sink"),
1094
+ "f",
1095
+ ))
1096
+
1097
+ if not parts:
1098
+ return _empty(out_schema)
1099
+ return (pl.concat(parts)
1100
+ .unique()
1101
+ .sort("p", "source", "sink", "f")
1102
+ .collect())
1103
+
1104
+
1105
+ def node_profile_filter(source: "InputSource", method: str) -> pl.DataFrame:
1106
+ """Filter ``node__profile`` by ``profile_method``.
1107
+
1108
+ Schema: ``[n, f]``. Column ``f`` is flextool's name for the
1109
+ profile column.
1110
+ """
1111
+ df = _try_param(source, "node__profile", "profile_method")
1112
+ if df is None:
1113
+ return _empty({"n": pl.Utf8, "f": pl.Utf8})
1114
+ return (df.lazy()
1115
+ .filter(pl.col("value") == method)
1116
+ .select(alias_to_axis("node", "n"),
1117
+ alias_to_axis("profile", "f"))
1118
+ .sort("n", "f")
1119
+ .collect())
1120
+
1121
+
1122
+ def node_profile_upper(source: "InputSource") -> pl.DataFrame:
1123
+ return node_profile_filter(source, "upper_limit")
1124
+
1125
+
1126
+ def node_profile_lower(source: "InputSource") -> pl.DataFrame:
1127
+ return node_profile_filter(source, "lower_limit")
1128
+
1129
+
1130
+ def node_profile_fixed(source: "InputSource") -> pl.DataFrame:
1131
+ return node_profile_filter(source, "fixed")
1132
+
1133
+
1134
+ # ---------------------------------------------------------------------------
1135
+ # §1.10 — Online / min_load Projections
1136
+ # ---------------------------------------------------------------------------
1137
+
1138
+ def process_online(source: "InputSource") -> pl.DataFrame:
1139
+ """Units with ``startup_method ∈ {linear, integer}``. Schema: ``[p]``."""
1140
+ df = _try_param(source, "unit", "startup_method")
1141
+ if df is None:
1142
+ return _empty({"p": pl.Utf8})
1143
+ return (df.lazy()
1144
+ .filter(pl.col("value").is_in(["linear", "integer"]))
1145
+ .select(alias_to_axis("name", "p"))
1146
+ .sort("p")
1147
+ .collect())
1148
+
1149
+
1150
+ def process_online_linear(source: "InputSource") -> pl.DataFrame:
1151
+ df = _try_param(source, "unit", "startup_method")
1152
+ if df is None:
1153
+ return _empty({"p": pl.Utf8})
1154
+ return (df.lazy()
1155
+ .filter(pl.col("value") == "linear")
1156
+ .select(alias_to_axis("name", "p"))
1157
+ .sort("p")
1158
+ .collect())
1159
+
1160
+
1161
+ def process_online_integer(source: "InputSource") -> pl.DataFrame:
1162
+ df = _try_param(source, "unit", "startup_method")
1163
+ if df is None:
1164
+ return _empty({"p": pl.Utf8})
1165
+ return (df.lazy()
1166
+ .filter(pl.col("value") == "integer")
1167
+ .select(alias_to_axis("name", "p"))
1168
+ .sort("p")
1169
+ .collect())
1170
+
1171
+
1172
+ def process_minload(source: "InputSource") -> pl.DataFrame:
1173
+ """Units with ``min_load > 0``. Schema: ``[p]``.
1174
+
1175
+ ``min_load`` defaults to 0.0 (per §5.3) — entities with the default
1176
+ are filtered out.
1177
+ """
1178
+ df = _try_param(source, "unit", "min_load")
1179
+ if df is None:
1180
+ return _empty({"p": pl.Utf8})
1181
+ return (df.lazy()
1182
+ .filter(pl.col("value") > 0)
1183
+ .select(alias_to_axis("name", "p"))
1184
+ .sort("p")
1185
+ .collect())
1186
+
1187
+
1188
+ def process_min_load_eff(source: "InputSource") -> pl.DataFrame:
1189
+ """Units with ``conversion_method='min_load_efficiency'``. Schema: ``[p]``."""
1190
+ df = _try_param(source, "unit", "conversion_method")
1191
+ if df is None:
1192
+ return _empty({"p": pl.Utf8})
1193
+ return (df.lazy()
1194
+ .filter(pl.col("value") == "min_load_efficiency")
1195
+ .select(alias_to_axis("name", "p"))
1196
+ .sort("p")
1197
+ .collect())
1198
+
1199
+
1200
+ # ---------------------------------------------------------------------------
1201
+ # §1.11 — Storage Projections
1202
+ # ---------------------------------------------------------------------------
1203
+
1204
+ def nodeState(source: "InputSource") -> pl.DataFrame:
1205
+ """Nodes with ``node_type='storage'``. Schema: ``[n]``.
1206
+ """
1207
+ df = _try_param(source, "node", "node_type")
1208
+ if df is None:
1209
+ return _empty({"n": pl.Utf8})
1210
+ return (df.lazy()
1211
+ .filter(pl.col("value") == "storage")
1212
+ .pipe(rename_to_axis, {"name": "n"})
1213
+ .select("n")
1214
+ .sort("n")
1215
+ .collect())
1216
+
1217
+
1218
+ def storage_bind_filter(source: "InputSource", method: str) -> pl.DataFrame:
1219
+ """Filter ``node`` set by ``storage_binding_method``."""
1220
+ df = _try_param(source, "node", "storage_binding_method")
1221
+ if df is None:
1222
+ return _empty({"n": pl.Utf8})
1223
+ return (df.lazy()
1224
+ .filter(pl.col("value") == method)
1225
+ .pipe(rename_to_axis, {"name": "n"})
1226
+ .select("n")
1227
+ .sort("n")
1228
+ .collect())
1229
+
1230
+
1231
+ def storage_bind_within_timeblock(source: "InputSource") -> pl.DataFrame:
1232
+ return storage_bind_filter(source, "bind_within_timeblock")
1233
+
1234
+
1235
+ def storage_bind_forward_only(source: "InputSource") -> pl.DataFrame:
1236
+ return storage_bind_filter(source, "forward_only")
1237
+
1238
+
1239
+ def storage_bind_within_solve(source: "InputSource") -> pl.DataFrame:
1240
+ return storage_bind_filter(source, "bind_within_solve")
1241
+
1242
+
1243
+ def storage_bind_within_solve_blended_weights(source: "InputSource") -> pl.DataFrame:
1244
+ return storage_bind_filter(source, "bind_within_solve_blended_weights")
1245
+
1246
+
1247
+ def storage_fix_start(source: "InputSource") -> pl.DataFrame:
1248
+ """Nodes with ``storage_start_end_method='fix_start'``. Schema: ``[n]``."""
1249
+ df = _try_param(source, "node", "storage_start_end_method")
1250
+ if df is None:
1251
+ return _empty({"n": pl.Utf8})
1252
+ return (df.lazy()
1253
+ .filter(pl.col("value") == "fix_start")
1254
+ .pipe(rename_to_axis, {"name": "n"})
1255
+ .select("n")
1256
+ .sort("n")
1257
+ .collect())
1258
+
1259
+
1260
+ def n_fix_storage_quantity(source: "InputSource") -> pl.DataFrame:
1261
+ """Nodes with ``storage_nested_fix_method='fix_quantity'``. Schema: ``[n]``."""
1262
+ df = _try_param(source, "node", "storage_nested_fix_method")
1263
+ if df is None:
1264
+ return _empty({"n": pl.Utf8})
1265
+ return (df.lazy()
1266
+ .filter(pl.col("value") == "fix_quantity")
1267
+ .pipe(rename_to_axis, {"name": "n"})
1268
+ .select("n")
1269
+ .sort("n")
1270
+ .collect())
1271
+
1272
+
1273
+ # ---------------------------------------------------------------------------
1274
+ # §1.14 — Group-level slack Projections
1275
+ # ---------------------------------------------------------------------------
1276
+
1277
+ def _group_yes(source: "InputSource", parameter_name: str) -> pl.DataFrame:
1278
+ """Distinct groups where the boolean param is ``yes``. Schema: ``[g]``."""
1279
+ df = _try_param(source, "group", parameter_name)
1280
+ if df is None:
1281
+ return _empty({"g": pl.Utf8})
1282
+ return (df.lazy()
1283
+ .filter(pl.col("value") == "yes")
1284
+ .select(alias_to_axis("name", "g"))
1285
+ .unique()
1286
+ .sort("g")
1287
+ .collect())
1288
+
1289
+
1290
+ def _group_param_equals(
1291
+ source: "InputSource", parameter_name: str, match: str
1292
+ ) -> pl.DataFrame:
1293
+ """Distinct groups where ``parameter_name`` equals ``match``. Schema: ``[g]``."""
1294
+ df = _try_param(source, "group", parameter_name)
1295
+ if df is None:
1296
+ return _empty({"g": pl.Utf8})
1297
+ return (df.lazy()
1298
+ .filter(pl.col("value") == match)
1299
+ .select(alias_to_axis("name", "g"))
1300
+ .unique()
1301
+ .sort("g")
1302
+ .collect())
1303
+
1304
+
1305
+ def groupCapacityMargin(source: "InputSource") -> pl.DataFrame:
1306
+ return _group_param_equals(source, "capacity_margin_method", "manual")
1307
+
1308
+
1309
+ def groupInertia(source: "InputSource") -> pl.DataFrame:
1310
+ return _group_yes(source, "has_inertia")
1311
+
1312
+
1313
+ def groupNonSync(source: "InputSource") -> pl.DataFrame:
1314
+ return _group_yes(source, "has_non_synchronous")
1315
+
1316
+
1317
+ def groupStochastic(source: "InputSource") -> pl.DataFrame:
1318
+ return _group_yes(source, "include_stochastics")
1319
+
1320
+
1321
+ def group_node(source: "InputSource") -> pl.DataFrame:
1322
+ """Membership ``group__node``. Schema: ``[g, n]``."""
1323
+ df = _try_entities(source, "group__node")
1324
+ if df is None:
1325
+ return _empty({"g": pl.Utf8, "n": pl.Utf8})
1326
+ return (df.lazy()
1327
+ .pipe(rename_to_axis, {"group": "g", "node": "n"})
1328
+ .select("g", "n")
1329
+ .sort("g", "n")
1330
+ .collect())
1331
+
1332
+
1333
+ def process_unit(source: "InputSource") -> pl.DataFrame:
1334
+ """Unit-typed processes (excludes connections). Schema: ``[p]``."""
1335
+ df = _try_entities(source, "unit")
1336
+ if df is None:
1337
+ return _empty({"p": pl.Utf8})
1338
+ return (df.lazy()
1339
+ .pipe(rename_to_axis, {"name": "p"})
1340
+ .select("p")
1341
+ .sort("p")
1342
+ .collect())
1343
+
1344
+
1345
+ def process_sink_inertia(source: "InputSource") -> pl.DataFrame:
1346
+ """Output-arcs with non-zero ``inertia_constant``. Schema: ``[p, sink]``."""
1347
+ df = _try_param(source, "unit__outputNode", "inertia_constant")
1348
+ if df is None:
1349
+ return _empty({"p": pl.Utf8, "sink": pl.Utf8})
1350
+ return (df.lazy()
1351
+ .filter(pl.col("value") != 0)
1352
+ .pipe(rename_to_axis, {"unit": "p", "node": "sink"})
1353
+ .select("p", "sink")
1354
+ .sort("p", "sink")
1355
+ .collect())
1356
+
1357
+
1358
+ def process_source_inertia(source: "InputSource") -> pl.DataFrame:
1359
+ """Input-arcs with non-zero ``inertia_constant``. Schema: ``[p, source]``."""
1360
+ df = _try_param(source, "unit__inputNode", "inertia_constant")
1361
+ if df is None:
1362
+ return _empty({"p": pl.Utf8, "source": pl.Utf8})
1363
+ return (df.lazy()
1364
+ .filter(pl.col("value") != 0)
1365
+ .pipe(rename_to_axis, {"unit": "p", "node": "source"})
1366
+ .select("p", "source")
1367
+ .sort("p", "source")
1368
+ .collect())
1369
+
1370
+
1371
+ def process_sink_nonSync(source: "InputSource") -> pl.DataFrame:
1372
+ """Output-arcs with ``is_non_synchronous='yes'``. Schema: ``[p, sink]``."""
1373
+ df = _try_param(source, "unit__outputNode", "is_non_synchronous")
1374
+ if df is None:
1375
+ return _empty({"p": pl.Utf8, "sink": pl.Utf8})
1376
+ return (df.lazy()
1377
+ .filter(pl.col("value") == "yes")
1378
+ .pipe(rename_to_axis, {"unit": "p", "node": "sink"})
1379
+ .select("p", "sink")
1380
+ .sort("p", "sink")
1381
+ .collect())
1382
+
1383
+
1384
+ # ---------------------------------------------------------------------------
1385
+ # §1.15 — Reserves
1386
+ # ---------------------------------------------------------------------------
1387
+
1388
+ def reserve_upDown_group(source: "InputSource") -> pl.DataFrame:
1389
+ """Distinct ``(r, ud, g)`` tuples for reserves with a populated
1390
+ ``reserve_method`` (the active reserve subset; entities without a
1391
+ method are silently dropped, mirroring flextool's preprocessing).
1392
+
1393
+ Schema: ``[r, ud, g]``.
1394
+ """
1395
+ df = _try_param(source, "reserve__upDown__group", "reserve_method")
1396
+ if df is None:
1397
+ return _empty({"r": pl.Utf8, "ud": pl.Utf8, "g": pl.Utf8})
1398
+ return (df.lazy()
1399
+ .pipe(rename_to_axis, {"reserve": "r", "upDown": "ud", "group": "g"})
1400
+ .select("r", "ud", "g")
1401
+ .unique()
1402
+ .sort("r", "ud", "g")
1403
+ .collect())
1404
+
1405
+
1406
+ _RESERVE_METHOD_TO_FIELD = {
1407
+ "timeseries": "timeseries_only",
1408
+ "dynamic": "dynamic",
1409
+ "n_1": "large_failure",
1410
+ }
1411
+
1412
+
1413
+ def reserve_upDown_group_method(source: "InputSource", method: str) -> pl.DataFrame:
1414
+ """Filter ``reserve__upDown__group`` by ``reserve_method`` value.
1415
+
1416
+ ``method`` is the polar_high partition name (``timeseries`` / ``dynamic``
1417
+ / ``n_1``); the underlying Spine value is mapped via
1418
+ ``_RESERVE_METHOD_TO_FIELD``.
1419
+ """
1420
+ spine_method = _RESERVE_METHOD_TO_FIELD.get(method, method)
1421
+ df = _try_param(source, "reserve__upDown__group", "reserve_method")
1422
+ if df is None:
1423
+ return _empty({"r": pl.Utf8, "ud": pl.Utf8, "g": pl.Utf8,
1424
+ "method": pl.Utf8})
1425
+ return (df.lazy()
1426
+ .filter(pl.col("value") == spine_method)
1427
+ .pipe(rename_to_axis, {"reserve": "r", "upDown": "ud", "group": "g",
1428
+ "value": "method"})
1429
+ .select("r", "ud", "g", "method")
1430
+ .sort("r", "ud", "g")
1431
+ .collect())
1432
+
1433
+
1434
+ def process_reserve_upDown_node_active(source: "InputSource") -> pl.DataFrame:
1435
+ """Union of ``reserve__upDown__unit__node`` and
1436
+ ``reserve__upDown__connection__node`` membership.
1437
+
1438
+ Schema: ``[p, r, ud, n]``.
1439
+ """
1440
+ parts: list[pl.LazyFrame] = []
1441
+ u = _try_entities(source, "reserve__upDown__unit__node")
1442
+ if u is not None:
1443
+ parts.append(u.lazy().select(
1444
+ alias_to_axis("unit", "p"),
1445
+ alias_to_axis("reserve", "r"),
1446
+ alias_to_axis("upDown", "ud"),
1447
+ alias_to_axis("node", "n"),
1448
+ ))
1449
+ c = _try_entities(source, "reserve__upDown__connection__node")
1450
+ if c is not None:
1451
+ parts.append(c.lazy().select(
1452
+ alias_to_axis("connection", "p"),
1453
+ alias_to_axis("reserve", "r"),
1454
+ alias_to_axis("upDown", "ud"),
1455
+ alias_to_axis("node", "n"),
1456
+ ))
1457
+ if not parts:
1458
+ return _empty({"p": pl.Utf8, "r": pl.Utf8, "ud": pl.Utf8, "n": pl.Utf8})
1459
+ return (pl.concat(parts).unique().sort("p", "r", "ud", "n").collect())
1460
+
1461
+
1462
+ def _process_reserve_upDown_node_filter(source: "InputSource",
1463
+ parameter_name: str) -> pl.DataFrame:
1464
+ """Union the per-(p, r, ud, n) parameter across the two relationship
1465
+ classes, filter to non-zero values, project to membership.
1466
+ """
1467
+ parts: list[pl.LazyFrame] = []
1468
+ u = _try_param(source, "reserve__upDown__unit__node", parameter_name)
1469
+ if u is not None:
1470
+ parts.append(u.lazy()
1471
+ .filter(pl.col("value") != 0)
1472
+ .select(
1473
+ alias_to_axis("unit", "p"),
1474
+ alias_to_axis("reserve", "r"),
1475
+ alias_to_axis("upDown", "ud"),
1476
+ alias_to_axis("node", "n"),
1477
+ ))
1478
+ c = _try_param(source, "reserve__upDown__connection__node", parameter_name)
1479
+ if c is not None:
1480
+ parts.append(c.lazy()
1481
+ .filter(pl.col("value") != 0)
1482
+ .select(
1483
+ alias_to_axis("connection", "p"),
1484
+ alias_to_axis("reserve", "r"),
1485
+ alias_to_axis("upDown", "ud"),
1486
+ alias_to_axis("node", "n"),
1487
+ ))
1488
+ if not parts:
1489
+ return _empty({"p": pl.Utf8, "r": pl.Utf8, "ud": pl.Utf8, "n": pl.Utf8})
1490
+ return (pl.concat(parts).unique().sort("p", "r", "ud", "n").collect())
1491
+
1492
+
1493
+ def process_reserve_upDown_node_increase_reserve_ratio(source: "InputSource") -> pl.DataFrame:
1494
+ return _process_reserve_upDown_node_filter(source, "increase_reserve_ratio")
1495
+
1496
+
1497
+ def process_reserve_upDown_node_large_failure_ratio(source: "InputSource") -> pl.DataFrame:
1498
+ return _process_reserve_upDown_node_filter(source, "large_failure_ratio")
1499
+
1500
+
1501
+ # ---------------------------------------------------------------------------
1502
+ # §1.16 — Cumulative / group-invest Projections
1503
+ # ---------------------------------------------------------------------------
1504
+
1505
+ def _e_param_set(source: "InputSource", parameter_name: str,
1506
+ positive_only: bool = True) -> pl.DataFrame:
1507
+ """Distinct entities (across unit/node/connection) where the
1508
+ parameter has a (non-zero, positive if ``positive_only``) value.
1509
+
1510
+ Schema: ``[e]``.
1511
+ """
1512
+ parts: list[pl.LazyFrame] = []
1513
+ for cls in ("unit", "node", "connection"):
1514
+ df = _try_param(source, cls, parameter_name)
1515
+ if df is None:
1516
+ continue
1517
+ flt = df.lazy()
1518
+ if positive_only:
1519
+ flt = flt.filter(pl.col("value") > 0)
1520
+ parts.append(flt.select(alias_to_axis("name", "e")))
1521
+ if not parts:
1522
+ return _empty({"e": pl.Utf8})
1523
+ return pl.concat(parts).unique().sort("e").collect()
1524
+
1525
+
1526
+ # Mirror ``invest_total_sets.py:25-32``.
1527
+ _INVEST_METHODS_INVEST_TOTAL: frozenset[str] = frozenset((
1528
+ "invest_total", "invest_period_total",
1529
+ "invest_retire_total", "invest_retire_period_total",
1530
+ ))
1531
+ _INVEST_METHODS_DIVEST_TOTAL: frozenset[str] = frozenset((
1532
+ "retire_total", "retire_period_total",
1533
+ "invest_retire_total", "invest_retire_period_total",
1534
+ ))
1535
+
1536
+
1537
+ def _e_invest_method_filter(source: "InputSource",
1538
+ allowed: frozenset[str]) -> pl.DataFrame:
1539
+ """Distinct entities (across unit/node/connection) whose
1540
+ ``invest_method`` is in ``allowed``. Schema: ``[e]``.
1541
+
1542
+ Mirrors ``invest_total_sets.py:74-90``: ``e_invest_total`` is the
1543
+ intersection of ``entityInvest`` (allowed-invest method enum) with
1544
+ the ``INVEST_TOTAL`` enum subset.
1545
+ """
1546
+ parts: list[pl.LazyFrame] = []
1547
+ for cls in ("unit", "node", "connection"):
1548
+ df = _try_param(source, cls, "invest_method")
1549
+ if df is None:
1550
+ continue
1551
+ parts.append(df.lazy()
1552
+ .filter(pl.col("value").is_in(list(allowed)))
1553
+ .select(alias_to_axis("name", "e")))
1554
+ if not parts:
1555
+ return _empty({"e": pl.Utf8})
1556
+ return pl.concat(parts).unique().sort("e").collect()
1557
+
1558
+
1559
+ def e_invest_total(source: "InputSource") -> pl.DataFrame:
1560
+ """Entities whose ``invest_method`` ∈ INVEST_TOTAL enum subset.
1561
+
1562
+ Mirrors flextool's ``invest_total_sets.write_invest_total_sets`` —
1563
+ filters ``entity__invest_method`` by the four
1564
+ ``invest_total / invest_period_total / invest_retire_total /
1565
+ invest_retire_period_total`` enum values.
1566
+ """
1567
+ return _e_invest_method_filter(source, _INVEST_METHODS_INVEST_TOTAL)
1568
+
1569
+
1570
+ def e_divest_total(source: "InputSource") -> pl.DataFrame:
1571
+ """Entities whose ``invest_method`` ∈ DIVEST_TOTAL enum subset."""
1572
+ return _e_invest_method_filter(source, _INVEST_METHODS_DIVEST_TOTAL)
1573
+
1574
+
1575
+ def ed_invest_cumulative(source: "InputSource",
1576
+ ed_invest: pl.DataFrame | None = None) -> pl.DataFrame:
1577
+ """Subset of ``ed_invest`` whose entity carries
1578
+ ``cumulative_max_capacity > 0`` OR ``cumulative_min_capacity > 0``
1579
+ (the cumulative-cap rows). Schema: ``[e, d]``.
1580
+
1581
+ NOTE: ``ed_invest`` is itself Derived (Γ.3); this Projection only
1582
+ runs when ``ed_invest`` is supplied externally — without it we
1583
+ cannot resolve the (e, d) tuples. When ``ed_invest`` is None we
1584
+ return an empty frame and the caller (CSV-side) keeps its native
1585
+ value.
1586
+ """
1587
+ if ed_invest is None or ed_invest.height == 0:
1588
+ return _empty({"e": pl.Utf8, "d": pl.Utf8})
1589
+
1590
+ parts: list[pl.LazyFrame] = []
1591
+ for parameter_name in ("cumulative_max_capacity", "cumulative_min_capacity"):
1592
+ for cls in ("unit", "node", "connection"):
1593
+ df = _try_param(source, cls, parameter_name)
1594
+ if df is None:
1595
+ continue
1596
+ parts.append(df.lazy()
1597
+ .filter(pl.col("value") > 0)
1598
+ .select(alias_to_axis("name", "e")))
1599
+
1600
+ if not parts:
1601
+ return _empty({"e": pl.Utf8, "d": pl.Utf8})
1602
+ cum_e = pl.concat(parts).unique().collect()
1603
+ return (ed_invest.lazy()
1604
+ .join(cum_e.lazy(), on="e", how="inner")
1605
+ .sort("e", "d")
1606
+ .collect())
1607
+
1608
+
1609
+ def group_entity(source: "InputSource") -> pl.DataFrame:
1610
+ """Union of ``group__node``, ``group__unit``, ``group__connection``.
1611
+
1612
+ Schema: ``[g, e]``.
1613
+ """
1614
+ parts: list[pl.LazyFrame] = []
1615
+ _enums = get_global_axis_enums()
1616
+ for cls, dim in (("group__node", "node"),
1617
+ ("group__unit", "unit"),
1618
+ ("group__connection", "connection")):
1619
+ df = _try_entities(source, cls)
1620
+ if df is None:
1621
+ continue
1622
+ parts.append(df.lazy().select(
1623
+ alias_to_axis("group", "g"),
1624
+ # cast across class-specific enum into the entity-union ``e``
1625
+ # enum so concat across (group__node, group__unit,
1626
+ # group__connection) lines up on a single dtype.
1627
+ alias_to_axis(pl.col(dim), "e"),
1628
+ ))
1629
+ if not parts:
1630
+ return _empty({"g": pl.Utf8, "e": pl.Utf8})
1631
+ out = pl.concat(parts).unique().sort("g", "e").collect()
1632
+ if _enums is not None:
1633
+ out = cast_frame_axes(out, _enums)
1634
+ return out
1635
+
1636
+
1637
+ def group_process_node(source: "InputSource") -> pl.DataFrame:
1638
+ """Union of ``group__unit__node`` and ``group__connection__node``.
1639
+
1640
+ Schema: ``[g, p, n]``.
1641
+ """
1642
+ parts: list[pl.LazyFrame] = []
1643
+ u = _try_entities(source, "flowGroup__unit__node")
1644
+ if u is not None:
1645
+ parts.append(
1646
+ u.lazy().pipe(rename_to_axis,
1647
+ {"flowGroup": "g", "unit": "p", "node": "n"})
1648
+ .select("g", "p", "n"),
1649
+ )
1650
+ c = _try_entities(source, "flowGroup__connection__node")
1651
+ if c is not None:
1652
+ parts.append(
1653
+ c.lazy().pipe(rename_to_axis,
1654
+ {"flowGroup": "g", "connection": "p", "node": "n"})
1655
+ .select("g", "p", "n"),
1656
+ )
1657
+ if not parts:
1658
+ return _empty({"g": pl.Utf8, "p": pl.Utf8, "n": pl.Utf8})
1659
+ return pl.concat(parts).unique().sort("g", "p", "n").collect()
1660
+
1661
+
1662
+ def _g_param_set(source: "InputSource", parameter_name: str,
1663
+ positive_only: bool = True) -> pl.DataFrame:
1664
+ """Distinct groups with a (positive) value for the named parameter.
1665
+ Handles both scalar and 1d_map(period) shapes. Schema: ``[g]``.
1666
+ """
1667
+ df = _try_param(source, "group", parameter_name)
1668
+ if df is None:
1669
+ return _empty({"g": pl.Utf8})
1670
+ flt = df.lazy()
1671
+ if positive_only:
1672
+ flt = flt.filter(pl.col("value") > 0)
1673
+ return (flt.select(alias_to_axis("name", "g"))
1674
+ .unique()
1675
+ .sort("g")
1676
+ .collect())
1677
+
1678
+
1679
+ def g_invest_total(source: "InputSource") -> pl.DataFrame:
1680
+ return _g_param_set(source, "invest_max_total")
1681
+
1682
+
1683
+ def g_divest_total(source: "InputSource") -> pl.DataFrame:
1684
+ return _g_param_set(source, "retire_max_total")
1685
+
1686
+
1687
+ def g_invest_cumulative(source: "InputSource") -> pl.DataFrame:
1688
+ return _g_param_set(source, "invest_max_cumulative")
1689
+
1690
+
1691
+ # NB: the instant-flow constraint support (``gdt_*InstantFlow``) used to
1692
+ # live here as a raw-source projection, but it mis-detected the index
1693
+ # axis by column name — silently dropping Spine's silent-default ``"x"``
1694
+ # index_name and constant maps, and crashing on pure period maps (no
1695
+ # ``t`` column to select). The ``(g, d, t)`` support is now derived
1696
+ # directly from the resolved ``pdt_*_instant_flow`` cap inside
1697
+ # ``_cumulative_invest._emit_instant_flow``, which routes shape detection
1698
+ # through ``_param_shapes.resolve_param_shape``.
1699
+
1700
+
1701
+ # ---------------------------------------------------------------------------
1702
+ # §1.17 — Delayed processes
1703
+ # ---------------------------------------------------------------------------
1704
+
1705
+ def process_delayed(source: "InputSource") -> pl.DataFrame:
1706
+ """Units / connections with a non-zero ``delay`` Map.
1707
+
1708
+ ``delay`` is a 1d_map(td) parameter; non-empty rows yield
1709
+ membership. Schema: ``[p]``.
1710
+ """
1711
+ parts: list[pl.LazyFrame] = []
1712
+ for cls in ("unit", "connection"):
1713
+ df = _try_param(source, cls, "delay")
1714
+ if df is None:
1715
+ continue
1716
+ parts.append(df.lazy()
1717
+ .filter(pl.col("value") != 0)
1718
+ .select(alias_to_axis("name", "p")))
1719
+ if not parts:
1720
+ return _empty({"p": pl.Utf8})
1721
+ return pl.concat(parts).unique().sort("p").collect()
1722
+
1723
+
1724
+ def process_source_delayed(source: "InputSource",
1725
+ pss: pl.DataFrame | None = None,
1726
+ process_delayed_set: pl.DataFrame | None = None) -> pl.DataFrame:
1727
+ """Source-side arcs of delayed processes. Schema: ``[p, source]``."""
1728
+ if pss is None:
1729
+ pss = process_source_sink(source)
1730
+ if process_delayed_set is None:
1731
+ process_delayed_set = process_delayed(source)
1732
+ if pss.height == 0 or process_delayed_set.height == 0:
1733
+ return _empty({"p": pl.Utf8, "source": pl.Utf8})
1734
+ return (pss.lazy()
1735
+ .join(process_delayed_set.lazy(), on="p", how="inner")
1736
+ .select("p", "source")
1737
+ .unique()
1738
+ .sort("p", "source")
1739
+ .collect())
1740
+
1741
+
1742
+ def process_source_undelayed(source: "InputSource",
1743
+ pss: pl.DataFrame | None = None,
1744
+ process_delayed_set: pl.DataFrame | None = None) -> pl.DataFrame:
1745
+ """Source-side arcs of NON-delayed processes. Schema: ``[p, source]``."""
1746
+ if pss is None:
1747
+ pss = process_source_sink(source)
1748
+ if process_delayed_set is None:
1749
+ process_delayed_set = process_delayed(source)
1750
+ if pss.height == 0:
1751
+ return _empty({"p": pl.Utf8, "source": pl.Utf8})
1752
+ return (pss.lazy()
1753
+ .join(process_delayed_set.lazy(), on="p", how="anti")
1754
+ .select("p", "source")
1755
+ .unique()
1756
+ .sort("p", "source")
1757
+ .collect())
1758
+
1759
+
1760
+ def process_source_sink_delayed(source: "InputSource",
1761
+ pss: pl.DataFrame | None = None,
1762
+ process_delayed_set: pl.DataFrame | None = None) -> pl.DataFrame:
1763
+ """Full pss tuples of delayed processes. Schema: ``[p, source, sink]``."""
1764
+ if pss is None:
1765
+ pss = process_source_sink(source)
1766
+ if process_delayed_set is None:
1767
+ process_delayed_set = process_delayed(source)
1768
+ if pss.height == 0 or process_delayed_set.height == 0:
1769
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
1770
+ return (pss.lazy()
1771
+ .join(process_delayed_set.lazy(), on="p", how="inner")
1772
+ .sort("p", "source", "sink")
1773
+ .collect())
1774
+
1775
+
1776
+ def process_source_sink_undelayed(source: "InputSource",
1777
+ pss: pl.DataFrame | None = None,
1778
+ process_delayed_set: pl.DataFrame | None = None) -> pl.DataFrame:
1779
+ """Full pss tuples of NON-delayed processes. Schema: ``[p, source, sink]``."""
1780
+ if pss is None:
1781
+ pss = process_source_sink(source)
1782
+ if process_delayed_set is None:
1783
+ process_delayed_set = process_delayed(source)
1784
+ if pss.height == 0:
1785
+ return _empty({"p": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8})
1786
+ return (pss.lazy()
1787
+ .join(process_delayed_set.lazy(), on="p", how="anti")
1788
+ .sort("p", "source", "sink")
1789
+ .collect())
1790
+
1791
+
1792
+ # ---------------------------------------------------------------------------
1793
+ # §1.18 — DC power flow
1794
+ # ---------------------------------------------------------------------------
1795
+
1796
+ def connection_dc_power_flow(source: "InputSource") -> pl.DataFrame:
1797
+ """Connections with ``is_DC='yes'``. Schema: ``[p]``."""
1798
+ df = _try_param(source, "connection", "is_DC")
1799
+ if df is None:
1800
+ return _empty({"p": pl.Utf8})
1801
+ return (df.lazy()
1802
+ .filter(pl.col("value") == "yes")
1803
+ .select(alias_to_axis("name", "p"))
1804
+ .sort("p")
1805
+ .collect())
1806
+
1807
+
1808
+ # ---------------------------------------------------------------------------
1809
+ # §1.19 — Commodity ladder Projections
1810
+ # ---------------------------------------------------------------------------
1811
+
1812
+ _LADDER_METHODS = ("price_ladder_annual", "price_ladder_cumulative")
1813
+
1814
+
1815
+ def commodity_with_ladder(source: "InputSource") -> pl.DataFrame:
1816
+ """Commodities with ``price_method`` in the ladder family. Schema: ``[c]``."""
1817
+ df = _try_param(source, "commodity", "price_method")
1818
+ if df is None:
1819
+ return _empty({"c": pl.Utf8})
1820
+ return (df.lazy()
1821
+ .filter(pl.col("value").is_in(_LADDER_METHODS))
1822
+ .select(alias_to_axis("name", "c"))
1823
+ .sort("c")
1824
+ .collect())
1825
+
1826
+
1827
+ def commodity_with_ladder_annual(source: "InputSource") -> pl.DataFrame:
1828
+ df = _try_param(source, "commodity", "price_method")
1829
+ if df is None:
1830
+ return _empty({"c": pl.Utf8})
1831
+ return (df.lazy()
1832
+ .filter(pl.col("value") == "price_ladder_annual")
1833
+ .select(alias_to_axis("name", "c"))
1834
+ .sort("c")
1835
+ .collect())
1836
+
1837
+
1838
+ def commodity_with_ladder_cumulative(source: "InputSource") -> pl.DataFrame:
1839
+ df = _try_param(source, "commodity", "price_method")
1840
+ if df is None:
1841
+ return _empty({"c": pl.Utf8})
1842
+ return (df.lazy()
1843
+ .filter(pl.col("value") == "price_ladder_cumulative")
1844
+ .select(alias_to_axis("name", "c"))
1845
+ .sort("c")
1846
+ .collect())
1847
+
1848
+
1849
+ def commodity__tier_ann(source: "InputSource") -> pl.DataFrame:
1850
+ """Distinct ``(c, i)`` pairs from the annual price ladder.
1851
+
1852
+ Schema: ``[c, i]``. ``commodity_ladder_annual`` is a structured
1853
+ Map(period → Map(tier → Map({"price","quantity"} → f64))). The
1854
+ SpineDbReader split this into ``p_ladder_ann_price`` /
1855
+ ``p_ladder_ann_quantity`` keyed on ``(commodity, period, tier)``;
1856
+ we project to distinct ``(c, tier)`` pairs.
1857
+ """
1858
+ # Use the resolved Direct frame as the source — try price first
1859
+ # then quantity (either suffices for tier membership).
1860
+ for parameter_name in ("price_ladder_annual",):
1861
+ df = _try_param(source, "commodity", parameter_name)
1862
+ if df is None:
1863
+ continue
1864
+ cols = df.columns
1865
+ if "tier" in cols and "name" in cols:
1866
+ return (df.lazy()
1867
+ .pipe(rename_to_axis, {"name": "c", "tier": "i"})
1868
+ .select("c", "i")
1869
+ .unique()
1870
+ .sort("c", "i")
1871
+ .collect())
1872
+ return _empty({"c": pl.Utf8, "i": pl.Utf8})
1873
+
1874
+
1875
+ def commodity__tier_cum(source: "InputSource") -> pl.DataFrame:
1876
+ """Distinct ``(c, i)`` pairs from the cumulative price ladder."""
1877
+ for parameter_name in ("price_ladder_cumulative",):
1878
+ df = _try_param(source, "commodity", parameter_name)
1879
+ if df is None:
1880
+ continue
1881
+ cols = df.columns
1882
+ if "tier" in cols and "name" in cols:
1883
+ return (df.lazy()
1884
+ .pipe(rename_to_axis, {"name": "c", "tier": "i"})
1885
+ .select("c", "i")
1886
+ .unique()
1887
+ .sort("c", "i")
1888
+ .collect())
1889
+ return _empty({"c": pl.Utf8, "i": pl.Utf8})
1890
+
1891
+
1892
+ # ---------------------------------------------------------------------------
1893
+ # Catalog (used by parity tests)
1894
+ # ---------------------------------------------------------------------------
1895
+
1896
+ # Reclassified-Derived list (audit row says "P" but the helper actually
1897
+ # requires method-aware preprocessing — see flextool's
1898
+ # preprocessing/process_arc_unions.py 10-way union of method-specific
1899
+ # arc tuples. Flagged for Γ.3.):
1900
+ # process_source_sink, process_source_sink_eff, process_source_sink_noEff
1901
+ # flow_to_commodity, flow_from_commodity_eff, flow_from_commodity_noEff
1902
+ # flow_from_co2_priced/_capped (use the eff/noEff partitions)
1903
+ # process_indirect, process_input_flows, process_output_flows, process_indirect_dt
1904
+ # process_source_delayed, process_source_undelayed,
1905
+ # process_source_sink_delayed, process_source_sink_undelayed,
1906
+ # pdt_online_linear / pdt_online_integer (depend on p_online_dt — Derived)
1907
+ # p_online_dt itself (block-aware — Derived)
1908
+ #
1909
+ # These are still implemented in this module (taking pre-built ``pss`` /
1910
+ # ``process_indirect_set`` / ``process_delayed_set`` arguments) so they
1911
+ # can be invoked once the prerequisite Derived frames land in Γ.3 — but
1912
+ # they are NOT wired into ``apply_projection_params``.
1913
+
1914
+ # Mapping: FlexData field → callable taking only ``source``. These are
1915
+ # clean, source-only Projections — no dependencies on Derived state.
1916
+ SIMPLE_PROJECTIONS: dict[str, callable] = {
1917
+ "nodeBalance": nodeBalance,
1918
+ "nodeState": nodeState,
1919
+ "process_unit": process_unit,
1920
+ "process_minload": process_minload,
1921
+ "process_min_load_eff": process_min_load_eff,
1922
+ "process_online": process_online,
1923
+ "process_online_linear": process_online_linear,
1924
+ "process_online_integer": process_online_integer,
1925
+ "process_delayed": process_delayed,
1926
+ "storage_bind_within_timeblock": storage_bind_within_timeblock,
1927
+ "storage_bind_forward_only": storage_bind_forward_only,
1928
+ "storage_bind_within_solve": storage_bind_within_solve,
1929
+ # Phase C — ``storage_bind_within_solve_blended_weights`` is now
1930
+ # derived authoritatively in ``input._load_storage`` from the
1931
+ # per-solve ``solve_data/node__storage_binding_method`` CSV, so
1932
+ # that the cascade-applied silent downgrade
1933
+ # (``_native_run_model._downgrade_rp_methods_for_non_rp_solve``)
1934
+ # is honoured instead of being overwritten by the DB-source
1935
+ # projection. The helper :func:`storage_bind_within_solve_blended_weights`
1936
+ # remains exported below for callers that want raw DB-source
1937
+ # projection (e.g. diagnostics), but is no longer wired here.
1938
+ "storage_fix_start": storage_fix_start,
1939
+ "n_fix_storage_quantity": n_fix_storage_quantity,
1940
+ "groupCapacityMargin": groupCapacityMargin,
1941
+ "groupInertia": groupInertia,
1942
+ "groupNonSync": groupNonSync,
1943
+ "groupStochastic": groupStochastic,
1944
+ "group_node": group_node,
1945
+ "group_entity": group_entity,
1946
+ "group_process_node": group_process_node,
1947
+ "group_co2_max_period": group_co2_max_period,
1948
+ "process_sink_inertia": process_sink_inertia,
1949
+ "process_source_inertia": process_source_inertia,
1950
+ "process_sink_nonSync": process_sink_nonSync,
1951
+ "node_profile_upper": node_profile_upper,
1952
+ "node_profile_lower": node_profile_lower,
1953
+ "node_profile_fixed": node_profile_fixed,
1954
+ # Δ.29 — process_profile_upper / _lower / _fixed. The (p, source, sink, f)
1955
+ # tuple the slow path's ``_load_profiles`` reconstructs from
1956
+ # ``process__source__sink__profile__profile_method.csv`` (itself a
1957
+ # 4-way union over ``profileProcess`` connection/direct + indirect
1958
+ # source/sink helpers — preprocessing/process_arc_unions.py:1837-1851)
1959
+ # is exactly what ``_profile_method_arc`` rebuilds via the
1960
+ # ``unit__node__profile`` × ``unit__inputNode`` / ``unit__outputNode``
1961
+ # join. Direct units with a single side (no source OR no sink) — the
1962
+ # common VRE case (wind_X with method_1way_1var_off, no source) —
1963
+ # land on the output-side branch (``source=unit, sink=node``) which
1964
+ # matches the ``no_source_arcs`` partition of
1965
+ # :func:`process_source_sink_canonical`. Connection-side profiles
1966
+ # are covered via ``connection__node__profile`` × ``connection__node__node``.
1967
+ # Without this wiring on the fast path the wind upper-limit cap is
1968
+ # absent → the LP satisfies demand from unconstrained wind → coal
1969
+ # flow stays at 0 → obj=0 (Δ.28's gap diagnosis).
1970
+ "process_profile_upper": process_profile_upper,
1971
+ "process_profile_lower": process_profile_lower,
1972
+ "process_profile_fixed": process_profile_fixed,
1973
+ "reserve_upDown_group": reserve_upDown_group,
1974
+ # process_reserve_upDown_node_active / _increase_reserve_ratio /
1975
+ # _large_failure_ratio are reclassified Derived: their CSV-side
1976
+ # values are intersected with the per-(d, t) reservation > 0
1977
+ # "active" set, which itself depends on pdtReserve_upDown_group_
1978
+ # reservation (a Direct-Param-fed cascade, Derived in our scope).
1979
+ # See preprocessing/reserve_calc_params.py:237 for the gate.
1980
+ "e_invest_total": e_invest_total,
1981
+ "e_divest_total": e_divest_total,
1982
+ "g_invest_total": g_invest_total,
1983
+ "g_divest_total": g_divest_total,
1984
+ "g_invest_cumulative": g_invest_cumulative,
1985
+ "connection_dc_power_flow": connection_dc_power_flow,
1986
+ "commodity_with_ladder": commodity_with_ladder,
1987
+ "commodity_with_ladder_annual": commodity_with_ladder_annual,
1988
+ "commodity_with_ladder_cumulative": commodity_with_ladder_cumulative,
1989
+ "commodity__tier_ann": commodity__tier_ann,
1990
+ "commodity__tier_cum": commodity__tier_cum,
1991
+ # Δ.17c Gap D — process_source_sink × ramp_method partitions.
1992
+ "process_source_sink_ramp_limit_source_up":
1993
+ process_source_sink_ramp_limit_source_up,
1994
+ "process_source_sink_ramp_limit_source_down":
1995
+ process_source_sink_ramp_limit_source_down,
1996
+ "process_source_sink_ramp_limit_sink_up":
1997
+ process_source_sink_ramp_limit_sink_up,
1998
+ "process_source_sink_ramp_limit_sink_down":
1999
+ process_source_sink_ramp_limit_sink_down,
2000
+ "process_source_sink_ramp_cost":
2001
+ process_source_sink_ramp_cost,
2002
+ # method_2way_1var_off arcs — need a non-negative reverse auxiliary
2003
+ # ``v_flow_back`` to carry sink→source flow (the engine emits
2004
+ # ``v_flow ≥ 0`` forward-only). Union of DC + non-DC arcs.
2005
+ "process_source_sink_2way_1var":
2006
+ process_source_sink_2way_1var,
2007
+ }
2008
+
2009
+
2010
+ def apply_projection_params(source: "InputSource",
2011
+ flex_data: object) -> None:
2012
+ """Apply the DB-direct construction for Γ.2 Projection Params,
2013
+ mutating ``flex_data`` in place.
2014
+
2015
+ Δ.12b — defensive ``try / except: continue`` removed; helper
2016
+ exceptions propagate. The
2017
+ ``if v is None or v.height == 0: continue`` guard is retained:
2018
+ SIMPLE_PROJECTIONS' SET-frame helpers may return empty when the
2019
+ source has rows for a class but none match the projection's
2020
+ membership filter, in which case we want to keep the seed's
2021
+ populated value rather than zero it out. Per-helper coverage
2022
+ audits (Δ.12-drop preparation) will fold this into helper-side
2023
+ "produce-or-raise" once each helper's empty-vs-None semantics
2024
+ are pinned.
2025
+ """
2026
+ # First pass: simple no-deps projections.
2027
+ for field_name, fn in SIMPLE_PROJECTIONS.items():
2028
+ v = fn(source)
2029
+ if v is None or (hasattr(v, "height") and v.height == 0):
2030
+ continue
2031
+ setattr(flex_data, field_name, v)
2032
+
2033
+ # Second pass: projections that need ``dt`` from the CSV-loaded
2034
+ # FlexData — they overlay the CSV value of the dependent fields.
2035
+ dt = getattr(flex_data, "dt", None)
2036
+
2037
+ # cdt_eq / cdt_le / cdt_ge — only emit when DB has constraints.
2038
+ if dt is not None:
2039
+ for sense, field_name in (("equal", "cdt_eq"),
2040
+ ("less_than", "cdt_le"),
2041
+ ("greater_than", "cdt_ge")):
2042
+ cdt = cdt_filter(source, sense, dt)
2043
+ if cdt is not None and cdt.height > 0:
2044
+ setattr(flex_data, field_name, cdt)
2045
+
2046
+ # Reserve-method partitions — applied unconditionally; emit only
2047
+ # when the DB-side computes a non-empty frame.
2048
+ for method, field_name in (
2049
+ ("timeseries", "reserve_upDown_group_method_timeseries"),
2050
+ ("dynamic", "reserve_upDown_group_method_dynamic"),
2051
+ ("n_1", "reserve_upDown_group_method_n_1"),
2052
+ ):
2053
+ v = reserve_upDown_group_method(source, method)
2054
+ if v is not None and v.height > 0:
2055
+ setattr(flex_data, field_name, v)
2056
+ return {}