flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1515 @@
1
+ """
2
+ blocks.py — Per-entity temporal block derivation and overlap set.
3
+
4
+ Purpose:
5
+
6
+ * Take group-level ``new_stepduration`` (v51 schema) and derive a
7
+ **block** for every node, unit and connection. A block is an
8
+ identifier that pairs the entity with a particular stepduration
9
+ schedule in the solve. When no group sets ``new_stepduration`` the
10
+ derivation collapses to a single block named ``"default"``, which is
11
+ bit-identical to pre-v51 behaviour.
12
+ * Precompute the cross-resolution **overlap set** — the mapping from
13
+ each (coarse block, coarse timestep) to the (fine block, fine
14
+ timestep) tuples it contains, with the area-fraction the fine step
15
+ contributes to the coarse row. The generalised node balance
16
+ consumes this set.
17
+ * Emit CSVs in ``solve_data/`` so downstream consumers can
18
+ index by block without re-running the derivation:
19
+
20
+ * ``entity_block.csv`` — one row per node.
21
+ * ``process_side_block.csv`` — two rows per process (source + sink).
22
+ * ``block_step_duration.csv`` — per (block, period, step) step
23
+ duration, analogous to
24
+ ``steps_in_use.csv``.
25
+ * ``overlap_set.csv`` — (period, block_coarse, step_coarse,
26
+ block_fine, step_fine, fraction).
27
+ * ``block_step_previous.csv`` — per-block predecessor relations
28
+ analogous to ``step_previous.csv``
29
+ (Agent 1.4): for each
30
+ (block, period, step) row the
31
+ corresponding
32
+ ``step_previous``,
33
+ ``step_previous_within_timeset``,
34
+ ``period_previous`` and
35
+ ``step_previous_within_solve``.
36
+ * ``block_period_time_first.csv`` — per-block first step of each
37
+ period (Agent 1.4). Analogous
38
+ to ``first_timesteps.csv``.
39
+ * ``block_period_time_last.csv`` — per-block last step of each
40
+ period (Agent 1.4). Analogous
41
+ to ``last_timesteps.csv``.
42
+
43
+ These CSVs are consumed by the polars LP build.
44
+
45
+ Design notes
46
+ ------------
47
+ * Block naming: an entity in a resolution-group takes that group's
48
+ name; everything else is in ``"default"``.
49
+ * Step duration of a block: the resolution-group's
50
+ ``new_stepduration`` if set; else the solve's new_stepduration
51
+ (v50); else the finest step from the solve's timeline. The
52
+ ``"default"`` block always takes the solve-level value (or the
53
+ raw timeline when unset).
54
+ * Per-process block: an explicit ``group__unit`` /
55
+ ``group__connection`` membership in a resolution-group overrides
56
+ both sides to that block. Otherwise the source-side block follows
57
+ the source node and the sink-side block follows the sink node.
58
+ For direct (1var) methods the two sides agree in practice — when
59
+ they don't the finer side's resolution dominates (Agent 1.2 will
60
+ express this in the model).
61
+ * Overlap set: for **aligned subsets** (all coarse stepdurations are
62
+ integer multiples of the finest in the solve, and the finer timeline
63
+ cleanly partitions each coarse row), the fraction is always 1.0 and
64
+ non-matching rows are omitted. A ``NotImplementedError`` is raised
65
+ for the non-aligned case; the three-region LH2 test case is aligned
66
+ by construction.
67
+ """
68
+ from __future__ import annotations
69
+
70
+ import csv
71
+ from collections import defaultdict
72
+ from dataclasses import dataclass, field
73
+ from pathlib import Path
74
+ from typing import TYPE_CHECKING, Iterable
75
+
76
+ import polars as pl
77
+
78
+ from flextool.engine_polars._solve_state import FlexToolConfigError
79
+ from flextool.engine_polars._emit_provider_io import _emit
80
+ from flextool.engine_polars.input import _provider_open
81
+
82
+
83
+ # ---------------------------------------------------------------------------
84
+ # Single-frame writer (canonical _write helper)
85
+ # ---------------------------------------------------------------------------
86
+
87
+
88
+ if TYPE_CHECKING: # pragma: no cover — import cycle guard only
89
+ from flextool.engine_polars._solve_config import SolveConfig
90
+ from flextool.engine_polars._timeline import TimelineConfig
91
+
92
+
93
+ # ---------------------------------------------------------------------------
94
+ # Public types
95
+ # ---------------------------------------------------------------------------
96
+
97
+
98
+ DEFAULT_BLOCK = "default"
99
+
100
+
101
+ # Direct (1var) and indirect (nvar) ct_method classification — matches
102
+ # ``set method_direct`` / ``set method_indirect`` in flextool_base.dat.
103
+ # Users write ct_method values in their DB (constant_efficiency, regular,
104
+ # …); the per-(ct, startup, fork) → method resolution happens in
105
+ # ``_write_process_method``. For block derivation we only need the
106
+ # single-var vs two-var distinction. Classification is at the ct_method
107
+ # level; an ``exact`` unit with fork_yes becomes ``method_2way_nvar_off``
108
+ # — still indirect — so the ct_method → direct/indirect mapping below
109
+ # holds regardless of fork.
110
+ _CT_METHOD_DIRECT: frozenset[str] = frozenset({
111
+ "constant_efficiency",
112
+ "min_load_efficiency",
113
+ "no_losses_no_variable_cost",
114
+ "variable_cost_only",
115
+ "unidirectional",
116
+ })
117
+ _CT_METHOD_INDIRECT: frozenset[str] = frozenset({
118
+ "regular",
119
+ "exact",
120
+ })
121
+
122
+
123
+ def _is_direct(ct_method: str) -> bool:
124
+ """Return True for 1var-per-process (direct) methods.
125
+
126
+ Unknown ct_method values default to ``True`` — i.e. treat them as
127
+ direct — because the block-assignment logic for direct methods is
128
+ strictly more conservative (it picks the finer adjacent block),
129
+ keeping the per-side blocks identical for mis-classified processes.
130
+ """
131
+ return ct_method in _CT_METHOD_DIRECT or ct_method not in _CT_METHOD_INDIRECT
132
+
133
+
134
+ @dataclass
135
+ class BlockAssignments:
136
+ """Per-entity block mappings for a single solve.
137
+
138
+ Attributes
139
+ ----------
140
+ node_block : dict[str, str]
141
+ node_name → block_name.
142
+ process_block_in : dict[str, str]
143
+ process_name → source-side block_name.
144
+ process_block_out : dict[str, str]
145
+ process_name → sink-side block_name.
146
+ process_block : dict[str, str]
147
+ process_name → process's unified block. Equals the process's
148
+ explicit resolution-group block if set, else the finer of
149
+ ``process_block_in`` / ``process_block_out``. Used to index
150
+ UC / startup / shutdown variables and their constraints so
151
+ that unit commitment lives at the process's own temporal
152
+ resolution. In the degenerate case (all entities on
153
+ ``DEFAULT_BLOCK``) this equals ``DEFAULT_BLOCK`` for every
154
+ process.
155
+ block_step_duration : dict[str, float]
156
+ block_name → hours per step. ``DEFAULT_BLOCK`` always present.
157
+ """
158
+ node_block: dict[str, str] = field(default_factory=dict)
159
+ process_block_in: dict[str, str] = field(default_factory=dict)
160
+ process_block_out: dict[str, str] = field(default_factory=dict)
161
+ process_block: dict[str, str] = field(default_factory=dict)
162
+ block_step_duration: dict[str, float] = field(default_factory=dict)
163
+
164
+
165
+ @dataclass
166
+ class OverlapSet:
167
+ """Cross-resolution overlap rows.
168
+
169
+ Each row is ``(period, block_coarse, step_coarse, block_fine,
170
+ step_fine, fraction)``. Fractions follow the aligned-subsets
171
+ convention: always 1.0 when present, non-matching tuples omitted.
172
+ """
173
+ rows: list[tuple[str, str, str, str, str, float]] = field(default_factory=list)
174
+
175
+
176
+ @dataclass
177
+ class BlockTimelines:
178
+ """Per-block (period → [(step, duration_hours)]) timelines.
179
+
180
+ Used internally to emit ``block_step_duration.csv`` and to build
181
+ the overlap set.
182
+ """
183
+ per_block: dict[str, dict[str, list[tuple[str, float]]]] = field(default_factory=dict)
184
+
185
+
186
+ @dataclass
187
+ class BlockPredecessors:
188
+ """Per-block predecessor relations (Agent 1.4).
189
+
190
+ One row per ``(block, period, step)`` in each block's timeline,
191
+ carrying the successor→predecessor mapping analogous to
192
+ ``dtttdt`` / ``step_previous.csv``. In the degenerate case (only
193
+ the ``"default"`` block) the row set equals the solve's existing
194
+ ``dtttdt`` with the ``"default"`` tag prepended.
195
+
196
+ Row schema: ``(block, period, step, step_previous,
197
+ step_previous_within_timeset, period_previous,
198
+ step_previous_within_solve)``.
199
+ """
200
+ rows: list[tuple[str, str, str, str, str, str, str]] = field(default_factory=list)
201
+
202
+
203
+ @dataclass
204
+ class BlockBoundaries:
205
+ """Per-block first / last step of each period (Agent 1.4).
206
+
207
+ ``first`` and ``last`` are lists of ``(block, period, step)`` rows,
208
+ covering every block in ``BlockAssignments.block_step_duration``.
209
+ In the degenerate case ``first`` and ``last`` for the ``"default"``
210
+ block equal the solve's ``period__time_first`` / ``period__time_last``
211
+ with the ``"default"`` tag prepended.
212
+ """
213
+ first: list[tuple[str, str, str]] = field(default_factory=list)
214
+ last: list[tuple[str, str, str]] = field(default_factory=list)
215
+
216
+
217
+ # ---------------------------------------------------------------------------
218
+ # Validation
219
+ # ---------------------------------------------------------------------------
220
+
221
+
222
+ def validate_group_membership(
223
+ group_unit: Iterable[tuple[str, str]],
224
+ group_connection: Iterable[tuple[str, str]],
225
+ group_node: Iterable[tuple[str, str]],
226
+ resolution_groups: dict[str, float],
227
+ decomposition_groups: dict[str, str],
228
+ reserve_upDown_group: Iterable[tuple[str, str, str]] | None = None,
229
+ process_reserve_upDown_node: Iterable[tuple[str, str, str, str]] | None = None,
230
+ ) -> None:
231
+ """Reject configurations with ambiguous block / region membership.
232
+
233
+ Rules
234
+ -----
235
+ Each entity (node / unit / connection) must be in **at most one**
236
+ resolution-group (a group with ``new_stepduration`` set) and
237
+ **at most one** decomposition-group (a group with
238
+ ``decomposition_method != 'none'``). Membership in any number of
239
+ regular groups (CO2 caps, reserves, inertia …) is unconstrained.
240
+
241
+ Agent 1.7 additional rule: V1 restricts reserves to the default
242
+ (finest) temporal block. Any entity that participates in a reserve
243
+ (reserve-group member nodes, or processes listed in
244
+ ``process_reserve_upDown_node``) must therefore NOT sit in a
245
+ resolution-group. Mixing reserves with coarse-resolution
246
+ participants creates subtle semantic issues around energy vs. power
247
+ aggregation; V1 forbids the configuration outright so the reserve
248
+ constraints in ``flextool.mod`` can stay at the fine ``dt`` index.
249
+
250
+ Args:
251
+ group_unit: ``(group, unit)`` membership tuples.
252
+ group_connection: ``(group, connection)`` membership tuples.
253
+ group_node: ``(group, node)`` membership tuples.
254
+ resolution_groups: group_name → new_stepduration hours. Only
255
+ keys matter (the set of resolution-groups).
256
+ decomposition_groups: group_name → decomposition_method value.
257
+ Only groups with a value other than ``"none"`` count as
258
+ decomposition-groups.
259
+ reserve_upDown_group: ``(reserve, upDown, group)`` rows defining
260
+ active reserves. When the iterable is empty the reserve
261
+ rule is a no-op.
262
+ process_reserve_upDown_node: ``(process, reserve, upDown, node)``
263
+ rows of processes that participate in reserves.
264
+
265
+ Raises:
266
+ FlexToolConfigError: if any entity is in two or more
267
+ resolution-groups, two or more decomposition-groups, or
268
+ is a reserve participant while sitting in a resolution-
269
+ group (Agent 1.7 V1 rule). The message names the entity
270
+ and both conflicting groups.
271
+ """
272
+ res_set = set(resolution_groups.keys())
273
+ decomp_set = {g for g, m in decomposition_groups.items() if m and m != "none"}
274
+
275
+ # Build entity → list[group_in_resolution / in_decomposition] maps.
276
+ entity_res: dict[str, list[str]] = defaultdict(list)
277
+ entity_decomp: dict[str, list[str]] = defaultdict(list)
278
+
279
+ for members in (group_unit, group_connection, group_node):
280
+ for g, e in members:
281
+ if g in res_set:
282
+ entity_res[e].append(g)
283
+ if g in decomp_set:
284
+ entity_decomp[e].append(g)
285
+
286
+ errors: list[str] = []
287
+ for e, gs in entity_res.items():
288
+ if len(gs) > 1:
289
+ errors.append(
290
+ f"entity '{e}' is a member of multiple resolution-groups "
291
+ f"({sorted(set(gs))}); each entity may belong to at most "
292
+ f"one group with new_stepduration set."
293
+ )
294
+ for e, gs in entity_decomp.items():
295
+ if len(gs) > 1:
296
+ errors.append(
297
+ f"entity '{e}' is a member of multiple decomposition-groups "
298
+ f"({sorted(set(gs))}); each entity may belong to at most "
299
+ f"one group with decomposition_method != 'none'."
300
+ )
301
+
302
+ # Agent 1.7: reserve-block compatibility. Collect the set of
303
+ # reserve-participating entities (nodes whose reserve-groups are
304
+ # active, plus the processes and their nodes listed in
305
+ # ``process_reserve_upDown_node``) and reject any entity whose
306
+ # resolution-group is set (entity_res[e] non-empty).
307
+ reserve_groups: set[str] = set()
308
+ if reserve_upDown_group is not None:
309
+ for row in reserve_upDown_group:
310
+ if len(row) >= 3:
311
+ reserve_groups.add(row[2])
312
+ reserve_entities: set[str] = set()
313
+ # Nodes carried into a reserve via group membership.
314
+ for g, n in group_node:
315
+ if g in reserve_groups:
316
+ reserve_entities.add(n)
317
+ # Processes (and their nodes) explicitly listed as reserve
318
+ # participants.
319
+ if process_reserve_upDown_node is not None:
320
+ for row in process_reserve_upDown_node:
321
+ if len(row) >= 4:
322
+ reserve_entities.add(row[0]) # process
323
+ reserve_entities.add(row[3]) # node
324
+
325
+ for e in sorted(reserve_entities):
326
+ gs = entity_res.get(e, [])
327
+ if gs:
328
+ errors.append(
329
+ f"entity '{e}' participates in reserves but is a member of "
330
+ f"resolution-group(s) {sorted(set(gs))}; V1 restricts "
331
+ f"reserves to the default (finest) block — move '{e}' out "
332
+ f"of the resolution-group or drop the reserve participation."
333
+ )
334
+
335
+ if errors:
336
+ raise FlexToolConfigError("; ".join(errors))
337
+
338
+
339
+ # ---------------------------------------------------------------------------
340
+ # Block derivation
341
+ # ---------------------------------------------------------------------------
342
+
343
+
344
+ def _solve_step_duration(
345
+ solve: str,
346
+ solve_config: "SolveConfig | None",
347
+ timeline_config: "TimelineConfig | None",
348
+ ) -> float:
349
+ """Return the default-block step duration for *solve* in hours.
350
+
351
+ Priority (matches existing pre-v51 behaviour):
352
+
353
+ 1. ``solve.new_stepduration`` (v50 parameter).
354
+ 2. The smallest step duration that actually appears in any of the
355
+ solve's timelines. This catches the pre-v50 case where the
356
+ timeline already encodes the desired resolution.
357
+
358
+ When neither is available (unit-test fixture with only the bare
359
+ block dicts) falls back to 1.0. The return value is only used for
360
+ the ``DEFAULT_BLOCK`` entry in ``block_step_duration.csv`` and the
361
+ identity overlap rows; it does not participate in LP coefficients
362
+ in Agent 1.1.
363
+ """
364
+ if solve_config is not None:
365
+ pass # type: ignore
366
+ # Solve-level new_stepduration (v50)
367
+ if timeline_config is not None:
368
+ raw = timeline_config.new_step_durations.get(solve)
369
+ if raw is not None:
370
+ try:
371
+ return float(raw)
372
+ except (TypeError, ValueError):
373
+ pass
374
+
375
+ # Fall back to the finest step in the solve's timelines.
376
+ if timeline_config is not None and solve_config is not None:
377
+ timesets_used = solve_config.timesets_used_by_solves.get(solve, [])
378
+ durations: list[float] = []
379
+ for _period, ts in timesets_used:
380
+ tl = timeline_config.timesets__timeline.get(ts)
381
+ if not tl:
382
+ continue
383
+ for step, dur in timeline_config.timelines.get(tl, []):
384
+ try:
385
+ durations.append(float(dur))
386
+ except (TypeError, ValueError):
387
+ continue
388
+ if durations:
389
+ return min(durations)
390
+ return 1.0
391
+
392
+
393
+ def _assign_entity_to_group(
394
+ entity: str,
395
+ memberships: Iterable[tuple[str, str]],
396
+ resolution_groups: dict[str, float],
397
+ ) -> str | None:
398
+ """Return the resolution-group *entity* belongs to, or None.
399
+
400
+ Linear scan kept for backwards compatibility with callers that pass a
401
+ raw membership iterable. Hot per-entity-loop callers should build a
402
+ lookup once via :func:`_build_membership_index` instead — the linear
403
+ scan is O(M) per entity and turns the surrounding loop into O(N × M)
404
+ when called inside a `for entity in entities:` block.
405
+ """
406
+ for g, e in memberships:
407
+ if e == entity and g in resolution_groups:
408
+ return g
409
+ return None
410
+
411
+
412
+ def _build_membership_index(
413
+ memberships: Iterable[tuple[str, str]],
414
+ resolution_groups: dict[str, float],
415
+ ) -> dict[str, str]:
416
+ """Pre-build {entity -> first resolution_group} so per-entity lookups
417
+ in `derive_blocks` go from O(M) linear scan to O(1) dict lookup.
418
+
419
+ Preserves :func:`_assign_entity_to_group`'s "first match wins"
420
+ semantics: when an entity appears in multiple resolution-groups, the
421
+ first occurrence in *memberships* iteration order wins, matching the
422
+ behaviour of the linear-scan helper.
423
+ """
424
+ idx: dict[str, str] = {}
425
+ for g, e in memberships:
426
+ if e in idx:
427
+ continue
428
+ if g in resolution_groups:
429
+ idx[e] = g
430
+ return idx
431
+
432
+
433
+ def derive_blocks(
434
+ solve: str,
435
+ solve_config: "SolveConfig | None",
436
+ timeline_config: "TimelineConfig | None",
437
+ nodes: Iterable[str],
438
+ units: Iterable[str],
439
+ connections: Iterable[str],
440
+ resolution_groups: dict[str, float],
441
+ group_unit: Iterable[tuple[str, str]],
442
+ group_connection: Iterable[tuple[str, str]],
443
+ group_node: Iterable[tuple[str, str]],
444
+ process_source_sink: Iterable[tuple[str, str, str]],
445
+ process_ct_method: dict[str, str],
446
+ ) -> BlockAssignments:
447
+ """Compute per-entity block assignments for *solve*.
448
+
449
+ Args:
450
+ solve: Solve name (for ``new_stepduration`` lookup).
451
+ solve_config: Live SolveConfig (optional — None in unit tests).
452
+ timeline_config: Live TimelineConfig (optional — None in unit
453
+ tests).
454
+ nodes: Iterable of node names in the solve.
455
+ units: Iterable of unit names in the solve.
456
+ connections: Iterable of connection names in the solve.
457
+ resolution_groups: group_name → hours (``new_stepduration``).
458
+ Used as the set of resolution-groups and for
459
+ ``block_step_duration`` lookup.
460
+ group_unit: ``(group, unit)`` membership rows.
461
+ group_connection: ``(group, connection)`` membership rows.
462
+ group_node: ``(group, node)`` membership rows.
463
+ process_source_sink: ``(process, source_node, sink_node)`` rows.
464
+ A process can appear multiple times for a multi-source or
465
+ multi-sink setup (fork_yes). The first source/sink seen is
466
+ canonical for block assignment — per-flow blocks are
467
+ derived from node_block in Agent 1.3.
468
+ process_ct_method: process_name → ct_method string (raw
469
+ user-facing values: constant_efficiency, regular, exact,
470
+ …). Drives direct vs indirect classification.
471
+
472
+ Returns:
473
+ BlockAssignments with node_block, process_block_in,
474
+ process_block_out and block_step_duration populated.
475
+ """
476
+ gu = list(group_unit)
477
+ gc = list(group_connection)
478
+ gn = list(group_node)
479
+
480
+ # Step-duration table for all blocks seen so far. Always includes
481
+ # the default block.
482
+ default_step_duration = _solve_step_duration(
483
+ solve, solve_config, timeline_config
484
+ )
485
+ block_step_duration: dict[str, float] = {DEFAULT_BLOCK: default_step_duration}
486
+ for g, dur in resolution_groups.items():
487
+ try:
488
+ block_step_duration[g] = float(dur)
489
+ except (TypeError, ValueError):
490
+ continue
491
+
492
+ # --- Node block ----------------------------------------------------
493
+ # Pre-build node→group lookup once: was O(|nodes| × |gn|) linear scans
494
+ # per node (quadratic in entity count for models where ~every node is
495
+ # tagged). Now O(|gn|) once + O(|nodes|) lookups.
496
+ node_membership_index = _build_membership_index(gn, resolution_groups)
497
+ node_block: dict[str, str] = {}
498
+ for n in nodes:
499
+ g = node_membership_index.get(n)
500
+ node_block[n] = g if g is not None else DEFAULT_BLOCK
501
+
502
+ # --- Process block(s) ----------------------------------------------
503
+ # First source and first sink per process (preserve iteration order).
504
+ first_source: dict[str, str] = {}
505
+ first_sink: dict[str, str] = {}
506
+ for p, s, k in process_source_sink:
507
+ first_source.setdefault(p, s)
508
+ first_sink.setdefault(p, k)
509
+
510
+ process_block_in: dict[str, str] = {}
511
+ process_block_out: dict[str, str] = {}
512
+ process_block: dict[str, str] = {}
513
+
514
+ def _finer(a: str, b: str) -> str:
515
+ """Pick the block with the shorter step duration.
516
+
517
+ Ties broken in favour of *a* to keep the iteration-order
518
+ determinism callers rely on.
519
+ """
520
+ da = block_step_duration.get(a, default_step_duration)
521
+ db = block_step_duration.get(b, default_step_duration)
522
+ return a if da <= db else b
523
+
524
+ # Pre-build process→group lookup once across both unit and connection
525
+ # membership tables. Was an O((|units|+|connections|) × (|gu|+|gc|))
526
+ # nested scan with `list(gu) + list(gc)` rebuilt every iteration.
527
+ process_memberships: list[tuple[str, str]] = list(gu) + list(gc)
528
+ process_membership_index = _build_membership_index(
529
+ process_memberships, resolution_groups
530
+ )
531
+
532
+ for p in list(units) + list(connections):
533
+ # Check explicit unit / connection membership first (same code path —
534
+ # both lists get scanned).
535
+ explicit_block = process_membership_index.get(p)
536
+ if explicit_block is not None:
537
+ process_block_in[p] = explicit_block
538
+ process_block_out[p] = explicit_block
539
+ process_block[p] = explicit_block
540
+ continue
541
+
542
+ src = first_source.get(p)
543
+ snk = first_sink.get(p)
544
+ src_block = node_block.get(src, DEFAULT_BLOCK) if src else DEFAULT_BLOCK
545
+ snk_block = node_block.get(snk, DEFAULT_BLOCK) if snk else DEFAULT_BLOCK
546
+
547
+ ct = process_ct_method.get(p, "")
548
+ if _is_direct(ct):
549
+ # One flow variable per process; put it on the finer side.
550
+ finer = _finer(src_block, snk_block)
551
+ process_block_in[p] = finer
552
+ process_block_out[p] = finer
553
+ # Agent 1.6: process's unified block for UC / ramp / profile.
554
+ # Direct methods already collapsed both sides to the finer
555
+ # one, so the process block matches.
556
+ process_block[p] = finer
557
+ else:
558
+ # Indirect (nvar) — flows already split by side.
559
+ process_block_in[p] = src_block
560
+ process_block_out[p] = snk_block
561
+ # Agent 1.6: process's unified block for UC / ramp / profile.
562
+ # No explicit process-level resolution-group override, so
563
+ # fall back to the finer of the two sides.
564
+ process_block[p] = _finer(src_block, snk_block)
565
+
566
+ return BlockAssignments(
567
+ node_block=node_block,
568
+ process_block_in=process_block_in,
569
+ process_block_out=process_block_out,
570
+ process_block=process_block,
571
+ block_step_duration=block_step_duration,
572
+ )
573
+
574
+
575
+ # ---------------------------------------------------------------------------
576
+ # Per-block timelines
577
+ # ---------------------------------------------------------------------------
578
+
579
+
580
+ def _build_block_timelines(
581
+ solve: str,
582
+ solve_config: "SolveConfig | None",
583
+ timeline_config: "TimelineConfig | None",
584
+ block_assignments: BlockAssignments,
585
+ active_time_list: dict[str, list] | None = None,
586
+ ) -> BlockTimelines:
587
+ """Build per-block timelines for *solve*.
588
+
589
+ Each block's timeline is a dict of ``period → [(step, duration)]``
590
+ at that block's stepduration. For the default block this is the
591
+ same list of (timestep, duration) rows the rest of the solve data
592
+ already uses (``steps_in_use.csv``). For a resolution-group block,
593
+ the timeline is aggregated to that block's step count.
594
+
595
+ Aggregation rule
596
+ ----------------
597
+ Every timestep in the default timeline is assigned to the coarse
598
+ block row whose step duration it contributes to. Currently this
599
+ requires the coarse step duration to be an integer multiple of the
600
+ default step duration (aligned subsets). The non-aligned case (e.g.
601
+ default=2h, coarse=3h) is surfaced as ``NotImplementedError`` —
602
+ Agent 1.3 will revisit it when the model actually consumes these
603
+ fractions.
604
+ """
605
+ per_block: dict[str, dict[str, list[tuple[str, float]]]] = {}
606
+
607
+ # --- Source timeline (default block) ---------------------------------
608
+ # Priority: active_time_list from the orchestration loop (has the
609
+ # trimmed per-solve windowing); otherwise reconstruct from the
610
+ # timeline_config.
611
+ block_assignments.block_step_duration[DEFAULT_BLOCK]
612
+ default_periods: dict[str, list[tuple[str, float]]] = {}
613
+
614
+ if active_time_list is not None:
615
+ for period, rows in active_time_list.items():
616
+ default_periods[period] = [
617
+ (entry.timestep, float(entry.duration)) for entry in rows
618
+ ]
619
+ elif solve_config is not None and timeline_config is not None:
620
+ for period, ts in solve_config.timesets_used_by_solves.get(solve, []):
621
+ tl = timeline_config.timesets__timeline.get(ts)
622
+ if not tl:
623
+ continue
624
+ tl_rows = timeline_config.timelines.get(tl, [])
625
+ default_periods.setdefault(period, [])
626
+ for step, dur in tl_rows:
627
+ default_periods[period].append((step, float(dur)))
628
+ else:
629
+ # Unit-test mode: no timelines known. Caller must set them.
630
+ pass
631
+
632
+ per_block[DEFAULT_BLOCK] = default_periods
633
+
634
+ # --- Aggregate per-resolution-group block ----------------------------
635
+ for block, dur in block_assignments.block_step_duration.items():
636
+ if block == DEFAULT_BLOCK:
637
+ continue
638
+ # Skip blocks that no entity is assigned to (would produce
639
+ # empty rows); consumers still get the duration via
640
+ # block_step_duration.
641
+ if (
642
+ block not in block_assignments.node_block.values()
643
+ and block not in block_assignments.process_block_in.values()
644
+ and block not in block_assignments.process_block_out.values()
645
+ and block not in block_assignments.process_block.values()
646
+ ):
647
+ continue
648
+ block_rows: dict[str, list[tuple[str, float]]] = {}
649
+ for period, rows in default_periods.items():
650
+ aggregated = _aggregate_timeline(rows, coarse_duration=float(dur))
651
+ block_rows[period] = aggregated
652
+ per_block[block] = block_rows
653
+
654
+ return BlockTimelines(per_block=per_block)
655
+
656
+
657
+ def _aggregate_timeline(
658
+ rows: list[tuple[str, float]], coarse_duration: float,
659
+ ) -> list[tuple[str, float]]:
660
+ """Aggregate a fine-resolution timeline to *coarse_duration* hours.
661
+
662
+ Emits one coarse row per accumulated block of fine rows whose
663
+ durations sum to exactly ``coarse_duration``. The coarse row's
664
+ label is the label of the first fine step it covers (matches the
665
+ pattern used by ``create_timeline_from_timestep_duration``).
666
+
667
+ Raises ``NotImplementedError`` when a block of fine rows cannot be
668
+ cleanly packed — i.e. the coarse duration is not an integer multiple
669
+ of the fine row it straddles. The three-region LH2 example is
670
+ aligned-subsets by construction; non-aligned support is deferred.
671
+ """
672
+ out: list[tuple[str, float]] = []
673
+ i = 0
674
+ n = len(rows)
675
+ eps = 1e-9
676
+ while i < n:
677
+ acc = 0.0
678
+ first_label = rows[i][0]
679
+ j = i
680
+ while j < n and acc + float(rows[j][1]) <= coarse_duration + eps:
681
+ acc += float(rows[j][1])
682
+ j += 1
683
+ if abs(acc - coarse_duration) <= eps:
684
+ break
685
+ if abs(acc - coarse_duration) > eps:
686
+ raise NotImplementedError(
687
+ f"Non-aligned subset: fine rows starting at "
688
+ f"'{first_label}' accumulate to {acc}h but coarse "
689
+ f"block stepduration is {coarse_duration}h. "
690
+ f"Non-integer multiples of the default step are not "
691
+ f"supported yet (aligned-subsets only)."
692
+ )
693
+ out.append((first_label, coarse_duration))
694
+ i = j
695
+ return out
696
+
697
+
698
+ # ---------------------------------------------------------------------------
699
+ # Overlap set
700
+ # ---------------------------------------------------------------------------
701
+
702
+
703
+ def derive_overlap_set(
704
+ solve: str,
705
+ block_assignments: BlockAssignments,
706
+ block_timelines: BlockTimelines | None = None,
707
+ solve_config: "SolveConfig | None" = None,
708
+ timeline_config: "TimelineConfig | None" = None,
709
+ active_time_list: dict[str, list] | None = None,
710
+ ) -> OverlapSet:
711
+ """Compute the overlap rows between every pair of blocks in the solve.
712
+
713
+ Row schema: ``(period, block_coarse, step_coarse, block_fine,
714
+ step_fine, fraction)``.
715
+
716
+ The aligned-subsets case emits fraction = 1.0 for every fine step
717
+ that falls within a coarse step (same calendar moment), and omits
718
+ non-matching rows. The degenerate case — only the default block
719
+ exists — emits identity rows ``(d, "default", t, "default", t,
720
+ 1.0)``.
721
+
722
+ Args:
723
+ solve: Solve name.
724
+ block_assignments: From :func:`derive_blocks`.
725
+ block_timelines: Optional precomputed timelines; when ``None``
726
+ this function calls :func:`_build_block_timelines`.
727
+ solve_config / timeline_config: Passed through to the timeline
728
+ builder when *block_timelines* is ``None``.
729
+ active_time_list: Passed through to the timeline builder;
730
+ typically the ``active_time_lists[solve]`` value from the
731
+ orchestration loop.
732
+ """
733
+ if block_timelines is None:
734
+ block_timelines = _build_block_timelines(
735
+ solve=solve,
736
+ solve_config=solve_config,
737
+ timeline_config=timeline_config,
738
+ block_assignments=block_assignments,
739
+ active_time_list=active_time_list,
740
+ )
741
+ per_block = block_timelines.per_block
742
+ default_rows = per_block.get(DEFAULT_BLOCK, {})
743
+
744
+ rows: list[tuple[str, str, str, str, str, float]] = []
745
+
746
+ # Degenerate case: only the default block exists → identity rows.
747
+ non_default = [b for b in per_block if b != DEFAULT_BLOCK]
748
+ if not non_default:
749
+ for period, pr in default_rows.items():
750
+ for step, _dur in pr:
751
+ rows.append((period, DEFAULT_BLOCK, step, DEFAULT_BLOCK, step, 1.0))
752
+ return OverlapSet(rows=rows)
753
+
754
+ # Default-against-default identity rows are always emitted too so
755
+ # that constraints indexed on ``(b_coarse, b_fine) = (default,
756
+ # default)`` have the right coverage.
757
+ for period, pr in default_rows.items():
758
+ for step, _dur in pr:
759
+ rows.append((period, DEFAULT_BLOCK, step, DEFAULT_BLOCK, step, 1.0))
760
+
761
+ # For every resolution-group block, emit coarse↔fine rows. We
762
+ # consider "fine" = default (and any block strictly finer than
763
+ # coarse); "coarse" = the resolution-group block itself. Every
764
+ # fine step that falls inside a coarse step becomes an overlap row.
765
+ #
766
+ # Agent 1.9 fix: also emit *self-identity* rows for every
767
+ # non-default block (e.g. ``(d, daily, t, daily, t, 1.0)``). The
768
+ # generalised node-balance constraints in ``flextool.mod`` look up
769
+ # ``(d, b_c, tc, b_f, t_f) in overlap`` with ``b_c = b_n`` (node
770
+ # block) and ``b_f`` = the process-side block. When both the node
771
+ # AND the process side live on the same coarse block (e.g. an H2
772
+ # node and a liquefier flow both at ``daily``) the self-identity
773
+ # row is what carries the contribution. Agent 1.1 only emitted
774
+ # ``(default, default)`` self-identity which is fine for the
775
+ # degenerate case but silently zeroes out coarse-coarse couplings.
776
+ for coarse in non_default:
777
+ coarse_rows = per_block.get(coarse, {})
778
+ # Self-identity: every (period, step) on the coarse block maps
779
+ # onto itself with fraction 1.0. Without this row, a
780
+ # daily→daily flow would produce no contribution to the daily
781
+ # node balance.
782
+ for period, c_rows in coarse_rows.items():
783
+ for coarse_step, _coarse_dur in c_rows:
784
+ rows.append(
785
+ (period, coarse, coarse_step, coarse, coarse_step, 1.0)
786
+ )
787
+ fine_rows_per_period = default_rows
788
+ for period, c_rows in coarse_rows.items():
789
+ # Walk the fine timeline in parallel, packing fine rows
790
+ # into the coarse row until its duration is met.
791
+ fine = fine_rows_per_period.get(period, [])
792
+ f_idx = 0
793
+ for coarse_step, coarse_dur in c_rows:
794
+ remaining = float(coarse_dur)
795
+ eps = 1e-9
796
+ while remaining > eps and f_idx < len(fine):
797
+ fine_step, fine_dur = fine[f_idx]
798
+ # Aligned-subsets assumption: every fine row is fully
799
+ # contained in the current coarse row, so we emit the
800
+ # 1.0 overlap and advance.
801
+ rows.append(
802
+ (period, coarse, coarse_step, DEFAULT_BLOCK, fine_step, 1.0)
803
+ )
804
+ # Symmetric row: default↔coarse mapping for callers
805
+ # that query by fine block.
806
+ rows.append(
807
+ (period, DEFAULT_BLOCK, fine_step, coarse, coarse_step, 1.0)
808
+ )
809
+ remaining -= float(fine_dur)
810
+ f_idx += 1
811
+ if remaining > eps:
812
+ raise NotImplementedError(
813
+ f"overlap_set: coarse step '{coarse_step}' on "
814
+ f"block '{coarse}' left {remaining}h unmatched "
815
+ f"at end of fine timeline — non-aligned subsets "
816
+ f"are not supported yet."
817
+ )
818
+
819
+ # Pairs of resolution-group blocks (coarse vs finer-coarse): emit
820
+ # an overlap row for every shared fine step. Handles e.g. a 24h
821
+ # block coexisting with a 6h block. Implemented by intersecting
822
+ # each block's coverage against the default-block steps.
823
+ if len(non_default) >= 2:
824
+ # Build (period, step) → block coverage for each non-default
825
+ # block so we can cross-reference.
826
+ coverage: dict[str, dict[str, dict[str, str]]] = {}
827
+ # coverage[block][period][fine_step] = coarse_step_that_covers_it
828
+ for block in non_default:
829
+ coverage[block] = {}
830
+ b_rows = per_block.get(block, {})
831
+ for period, c_rows in b_rows.items():
832
+ coverage[block][period] = {}
833
+ fine = default_rows.get(period, [])
834
+ f_idx = 0
835
+ for coarse_step, coarse_dur in c_rows:
836
+ remaining = float(coarse_dur)
837
+ eps = 1e-9
838
+ while remaining > eps and f_idx < len(fine):
839
+ fine_step, fine_dur = fine[f_idx]
840
+ coverage[block][period][fine_step] = coarse_step
841
+ remaining -= float(fine_dur)
842
+ f_idx += 1
843
+ for a in non_default:
844
+ for b in non_default:
845
+ if a == b:
846
+ continue
847
+ for period, a_cov in coverage[a].items():
848
+ b_cov = coverage[b].get(period, {})
849
+ for fine_step, a_step in a_cov.items():
850
+ b_step = b_cov.get(fine_step)
851
+ if b_step is None:
852
+ continue
853
+ # Identify which is coarser by step-duration.
854
+ a_dur = block_assignments.block_step_duration.get(a, 0.0)
855
+ b_dur = block_assignments.block_step_duration.get(b, 0.0)
856
+ # Only emit coarse→fine pairs to avoid duplication.
857
+ if a_dur > b_dur:
858
+ rows.append((period, a, a_step, b, b_step, 1.0))
859
+ elif a_dur == b_dur and a < b:
860
+ # Same duration, both cover the same fine
861
+ # steps — emit the canonical direction.
862
+ rows.append((period, a, a_step, b, b_step, 1.0))
863
+ # Deduplicate while preserving order: two different fine steps
864
+ # contained in the same coarse pair yields the same (a_step,
865
+ # b_step) pair every iteration.
866
+ seen: set[tuple] = set()
867
+ deduped: list[tuple[str, str, str, str, str, float]] = []
868
+ for row in rows:
869
+ if row in seen:
870
+ continue
871
+ seen.add(row)
872
+ deduped.append(row)
873
+ return OverlapSet(rows=deduped)
874
+
875
+
876
+ # ---------------------------------------------------------------------------
877
+ # Per-block predecessors + boundaries (Agent 1.4)
878
+ # ---------------------------------------------------------------------------
879
+
880
+
881
+ def derive_block_predecessors(
882
+ solve: str,
883
+ block_assignments: BlockAssignments,
884
+ block_timelines: BlockTimelines,
885
+ default_jump_list: Iterable[tuple] | None = None,
886
+ ) -> BlockPredecessors:
887
+ """Compute per-block predecessor relations analogous to ``dtttdt``.
888
+
889
+ For the ``"default"`` block the predecessor rows are taken straight
890
+ from *default_jump_list* — the orchestration-level
891
+ ``jump_list[solve]`` produced by :func:`make_step_jump` — and
892
+ tagged with the ``"default"`` block label. This guarantees the
893
+ degenerate case is bit-identical to pre-v51 behaviour: every row
894
+ carries exactly the same ``(step_previous,
895
+ step_previous_within_timeset, period_previous,
896
+ step_previous_within_solve)`` values consumed from
897
+ ``step_previous.csv``.
898
+
899
+ For every non-default block we derive the predecessor relations
900
+ from that block's own aggregated timeline (``block_timelines``).
901
+ The block's timeline is a per-period ordered list of
902
+ ``(step, duration)`` rows; the predecessor of step ``i`` in
903
+ period ``p`` is step ``i - 1`` in the same period (``jump = 1``),
904
+ except the first step of the first period which wraps cyclically
905
+ to the last step of the last period (``jump = -N + 1``), and the
906
+ first step of any subsequent period which points at the last step
907
+ of the previous period. This mirrors the ``make_step_jump``
908
+ behaviour — block aggregation preserves the simple cyclic pattern
909
+ because the aligned-subsets assumption collapses every block
910
+ internally to one monotone timestep sequence per period.
911
+
912
+ Args:
913
+ solve: Solve name (unused for now — kept for symmetry with
914
+ sibling functions).
915
+ block_assignments: From :func:`derive_blocks`.
916
+ block_timelines: From :func:`_build_block_timelines`.
917
+ default_jump_list: Iterable of 7-tuples as produced by
918
+ :func:`flextool.engine_polars._timeline.make_step_jump`.
919
+ When ``None`` the default-block rows fall back to the
920
+ simple cyclic pattern used for non-default blocks — this
921
+ only matters for unit tests; the orchestration loop always
922
+ passes the real list.
923
+
924
+ Returns:
925
+ BlockPredecessors. ``rows`` preserves iteration order:
926
+ default block first, then resolution-group blocks in insertion
927
+ order.
928
+ """
929
+ rows: list[tuple[str, str, str, str, str, str, str]] = []
930
+
931
+ # --- Default block ------------------------------------------------------
932
+ if default_jump_list is not None:
933
+ for entry in default_jump_list:
934
+ # jump_list entries are 7-tuples (period, step, previous,
935
+ # previous_within_timeset, previous_period,
936
+ # previous_within_solve, jump). Re-project onto the block-
937
+ # tagged 7-tuple (block, period, step, previous,
938
+ # previous_within_timeset, previous_period,
939
+ # previous_within_solve).
940
+ period = entry[0]
941
+ step = entry[1]
942
+ previous = entry[2]
943
+ previous_within_timeset = entry[3]
944
+ previous_period = entry[4]
945
+ previous_within_solve = entry[5]
946
+ rows.append(
947
+ (
948
+ DEFAULT_BLOCK,
949
+ period,
950
+ step,
951
+ previous,
952
+ previous_within_timeset,
953
+ previous_period,
954
+ previous_within_solve,
955
+ )
956
+ )
957
+ else:
958
+ default_rows = block_timelines.per_block.get(DEFAULT_BLOCK, {})
959
+ rows.extend(_cyclic_block_predecessors(DEFAULT_BLOCK, default_rows))
960
+
961
+ # --- Non-default blocks -------------------------------------------------
962
+ for block in block_assignments.block_step_duration:
963
+ if block == DEFAULT_BLOCK:
964
+ continue
965
+ block_rows_per_period = block_timelines.per_block.get(block, {})
966
+ if not block_rows_per_period:
967
+ # Block declared in block_step_duration but no entity is on
968
+ # it — skip to avoid empty period rows.
969
+ continue
970
+ rows.extend(_cyclic_block_predecessors(block, block_rows_per_period))
971
+
972
+ return BlockPredecessors(rows=rows)
973
+
974
+
975
+ def _cyclic_block_predecessors(
976
+ block: str,
977
+ per_period_rows: dict[str, list[tuple[str, float]]],
978
+ ) -> list[tuple[str, str, str, str, str, str, str]]:
979
+ """Cyclic predecessor pattern over a block's aggregated timeline.
980
+
981
+ Cycles within the whole block's timeline: the first step of the
982
+ first period wraps to the last step of the last period. Within a
983
+ period interior steps point at the preceding step (jump=1), and
984
+ the first step of non-first periods points at the last step of
985
+ the previous period (jump across periods).
986
+
987
+ The ``step_previous_within_timeset`` column follows the pre-v51
988
+ convention: it equals ``step_previous`` for interior steps
989
+ (rows where the previous step is the immediately adjacent row in
990
+ the same period). For first-of-period rows — where the jump
991
+ crosses a period boundary — it's pinned at the current period's
992
+ block-last step (matching the ``block_last.timestep`` projection
993
+ used by :func:`make_step_jump`). In the degenerate case this
994
+ column is populated from the real ``jump_list`` instead of this
995
+ fallback, so the bit-identical promise still holds.
996
+ """
997
+ periods = list(per_period_rows.keys())
998
+ if not periods:
999
+ return []
1000
+ first_period = periods[0]
1001
+ last_period = periods[-1]
1002
+
1003
+ out: list[tuple[str, str, str, str, str, str, str]] = []
1004
+ for pi, period in enumerate(periods):
1005
+ rows = per_period_rows[period]
1006
+ if not rows:
1007
+ continue
1008
+ block_last_step = rows[-1][0]
1009
+ for si, (step, _dur) in enumerate(rows):
1010
+ if si > 0:
1011
+ prev_step = rows[si - 1][0]
1012
+ prev_within_ts = prev_step
1013
+ prev_period = period
1014
+ prev_within_solve = prev_step
1015
+ else:
1016
+ # First step of the period — cross-period predecessor.
1017
+ if period == first_period:
1018
+ # Wrap to last period's last step (cyclic).
1019
+ prev_period_rows = per_period_rows[last_period]
1020
+ prev_step = prev_period_rows[-1][0] if prev_period_rows else step
1021
+ prev_period = last_period
1022
+ prev_within_solve = prev_step
1023
+ prev_within_ts = block_last_step
1024
+ else:
1025
+ prev_period = periods[pi - 1]
1026
+ prev_rows = per_period_rows[prev_period]
1027
+ prev_step = prev_rows[-1][0] if prev_rows else step
1028
+ prev_within_solve = prev_step
1029
+ prev_within_ts = block_last_step
1030
+ out.append(
1031
+ (
1032
+ block,
1033
+ period,
1034
+ step,
1035
+ prev_step,
1036
+ prev_within_ts,
1037
+ prev_period,
1038
+ prev_within_solve,
1039
+ )
1040
+ )
1041
+ return out
1042
+
1043
+
1044
+ def derive_block_boundaries(
1045
+ block_assignments: BlockAssignments,
1046
+ block_timelines: BlockTimelines,
1047
+ ) -> BlockBoundaries:
1048
+ """Compute per-block first and last step of each period.
1049
+
1050
+ Mirrors the solve-level ``period__time_first`` /
1051
+ ``period__time_last`` sets, but keyed at the block level. In the
1052
+ degenerate (default-block-only) case ``first`` / ``last`` for
1053
+ ``"default"`` equal the pre-v51 sets with the block tag prepended.
1054
+ """
1055
+ first: list[tuple[str, str, str]] = []
1056
+ last: list[tuple[str, str, str]] = []
1057
+ for block in block_assignments.block_step_duration:
1058
+ rows_per_period = block_timelines.per_block.get(block, {})
1059
+ for period, rows in rows_per_period.items():
1060
+ if not rows:
1061
+ continue
1062
+ first.append((block, period, rows[0][0]))
1063
+ last.append((block, period, rows[-1][0]))
1064
+ return BlockBoundaries(first=first, last=last)
1065
+
1066
+
1067
+ # ---------------------------------------------------------------------------
1068
+ # CSV emission
1069
+ # ---------------------------------------------------------------------------
1070
+
1071
+
1072
+ def _entity_block_frame(block_assignments: BlockAssignments) -> pl.DataFrame:
1073
+ rows = list(block_assignments.node_block.items())
1074
+ return pl.DataFrame(
1075
+ {
1076
+ "entity": [r[0] for r in rows],
1077
+ "block": [r[1] for r in rows],
1078
+ },
1079
+ schema={"entity": pl.Utf8, "block": pl.Utf8},
1080
+ )
1081
+
1082
+
1083
+ def _process_side_block_frame(
1084
+ block_assignments: BlockAssignments,
1085
+ ) -> pl.DataFrame:
1086
+ procs: list[str] = []
1087
+ sides: list[str] = []
1088
+ blocks: list[str] = []
1089
+ for process, block in block_assignments.process_block_in.items():
1090
+ procs.append(process)
1091
+ sides.append("source")
1092
+ blocks.append(block)
1093
+ for process, block in block_assignments.process_block_out.items():
1094
+ procs.append(process)
1095
+ sides.append("sink")
1096
+ blocks.append(block)
1097
+ return pl.DataFrame(
1098
+ {"process": procs, "side": sides, "block": blocks},
1099
+ schema={"process": pl.Utf8, "side": pl.Utf8, "block": pl.Utf8},
1100
+ )
1101
+
1102
+
1103
+ def _process_block_frame(
1104
+ block_assignments: BlockAssignments,
1105
+ ) -> pl.DataFrame:
1106
+ rows = list(block_assignments.process_block.items())
1107
+ return pl.DataFrame(
1108
+ {
1109
+ "process": [r[0] for r in rows],
1110
+ "block": [r[1] for r in rows],
1111
+ },
1112
+ schema={"process": pl.Utf8, "block": pl.Utf8},
1113
+ )
1114
+
1115
+
1116
+ def _block_step_duration_frame(
1117
+ block_timelines: BlockTimelines | None,
1118
+ ) -> pl.DataFrame:
1119
+ blocks: list[str] = []
1120
+ periods: list[str] = []
1121
+ steps: list[str] = []
1122
+ durs: list[float] = []
1123
+ if block_timelines is not None:
1124
+ for block, period_rows in block_timelines.per_block.items():
1125
+ for period, rows in period_rows.items():
1126
+ for step, dur in rows:
1127
+ blocks.append(block)
1128
+ periods.append(period)
1129
+ steps.append(step)
1130
+ durs.append(float(dur))
1131
+ return pl.DataFrame(
1132
+ {
1133
+ "block": blocks, "period": periods,
1134
+ "step": steps, "step_duration": durs,
1135
+ },
1136
+ schema={
1137
+ "block": pl.Utf8, "period": pl.Utf8,
1138
+ "step": pl.Utf8, "step_duration": pl.Float64,
1139
+ },
1140
+ )
1141
+
1142
+
1143
+ def _overlap_set_frame(overlap_set: OverlapSet) -> pl.DataFrame:
1144
+ rows = overlap_set.rows
1145
+ return pl.DataFrame(
1146
+ {
1147
+ "period": [r[0] for r in rows],
1148
+ "block_coarse": [r[1] for r in rows],
1149
+ "step_coarse": [r[2] for r in rows],
1150
+ "block_fine": [r[3] for r in rows],
1151
+ "step_fine": [r[4] for r in rows],
1152
+ "fraction": [r[5] for r in rows],
1153
+ },
1154
+ schema={
1155
+ "period": pl.Utf8, "block_coarse": pl.Utf8,
1156
+ "step_coarse": pl.Utf8, "block_fine": pl.Utf8,
1157
+ "step_fine": pl.Utf8, "fraction": pl.Float64,
1158
+ },
1159
+ )
1160
+
1161
+
1162
+ def _block_step_previous_frame(
1163
+ block_predecessors: BlockPredecessors | None,
1164
+ ) -> pl.DataFrame:
1165
+ rows = block_predecessors.rows if block_predecessors is not None else []
1166
+ return pl.DataFrame(
1167
+ {
1168
+ "block": [r[0] for r in rows],
1169
+ "period": [r[1] for r in rows],
1170
+ "step": [r[2] for r in rows],
1171
+ "step_previous": [r[3] for r in rows],
1172
+ "step_previous_within_timeset": [r[4] for r in rows],
1173
+ "period_previous": [r[5] for r in rows],
1174
+ "step_previous_within_solve": [r[6] for r in rows],
1175
+ },
1176
+ schema={
1177
+ "block": pl.Utf8, "period": pl.Utf8, "step": pl.Utf8,
1178
+ "step_previous": pl.Utf8,
1179
+ "step_previous_within_timeset": pl.Utf8,
1180
+ "period_previous": pl.Utf8,
1181
+ "step_previous_within_solve": pl.Utf8,
1182
+ },
1183
+ )
1184
+
1185
+
1186
+ def _block_period_time_frame(
1187
+ rows: list[tuple[str, str, str]],
1188
+ ) -> pl.DataFrame:
1189
+ return pl.DataFrame(
1190
+ {
1191
+ "block": [r[0] for r in rows],
1192
+ "period": [r[1] for r in rows],
1193
+ "step": [r[2] for r in rows],
1194
+ },
1195
+ schema={"block": pl.Utf8, "period": pl.Utf8, "step": pl.Utf8},
1196
+ )
1197
+
1198
+
1199
+ def emit_block_data(
1200
+ block_assignments: BlockAssignments,
1201
+ overlap_set: OverlapSet,
1202
+ block_timelines: BlockTimelines | None,
1203
+ solve_data_dir: Path,
1204
+ block_predecessors: BlockPredecessors | None = None,
1205
+ block_boundaries: BlockBoundaries | None = None,
1206
+ *, provider,
1207
+ ) -> None:
1208
+ """Provider-emitting twin of :func:`write_block_data`.
1209
+
1210
+ Emits the eight block frames under ``solve_data/<basename>`` keys
1211
+ via :func:`_emit` (dual-key registration: basename and
1212
+ parent/basename). *solve_data_dir* is retained for signature parity
1213
+ with :func:`write_block_data`; the Provider is the only sink.
1214
+ """
1215
+ del solve_data_dir
1216
+ _emit(provider, "solve_data/entity_block.csv",
1217
+ _entity_block_frame(block_assignments))
1218
+ _emit(provider, "solve_data/process_side_block.csv",
1219
+ _process_side_block_frame(block_assignments))
1220
+ _emit(provider, "solve_data/process_block.csv",
1221
+ _process_block_frame(block_assignments))
1222
+ _emit(provider, "solve_data/block_step_duration.csv",
1223
+ _block_step_duration_frame(block_timelines))
1224
+ _emit(provider, "solve_data/overlap_set.csv",
1225
+ _overlap_set_frame(overlap_set))
1226
+ _emit(provider, "solve_data/block_step_previous.csv",
1227
+ _block_step_previous_frame(block_predecessors))
1228
+ first_rows = (
1229
+ block_boundaries.first if block_boundaries is not None else []
1230
+ )
1231
+ last_rows = (
1232
+ block_boundaries.last if block_boundaries is not None else []
1233
+ )
1234
+ _emit(provider, "solve_data/block_period_time_first.csv",
1235
+ _block_period_time_frame(first_rows))
1236
+ _emit(provider, "solve_data/block_period_time_last.csv",
1237
+ _block_period_time_frame(last_rows))
1238
+
1239
+
1240
+ # ---------------------------------------------------------------------------
1241
+ # Orchestration hook
1242
+ # ---------------------------------------------------------------------------
1243
+
1244
+
1245
+ def _cell_str(value: "object | None") -> str:
1246
+ """Reproduce a ``csv.reader`` cell string for a native frame value.
1247
+
1248
+ ``DataFrame.write_csv`` renders ``null`` as the empty string and every
1249
+ other scalar as its textual form; ``csv.reader`` then reads those
1250
+ strings back. Mirror that here so the row-of-strings shape stays
1251
+ byte-identical to the legacy CSV round-trip the block-assembly code
1252
+ indexes positionally and compares against.
1253
+ """
1254
+ return "" if value is None else str(value)
1255
+
1256
+
1257
+ def _read_input_rows(
1258
+ inp: Path,
1259
+ basename: str,
1260
+ provider: "object | None" = None,
1261
+ ) -> list[list[str]]:
1262
+ """Read rows from ``inp/<basename>.csv``, falling back to *provider*.
1263
+
1264
+ Returns a list of row lists with the header stripped. When the
1265
+ Provider is supplied and the disk file is absent / empty, looks up
1266
+ the frame under the canonical ``input/<stem>`` key the cascade uses.
1267
+
1268
+ Returning the same row-of-strings shape as the legacy ``csv.reader``
1269
+ loops keeps the existing assembly code byte-identical.
1270
+ """
1271
+ path = inp / basename
1272
+ stem = basename[:-4] if basename.endswith(".csv") else basename
1273
+ # Provider arm: read the in-memory frame natively instead of
1274
+ # round-tripping through ``DataFrame.write_csv`` + ``csv.reader``.
1275
+ # Try the canonical ``input/<stem>`` first (post-Δ.20 input writer),
1276
+ # then the bare ``<stem>`` for the legacy alias; first non-None
1277
+ # frame wins. ``provider.get`` returns DATA rows only (no header),
1278
+ # so no header is skipped. Frames never carry empty rows, so the
1279
+ # legacy ``if row`` blank-row filter is a no-op here. When neither
1280
+ # key resolves, fall through to the unchanged disk-fallback arm.
1281
+ if provider is not None:
1282
+ for key in (f"input/{stem}", stem):
1283
+ try:
1284
+ df = provider.get(key)
1285
+ except Exception: # pragma: no cover — defensive only
1286
+ df = None
1287
+ if df is None:
1288
+ continue
1289
+ return [[_cell_str(c) for c in row] for row in df.iter_rows()]
1290
+ fh = _provider_open(None, basename, path)
1291
+ if fh is None:
1292
+ return []
1293
+ with fh as f:
1294
+ reader = csv.reader(f)
1295
+ next(reader, None)
1296
+ return [row for row in reader if row]
1297
+
1298
+
1299
+ def emit_block_data_for_solve(
1300
+ solve: str,
1301
+ solve_config: "SolveConfig",
1302
+ timeline_config: "TimelineConfig",
1303
+ work_folder: Path,
1304
+ active_time_list: dict[str, list] | None = None,
1305
+ default_jump_list: Iterable[tuple] | None = None,
1306
+ provider: "object | None" = None,
1307
+ *, emit_provider,
1308
+ ) -> BlockAssignments:
1309
+ """Provider-emitting twin of :func:`write_block_data_for_solve`.
1310
+
1311
+ Mirrors the orchestration hook end-to-end but routes the final
1312
+ eight CSV emissions through :func:`emit_block_data` (Provider) rather
1313
+ than :func:`write_block_data` (disk).
1314
+
1315
+ The ``provider`` keyword is retained as the *input* Provider —
1316
+ :func:`_read_input_rows` consults it for ``input/*.csv`` lookups.
1317
+ The new keyword-only ``emit_provider`` is the *output* Provider into
1318
+ which the eight block frames are emitted.
1319
+ """
1320
+ wf = Path(work_folder)
1321
+ inp = wf / "input"
1322
+
1323
+ nodes: list[str] = []
1324
+ units: list[str] = []
1325
+ connections: list[str] = []
1326
+ for row in _read_input_rows(inp, "node.csv", provider):
1327
+ if row:
1328
+ nodes.append(row[0])
1329
+ for row in _read_input_rows(inp, "process_unit.csv", provider):
1330
+ if row:
1331
+ units.append(row[0])
1332
+ for row in _read_input_rows(inp, "process_connection.csv", provider):
1333
+ if row:
1334
+ connections.append(row[0])
1335
+ if not nodes and not units and not connections:
1336
+ for row in _read_input_rows(inp, "entity.csv", provider):
1337
+ if row:
1338
+ nodes.append(row[0])
1339
+
1340
+ resolution_groups: dict[str, float] = {}
1341
+ p_group_rows = _read_input_rows(inp, "p_group.csv", provider)
1342
+ p_group_csv = inp / "p_group.csv"
1343
+ p_group_header: list[str] | None = None
1344
+ # Provider arm: the frame carries the column NAMES, so the "header"
1345
+ # is ``list(df.columns)`` — exactly what ``next(csv.reader(f), None)``
1346
+ # returned for the header row. Same key-candidate order
1347
+ # (``input/p_group`` before bare ``p_group``); first non-None frame
1348
+ # wins. When no frame resolves, fall to the unchanged disk arm.
1349
+ if provider is not None:
1350
+ for key in ("input/p_group", "p_group"):
1351
+ try:
1352
+ df = provider.get(key)
1353
+ except Exception: # pragma: no cover — defensive only
1354
+ df = None
1355
+ if df is not None:
1356
+ p_group_header = list(df.columns)
1357
+ break
1358
+ if p_group_header is None:
1359
+ p_group_fh = _provider_open(None, "p_group", p_group_csv)
1360
+ if p_group_fh is not None:
1361
+ with p_group_fh as f:
1362
+ p_group_header = next(csv.reader(f), None)
1363
+ if p_group_header is None:
1364
+ p_group_header = ["group", "groupParam", "p_group"]
1365
+
1366
+ def _row_dict(row: list[str], header: list[str]) -> dict[str, str]:
1367
+ return {h: (row[i] if i < len(row) else "")
1368
+ for i, h in enumerate(header)}
1369
+
1370
+ for row in p_group_rows:
1371
+ r = _row_dict(row, p_group_header)
1372
+ if r.get("groupParam") == "new_stepduration":
1373
+ try:
1374
+ resolution_groups[r["group"]] = float(r["p_group"])
1375
+ except (TypeError, ValueError):
1376
+ continue
1377
+ decomposition_groups: dict[str, str] = {}
1378
+ p_group_decomp_csv = inp / "p_group_decomposition.csv"
1379
+ p_group_decomp_header: list[str] | None = None
1380
+ # Same Provider-first / disk-fallback shape as the ``p_group`` header
1381
+ # probe above: the frame's column NAMES are the header. Same
1382
+ # key-candidate order; first non-None frame wins; otherwise fall to
1383
+ # the unchanged disk arm.
1384
+ if provider is not None:
1385
+ for key in ("input/p_group_decomposition", "p_group_decomposition"):
1386
+ try:
1387
+ df = provider.get(key)
1388
+ except Exception: # pragma: no cover — defensive only
1389
+ df = None
1390
+ if df is not None:
1391
+ p_group_decomp_header = list(df.columns)
1392
+ break
1393
+ if p_group_decomp_header is None:
1394
+ p_group_decomp_fh = _provider_open(
1395
+ None, "p_group_decomposition", p_group_decomp_csv,
1396
+ )
1397
+ if p_group_decomp_fh is not None:
1398
+ with p_group_decomp_fh as f:
1399
+ p_group_decomp_header = next(csv.reader(f), None)
1400
+ if p_group_decomp_header is None:
1401
+ p_group_decomp_header = ["group", "groupParam", "p_group"]
1402
+ for row in _read_input_rows(inp, "p_group_decomposition.csv", provider):
1403
+ r = _row_dict(row, p_group_decomp_header)
1404
+ if r.get("groupParam") == "decomposition_method":
1405
+ decomposition_groups[r["group"]] = str(r["p_group"])
1406
+ if not decomposition_groups:
1407
+ for row in p_group_rows:
1408
+ r = _row_dict(row, p_group_header)
1409
+ if r.get("groupParam") == "decomposition_method":
1410
+ decomposition_groups[r["group"]] = str(r["p_group"])
1411
+
1412
+ group_node: list[tuple[str, str]] = []
1413
+ group_process: list[tuple[str, str]] = []
1414
+ for row in _read_input_rows(inp, "group__node.csv", provider):
1415
+ if len(row) >= 2:
1416
+ group_node.append((row[0], row[1]))
1417
+ for row in _read_input_rows(inp, "group__process.csv", provider):
1418
+ if len(row) >= 2:
1419
+ group_process.append((row[0], row[1]))
1420
+
1421
+ unit_set = set(units)
1422
+ conn_set = set(connections)
1423
+ group_unit = [(g, p) for g, p in group_process if p in unit_set]
1424
+ group_connection = [(g, p) for g, p in group_process if p in conn_set]
1425
+
1426
+ reserve_upDown_group: list[tuple[str, str, str]] = []
1427
+ for row in _read_input_rows(inp, "reserve__upDown__group__method.csv",
1428
+ provider):
1429
+ if len(row) >= 4 and row[3] != "no_reserve":
1430
+ reserve_upDown_group.append((row[0], row[1], row[2]))
1431
+ process_reserve_upDown_node: list[tuple[str, str, str, str]] = []
1432
+ for row in _read_input_rows(inp, "process__reserve__upDown__node.csv",
1433
+ provider):
1434
+ if len(row) >= 4:
1435
+ process_reserve_upDown_node.append(
1436
+ (row[0], row[1], row[2], row[3])
1437
+ )
1438
+
1439
+ validate_group_membership(
1440
+ group_unit, group_connection, group_node,
1441
+ resolution_groups, decomposition_groups,
1442
+ reserve_upDown_group=reserve_upDown_group,
1443
+ process_reserve_upDown_node=process_reserve_upDown_node,
1444
+ )
1445
+
1446
+ process_source_sink: list[tuple[str, str, str]] = []
1447
+ sources: dict[str, list[str]] = defaultdict(list)
1448
+ sinks: dict[str, list[str]] = defaultdict(list)
1449
+ for row in _read_input_rows(inp, "process__source.csv", provider):
1450
+ if len(row) >= 2:
1451
+ sources[row[0]].append(row[1])
1452
+ for row in _read_input_rows(inp, "process__sink.csv", provider):
1453
+ if len(row) >= 2:
1454
+ sinks[row[0]].append(row[1])
1455
+ for p in set(list(sources.keys()) + list(sinks.keys())):
1456
+ src_list = sources.get(p, [""])
1457
+ snk_list = sinks.get(p, [""])
1458
+ process_source_sink.append((p, src_list[0], snk_list[0]))
1459
+
1460
+ process_ct_method: dict[str, str] = {}
1461
+ for row in _read_input_rows(inp, "process__ct_method.csv", provider):
1462
+ if len(row) >= 2:
1463
+ process_ct_method[row[0]] = row[1]
1464
+
1465
+ block_assignments = derive_blocks(
1466
+ solve=solve,
1467
+ solve_config=solve_config,
1468
+ timeline_config=timeline_config,
1469
+ nodes=nodes,
1470
+ units=units,
1471
+ connections=connections,
1472
+ resolution_groups=resolution_groups,
1473
+ group_unit=group_unit,
1474
+ group_connection=group_connection,
1475
+ group_node=group_node,
1476
+ process_source_sink=process_source_sink,
1477
+ process_ct_method=process_ct_method,
1478
+ )
1479
+
1480
+ block_timelines = _build_block_timelines(
1481
+ solve=solve,
1482
+ solve_config=solve_config,
1483
+ timeline_config=timeline_config,
1484
+ block_assignments=block_assignments,
1485
+ active_time_list=active_time_list,
1486
+ )
1487
+
1488
+ overlap = derive_overlap_set(
1489
+ solve=solve,
1490
+ block_assignments=block_assignments,
1491
+ block_timelines=block_timelines,
1492
+ )
1493
+
1494
+ block_predecessors = derive_block_predecessors(
1495
+ solve=solve,
1496
+ block_assignments=block_assignments,
1497
+ block_timelines=block_timelines,
1498
+ default_jump_list=default_jump_list,
1499
+ )
1500
+ block_boundaries = derive_block_boundaries(
1501
+ block_assignments=block_assignments,
1502
+ block_timelines=block_timelines,
1503
+ )
1504
+
1505
+ emit_block_data(
1506
+ block_assignments=block_assignments,
1507
+ overlap_set=overlap,
1508
+ block_timelines=block_timelines,
1509
+ solve_data_dir=wf / "solve_data",
1510
+ block_predecessors=block_predecessors,
1511
+ block_boundaries=block_boundaries,
1512
+ provider=emit_provider,
1513
+ )
1514
+
1515
+ return block_assignments