flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1315 @@
1
+ """
2
+ Solve-to-solve handoff CSV writers.
3
+
4
+ These writers produce the six CSVs carrying state from the previous
5
+ solve, written from the live ``Highs`` instance + the input /
6
+ parameter data.
7
+
8
+ The six handoff files (per ``ARCHITECTURE.md`` "Solve-to-solve
9
+ handoff"):
10
+
11
+ 1. solve_data/p_entity_period_existing_capacity.csv
12
+ 2. solve_data/p_entity_divested.csv
13
+ 3. solve_data/fix_storage_quantity.csv
14
+ 4. solve_data/fix_storage_price.csv
15
+ 5. solve_data/fix_storage_usage.csv
16
+ 6. solve_data/p_roll_continue_state.csv
17
+
18
+ All six are implemented. #5 (``fix_storage_usage``) uses a simplified
19
+ ``v_flow × unitsize`` formula that is exact for ``method_nvar`` and for
20
+ ``method_1var_per_way`` with trivial coefficients / no min_load_efficiency
21
+ — enough for the typical battery / inverter topology. See
22
+ :func:`write_fix_storage_usage` for the caveat on more exotic process
23
+ methods.
24
+
25
+ Sourcing strategy
26
+ -----------------
27
+ The writers source SOLVER values directly from ``highspy.Highs``:
28
+
29
+ * variable values via :func:`extract_variable` (already used by
30
+ ``read_highs_solution.py``).
31
+ * constraint duals — same machinery, with ``source="row_dual"``.
32
+
33
+ PARAMETER values are read from ``input/`` (write-once) and
34
+ ``solve_data/`` (per-solve) — both pure parameter snapshots (input
35
+ data after model-side derivations).
36
+ """
37
+ from __future__ import annotations
38
+
39
+ import logging
40
+ from pathlib import Path
41
+ from typing import TYPE_CHECKING
42
+
43
+ import pandas as pd
44
+ import polars as pl
45
+
46
+ from flextool.process_outputs.read_highs_solution import (
47
+ _actual_solve_name,
48
+ _load_realized_periods,
49
+ _load_realized_set,
50
+ extract_variable,
51
+ )
52
+
53
+ if TYPE_CHECKING:
54
+ import highspy
55
+ from flextool.engine_polars._solve_handoff import SolveHandoff
56
+ from flextool.engine_polars.input import FlexData
57
+ from flextool.engine_polars._output_writer import OutputWriterState
58
+
59
+ _logger = logging.getLogger(__name__)
60
+
61
+ # Fallback inverse of scale_the_objective (multiply duals by 1e6 to
62
+ # undo a 1e-6 scaling). The live per-solve value is read via
63
+ # :func:`flextool.process_outputs.read_highs_solution._resolve_inv_scale_the_objective`
64
+ # — this constant only applies when the CSV can't be read.
65
+ _INV_SCALE_THE_OBJECTIVE = 1e6
66
+
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # Provider-aware lookup helper — Provider-first, then ``None`` (caller
70
+ # falls back to its own disk read). Provider key uses the
71
+ # parent-qualified convention (``"<parent>/<basename>"`` without
72
+ # ``.csv``).
73
+
74
+ def _provider_lookup_df(provider: "object | None", path: "Path | str"):
75
+ """Return the polars frame for *path* sourced from the Provider, or
76
+ ``None`` when the Provider doesn't carry it.
77
+ """
78
+ p = Path(path)
79
+ parent = p.parent.name
80
+ stem = p.stem
81
+ name = f"{parent}/{stem}" if parent else stem
82
+ if provider is not None and provider.has(name):
83
+ return provider.get(name)
84
+ return None
85
+
86
+
87
+ # ---------------------------------------------------------------------------
88
+ # Parameter / set loaders
89
+ #
90
+ # Parameter CSVs come from ``input/`` (write-once) or ``solve_data/``
91
+ # (per-solve). These small helpers normalise their varied shapes into
92
+ # Python dicts.
93
+ # ---------------------------------------------------------------------------
94
+
95
+
96
+ def _load_unitsize(work_folder: Path) -> dict[str, float]:
97
+ """Return ``{entity: unitsize}`` from ``input/p_entity_unitsize.csv``.
98
+
99
+ The CSV is wide: header row = entity names, second row labelled
100
+ ``value`` = the unitsize per entity.
101
+ """
102
+ path = work_folder / "input" / "p_entity_unitsize.csv"
103
+ df = pd.read_csv(path, index_col=0)
104
+ return df.loc["value"].astype(float).to_dict()
105
+
106
+
107
+ def _load_pre_existing(
108
+ work_folder: Path,
109
+ *,
110
+ provider: "object | None" = None,
111
+ ) -> dict[tuple[str, str], float]:
112
+ """Return ``{(period, entity): value}`` for ``p_entity_pre_existing``.
113
+
114
+ Provider-first: the canonical in-memory frame at
115
+ ``solve_data/p_entity_pre_existing`` (populated by
116
+ ``_emit_chain_params.emit_p_entity_pre_existing``) is tall —
117
+ ``(entity, period, value)`` — and is what every consumer should
118
+ use on the in-memory cascade.
119
+
120
+ Falls back to the per-solve appendage CSV
121
+ ``solve_data/solve__p_entity_pre_existing.csv`` (wide
122
+ ``(solve, period) × entity_columns``) when the Provider doesn't
123
+ have it. On the current in-memory cascade that CSV is typically
124
+ absent — in which case we silently return ``{}`` rather than
125
+ raising ``FileNotFoundError``. ``write_p_entity_period_existing_capacity``
126
+ treats an empty pre_existing as "no pre-existing capacity recorded
127
+ for this solve", which is the correct semantics when nothing
128
+ upstream produced it.
129
+ """
130
+ if provider is not None and provider.has("solve_data/p_entity_pre_existing"):
131
+ df_pl = provider.get("solve_data/p_entity_pre_existing")
132
+ if df_pl is None or df_pl.is_empty():
133
+ return {}
134
+ # Tall ``(entity, period, value)`` schema — ``value`` is a
135
+ # repr-encoded string per ``_ed_value_frame``. Convert
136
+ # directly to the ``{(period, entity): float}`` dict the
137
+ # downstream consumers expect.
138
+ out: dict[tuple[str, str], float] = {}
139
+ for row in df_pl.iter_rows(named=True):
140
+ period_v = row.get("period")
141
+ entity_v = row.get("entity")
142
+ value_v = row.get("value")
143
+ if period_v is None or entity_v is None or value_v is None:
144
+ continue
145
+ try:
146
+ out[(str(period_v), str(entity_v))] = float(value_v)
147
+ except (TypeError, ValueError):
148
+ continue
149
+ return out
150
+
151
+ # Disk fallback — legacy pre-Δ.31 CSV.
152
+ path = work_folder / "solve_data" / "solve__p_entity_pre_existing.csv"
153
+ if not path.exists():
154
+ return {}
155
+ df = pd.read_csv(path, index_col=[0, 1])
156
+ if df.empty:
157
+ return {}
158
+ df = df.droplevel(0) # drop solve level → period only
159
+ out: dict[tuple[str, str], float] = {}
160
+ for period, row in df.iterrows():
161
+ for entity, value in row.items():
162
+ if pd.notna(value):
163
+ out[(str(period), str(entity))] = float(value)
164
+ return out
165
+
166
+
167
+ def _load_prior_existing(
168
+ work_folder: Path,
169
+ *, prior_handoff: "SolveHandoff | None" = None,
170
+ ) -> tuple[dict[tuple[str, str], float], dict[tuple[str, str], float]]:
171
+ """Return prior ``p_entity_period_existing_capacity`` + ``_invested_capacity``.
172
+
173
+ With ``prior_handoff`` populated, read the two dicts from
174
+ ``realized_existing`` / ``realized_invest`` (in-memory consume side).
175
+ Else fall back to the previous solve's
176
+ ``solve_data/p_entity_period_existing_capacity.csv`` (file fallback).
177
+ For the first solve neither is available → return empty dicts.
178
+ """
179
+ if prior_handoff is not None and (
180
+ prior_handoff.realized_existing is not None
181
+ or prior_handoff.realized_invest is not None
182
+ ):
183
+ existing: dict[tuple[str, str], float] = {}
184
+ invested: dict[tuple[str, str], float] = {}
185
+ if prior_handoff.realized_existing is not None:
186
+ for r in prior_handoff.realized_existing.iter_rows(named=True):
187
+ existing[(str(r["entity"]), str(r["period"]))] = float(r["value"])
188
+ if prior_handoff.realized_invest is not None:
189
+ for r in prior_handoff.realized_invest.iter_rows(named=True):
190
+ invested[(str(r["entity"]), str(r["period"]))] = float(r["value"])
191
+ return existing, invested
192
+ path = work_folder / "solve_data" / "p_entity_period_existing_capacity.csv"
193
+ if not path.exists():
194
+ return {}, {}
195
+ df = pd.read_csv(path)
196
+ if df.empty:
197
+ return {}, {}
198
+ existing = {}
199
+ invested = {}
200
+ for _, row in df.iterrows():
201
+ key = (str(row["entity"]), str(row["period"]))
202
+ existing[key] = float(row["p_entity_period_existing_capacity"])
203
+ invested[key] = float(row["p_entity_period_invested_capacity"])
204
+ return existing, invested
205
+
206
+
207
+ def _load_prior_divested(
208
+ work_folder: Path,
209
+ *, prior_handoff: "SolveHandoff | None" = None,
210
+ ) -> dict[str, float]:
211
+ """Return ``{entity: cumulative_divested}`` from prior solve, or empty.
212
+
213
+ Reads from ``prior_handoff.divest_cumulative`` when populated; else
214
+ falls back to ``solve_data/p_entity_divested.csv``."""
215
+ if prior_handoff is not None and prior_handoff.divest_cumulative is not None:
216
+ return {
217
+ str(r["entity"]): float(r["value"])
218
+ for r in prior_handoff.divest_cumulative.iter_rows(named=True)
219
+ }
220
+ path = work_folder / "solve_data" / "p_entity_divested.csv"
221
+ if not path.exists():
222
+ return {}
223
+ df = pd.read_csv(path)
224
+ if df.empty or "p_entity_divested" not in df.columns:
225
+ return {}
226
+ return {
227
+ str(row["entity"]): float(row["p_entity_divested"])
228
+ for _, row in df.iterrows()
229
+ }
230
+
231
+
232
+ def _load_storage_fix_methods(
233
+ work_folder: Path, target_method: str,
234
+ ) -> set[str]:
235
+ """Return ``{node, ...}`` whose nested fix method == *target_method*.
236
+
237
+ Reads ``input/node__storage_nested_fix_method.csv`` (long format
238
+ ``node, storage_nested_fix_method``). Empty when the file doesn't
239
+ exist or no node has that method.
240
+ """
241
+ path = work_folder / "input" / "node__storage_nested_fix_method.csv"
242
+ if not path.exists():
243
+ return set()
244
+ df = pd.read_csv(path)
245
+ if df.empty:
246
+ return set()
247
+ return set(
248
+ df.loc[df["storage_nested_fix_method"] == target_method, "node"]
249
+ .astype(str)
250
+ .tolist()
251
+ )
252
+
253
+
254
+ def _load_node_state(work_folder: Path) -> set[str]:
255
+ """Return the set of nodes that maintain a state variable.
256
+
257
+ Derived from ``input/p_node_type.csv``: rows whose ``p_node_type``
258
+ equals ``'storage'``. Nodes absent from the file use the mod's
259
+ default (``'balance'``), which is not a storage node.
260
+ """
261
+ path = work_folder / "input" / "p_node_type.csv"
262
+ if not path.exists():
263
+ return set()
264
+ df = pd.read_csv(path)
265
+ if df.empty or "p_node_type" not in df.columns:
266
+ return set()
267
+ storage = df.loc[df["p_node_type"].astype(str) == "storage", "node"]
268
+ return set(storage.astype(str).tolist())
269
+
270
+
271
+ def _load_entity(work_folder: Path) -> set[str]:
272
+ """Return the full ``entity`` set from ``input/entity.csv``."""
273
+ path = work_folder / "input" / "entity.csv"
274
+ if not path.exists():
275
+ return set()
276
+ df = pd.read_csv(path)
277
+ if df.empty or len(df.columns) == 0:
278
+ return set()
279
+ return set(df.iloc[:, 0].astype(str).tolist())
280
+
281
+
282
+ def _load_entity_divest(work_folder: Path) -> set[str]:
283
+ """Return ``entityDivest`` — entities allowed to divest.
284
+
285
+ Sourced from ``solve_data/entityDivest.csv`` (phase 1 dump);
286
+ matches the ``setof {(e,m) in entity__invest_method : m not in
287
+ divest_method_not_allowed} (e)`` derivation in the model.
288
+ """
289
+ path = work_folder / "solve_data" / "entityDivest.csv"
290
+ if not path.exists():
291
+ return set()
292
+ df = pd.read_csv(path)
293
+ if df.empty or len(df.columns) == 0:
294
+ return set()
295
+ return set(df.iloc[:, 0].astype(str).tolist())
296
+
297
+
298
+ def _load_realized_period_time_last(
299
+ work_folder: Path,
300
+ ) -> list[tuple[str, str]]:
301
+ """Return one ``(period, time)`` pair per realized period — the LAST
302
+ realized timestep within that period.
303
+
304
+ Derived from ``solve_data/realized_dispatch.csv`` (period, step) by
305
+ taking the last row for each period (rows are written in dispatch
306
+ order).
307
+ """
308
+ path = work_folder / "solve_data" / "realized_dispatch.csv"
309
+ if not path.exists():
310
+ return []
311
+ df = pd.read_csv(path)
312
+ if df.empty:
313
+ return []
314
+ time_col = "step" if "step" in df.columns else "time"
315
+ return [
316
+ (str(period), str(group[time_col].iloc[-1]))
317
+ for period, group in df.groupby("period", sort=False)
318
+ ]
319
+
320
+
321
+ def _load_complete_period_share_of_year(
322
+ work_folder: Path,
323
+ ) -> dict[str, float]:
324
+ """Return ``{period: share_of_year}`` from
325
+ ``solve_data/complete_period_share_of_year.csv``."""
326
+ path = work_folder / "solve_data" / "complete_period_share_of_year.csv"
327
+ if not path.exists():
328
+ return {}
329
+ df = pd.read_csv(path)
330
+ # Expected layout: solve,period,value or period,value
331
+ period_col = "period"
332
+ value_col = [c for c in df.columns if c not in ("solve", "period")][0]
333
+ return dict(zip(df[period_col].astype(str), df[value_col].astype(float)))
334
+
335
+
336
+ def _load_inflation_factor_operations_yearly(
337
+ work_folder: Path,
338
+ ) -> dict[str, float]:
339
+ """Return ``{period: inflation_factor}`` from
340
+ ``solve_data/solve__p_inflation_factor_operations_yearly.csv``."""
341
+ path = work_folder / "solve_data" / "solve__p_inflation_factor_operations_yearly.csv"
342
+ if not path.exists():
343
+ return {}
344
+ df = pd.read_csv(path)
345
+ period_col = "period"
346
+ value_col = [c for c in df.columns if c not in ("solve", "period")][0]
347
+ return dict(zip(df[period_col].astype(str), df[value_col].astype(float)))
348
+
349
+
350
+ def _load_step_duration(work_folder: Path) -> dict[tuple[str, str], float]:
351
+ """Return ``{(period, time): step_duration}`` from ``solve_data/steps_in_use.csv``."""
352
+ path = work_folder / "solve_data" / "steps_in_use.csv"
353
+ if not path.exists():
354
+ return {}
355
+ df = pd.read_csv(path)
356
+ time_col = "step" if "step" in df.columns else "time"
357
+ dur_col = "step_duration"
358
+ return {
359
+ (str(p), str(t)): float(d)
360
+ for p, t, d in zip(df["period"], df[time_col], df[dur_col])
361
+ }
362
+
363
+
364
+ def _is_first_solve(work_folder: Path) -> bool:
365
+ """True iff this is the first solve in the model.
366
+
367
+ Source: ``solve_data/p_model.csv`` (long ``modelParam,p_model`` pairs)
368
+ written by ``write_solve_status``. Reads ``p_model['solveFirst']``.
369
+ """
370
+ path = work_folder / "solve_data" / "p_model.csv"
371
+ if not path.exists():
372
+ return True
373
+ df = pd.read_csv(path)
374
+ matches = df.loc[df["modelParam"] == "solveFirst", "p_model"]
375
+ if matches.empty:
376
+ return True
377
+ return bool(int(matches.iloc[0]))
378
+
379
+
380
+ # ---------------------------------------------------------------------------
381
+ # Phase G — in-memory resolvers
382
+ #
383
+ # Each ``_resolve_*`` prefers a value from the in-memory ``FlexData``
384
+ # (or an explicit kwarg passed by the cascade) and falls back to the
385
+ # matching ``_load_*`` disk reader. Callers thread ``flex_data`` /
386
+ # ``is_first_solve`` from the cascade so the per-iter file reads listed
387
+ # in ``specs/in_memory_carriers_audit.md`` (12 readers in this module)
388
+ # disappear from the hot path while the disk fallback stays intact for
389
+ # unit tests / synthesized callers that never instantiate FlexData.
390
+ # ---------------------------------------------------------------------------
391
+
392
+
393
+ def _resolve_unitsize(
394
+ work_folder: Path, flex_data: "FlexData | None" = None,
395
+ ) -> dict[str, float]:
396
+ """``{entity: unitsize}`` from ``flex_data.p_all_entity_unitsize``.
397
+
398
+ Phase G — when ``flex_data`` is supplied, trust the in-memory
399
+ carrier even if ``p_all_entity_unitsize`` is ``None`` (returns
400
+ empty dict, same as the disk fallback would on a missing file)."""
401
+ if flex_data is not None:
402
+ param = getattr(flex_data, "p_all_entity_unitsize", None)
403
+ if param is None:
404
+ return {}
405
+ try:
406
+ f = param.frame
407
+ entity_col = f.columns[0]
408
+ return dict(zip(
409
+ f[entity_col].cast(str).to_list(),
410
+ f["value"].cast(float).to_list(),
411
+ ))
412
+ except Exception: # noqa: BLE001
413
+ pass
414
+ return _load_unitsize(work_folder)
415
+
416
+
417
+ def _resolve_pre_existing(
418
+ work_folder: Path, flex_data: "FlexData | None" = None,
419
+ *, provider: "object | None" = None,
420
+ ) -> dict[tuple[str, str], float]:
421
+ """``{(period, entity): value}`` — pre-existing capacity from FlexData.
422
+
423
+ Prefers ``flex_data.p_entity_period_existing_capacity`` (Param) when
424
+ populated; this is the same data that ``solve__p_entity_pre_existing.csv``
425
+ snapshots after the parent's overlay applies. Falls back to the disk
426
+ CSV when the in-memory carrier is absent (e.g. unit-test paths).
427
+ """
428
+ # No direct in-memory equivalent on FlexData; fall back to disk.
429
+ # (The audit's "wire kwarg" suggestion would require surfacing
430
+ # ``solve__p_entity_pre_existing.csv`` as a FlexData field — deferred.)
431
+ return _load_pre_existing(work_folder, provider=provider)
432
+
433
+
434
+ def _resolve_storage_fix_methods(
435
+ work_folder: Path, target_method: str,
436
+ flex_data: "FlexData | None" = None,
437
+ ) -> set[str]:
438
+ """``{node, ...}`` whose nested fix method == *target_method*.
439
+
440
+ Phase G — when ``flex_data`` is supplied, treat ``flex_data
441
+ .node__storage_nested_fix_method`` as the authoritative source.
442
+ ``None`` means "no such CSV existed when FlexData was built" — the
443
+ on-disk fallback would return an empty set in that case, so we
444
+ short-circuit here without re-reading the file.
445
+ """
446
+ if flex_data is not None:
447
+ f = getattr(flex_data, "node__storage_nested_fix_method", None)
448
+ if f is None:
449
+ return set()
450
+ try:
451
+ sub = f.filter(pl.col("method") == target_method)
452
+ return set(sub["node"].cast(str).to_list())
453
+ except Exception: # noqa: BLE001
454
+ pass
455
+ return _load_storage_fix_methods(work_folder, target_method)
456
+
457
+
458
+ def _resolve_node_state(
459
+ work_folder: Path, flex_data: "FlexData | None" = None,
460
+ ) -> set[str]:
461
+ """``{node, ...}`` carrying a state variable (``nodeState`` set).
462
+
463
+ Phase G — when ``flex_data`` is supplied we trust the in-memory
464
+ ``nodeState`` set even if it's ``None`` (which means: no storage
465
+ nodes in this model, same answer the disk fallback would give from
466
+ a missing ``p_node_type.csv``).
467
+ """
468
+ if flex_data is not None:
469
+ f = getattr(flex_data, "nodeState", None)
470
+ if f is None:
471
+ return set()
472
+ try:
473
+ col = f.columns[0]
474
+ return set(f[col].cast(str).to_list())
475
+ except Exception: # noqa: BLE001
476
+ pass
477
+ return _load_node_state(work_folder)
478
+
479
+
480
+ def _resolve_entity(
481
+ work_folder: Path, flex_data: "FlexData | None" = None,
482
+ ) -> set[str]:
483
+ """Full entity set (every process + connection + node).
484
+
485
+ Phase G — when ``flex_data`` is supplied, trust the in-memory
486
+ entity carrier (use ``p_all_entity_unitsize``'s entity column as the
487
+ canonical set)."""
488
+ if flex_data is not None:
489
+ param = getattr(flex_data, "p_all_entity_unitsize", None)
490
+ if param is None:
491
+ return set()
492
+ try:
493
+ f = param.frame
494
+ entity_col = f.columns[0]
495
+ return set(f[entity_col].cast(str).to_list())
496
+ except Exception: # noqa: BLE001
497
+ pass
498
+ return _load_entity(work_folder)
499
+
500
+
501
+ def _resolve_entity_divest(
502
+ work_folder: Path, flex_data: "FlexData | None" = None,
503
+ ) -> set[str]:
504
+ """``entityDivest`` — entities allowed to divest. Phase G trusts
505
+ ``flex_data.ed_divest_set`` when supplied (``None`` ⇒ empty set,
506
+ same as missing CSV)."""
507
+ if flex_data is not None:
508
+ f = getattr(flex_data, "ed_divest_set", None)
509
+ if f is None:
510
+ return set()
511
+ try:
512
+ entity_col = f.columns[0]
513
+ return set(f[entity_col].cast(str).to_list())
514
+ except Exception: # noqa: BLE001
515
+ pass
516
+ return _load_entity_divest(work_folder)
517
+
518
+
519
+ def _resolve_realized_period_time_last(
520
+ work_folder: Path, flex_data: "FlexData | None" = None,
521
+ ) -> list[tuple[str, str]]:
522
+ """One ``(period, time)`` pair per realized period — the LAST step in
523
+ each. Prefers ``flex_data.realized_dispatch`` (polars frame of
524
+ ``(period, step)``). When ``flex_data`` is supplied we trust the
525
+ in-memory state — empty/None means "no realized dispatch this
526
+ solve", same answer the disk fallback would give."""
527
+ if flex_data is not None:
528
+ rd = getattr(flex_data, "realized_dispatch", None)
529
+ if rd is None:
530
+ return []
531
+ try:
532
+ cols = rd.columns
533
+ time_col = "step" if "step" in cols else ("time" if "time" in cols else cols[1])
534
+ # Keep LAST step per period, preserving period order of first occurrence.
535
+ seen_order: list[str] = []
536
+ last_per: dict[str, str] = {}
537
+ for p, t in zip(rd["period"].cast(str).to_list(), rd[time_col].cast(str).to_list()):
538
+ if p not in last_per:
539
+ seen_order.append(p)
540
+ last_per[p] = t
541
+ return [(p, last_per[p]) for p in seen_order]
542
+ except Exception: # noqa: BLE001
543
+ pass
544
+ return _load_realized_period_time_last(work_folder)
545
+
546
+
547
+ def _resolve_complete_period_share_of_year(
548
+ work_folder: Path, flex_data: "FlexData | None" = None,
549
+ ) -> dict[str, float]:
550
+ """``{period: share_of_year}`` from ``flex_data.p_period_share``.
551
+ Phase G trusts the in-memory carrier when supplied."""
552
+ if flex_data is not None:
553
+ param = getattr(flex_data, "p_period_share", None)
554
+ if param is None:
555
+ return {}
556
+ try:
557
+ f = param.frame
558
+ period_col = f.columns[0]
559
+ return dict(zip(
560
+ f[period_col].cast(str).to_list(),
561
+ f["value"].cast(float).to_list(),
562
+ ))
563
+ except Exception: # noqa: BLE001
564
+ pass
565
+ return _load_complete_period_share_of_year(work_folder)
566
+
567
+
568
+ def _resolve_inflation_factor_operations_yearly(
569
+ work_folder: Path, flex_data: "FlexData | None" = None,
570
+ ) -> dict[str, float]:
571
+ """``{period: inflation_factor}`` from ``flex_data.p_inflation_op``.
572
+ Phase G trusts the in-memory carrier when supplied."""
573
+ if flex_data is not None:
574
+ param = getattr(flex_data, "p_inflation_op", None)
575
+ if param is None:
576
+ return {}
577
+ try:
578
+ f = param.frame
579
+ period_col = f.columns[0]
580
+ return dict(zip(
581
+ f[period_col].cast(str).to_list(),
582
+ f["value"].cast(float).to_list(),
583
+ ))
584
+ except Exception: # noqa: BLE001
585
+ pass
586
+ return _load_inflation_factor_operations_yearly(work_folder)
587
+
588
+
589
+ def _resolve_step_duration(
590
+ work_folder: Path, flex_data: "FlexData | None" = None,
591
+ ) -> dict[tuple[str, str], float]:
592
+ """``{(period, time): step_duration}`` from ``flex_data.p_step_duration``.
593
+ Phase G trusts the in-memory carrier when supplied."""
594
+ if flex_data is not None:
595
+ param = getattr(flex_data, "p_step_duration", None)
596
+ if param is None:
597
+ return {}
598
+ try:
599
+ f = param.frame
600
+ cols = f.columns
601
+ period_col = cols[0]
602
+ time_col = cols[1]
603
+ return {
604
+ (str(p), str(t)): float(v)
605
+ for p, t, v in zip(
606
+ f[period_col].cast(str).to_list(),
607
+ f[time_col].cast(str).to_list(),
608
+ f["value"].cast(float).to_list(),
609
+ )
610
+ }
611
+ except Exception: # noqa: BLE001
612
+ pass
613
+ return _load_step_duration(work_folder)
614
+
615
+
616
+ def _resolve_is_first_solve(
617
+ work_folder: Path, is_first_solve: bool | None = None,
618
+ ) -> bool:
619
+ """Cascade-supplied flag preferred; else read ``solve_data/p_model.csv``."""
620
+ if is_first_solve is not None:
621
+ return is_first_solve
622
+ return _is_first_solve(work_folder)
623
+
624
+
625
+ # ---------------------------------------------------------------------------
626
+ # Handoff writers
627
+ # ---------------------------------------------------------------------------
628
+
629
+
630
+ def write_p_entity_divested(
631
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
632
+ prior_handoff: "SolveHandoff | None" = None,
633
+ flex_data: "FlexData | None" = None,
634
+ is_first_solve: bool | None = None,
635
+ ) -> Path:
636
+ """Write ``solve_data/p_entity_divested.csv`` from v_divest + prior.
637
+
638
+ ``cumulative_divested[e] = prior_divested[e] + sum_d v_divest[e,d] * unitsize[e]``
639
+ over every divest period (sum_d means all (e,d) declared in ed_divest).
640
+
641
+ ``prior_handoff`` (when populated) replaces the on-disk read of the
642
+ parent solve's ``p_entity_divested.csv`` with the in-memory
643
+ ``divest_cumulative`` carrier. ``flex_data`` / ``is_first_solve``
644
+ are Phase G kwargs that route the entity / unitsize / first-solve
645
+ lookups through in-memory carriers — see the resolvers above.
646
+ """
647
+ out_path = work_folder / "solve_data" / "p_entity_divested.csv"
648
+
649
+ entities = _resolve_entity_divest(work_folder, flex_data)
650
+ unitsize = _resolve_unitsize(work_folder, flex_data)
651
+ first = _resolve_is_first_solve(work_folder, is_first_solve)
652
+ prior = (
653
+ {} if first
654
+ else _load_prior_divested(work_folder, prior_handoff=prior_handoff)
655
+ )
656
+ divest_df = extract_variable(
657
+ h, "v_divest", ("entity",), solve_name=solve_name, has_time=False,
658
+ )
659
+
660
+ # divest_df: rows (solve, period), columns = entity. Sum across periods.
661
+ if divest_df.empty:
662
+ per_entity_sum: dict[str, float] = {}
663
+ else:
664
+ per_entity_sum = divest_df.sum(axis=0).to_dict()
665
+
666
+ rows = []
667
+ for e in sorted(entities):
668
+ cumulative = prior.get(e, 0.0) + per_entity_sum.get(e, 0.0) * unitsize.get(e, 1.0)
669
+ rows.append((e, cumulative))
670
+
671
+ out = pd.DataFrame(rows, columns=["entity", "p_entity_divested"])
672
+ out.to_csv(out_path, index=False, float_format="%.8g")
673
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
674
+ return out_path
675
+
676
+
677
+ def write_fix_storage_quantity(
678
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
679
+ flex_data: "FlexData | None" = None,
680
+ is_first_solve: bool | None = None,
681
+ ) -> Path:
682
+ """Write ``solve_data/fix_storage_quantity.csv`` from v_state for fix_quantity nodes.
683
+
684
+ ``v_state[n, d, t].val * p_entity_unitsize[n]`` for every
685
+ ``(n, d, t)`` with ``n in fix_quantity_nodes`` and
686
+ ``(d, t) in dt_fix_storage_timesteps``.
687
+ """
688
+ out_path = work_folder / "solve_data" / "fix_storage_quantity.csv"
689
+
690
+ target_nodes = _resolve_storage_fix_methods(work_folder, "fix_quantity", flex_data)
691
+ fix_steps_path = work_folder / "solve_data" / "fix_storage_timesteps.csv"
692
+ fix_steps = _load_realized_set(fix_steps_path)
693
+ unitsize = _resolve_unitsize(work_folder, flex_data)
694
+ state_df = extract_variable(h, "v_state", ("node",), solve_name=solve_name)
695
+
696
+ rows: list[tuple[str, str, str, float]] = []
697
+ if not state_df.empty and target_nodes:
698
+ for (_solve, period, time), row in state_df.iterrows():
699
+ if fix_steps is not None and (period, time) not in fix_steps:
700
+ continue
701
+ for node in row.index:
702
+ if node not in target_nodes:
703
+ continue
704
+ rows.append((period, time, node, float(row[node]) * unitsize.get(node, 1.0)))
705
+
706
+ # Match phase-3 semantics: only OVERWRITE the file when this solve
707
+ # actually has fix-storage entries to record. An empty dispatch
708
+ # sub-solve must NOT clobber rows the upper storage solve wrote
709
+ # earlier (the next solve reads them back).
710
+ if not rows:
711
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
712
+ out_path.write_text("period,step,node,p_fix_storage_quantity\n")
713
+ _logger.info("wrote %s (header only — first solve)", out_path)
714
+ else:
715
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
716
+ return out_path
717
+
718
+ out = pd.DataFrame(rows, columns=["period", "step", "node", "p_fix_storage_quantity"])
719
+ out.to_csv(out_path, index=False, float_format="%.8g")
720
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
721
+ return out_path
722
+
723
+
724
+ def write_p_roll_continue_state(
725
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
726
+ flex_data: "FlexData | None" = None,
727
+ ) -> Path:
728
+ """Write ``solve_data/p_roll_continue_state.csv``.
729
+
730
+ For every ``n in nodeState`` and the LAST realized timestep of each
731
+ realized period, store ``v_state[n,d,t].val * p_entity_unitsize[n]``.
732
+ Output file has only the latest period's entries (the model writer
733
+ iterates and overwrites; the next read uses the final one).
734
+ """
735
+ out_path = work_folder / "solve_data" / "p_roll_continue_state.csv"
736
+
737
+ nodes = _resolve_node_state(work_folder, flex_data)
738
+ last_pairs = _resolve_realized_period_time_last(work_folder, flex_data)
739
+ unitsize = _resolve_unitsize(work_folder, flex_data)
740
+ state_df = extract_variable(h, "v_state", ("node",), solve_name=solve_name)
741
+
742
+ rows: list[tuple[str, float]] = []
743
+ if not state_df.empty and nodes and last_pairs:
744
+ # The mod re-opens the file inside the loop so only the LAST
745
+ # period's entries survive. Replicate by taking the last
746
+ # (period, time) pair.
747
+ last_period, last_time = last_pairs[-1]
748
+ try:
749
+ row = state_df.loc[(solve_name, last_period, last_time)]
750
+ for node in row.index:
751
+ if node not in nodes:
752
+ continue
753
+ rows.append((node, float(row[node]) * unitsize.get(node, 1.0)))
754
+ except KeyError:
755
+ pass
756
+
757
+ # Match phase-3 semantics — the mod only writes this file when both
758
+ # ``nodeState`` and ``realized_period__time_last`` are non-empty.
759
+ # When this solve has nothing to record, do NOT overwrite a prior
760
+ # solve's content.
761
+ if not rows:
762
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
763
+ return out_path
764
+
765
+ out = pd.DataFrame(rows, columns=["node", "p_roll_continue_state"])
766
+ out.to_csv(out_path, index=False, float_format="%.8g")
767
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
768
+ return out_path
769
+
770
+
771
+ def write_p_entity_period_existing_capacity(
772
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
773
+ prior_handoff: "SolveHandoff | None" = None,
774
+ flex_data: "FlexData | None" = None,
775
+ is_first_solve: bool | None = None,
776
+ provider: "object | None" = None,
777
+ csv_dump: bool = False,
778
+ ) -> Path:
779
+ """Write ``solve_data/p_entity_period_existing_capacity.csv``.
780
+
781
+ For each (entity, period) in ``ed_history_realized`` ∪ (entity ×
782
+ d_realize_invest), compute::
783
+
784
+ existing = (first_solve & period in period_first → p_entity_pre_existing[e,d])
785
+ + (not first_solve & (e,d) in history → prior_existing[e,d])
786
+ + ((e,d) in ed_invest & d in d_realize_invest → v_invest[e,d] * unitsize[e])
787
+ invested = (not first_solve & (e,d) in history → prior_invested[e,d])
788
+ + ((e,d) in ed_invest & d in d_realize_invest → v_invest[e,d] * unitsize[e])
789
+
790
+ The ``ed_history_realized`` set is read from
791
+ ``solve_data/solve__ed_invest.csv`` (entity, period) for the
792
+ "(e,d) in ed_invest" predicate, plus the prior history file's keys.
793
+ """
794
+ out_path = work_folder / "solve_data" / "p_entity_period_existing_capacity.csv"
795
+
796
+ unitsize = _resolve_unitsize(work_folder, flex_data)
797
+ pre_existing = _resolve_pre_existing(work_folder, flex_data, provider=provider)
798
+ first_solve = _resolve_is_first_solve(work_folder, is_first_solve)
799
+ prior_existing, prior_invested = (
800
+ ({}, {}) if first_solve
801
+ else _load_prior_existing(work_folder, prior_handoff=prior_handoff)
802
+ )
803
+
804
+ invest_df = extract_variable(
805
+ h, "v_invest", ("entity",), solve_name=solve_name, has_time=False,
806
+ provider=provider,
807
+ )
808
+
809
+ # Periods to include in the iteration set. For the FIRST solve the
810
+ # mod uses ``d_realize_invest ∪ d_fix_storage_period ∪
811
+ # d_realized_period`` (via ``ed_history_realized_first``); for later
812
+ # solves only ``d_realize_invest`` adds new keys (the realized /
813
+ # fix-storage periods only contribute on the first solve).
814
+ realize_invest = _load_realized_periods(
815
+ work_folder / "solve_data" / "realized_invest_periods_of_current_solve.csv",
816
+ provider=provider,
817
+ ) or set()
818
+ if first_solve:
819
+ # Step 1-e — Provider-aware: under the in-memory cascade the
820
+ # files aren't on disk but the per-sub-solve Provider has the
821
+ # frames. Transitional seed-funnel fallback for unplumbed
822
+ # callsites lives in :func:`_provider_lookup_df` below.
823
+ realized_periods: set[str] = set()
824
+ rd_path = work_folder / "solve_data" / "realized_dispatch.csv"
825
+ _seeded_rd = _provider_lookup_df(provider, rd_path)
826
+ if _seeded_rd is not None:
827
+ realized_periods.update(
828
+ _seeded_rd["period"].cast(str).unique().to_list()
829
+ )
830
+ elif rd_path.exists():
831
+ realized_periods.update(
832
+ pd.read_csv(rd_path)["period"].astype(str).unique()
833
+ )
834
+ fix_storage_periods: set[str] = set()
835
+ fs_path = work_folder / "solve_data" / "fix_storage_timesteps.csv"
836
+ _seeded_fs = _provider_lookup_df(provider, fs_path)
837
+ if _seeded_fs is not None:
838
+ if _seeded_fs.height > 0:
839
+ fix_storage_periods.update(
840
+ _seeded_fs["period"].cast(str).unique().to_list()
841
+ )
842
+ elif fs_path.exists():
843
+ fs_df = pd.read_csv(fs_path)
844
+ if not fs_df.empty:
845
+ fix_storage_periods.update(fs_df["period"].astype(str).unique())
846
+ iteration_periods = realize_invest | realized_periods | fix_storage_periods
847
+ else:
848
+ iteration_periods = set(realize_invest)
849
+
850
+ # period_first is written by Python orchestration to solve_data/.
851
+ # Fallback for the (unlikely) case it's missing: take min(realize_invest).
852
+ period_first = _load_realized_periods(
853
+ work_folder / "solve_data" / "period_first.csv",
854
+ provider=provider,
855
+ ) or ({min(realize_invest)} if realize_invest else set())
856
+
857
+ # ed_invest set (entity, period) — phase-1 dump. CSV layout is
858
+ # ``solve, entity, period`` (3 columns); we drop the solve column.
859
+ ed_invest: set[tuple[str, str]] = set()
860
+ ei_path = work_folder / "solve_data" / "solve__ed_invest.csv"
861
+ if ei_path.exists():
862
+ ei_df = pd.read_csv(ei_path)
863
+ if not ei_df.empty and {"entity", "period"}.issubset(ei_df.columns):
864
+ ed_invest = {
865
+ (str(r["entity"]), str(r["period"])) for _, r in ei_df.iterrows()
866
+ }
867
+ elif not ei_df.empty and len(ei_df.columns) >= 2:
868
+ # Fallback for files that don't carry headers
869
+ ed_invest = {
870
+ (str(r.iloc[-2]), str(r.iloc[-1])) for _, r in ei_df.iterrows()
871
+ }
872
+
873
+ # Iteration set: ed_history_realized ∪ (entity × d_realize_invest).
874
+ # On the FIRST solve, ed_history_realized = entity × realized_or_invest
875
+ # periods (within the same branch). On subsequent solves it grows
876
+ # with the prior-history file's keys. Approximation that matches
877
+ # phase 3 in the common cases the tests cover: union prior keys +
878
+ # all entities × realize_invest.
879
+ iter_keys: set[tuple[str, str]] = set(prior_existing.keys())
880
+ entities = _resolve_entity(work_folder, flex_data)
881
+ for e in entities:
882
+ for d in iteration_periods:
883
+ iter_keys.add((e, d))
884
+
885
+ rows: list[tuple[str, str, float, float]] = []
886
+ for e, d in sorted(iter_keys):
887
+ existing = 0.0
888
+ invested = 0.0
889
+ if first_solve and d in period_first:
890
+ existing += pre_existing.get((d, e), 0.0)
891
+ if not first_solve:
892
+ existing += prior_existing.get((e, d), 0.0)
893
+ invested += prior_invested.get((e, d), 0.0)
894
+ if (e, d) in ed_invest and d in realize_invest:
895
+ try:
896
+ v = float(invest_df.loc[(solve_name, d), e])
897
+ except KeyError:
898
+ v = 0.0
899
+ existing += v * unitsize.get(e, 1.0)
900
+ invested += v * unitsize.get(e, 1.0)
901
+ rows.append((e, d, existing, invested))
902
+
903
+ out = pd.DataFrame(
904
+ rows,
905
+ columns=[
906
+ "entity", "period",
907
+ "p_entity_period_existing_capacity",
908
+ "p_entity_period_invested_capacity",
909
+ ],
910
+ )
911
+ if csv_dump:
912
+ out.to_csv(out_path, index=False, float_format="%.8g")
913
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
914
+ return out_path
915
+
916
+
917
+ def write_fix_storage_price(
918
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
919
+ flex_data: "FlexData | None" = None,
920
+ is_first_solve: bool | None = None,
921
+ scale_the_objective: float | None = None,
922
+ ) -> Path:
923
+ """Write ``solve_data/fix_storage_price.csv``.
924
+
925
+ For each ``n`` with method ``fix_price`` and each
926
+ ``(d, t) in dt_fix_storage_timesteps``::
927
+
928
+ price = -nodeBalance_eq[c, n, d, t, ...].dual
929
+ / p_inflation_factor_operations_yearly[d]
930
+ * complete_period_share_of_year[d]
931
+ / scale_the_objective
932
+
933
+ The ``nodeBalance_eq`` constraint is indexed by 8 fields
934
+ ``(solve, node, period, time, t_prev, t_prev_within_timeset,
935
+ d_prev, t_prev_within_solve)``. We accept rows for any value of
936
+ the four "previous" indices — exactly one constraint exists per
937
+ ``(c, n, d, t)`` so the dual is well-defined.
938
+ """
939
+ from flextool.process_outputs.read_highs_solution import (
940
+ _resolve_inv_scale_the_objective,
941
+ )
942
+ out_path = work_folder / "solve_data" / "fix_storage_price.csv"
943
+
944
+ target_nodes = _resolve_storage_fix_methods(work_folder, "fix_price", flex_data)
945
+ fix_steps = _load_realized_set(
946
+ work_folder / "solve_data" / "fix_storage_timesteps.csv"
947
+ )
948
+ inflation = _resolve_inflation_factor_operations_yearly(work_folder, flex_data)
949
+ period_share = _resolve_complete_period_share_of_year(work_folder, flex_data)
950
+ # Agent 12: resolve live scale_the_objective from the per-solve CSV.
951
+ # Phase G: cascade-supplied ``scale_the_objective`` kwarg short-
952
+ # circuits the CSV read entirely.
953
+ inv_scale = _resolve_inv_scale_the_objective(
954
+ work_folder, scale_the_objective=scale_the_objective,
955
+ )
956
+
957
+ # Empty short-circuit — most scenarios have no fix_price nodes.
958
+ if not target_nodes or not fix_steps:
959
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
960
+ out_path.write_text("period,step,node,p_fix_storage_price\n")
961
+ _logger.info("wrote %s (header only — first solve)", out_path)
962
+ else:
963
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
964
+ return out_path
965
+
966
+ # nodeBalance_eq has 9 indices (Agent 1.4 added ``bn`` between
967
+ # ``node`` and ``period``). Use the generic extractor with 7
968
+ # col_names so the trailing two stay the (period, time) row index;
969
+ # then filter rows to fix_steps and entity-name match to
970
+ # target_nodes (col_names[1] == 'node'). In degenerate mode bn is
971
+ # always 'default'.
972
+ df = extract_variable(
973
+ h, "nodeBalance_eq",
974
+ col_names=("c", "node", "bn"),
975
+ solve_name=solve_name,
976
+ has_time=True,
977
+ source="row_dual",
978
+ value_scale=1.0, # we apply the full transform manually below
979
+ trailing_col_names=("t_prev", "t_prev_within_timeset",
980
+ "d_prev", "t_prev_within_solve"),
981
+ )
982
+
983
+ if df.empty:
984
+ # Same preserve-prior semantics as the empty short-circuit above.
985
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
986
+ out_path.write_text("period,step,node,p_fix_storage_price\n")
987
+ return out_path
988
+
989
+ rows: list[tuple[str, str, str, float]] = []
990
+ # df: row index (solve, period, time); columns MultiIndex
991
+ # (c, node, t_prev, t_prev_within_timeset, d_prev, t_prev_within_solve).
992
+ # The constraint uses ``c in solve_current`` so c == solve_name; pick
993
+ # the only matching slice. For each (period, time) ∈ fix_steps and
994
+ # each node ∈ target_nodes, sum the duals across the (typically
995
+ # single) "previous" combinations.
996
+ for (_s, period, time), row in df.iterrows():
997
+ if (period, time) not in fix_steps:
998
+ continue
999
+ for col, dual_val in row.items():
1000
+ node = col[1]
1001
+ if node not in target_nodes:
1002
+ continue
1003
+ scale = (
1004
+ -1.0
1005
+ / inflation.get(period, 1.0)
1006
+ * period_share.get(period, 1.0)
1007
+ * inv_scale
1008
+ )
1009
+ rows.append((period, time, node, float(dual_val) * scale))
1010
+
1011
+ if not rows:
1012
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
1013
+ out_path.write_text("period,step,node,p_fix_storage_price\n")
1014
+ else:
1015
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
1016
+ return out_path
1017
+
1018
+ out = pd.DataFrame(rows, columns=["period", "step", "node", "p_fix_storage_price"])
1019
+ out.to_csv(out_path, index=False, float_format="%.8g")
1020
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
1021
+ return out_path
1022
+
1023
+
1024
+ def write_fix_storage_usage(
1025
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
1026
+ flex_data: "FlexData | None" = None,
1027
+ is_first_solve: bool | None = None,
1028
+ ) -> Path:
1029
+ """Write ``solve_data/fix_storage_usage.csv``.
1030
+
1031
+ Net energy flow through each ``fix_usage`` storage node over the
1032
+ step::
1033
+
1034
+ usage[n, d, t] = (sum_{p: n is source} v_flow[p, n, *] × unitsize[p]
1035
+ - sum_{p: n is sink} v_flow[p, *, n] × unitsize[p])
1036
+ × step_duration[d, t]
1037
+
1038
+ This is the *simple* form of the model's ``r_storage_usage_dt`` —
1039
+ exact for ``method_nvar`` processes (line 5421 of the model) and
1040
+ for ``method_1var_per_way`` processes whose ``pdtProcess_slope == 1``
1041
+ and that have no ``min_load_efficiency`` and unit coefficients
1042
+ equal to 1. That covers the typical battery / storage-inverter
1043
+ topology; more exotic process methods connected to a ``fix_usage``
1044
+ node are NOT reproduced byte-for-byte here — the slope/section
1045
+ corrections in the full formula (lines 5389–5429) are skipped.
1046
+
1047
+ If your model pairs ``fix_usage`` nodes with min_load_efficiency or
1048
+ non-unity unit coefficients, fall back to ``--use-old-raw-csv`` for
1049
+ this file until the full flow derivation is ported over.
1050
+ """
1051
+ out_path = work_folder / "solve_data" / "fix_storage_usage.csv"
1052
+
1053
+ target_nodes = _resolve_storage_fix_methods(work_folder, "fix_usage", flex_data)
1054
+ fix_steps = _load_realized_set(
1055
+ work_folder / "solve_data" / "fix_storage_timesteps.csv"
1056
+ )
1057
+
1058
+ if not target_nodes or not fix_steps:
1059
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
1060
+ out_path.write_text("period,step,node,p_fix_storage_usage\n")
1061
+ else:
1062
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
1063
+ return out_path
1064
+
1065
+ unitsize = _resolve_unitsize(work_folder, flex_data)
1066
+ step_duration = _resolve_step_duration(work_folder, flex_data)
1067
+ flow_df = extract_variable(
1068
+ h, "v_flow", ("process", "source", "sink"), solve_name=solve_name,
1069
+ )
1070
+
1071
+ rows: list[tuple[str, str, str, float]] = []
1072
+ for (_solve, period, time), row in flow_df.iterrows():
1073
+ if (period, time) not in fix_steps:
1074
+ continue
1075
+ dt = step_duration.get((period, time), 1.0)
1076
+ per_node: dict[str, float] = {n: 0.0 for n in target_nodes}
1077
+ for (process, source, sink), v in row.items():
1078
+ us = unitsize.get(process, 1.0)
1079
+ if source in per_node:
1080
+ per_node[source] += float(v) * us # outflow from node
1081
+ if sink in per_node:
1082
+ per_node[sink] -= float(v) * us # inflow to node
1083
+ for node, net in per_node.items():
1084
+ if net == 0.0:
1085
+ continue
1086
+ rows.append((period, time, node, net * dt))
1087
+
1088
+ if not rows:
1089
+ if _resolve_is_first_solve(work_folder, is_first_solve) and not out_path.exists():
1090
+ out_path.write_text("period,step,node,p_fix_storage_usage\n")
1091
+ else:
1092
+ _logger.debug("skipped %s (empty; preserving prior content)", out_path)
1093
+ return out_path
1094
+
1095
+ out = pd.DataFrame(rows, columns=["period", "step", "node", "p_fix_storage_usage"])
1096
+ out.to_csv(out_path, index=False, float_format="%.8g")
1097
+ _logger.info("wrote %s (%d rows)", out_path, len(out))
1098
+ return out_path
1099
+
1100
+
1101
+ # ---------------------------------------------------------------------------
1102
+ # Cross-solve capacity accumulators
1103
+ #
1104
+ # These four writers are conceptually handoffs too — the file they emit
1105
+ # is the cumulative output-so-far across every solve that has run. On
1106
+ # the first solve they truncate + write the header; later solves append
1107
+ # only the rows whose period is not already in
1108
+ # ``solve_data/period_capacity.csv`` (so rolling windows don't write
1109
+ # the same period twice). Together they replicate the tail of phase 3
1110
+ # that produced ``solve_data/unit_capacity__period.csv`` etc. — so when
1111
+ # phase 3 is skipped, the user-facing capacity outputs stay intact.
1112
+ # ---------------------------------------------------------------------------
1113
+
1114
+
1115
+ def _load_entity_class_set(work_folder: Path, set_name: str) -> list[str]:
1116
+ """Return the ordered list of entities in ``input/<set_name>.csv``.
1117
+
1118
+ The resolved-set CSVs (``entity.csv``, ``process_unit.csv``,
1119
+ ``process_connection.csv``) are written by ``input_writer`` from
1120
+ the DB and live in ``input/``. The mod previously also re-emitted
1121
+ them under ``solve_data/`` via ``printf`` blocks; that redundant
1122
+ write was retired in the post-solve cleanup, so all consumers now
1123
+ read directly from ``input/``.
1124
+
1125
+ Special case: ``nodeState`` is derived from ``input/p_node_type.csv``
1126
+ (rows with ``p_node_type == 'storage'``), preserving the original
1127
+ node order.
1128
+ """
1129
+ if set_name == "nodeState":
1130
+ path = work_folder / "input" / "p_node_type.csv"
1131
+ if not path.exists():
1132
+ return []
1133
+ df = pd.read_csv(path)
1134
+ if df.empty or "p_node_type" not in df.columns:
1135
+ return []
1136
+ return df.loc[df["p_node_type"].astype(str) == "storage", "node"].astype(str).tolist()
1137
+ path = work_folder / "input" / f"{set_name}.csv"
1138
+ if not path.exists():
1139
+ return []
1140
+ df = pd.read_csv(path)
1141
+ if df.empty or len(df.columns) == 0:
1142
+ return []
1143
+ return df.iloc[:, 0].astype(str).tolist()
1144
+
1145
+
1146
+ def _load_unitsize_map(
1147
+ work_folder: Path, flex_data: "FlexData | None" = None,
1148
+ ) -> dict[str, float]:
1149
+ """Wrapper around :func:`_load_unitsize` that never raises on a
1150
+ missing file (some scenarios don't use any entities with unitsize).
1151
+
1152
+ Phase G — when ``flex_data`` is supplied, trust the in-memory
1153
+ ``p_all_entity_unitsize`` carrier (``None`` short-circuits to {} —
1154
+ same as the file fallback's missing-file branch)."""
1155
+ if flex_data is not None:
1156
+ param = getattr(flex_data, "p_all_entity_unitsize", None)
1157
+ if param is None:
1158
+ return {}
1159
+ try:
1160
+ f = param.frame
1161
+ entity_col = f.columns[0]
1162
+ return dict(zip(
1163
+ f[entity_col].cast(str).to_list(),
1164
+ f["value"].cast(float).to_list(),
1165
+ ))
1166
+ except Exception: # noqa: BLE001
1167
+ pass
1168
+ path = work_folder / "input" / "p_entity_unitsize.csv"
1169
+ if not path.exists():
1170
+ return {}
1171
+ df = pd.read_csv(path, index_col=0)
1172
+ return df.loc["value"].astype(float).to_dict() if "value" in df.index else {}
1173
+ def _load_period_capacity(work_folder: Path) -> set[str]:
1174
+ """Periods already output by a previous roll's capacity dump."""
1175
+ path = work_folder / "solve_data" / "period_capacity.csv"
1176
+ if not path.exists():
1177
+ return set()
1178
+ df = pd.read_csv(path)
1179
+ if df.empty or "period" not in df.columns:
1180
+ return set()
1181
+ return set(df["period"].astype(str))
1182
+
1183
+
1184
+ def _load_drdi(work_folder: Path, roll: str) -> list[str]:
1185
+ """Periods in this roll's ``d_realize_dispatch_or_invest`` set, in
1186
+ file order."""
1187
+ path = work_folder / "solve_data" / "d_realize_dispatch_or_invest.csv"
1188
+ if not path.exists():
1189
+ return []
1190
+ df = pd.read_csv(path, dtype=str)
1191
+ if df.empty:
1192
+ return []
1193
+ df = df[df["solve"] == roll]
1194
+ return df["period"].tolist()
1195
+ def _append_period_capacity(
1196
+ work_folder: Path, new_periods: list[str],
1197
+ writer_state: "OutputWriterState | None" = None,
1198
+ ) -> None:
1199
+ """Write ``solve_data/period_capacity.csv`` = prior_set ∪ new_periods.
1200
+
1201
+ Truncate + re-emit union so the next roll sees an up-to-date set.
1202
+
1203
+ When ``writer_state`` is supplied, source the prior set from the
1204
+ in-memory accumulator instead of re-reading the CSV.
1205
+ """
1206
+ path = work_folder / "solve_data" / "period_capacity.csv"
1207
+ if writer_state is not None:
1208
+ existing = set(writer_state.periods_already_emitted)
1209
+ else:
1210
+ existing = _load_period_capacity(work_folder)
1211
+ union = existing | set(new_periods)
1212
+ # Emit sorted for determinism (downstream must not rely on row order).
1213
+ with open(path, "w", encoding="utf-8") as f:
1214
+ f.write("period\n")
1215
+ for p in sorted(union):
1216
+ f.write(p + "\n")
1217
+ def _bump_period_capacity(
1218
+ work_folder: Path, solve_name: str,
1219
+ writer_state: "OutputWriterState | None" = None,
1220
+ ) -> None:
1221
+ """Accumulate this solve's realized periods into ``period_capacity.csv``.
1222
+
1223
+ Called once per solve at the end of :func:`write_all_handoffs`. This
1224
+ set used to let the per-period capacity dumps skip already-emitted
1225
+ periods; those writers were removed 2026-08-07, so the accumulator is
1226
+ now vestigial (kept pending a separate teardown of the
1227
+ ``period_capacity`` machinery, which ripples into orchestration and
1228
+ several tests). Still read via :func:`_load_period_capacity`.
1229
+
1230
+ When ``writer_state`` is supplied, ALSO push the newly
1231
+ accumulated periods into ``writer_state.periods_already_emitted``
1232
+ in-memory. The writer adapter previously re-read the file to refresh
1233
+ this set; with the in-memory dual update it can trust the accumulator.
1234
+ """
1235
+ roll = _actual_solve_name(work_folder, solve_name)
1236
+ new_periods = _load_drdi(work_folder, roll)
1237
+ _append_period_capacity(work_folder, new_periods, writer_state=writer_state)
1238
+ if writer_state is not None and new_periods:
1239
+ writer_state.periods_already_emitted.update(str(p) for p in new_periods)
1240
+
1241
+
1242
+ # ---------------------------------------------------------------------------
1243
+ # Orchestrator
1244
+ # ---------------------------------------------------------------------------
1245
+
1246
+
1247
+ def write_all_handoffs(
1248
+ h: "highspy.Highs", *, solve_name: str, work_folder: Path,
1249
+ prior_handoff: "SolveHandoff | None" = None,
1250
+ flex_data: "FlexData | None" = None,
1251
+ is_first_solve: bool | None = None,
1252
+ writer_state: "OutputWriterState | None" = None,
1253
+ scale_the_objective: float | None = None,
1254
+ provider: "object | None" = None,
1255
+ csv_dump: bool = False,
1256
+ ) -> list[Path]:
1257
+ """Write all six handoff files.
1258
+
1259
+ Each writer is independent — failure on one is logged and does not
1260
+ abort the rest, mirroring :func:`write_all_variables`.
1261
+
1262
+ ``prior_handoff`` (when provided) sources the prior-roll state
1263
+ (``realized_existing`` / ``realized_invest`` / ``divest_cumulative``)
1264
+ from the in-memory ``SolveHandoff`` instead of re-reading the
1265
+ parent solve's CSV outputs. Forwarded to the two writers that
1266
+ consume prior state; the remainder ignore it.
1267
+
1268
+ Phase G kwargs (additive, fully backward-compatible):
1269
+
1270
+ * ``flex_data`` — the cascade's in-memory FlexData; routes the per-
1271
+ writer ``_load_unitsize`` / ``_load_node_state`` / ``_load_entity``
1272
+ / ``_load_storage_fix_methods`` / period-share / inflation / step-
1273
+ duration / realized-dispatch lookups through in-memory carriers.
1274
+ CSV fallback preserved when not supplied.
1275
+ * ``is_first_solve`` — cascade-supplied first-solve flag; replaces
1276
+ the per-call ``solve_data/p_model.csv`` read in every writer.
1277
+ * ``writer_state`` — pushes newly-accumulated periods into the
1278
+ ``periods_already_emitted`` set in-memory + lets the per-period
1279
+ capacity writer skip the on-disk re-read of ``period_capacity.csv``.
1280
+
1281
+ See ``specs/in_memory_carriers_audit.md`` for the per-reader mapping.
1282
+ """
1283
+ # Writers that take additional kwargs beyond the base (h, solve_name,
1284
+ # work_folder). Phase G greatly expanded these — the dispatch table
1285
+ # below routes per-writer.
1286
+ written: list[Path] = []
1287
+ fd_kwargs = {"flex_data": flex_data, "is_first_solve": is_first_solve}
1288
+ dispatch: list[tuple[object, dict]] = [
1289
+ (write_p_entity_divested,
1290
+ {**fd_kwargs, "prior_handoff": prior_handoff}),
1291
+ (write_fix_storage_quantity, fd_kwargs),
1292
+ (write_p_roll_continue_state, {"flex_data": flex_data}),
1293
+ (write_p_entity_period_existing_capacity,
1294
+ {**fd_kwargs, "prior_handoff": prior_handoff, "provider": provider,
1295
+ "csv_dump": csv_dump}),
1296
+ (write_fix_storage_price,
1297
+ {**fd_kwargs, "scale_the_objective": scale_the_objective}),
1298
+ (write_fix_storage_usage, fd_kwargs),
1299
+ ]
1300
+ for fn, extra in dispatch:
1301
+ try:
1302
+ written.append(fn(
1303
+ h, solve_name=solve_name, work_folder=work_folder, **extra,
1304
+ ))
1305
+ except Exception as exc: # noqa: BLE001
1306
+ _logger.warning("handoff writer %s failed: %s", fn.__name__, exc)
1307
+ # Accumulate this solve's realized periods into ``period_capacity.csv``
1308
+ # for the next roll (vestigial: the per-period capacity dumps that
1309
+ # consumed this set were removed 2026-08-07; the accumulator is kept
1310
+ # pending a separate teardown of the ``period_capacity`` machinery).
1311
+ try:
1312
+ _bump_period_capacity(work_folder, solve_name, writer_state=writer_state)
1313
+ except Exception as exc: # noqa: BLE001
1314
+ _logger.warning("period_capacity accumulation failed: %s", exc)
1315
+ return written