flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,269 @@
1
+ import os
2
+ from spinedb_api import DatabaseMapping, from_database, to_database, SpineDBAPIError, Asterisk
3
+ from spinedb_api.exception import NothingToCommit
4
+ from flextool.update_flextool import FLEXTOOL_DB_VERSION
5
+
6
+ flextool_db_version = FLEXTOOL_DB_VERSION
7
+
8
+
9
+ def _import_legacy_timeset_timeline(db, input_path: str) -> None:
10
+ """Set the ``timeset.timeline`` parameter from a legacy Excel sheet.
11
+
12
+ Pre-v25 specification-format Excels store the timeset->timeline link as a
13
+ two-column ``timeblockSet_timeline`` sheet (column 0 = timeset name,
14
+ column 1 = timeline name; a header row, no alternative column). The
15
+ modern ``timeset_timeline`` sheet has a different layout, so this link is
16
+ not picked up by the generic spec mappings.
17
+
18
+ The v25 migration (``update_timestructure``) normally collapses the old
19
+ ``timeblockSet__timeline`` relationship into the ``timeset.timeline``
20
+ parameter, attaching each value to the alternative of that timeset's
21
+ duration. Mirror that here so a pre-v25 Excel imports directly into the
22
+ v25+ schema: for every ``(timeset, alternative)`` that has a
23
+ ``timeset_duration`` value, set ``timeline`` in the same alternative.
24
+ Without it the solve fails with "Failed to map timeset to timeline".
25
+ """
26
+ import openpyxl
27
+
28
+ try:
29
+ wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True)
30
+ except Exception:
31
+ return
32
+ try:
33
+ if "timeblockSet_timeline" not in wb.sheetnames:
34
+ return
35
+ timeset_to_timeline: dict[str, str] = {}
36
+ for i, row in enumerate(wb["timeblockSet_timeline"].iter_rows(values_only=True)):
37
+ if i == 0: # header row: (timeblockSet, timeline)
38
+ continue
39
+ if not row or row[0] is None or row[1] is None:
40
+ continue
41
+ timeset_to_timeline[str(row[0])] = str(row[1])
42
+ finally:
43
+ wb.close()
44
+
45
+ if not timeset_to_timeline:
46
+ return
47
+
48
+ # Already-present (entity, alternative) pairs — stay idempotent.
49
+ existing = {
50
+ (pv["entity_byname"], pv["alternative_name"])
51
+ for pv in db.find_parameter_values(
52
+ entity_class_name="timeset", parameter_definition_name="timeline"
53
+ )
54
+ }
55
+ added = 0
56
+ for pv in db.find_parameter_values(
57
+ entity_class_name="timeset", parameter_definition_name="timeset_duration"
58
+ ):
59
+ byname = pv["entity_byname"]
60
+ timeline_name = timeset_to_timeline.get(byname[0])
61
+ if timeline_name is None:
62
+ continue
63
+ key = (byname, pv["alternative_name"])
64
+ if key in existing:
65
+ continue
66
+ value, type_ = to_database(timeline_name)
67
+ try:
68
+ db.add_parameter_value(
69
+ entity_class_name="timeset",
70
+ parameter_definition_name="timeline",
71
+ entity_byname=byname,
72
+ alternative_name=pv["alternative_name"],
73
+ value=value,
74
+ type=type_,
75
+ )
76
+ existing.add(key)
77
+ added += 1
78
+ except Exception as e:
79
+ print(f" Could not set timeset.timeline for {byname[0]}: {e}")
80
+ if added:
81
+ print(
82
+ f"\nImported legacy timeset->timeline links "
83
+ f"({added} (timeset, alternative) pairs) from 'timeblockSet_timeline'."
84
+ )
85
+
86
+ def write_to_flextool_input_db(input_path, tabular_reader, target_db_url, input_type='excel',
87
+ migration_follows: bool = False):
88
+ """Write tabular data to FlexTool input database, reading one sheet/file at a time.
89
+
90
+ Args:
91
+ input_path: Path to Excel/ODS file or directory containing CSV files
92
+ tabular_reader: TabularReader instance with loaded specification
93
+ target_db_url: URL to target SQLite database
94
+ input_type: Either 'excel' or 'csv' to determine reading method
95
+ migration_follows: If True, accept a version mismatch because
96
+ the caller will migrate the database after import
97
+
98
+ This function reads sheets/files one at a time to minimize memory usage.
99
+ """
100
+ # Refuse specification-format Excels whose template layout predates the
101
+ # importer. Deferred import avoids a circular dependency with the package.
102
+ if input_type == 'excel':
103
+ from flextool.process_inputs import (
104
+ MIN_SUPPORTED_SPECIFICATION_VERSION,
105
+ read_specification_changelog_version,
106
+ unsupported_specification_message,
107
+ )
108
+ changelog_version = read_specification_changelog_version(input_path)
109
+ if (
110
+ changelog_version is not None
111
+ and changelog_version < MIN_SUPPORTED_SPECIFICATION_VERSION
112
+ ):
113
+ print(unsupported_specification_message(changelog_version))
114
+ exit(1)
115
+
116
+ if target_db_url.startswith('sqlite://') or target_db_url.startswith('http://'):
117
+ db_url = target_db_url
118
+ elif os.path.exists(target_db_url) and target_db_url.endswith(".sqlite"):
119
+ db_url = 'sqlite:///' + target_db_url
120
+ else:
121
+ print("No sqlite file at " + target_db_url)
122
+ exit(-1)
123
+
124
+ with DatabaseMapping(db_url, create = False, upgrade = True) as db:
125
+ sq = db.object_parameter_definition_sq
126
+ version_parameter = db.query(sq).filter(sq.c.object_class_name == "model").filter(sq.c.parameter_name == "version").one_or_none()
127
+ if version_parameter is None:
128
+ #if no version assume version 0
129
+ print(f"No FlexTool database version found. Needs to be proper FlexTool database at version: {flextool_db_version}")
130
+ exit(-1)
131
+ else:
132
+ version = from_database(version_parameter.default_value, version_parameter.default_type)
133
+ if version == flextool_db_version:
134
+ print(f"Valid FlexTool input database with correct version: {flextool_db_version}")
135
+ elif migration_follows:
136
+ print(f"Importing into a fresh FlexTool schema-v{int(version)} database "
137
+ "(a migration to the current version follows this import).")
138
+ else:
139
+ print(f"Wrong FlexTool input database version: {version}. Should be: {flextool_db_version}")
140
+ exit(-1)
141
+
142
+ # Get list of tables to process
143
+ selected_tables = tabular_reader.get_selected_table_names()
144
+ print(f"\nProcessing {len(selected_tables)} selected tables")
145
+
146
+ # Purge the database empty of data times
147
+ try:
148
+ db.remove_entity(id=Asterisk)
149
+ db.remove_alternative(id=Asterisk)
150
+ db.remove_scenario(id=Asterisk)
151
+ db.remove_parameter_value(id=Asterisk)
152
+ db.commit_session("Purged the db")
153
+ except Exception as e:
154
+ raise RuntimeError(f"Failed to purge the input database before writing. {e}")
155
+
156
+ # Process each table one at a time
157
+ for table_name in selected_tables:
158
+ print(f"\nProcessing table {table_name + ': ':<30}", end="")
159
+
160
+ # Read the sheet/file that contains the data
161
+ try:
162
+ if input_type == 'excel':
163
+ raw_df, column_types = tabular_reader.read_excel_sheet(input_path, table_name)
164
+ elif input_type == 'csv':
165
+ csv_file = os.path.join(input_path, f"{table_name}.csv")
166
+ if not os.path.exists(csv_file):
167
+ print(f" Warning: CSV file not found: {csv_file}")
168
+ continue
169
+ raw_df, column_types = tabular_reader.read_csv_file(csv_file)
170
+ else:
171
+ raise ValueError(f"Unknown input_type: {input_type}")
172
+
173
+ # Skip if sheet not found in the file
174
+ if raw_df is None:
175
+ print(f"Skipping {table_name} - no sheet found")
176
+ continue
177
+
178
+ # Skip if no data (e.g., sheet not selected)
179
+ if raw_df.empty:
180
+ print(f" Skipping {table_name} - no data")
181
+ continue
182
+
183
+ except Exception as e:
184
+ print(f" Error reading table '{table_name}': {e}")
185
+ continue # Continue to next table on read error
186
+
187
+ # Get mappings for this sheet
188
+ mappings = tabular_reader.get_table_mappings(table_name)
189
+ if not mappings:
190
+ raise ValueError(f"No specification found for sheet {table_name}")
191
+
192
+ table_options = tabular_reader.get_table_options(table_name)
193
+ if 'row' in table_options:
194
+ raw_df = raw_df.iloc[table_options['row']:]
195
+ if 'column' in table_options:
196
+ raw_df = raw_df.iloc[table_options['column']:]
197
+
198
+ # Process each mapping in the table
199
+ for mapping_name in mappings.keys():
200
+ print(f"{mapping_name:>25}", end="")
201
+
202
+ mapping_info = tabular_reader._parse_mapping(mappings[mapping_name])
203
+ if not mapping_info:
204
+ continue
205
+
206
+ # Read data and process the dataframe
207
+ try:
208
+ (data_df, ent_zip_list, ent_act_zip, scen_array, scen_alt_df) = tabular_reader._extract_data(raw_df, mapping_info, table_options, column_types, table_name, mapping_name)
209
+ except Exception as e:
210
+ print(
211
+ f"\n Error processing sheet '{table_name}', mapping "
212
+ f"'{mapping_name}': {type(e).__name__}: {e}"
213
+ )
214
+ continue # Continue to next mapping on processing error
215
+
216
+ if ent_zip_list:
217
+ tabular_reader._add_entities(ent_zip_list, db, table_name, mapping_name)
218
+
219
+ # Add alternatives to the database
220
+ if data_df is not None:
221
+ alt_array = data_df.columns.get_level_values('Alternative').unique().to_list()
222
+ tabular_reader._add_alternatives(alt_array, db, table_name, mapping_name)
223
+
224
+ # Write re-organised dataframe to database
225
+ if data_df is not None:
226
+ #try:
227
+ # check_type:
228
+ value_type = mapping_info['rules'].get('ParameterValueType')
229
+ if value_type:
230
+ value_type = value_type['value']
231
+ else:
232
+ value_type = 'constant'
233
+ tabular_reader._add_parameters(data_df, db, table_name, mapping_name, value_type)
234
+
235
+ # Ensure alternatives referenced by entity_alternatives exist
236
+ if ent_act_zip is not None:
237
+ ent_act_list = list(ent_act_zip)
238
+ ea_alts = list({alt for _cls, _ent, alt, _act in ent_act_list})
239
+ if ea_alts:
240
+ tabular_reader._add_alternatives(ea_alts, db, table_name, mapping_name)
241
+ tabular_reader._add_entity_alternatives(ent_act_list, db, table_name, mapping_name)
242
+
243
+ # Add scenarios to the database
244
+ if scen_array is not None:
245
+ for scen in scen_array:
246
+ try:
247
+ db.add_scenario(name=scen)
248
+ except Exception as e:
249
+ raise SpineDBAPIError(f'Could not add scenario {scen} to the database: {e}')
250
+
251
+ # Add scenario_alternatives to the database
252
+ if scen_alt_df is not None:
253
+ tabular_reader._add_scenario_alternatives(scen_alt_df, db, table_name, mapping_name)
254
+
255
+ # Backfill the timeset->timeline link from the legacy two-column sheet
256
+ # if present (pre-v25 Excels); modern files set it via the spec.
257
+ if input_type == 'excel':
258
+ _import_legacy_timeset_timeline(db, input_path)
259
+
260
+ # Commit the changes
261
+ try:
262
+ db.commit_session(f"Imported data from mapping '{mapping_name}'")
263
+ tabular_reader.logger.info(f"Successfully committed data and items from mapping {mapping_name}")
264
+ except NothingToCommit:
265
+ pass
266
+ except Exception as e:
267
+ raise SpineDBAPIError(f"Could not commit data and items based on {mapping_name}: {e}")
268
+
269
+ print("\nAll data processed successfully! (* = No data found for the mapping)")
@@ -0,0 +1,7 @@
1
+ """Output data processing: reads solver CSV results, post-processes, and writes outputs."""
2
+ from flextool.process_outputs.read_variables import read_variables
3
+ from flextool.process_outputs.read_parameters import read_parameters
4
+ from flextool.process_outputs.read_sets import read_sets
5
+ from flextool.process_outputs.process_results import post_process_results
6
+ from flextool.process_outputs.write_outputs import write_outputs
7
+ __all__ = ['read_variables', 'read_parameters', 'read_sets', 'post_process_results', 'write_outputs']
@@ -0,0 +1,55 @@
1
+ """Shared dt→d annualisation helper for the output-processing stage.
2
+
3
+ A single place that performs the *extensive* per-step → per-period→annual
4
+ aggregation so that no individual output site drifts from the cost-weighting
5
+ convention.
6
+
7
+ The LP objective annualises every per-timestep cost term with the
8
+ representative weight ``p_timestep_weight[d, t]`` (1.0 for a full / evenly
9
+ sampled timeline, a per-step share of the year for a representative timeset
10
+ with ``timeset_weights``). Annual *energy / extensive* outputs must use the
11
+ **same** weight so that the energy the model is scaled to serve, reports, and
12
+ costs all coincide. When ``w ≡ 1`` (the common full-timeline case) the result
13
+ is byte-identical to the unweighted formula.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import pandas as pd
19
+
20
+
21
+ def annualize_dt_to_d(
22
+ frame: pd.DataFrame,
23
+ timestep_weight: pd.Series,
24
+ complete_period_share_of_year: pd.Series,
25
+ step_duration: "pd.Series | None" = None,
26
+ *,
27
+ div_level: "int | None" = None,
28
+ ) -> pd.DataFrame:
29
+ """Weight a per-``(d, t)`` extensive frame and annualise it to per-period.
30
+
31
+ Steps (matching the objective's annualisation):
32
+
33
+ 1. If ``step_duration`` is given (the frame holds power, MW), multiply by
34
+ it to obtain energy per step (MWh). Inflow-type frames are already
35
+ MWh/step — pass ``step_duration=None``.
36
+ 2. Multiply by ``timestep_weight`` (the per-``(d, t)`` representative weight,
37
+ 1.0 when no/uniform ``timeset_weights``).
38
+ 3. Sum over the ``period`` level.
39
+ 4. Divide by ``complete_period_share_of_year`` (the uniform annualiser
40
+ ``Σ_t step_duration / 8760``).
41
+
42
+ ``timestep_weight`` and ``step_duration`` share the identical
43
+ ``(period, time)`` row index (both stripped of the ``solve`` level in
44
+ ``drop_levels``), so ``.mul(..., axis=0)`` broadcasts the same way.
45
+
46
+ ``div_level`` mirrors the per-site ``.div(..., axis=0, level=...)``
47
+ convention: pass ``level=1`` where the caller's frame has a multi-level
48
+ row index whose period sits at level 1 (e.g. ``out_node`` slacks).
49
+ """
50
+ f = frame if step_duration is None else frame.mul(step_duration, axis=0)
51
+ f = f.mul(timestep_weight, axis=0)
52
+ summed = f.groupby(level="period").sum()
53
+ if div_level is None:
54
+ return summed.div(complete_period_share_of_year, axis=0)
55
+ return summed.div(complete_period_share_of_year, axis=0, level=div_level)
@@ -0,0 +1,292 @@
1
+ """Δ.31 — In-memory FlexData → pandas helpers for read_parameters / read_sets.
2
+
3
+ These helpers translate the polars long-form FlexData fields into the
4
+ pandas wide-format shapes that the legacy CSV-based ``read_parameters``
5
+ and ``read_sets`` produced. They are stateless utilities; the
6
+ per-attribute mapping lives in
7
+ :mod:`flextool.process_outputs.read_parameters` and
8
+ :mod:`flextool.process_outputs.read_sets`.
9
+
10
+ Naming convention: dim columns in FlexData are short (``n``, ``p``,
11
+ ``d``, ``t``, ``g``, ``c``, ``e``, …). The legacy pandas namespace
12
+ spells them out in full (``node``, ``process``, ``period``, ``time``,
13
+ ``group``, ``commodity``, ``entity``, …). :data:`DIM_NAMES` is the
14
+ canonical translation table.
15
+
16
+ Failure mode: every helper raises (``KeyError``, ``ValueError``)
17
+ loudly when a FlexData field is absent or has an unexpected schema.
18
+ The legacy CSV path was tolerant of empty / missing files — the
19
+ in-memory replacement is strict so authoring bugs surface at the
20
+ call-site instead of producing silently-empty outputs.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ from typing import Sequence
25
+
26
+ import pandas as pd
27
+ import polars as pl
28
+
29
+
30
+ # ---------------------------------------------------------------------------
31
+ # Canonical dim-name translation table.
32
+ #
33
+ # FlexData uses short, model-internal names (``n``, ``p``, ``d``, ``t``).
34
+ # Legacy pandas output namespace spells them out so plot / write_outputs
35
+ # / out_*.py modules read them as ``period`` / ``time`` / ``node`` /
36
+ # ``process`` etc. Keep this map in sync with the per-field comments in
37
+ # :class:`flextool.engine_polars.input.FlexData`.
38
+ DIM_NAMES: dict[str, str] = {
39
+ "n": "node",
40
+ "p": "process",
41
+ "g": "group",
42
+ "c": "commodity",
43
+ "f": "profile",
44
+ "d": "period",
45
+ "t": "time",
46
+ "e": "entity",
47
+ "i": "tier",
48
+ "ud": "upDown",
49
+ "r": "reserve",
50
+ "source": "source",
51
+ "sink": "sink",
52
+ "method": "method",
53
+ "param": "param",
54
+ # Storage / time-helper extensions
55
+ "td": "td",
56
+ # Block / branch / lookback names that already match
57
+ "b_first": "b_first",
58
+ }
59
+
60
+
61
+ def long_dim(d: str) -> str:
62
+ """Return the long pandas dim-name for a FlexData short dim."""
63
+ if d in DIM_NAMES:
64
+ return DIM_NAMES[d]
65
+ return d
66
+
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # Frame → pandas helpers
70
+ # ---------------------------------------------------------------------------
71
+
72
+
73
+ def _to_pandas(frame_pl: "pl.DataFrame") -> pd.DataFrame:
74
+ """``polars.DataFrame.to_pandas()`` with a small empty-frame guard."""
75
+ if frame_pl is None:
76
+ raise ValueError("expected a polars DataFrame, got None")
77
+ return frame_pl.to_pandas()
78
+
79
+
80
+ def wide_per_entity(
81
+ frame_pl: "pl.DataFrame",
82
+ *,
83
+ row_dims: Sequence[str],
84
+ col_dim: str,
85
+ value: str = "value",
86
+ row_names: Sequence[str] | None = None,
87
+ col_name: str | None = None,
88
+ ) -> pd.DataFrame:
89
+ """Pivot a polars long-form frame to a single-level-column pandas
90
+ DataFrame.
91
+
92
+ Parameters
93
+ ----------
94
+ frame_pl : polars.DataFrame
95
+ Source frame; must carry ``row_dims + (col_dim, value)``.
96
+ row_dims : sequence of str
97
+ Dim columns that become the (multi-)row-index.
98
+ col_dim : str
99
+ Dim column that becomes the column header.
100
+ value : str, default "value"
101
+ Column carrying the numeric values.
102
+ row_names : sequence of str, optional
103
+ If supplied, the row index level names are set to these.
104
+ Otherwise the source dim names are used (translated via
105
+ :data:`DIM_NAMES`).
106
+ col_name : str, optional
107
+ Column index name (e.g. ``"node"``, ``"entity"``). Defaults
108
+ to the translated ``col_dim``.
109
+ """
110
+ pdf = _to_pandas(frame_pl)
111
+ pivoted = pdf.pivot(index=list(row_dims), columns=col_dim, values=value)
112
+ pivoted = pivoted.astype(float)
113
+ if row_names is not None:
114
+ pivoted.index.names = list(row_names)
115
+ else:
116
+ pivoted.index.names = [long_dim(d) for d in row_dims]
117
+ pivoted.columns.name = col_name if col_name is not None else long_dim(col_dim)
118
+ return pivoted
119
+
120
+
121
+ def wide_multi_col(
122
+ frame_pl: "pl.DataFrame",
123
+ *,
124
+ row_dims: Sequence[str],
125
+ col_dims: Sequence[str],
126
+ value: str = "value",
127
+ row_names: Sequence[str] | None = None,
128
+ col_names: Sequence[str] | None = None,
129
+ ) -> pd.DataFrame:
130
+ """Pivot a polars long-form frame to a wide-column pandas DataFrame
131
+ where the columns are a MultiIndex of ``col_dims``.
132
+
133
+ Used for ``flow_min`` / ``flow_max`` / ``process_source_sink_varCost`` /
134
+ ``reserve_upDown_group_reservation`` (column MultiIndex of 3 levels).
135
+ """
136
+ pdf = _to_pandas(frame_pl)
137
+ pivoted = pdf.pivot(index=list(row_dims), columns=list(col_dims), values=value)
138
+ pivoted = pivoted.astype(float)
139
+ if row_names is not None:
140
+ pivoted.index.names = list(row_names)
141
+ else:
142
+ pivoted.index.names = [long_dim(d) for d in row_dims]
143
+ if col_names is not None:
144
+ pivoted.columns.names = list(col_names)
145
+ else:
146
+ pivoted.columns.names = [long_dim(d) for d in col_dims]
147
+ return pivoted
148
+
149
+
150
+ def series_with_index(
151
+ frame_pl: "pl.DataFrame",
152
+ *,
153
+ dim: str,
154
+ value: str = "value",
155
+ name: str | None = None,
156
+ ) -> pd.Series:
157
+ """Polars long-form (single-dim) → pandas Series with a named index."""
158
+ pdf = _to_pandas(frame_pl)
159
+ s = pdf.set_index(dim)[value].astype(float)
160
+ s.index.name = name if name is not None else long_dim(dim)
161
+ return s
162
+
163
+
164
+ def series_from_two_dim(
165
+ frame_pl: "pl.DataFrame",
166
+ *,
167
+ dims: Sequence[str],
168
+ value: str = "value",
169
+ names: Sequence[str] | None = None,
170
+ ) -> pd.Series:
171
+ """Polars long-form (multi-dim) → pandas Series with MultiIndex."""
172
+ pdf = _to_pandas(frame_pl)
173
+ s = pdf.set_index(list(dims))[value].astype(float)
174
+ if names is not None:
175
+ s.index.names = list(names)
176
+ else:
177
+ s.index.names = [long_dim(d) for d in dims]
178
+ return s
179
+
180
+
181
+ def series_with_multi_index(
182
+ frame_pl: "pl.DataFrame",
183
+ *,
184
+ dims: Sequence[str],
185
+ value: str = "value",
186
+ names: Sequence[str] | None = None,
187
+ ) -> pd.Series:
188
+ """polars long-form → pandas Series whose index is a MultiIndex.
189
+
190
+ The legacy CSV path produced these for parameters with
191
+ ``header=[0, 1, 2], index_col=0`` and a single ``value`` row that
192
+ pandas exposed as a Series — e.g. ``process_sink_conversion_flow_coeff``,
193
+ ``reserve_upDown_group_penalty``. The Series's index carries the
194
+ multi-key tuple per cell; the in-memory equivalent is a Series with
195
+ a MultiIndex.
196
+ """
197
+ return series_from_two_dim(frame_pl, dims=dims, value=value, names=names)
198
+
199
+
200
+ # ---------------------------------------------------------------------------
201
+ # Set helpers (read_sets)
202
+ # ---------------------------------------------------------------------------
203
+
204
+
205
+ def to_index(frame_pl: "pl.DataFrame", *, dim: str, name: str | None = None) -> pd.Index:
206
+ """Polars (single-dim) → pandas Index with a named axis."""
207
+ pdf = _to_pandas(frame_pl.select(dim))
208
+ return pd.Index(pdf[dim].tolist(), name=name if name is not None else long_dim(dim))
209
+
210
+
211
+ def to_multi_index(
212
+ frame_pl: "pl.DataFrame",
213
+ *,
214
+ dims: Sequence[str],
215
+ names: Sequence[str] | None = None,
216
+ ) -> pd.MultiIndex:
217
+ """Polars long-form → pandas MultiIndex over the supplied dims.
218
+
219
+ The frame is restricted to the ``dims`` columns (de-duplicated /
220
+ not — left to the caller; we don't introduce extra ordering).
221
+ """
222
+ pdf = _to_pandas(frame_pl.select(list(dims)))
223
+ if names is None:
224
+ names = [long_dim(d) for d in dims]
225
+ return pd.MultiIndex.from_frame(pdf, names=list(names))
226
+
227
+
228
+ def empty_index(name: str | None = None, dtype: str = "object") -> pd.Index:
229
+ """Empty :class:`pd.Index` with a stable ``name`` / ``dtype``."""
230
+ return pd.Index([], dtype=dtype, name=name)
231
+
232
+
233
+ def empty_multi_index(names: Sequence[str]) -> pd.MultiIndex:
234
+ """Empty :class:`pd.MultiIndex` with ``len(names)`` levels."""
235
+ return pd.MultiIndex.from_arrays([[]] * len(names), names=list(names))
236
+
237
+
238
+ # ---------------------------------------------------------------------------
239
+ # Solve-name injection
240
+ # ---------------------------------------------------------------------------
241
+
242
+
243
+ def with_solve_column(frame_pl: "pl.DataFrame", solve_name: str) -> "pl.DataFrame":
244
+ """Return ``frame_pl`` with a leading ``solve`` column = ``solve_name``.
245
+
246
+ FlexData fields drop the ``solve`` column at load time (see
247
+ ``input.py:_read_long`` family). The legacy pandas namespace
248
+ keeps it as the leftmost index level. Inject it back in so the
249
+ pivoted DataFrame's row MultiIndex starts with ``solve``.
250
+ """
251
+ if frame_pl is None:
252
+ return None
253
+ return frame_pl.with_columns(pl.lit(solve_name).alias("solve"))
254
+
255
+
256
+ def add_solve_to_pandas(df: "pd.DataFrame | pd.Series", solve_name: str) -> "pd.DataFrame | pd.Series":
257
+ """Prepend a constant ``solve`` level to the row index.
258
+
259
+ Used when the frame already carries other dims and we want to
260
+ inject the solve level *after* pivoting (so wide-column pivots
261
+ don't wear the constant column unnecessarily).
262
+ """
263
+ if isinstance(df.index, pd.MultiIndex):
264
+ new = pd.MultiIndex.from_arrays(
265
+ [[solve_name] * len(df.index)] + [df.index.get_level_values(i) for i in range(df.index.nlevels)],
266
+ names=["solve"] + list(df.index.names),
267
+ )
268
+ else:
269
+ new = pd.MultiIndex.from_arrays(
270
+ [[solve_name] * len(df.index), list(df.index)],
271
+ names=["solve", df.index.name],
272
+ )
273
+ out = df.copy()
274
+ out.index = new
275
+ return out
276
+
277
+
278
+ __all__ = [
279
+ "DIM_NAMES",
280
+ "long_dim",
281
+ "wide_per_entity",
282
+ "wide_multi_col",
283
+ "series_with_index",
284
+ "series_from_two_dim",
285
+ "series_with_multi_index",
286
+ "to_index",
287
+ "to_multi_index",
288
+ "empty_index",
289
+ "empty_multi_index",
290
+ "with_solve_column",
291
+ "add_solve_to_pandas",
292
+ ]