flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,464 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import subprocess
5
+ import shutil
6
+ try:
7
+ from spinedb_api import import_data, DatabaseMapping
8
+ except ModuleNotFoundError:
9
+ exit("Cannot find the required Spine-Toolbox module. Check that the environment is activated and the toolbox is installed")
10
+ from spinedb_api.exception import NothingToCommit
11
+ from flextool._resources import package_data_path
12
+ from flextool.update_flextool.db_migration import migrate_database
13
+ from flextool.update_flextool.initialize_database import initialize_database
14
+ from flextool.update_flextool import canonical_databases
15
+
16
+
17
+ def _reinstall_if_needed():
18
+ """Re-install flextool and refresh its declared dependencies.
19
+
20
+ Runs ``pip install --upgrade [-e] <repo_root>`` after every
21
+ ``git pull``. Three reasons:
22
+
23
+ 1. **Editable installs need this too.** ``pip install -e .`` picks up
24
+ source-tree changes immediately but does NOT pull in new
25
+ dependencies that appeared in ``pyproject.toml``. Without this
26
+ reinstall, an editable user pulling a commit that adds (say)
27
+ ``polar-high`` to ``[project.dependencies]`` would see
28
+ ``ModuleNotFoundError`` on the next solver invocation.
29
+ 2. **``--upgrade``** re-resolves the install target and bumps any
30
+ direct dependency whose pyproject pin moved past the currently
31
+ installed version (e.g. ``highspy>=1.14`` floor bumps land
32
+ automatically). Replaces the previous bespoke
33
+ ``pip install --upgrade highspy>=1.14`` workaround.
34
+ 3. **NO ``--upgrade-strategy=eager``** — pip's default
35
+ ``only-if-needed`` semantics keep transitive dependencies stable
36
+ unless flextool's own pins force a move. This is deliberate:
37
+ eager upgrades churn the user's whole environment (numpy,
38
+ pandas, SQLAlchemy, etc.) and break other editable installs
39
+ (spinedb-api, toolbox extras) that pin to specific versions.
40
+
41
+ The install target is resolved from ``__file__`` rather than ``.``
42
+ so the re-install always targets THIS repo, regardless of the
43
+ caller's CWD (Spine Toolbox typically invokes from the project
44
+ directory, not the flextool repo root).
45
+ """
46
+ import sys
47
+ import os
48
+
49
+ # flextool/update_flextool/self_update.py → flextool/ → repo root.
50
+ repo_root = os.path.dirname(os.path.dirname(os.path.dirname(
51
+ os.path.abspath(__file__)
52
+ )))
53
+
54
+ result = subprocess.run(
55
+ [sys.executable, "-m", "pip", "show", "flextool"],
56
+ capture_output=True, text=True,
57
+ )
58
+ is_editable = "Editable project location" in result.stdout
59
+
60
+ print(
61
+ f"Refreshing flextool install and dependencies "
62
+ f"({'editable' if is_editable else 'non-editable'}, target={repo_root})..."
63
+ )
64
+ target = ["-e", repo_root] if is_editable else [repo_root]
65
+ completed = subprocess.run(
66
+ [sys.executable, "-m", "pip", "install", "--upgrade", *target],
67
+ )
68
+ if completed.returncode != 0:
69
+ flag = "-e " if is_editable else ""
70
+ print(
71
+ f"Warning: pip install failed. Run "
72
+ f"'pip install --upgrade {flag}{repo_root}' manually."
73
+ )
74
+
75
+
76
+ def _ensure_compatible_solver_stack():
77
+ """Verify the native solver stack (polars, highspy/HiGHS) runs on this
78
+ machine and, if a component crashes natively, re-install the compatible
79
+ build automatically.
80
+
81
+ Two known cases (see ``flextool.env_check``): the default ``polars``
82
+ wheel on a CPU lacking the SIMD baseline it was built for (cure:
83
+ ``polars-lts-cpu``), and ``highspy==1.14.0`` import-crashing on older
84
+ Windows (cure: ``highspy==1.13.1``). Both surface as a native crash —
85
+ a Windows access violation (exit ``3221225477``) or a POSIX
86
+ ``SIGILL``/``SIGSEGV``. We probe in a child process so the crash is
87
+ observable and pinpoint which component died, then apply its remedy.
88
+
89
+ Run unattended here (the user already opted into installs by choosing
90
+ "Update FlexTool"), but loudly: the banner names what is happening and
91
+ how to stop it, and everything prints to stdout so it lands in the
92
+ update log window for copy-paste.
93
+ """
94
+ import sys
95
+ from flextool import env_check
96
+
97
+ print("\nChecking that the solver libraries run on this computer...")
98
+ probe = env_check.probe_solver_stack()
99
+ print(env_check.diagnostics_report(probe))
100
+ if not probe.is_native_fault:
101
+ # OK, or an ordinary error (e.g. a package missing) that a build
102
+ # swap would not fix — leave it for the normal install machinery.
103
+ return
104
+
105
+ steps = env_check.remediation_steps(probe.failed_component, sys.executable)
106
+ if steps is None:
107
+ # A native crash with no package-level remedy (e.g. polar_high).
108
+ print("\n" + env_check.UNFIXABLE_HELP)
109
+ return
110
+
111
+ print("\n" + env_check.remediation_banner(probe.failed_component))
112
+ for step in steps:
113
+ print("\n$ " + " ".join(step))
114
+ completed = subprocess.run(step)
115
+ if completed.returncode != 0:
116
+ print(
117
+ f"Warning: '{' '.join(step)}' exited {completed.returncode}. "
118
+ f"Run it manually to finish the fix."
119
+ )
120
+ return
121
+
122
+ reprobe = env_check.probe_solver_stack()
123
+ print("\n" + reprobe.summary())
124
+ if not reprobe.ok:
125
+ print("\n" + env_check.UNFIXABLE_HELP)
126
+
127
+
128
+ def _sync_settings_param_defs(json_template, sqlite_path):
129
+ """Back-fill parameter DEFINITIONS from a settings template into an
130
+ EXISTING user settings DB.
131
+
132
+ The user-editable settings DBs (``output_settings.sqlite`` etc.) are only
133
+ created-if-absent, so a DB made by an older FlexTool version never gains
134
+ options added later (e.g. ``output-spinedb``). This imports just the
135
+ schema — entity classes + parameter definitions/types — so new options
136
+ appear in the editor, WITHOUT importing ``parameter_values`` (the user's
137
+ existing choices are never overwritten). Idempotent; non-fatal on error.
138
+ """
139
+ from flextool.update_flextool.export_database import keep_serialized_unparse
140
+ try:
141
+ with open(json_template) as json_file:
142
+ template = json.load(json_file)
143
+ payload = {
144
+ key: template[key]
145
+ for key in ("entity_classes", "parameter_definitions", "parameter_types")
146
+ if key in template
147
+ }
148
+ if not payload:
149
+ return
150
+ with DatabaseMapping('sqlite:///' + sqlite_path, create=False) as db:
151
+ (num, _log) = import_data(db, unparse_value=keep_serialized_unparse, **payload)
152
+ if num:
153
+ db.commit_session(
154
+ "flextool-update: back-fill settings parameter definitions"
155
+ )
156
+ except NothingToCommit:
157
+ pass
158
+ except Exception as exc: # noqa: BLE001
159
+ print(f"Warning: could not sync parameter definitions into "
160
+ f"{sqlite_path}: {exc}")
161
+
162
+
163
+ _PROJECT_FOLDER_TEMPLATE = (
164
+ "# FlexTool output project folder (user-local; not tracked by git).\n"
165
+ "# Write ONE path below — absolute, or relative to the FlexTool root —\n"
166
+ "# to direct this workflow's outputs (output_parquet/, results.sqlite,\n"
167
+ "# plots, and the per-project plot_settings.yaml) into that project folder.\n"
168
+ "# Leave blank to use the FlexTool root.\n"
169
+ "# Use projects sub-folders to allow access by FlexTool GUI as well.\n"
170
+ "# Using the project specific input_sources folder for the input data db\n"
171
+ "# makes it visible in the GUI too.\n"
172
+ "# e.g.: projects/Rivendell\n"
173
+ )
174
+
175
+
176
+ def ensure_runtime_files(verbose: bool = False) -> None:
177
+ """Idempotently materialize the per-installation runtime files that
178
+ FlexTool needs but does not track in git.
179
+
180
+ Every step is **create-if-missing**: an existing user-edited file is
181
+ never overwritten, so this is safe to call on every GUI launch *and*
182
+ as the seeding step of :func:`update_flextool`. Each group is wrapped
183
+ so that one failure (e.g. a locked DB) logs a warning and the rest
184
+ still run — a seeding hiccup must never block GUI startup.
185
+
186
+ Paths are interpreted relative to the current working directory (the
187
+ FlexTool workspace root), matching :func:`update_flextool` and the
188
+ GUI's ``get_projects_dir`` convention.
189
+
190
+ Covers: the root input/settings SQLites and their ``templates/``
191
+ copies, the canonical example DBs + XLSX templates, the user-facing
192
+ ``example_input.xlsx``, the ``solver_config/<solver>.opt`` files,
193
+ ``templates/project_folder.txt``, and ``results.sqlite``.
194
+
195
+ It deliberately does NOT perform the update-only operations (git
196
+ restore/pull, reinstall, the unconditional template refresh that
197
+ keeps artifacts schema-current after a pull, parameter-definition
198
+ back-fill into existing DBs, or DB-version migration) — those remain
199
+ in :func:`update_flextool`.
200
+ """
201
+ master_json = str(package_data_path("schemas/spinedb_schema.json"))
202
+ output_settings_json = str(package_data_path("schemas/output_settings_template.json"))
203
+ output_info_json = str(package_data_path("schemas/output_info_template.json"))
204
+ comparison_settings_json = str(package_data_path("schemas/comparison_settings_template.json"))
205
+
206
+ os.makedirs("templates", exist_ok=True)
207
+ os.makedirs("solver_config", exist_ok=True)
208
+
209
+ # 1. Root input/settings DBs + their templates/ copies, from the
210
+ # bundled package JSON. Created once; user edits then survive.
211
+ try:
212
+ for sqlite_rel, json_src in (
213
+ ("input_data.sqlite", master_json),
214
+ ("templates/input_data_template.sqlite", master_json),
215
+ ("output_settings.sqlite", output_settings_json),
216
+ ("output_info.sqlite", output_info_json),
217
+ ("comparison_settings.sqlite", comparison_settings_json),
218
+ ("templates/output_settings.sqlite", output_settings_json),
219
+ ("templates/output_info.sqlite", output_info_json),
220
+ ("templates/comparison_settings.sqlite", comparison_settings_json),
221
+ ):
222
+ if not os.path.exists(sqlite_rel):
223
+ if verbose:
224
+ print(f"Seeding {sqlite_rel}")
225
+ initialize_database(json_src, sqlite_rel)
226
+ except Exception as exc:
227
+ logging.warning("ensure_runtime_files: input/settings DB seeding failed: %s", exc)
228
+
229
+ # 2. Canonical example DBs, the XLSX templates derived from them, and
230
+ # the user-facing example_input.xlsx — all create-if-missing.
231
+ try:
232
+ canonical_databases.materialize(overwrite=False)
233
+ from flextool.export_to_tabular import export_to_excel
234
+ for sqlite_rel, xlsx_rel in (
235
+ ("templates/examples.sqlite", "templates/example_input_template.xlsx"),
236
+ ("templates/input_data_template.sqlite", "templates/empty_input_template.xlsx"),
237
+ ):
238
+ if os.path.exists(sqlite_rel) and not os.path.exists(xlsx_rel):
239
+ export_to_excel(f"sqlite:///{sqlite_rel}", xlsx_rel)
240
+ if not os.path.exists("example_input.xlsx") and os.path.exists("templates/example_input_template.xlsx"):
241
+ shutil.copy("templates/example_input_template.xlsx", "example_input.xlsx")
242
+ except Exception as exc:
243
+ logging.warning("ensure_runtime_files: example/template seeding failed: %s", exc)
244
+
245
+ # 3. Live solver options — HiGHS + the four commercial solvers — each
246
+ # seeded from its bundled <solver>.opt.template if missing. The
247
+ # runtime files are gitignored and user-editable (delete one to
248
+ # reset it to the bundled default).
249
+ try:
250
+ for _solver in ("highs", "gurobi", "cplex", "xpress", "copt"):
251
+ _runtime = f"solver_config/{_solver}.opt"
252
+ if not os.path.exists(_runtime):
253
+ shutil.copy(
254
+ str(package_data_path(f"solver_config/{_solver}.opt.template")),
255
+ _runtime,
256
+ )
257
+ except Exception as exc:
258
+ logging.warning("ensure_runtime_files: solver_config seeding failed: %s", exc)
259
+
260
+ # 4. templates/project_folder.txt — a commented guide; its CONTENTS are
261
+ # load-bearing for Spine Toolbox (the run Tool reads the first
262
+ # non-comment line as the output project folder). Gitignored.
263
+ try:
264
+ if not os.path.exists("templates/project_folder.txt"):
265
+ with open("templates/project_folder.txt", "w", encoding="utf-8") as fh:
266
+ fh.write(_PROJECT_FOLDER_TEMPLATE)
267
+ except Exception as exc:
268
+ logging.warning("ensure_runtime_files: project_folder.txt seeding failed: %s", exc)
269
+
270
+ # 5. results.sqlite (and its template), both create-if-missing.
271
+ try:
272
+ result_template_path = str(package_data_path("schemas/pre_v26/flextool_template_results_master.json"))
273
+ if not os.path.exists("templates/results_template.sqlite"):
274
+ initialize_result_database("templates/results_template.sqlite", result_template_path)
275
+ if not os.path.exists("results.sqlite") and os.path.exists("templates/results_template.sqlite"):
276
+ shutil.copy("templates/results_template.sqlite", "results.sqlite")
277
+ except Exception as exc:
278
+ logging.warning("ensure_runtime_files: results DB seeding failed: %s", exc)
279
+
280
+
281
+ def update_flextool(skip_git):
282
+
283
+ shutil.copy("./.spinetoolbox/project.json", "./.spinetoolbox/project_temp.json")
284
+ if not skip_git:
285
+ completed = subprocess.run(["git", "restore", "."])
286
+ if completed.returncode != 0:
287
+ print("Failed to restore version controlled files.")
288
+ exit(-1)
289
+
290
+ completed = subprocess.run(["git", "pull"])
291
+ if completed.returncode != 0:
292
+ print("Failed to get the new version.")
293
+ exit(-1)
294
+
295
+ # Re-install the package so that code changes from git pull take effect.
296
+ # Skip if the package is installed in editable mode (changes are live already).
297
+ _reinstall_if_needed()
298
+
299
+ # After (re)install, make sure the native solver stack actually runs on
300
+ # this machine; auto-fix the known incompatible-build cases (polars CPU
301
+ # baseline, highspy 1.14.0 on older Windows) if a component crashes.
302
+ _ensure_compatible_solver_stack()
303
+
304
+ migrate_project("./.spinetoolbox/project_temp.json","./.spinetoolbox/project.json")
305
+ os.remove("./.spinetoolbox/project_temp.json")
306
+
307
+ # Idempotently materialize all per-installation runtime files
308
+ # (root/settings DBs + templates, canonical examples + XLSX,
309
+ # example_input.xlsx, solver_config/*.opt, project_folder.txt,
310
+ # results.sqlite). Shared with the GUI startup path, so "an update
311
+ # brought a new file to copy" is handled by the same create-if-missing
312
+ # logic in both places — no duplication.
313
+ ensure_runtime_files()
314
+
315
+ # ----- Update-only refresh of EXISTING artifacts (after a git pull) ---
316
+ # These differ from the create-if-missing seeding above: they bring
317
+ # already-present files up to the current schema, so they must NOT run
318
+ # on every GUI launch (that is why they stay here, not in
319
+ # ensure_runtime_files).
320
+ output_settings_json = str(package_data_path("schemas/output_settings_template.json"))
321
+ output_info_json = str(package_data_path("schemas/output_info_template.json"))
322
+ comparison_settings_json = str(package_data_path("schemas/comparison_settings_template.json"))
323
+
324
+ # Back-fill parameter definitions added in newer versions into EXISTING
325
+ # user settings DBs. Options added later (e.g. output-spinedb) must be
326
+ # added to DBs made by older versions. Only schema is synced; user
327
+ # parameter values are untouched.
328
+ for sqlite_rel, json_src in (
329
+ ("output_settings.sqlite", output_settings_json),
330
+ ("output_info.sqlite", output_info_json),
331
+ ("comparison_settings.sqlite", comparison_settings_json),
332
+ ):
333
+ if os.path.exists(sqlite_rel):
334
+ _sync_settings_param_defs(json_src, sqlite_rel)
335
+
336
+ # Keep CWD-resident template SQLites up-to-date (Spine Toolbox refs):
337
+ # unconditional regen so a git pull's schema change is reflected.
338
+ for sqlite_rel, json_src in (
339
+ ("templates/output_settings.sqlite", output_settings_json),
340
+ ("templates/output_info.sqlite", output_info_json),
341
+ ("templates/comparison_settings.sqlite", comparison_settings_json),
342
+ ):
343
+ if os.path.exists(sqlite_rel):
344
+ os.remove(sqlite_rel)
345
+ initialize_database(json_src, sqlite_rel)
346
+
347
+ # Refresh XLSX templates from the canonical SQLites so they track the
348
+ # current schema (unconditional regen; the binaries are not in git).
349
+ from flextool.export_to_tabular import export_to_excel
350
+ for sqlite_rel, xlsx_rel in (
351
+ ("templates/examples.sqlite", "templates/example_input_template.xlsx"),
352
+ ("templates/input_data_template.sqlite", "templates/empty_input_template.xlsx"),
353
+ ):
354
+ if not os.path.exists(sqlite_rel):
355
+ continue
356
+ try:
357
+ export_to_excel(f"sqlite:///{sqlite_rel}", xlsx_rel)
358
+ except Exception as exc:
359
+ print(f"Warning: failed to generate {xlsx_rel}: {exc}")
360
+
361
+ # Migrate user-owned SQLites (canonical ones are already at current
362
+ # version because they were just materialized from the canonical
363
+ # JSONs, which migrate-all keeps at FLEXTOOL_DB_VERSION).
364
+ db_to_update = [
365
+ "input_data.sqlite",
366
+ "templates/input_data_template.sqlite",
367
+ ]
368
+
369
+ # add the database used in the input_data tool
370
+ with open("./.spinetoolbox/project.json") as json_file:
371
+ specifications = json.load(json_file)
372
+ path = specifications["items"]["Input data"]["url"]["database"]["path"]
373
+ db_to_update.append(path)
374
+
375
+ # migrate the databases to new version
376
+ for i in db_to_update:
377
+ migrate_database(i)
378
+
379
+ # Refresh the results template (unconditional) and ensure results.sqlite
380
+ # exists (ensure_runtime_files seeds it; this is a belt-and-braces copy
381
+ # for the case it was removed since).
382
+ result_template_path = str(package_data_path("schemas/pre_v26/flextool_template_results_master.json"))
383
+ if os.path.exists('templates/results_template.sqlite'):
384
+ os.remove('templates/results_template.sqlite')
385
+ initialize_result_database('templates/results_template.sqlite', result_template_path)
386
+
387
+ if not os.path.exists("results.sqlite"):
388
+ shutil.copy("templates/results_template.sqlite", "results.sqlite")
389
+ #update result parameter definitions
390
+ #get template JSON. This can be the master or old template if conflicting migrations in between
391
+ with open (result_template_path) as json_file:
392
+ template = json.load(json_file)
393
+ with DatabaseMapping('sqlite:///' + 'results.sqlite', create = False, upgrade = True) as db:
394
+ #these update the old descriptions, but wont remove them or change names (the new name is created, but old stays)
395
+ (num,log) = import_data(db, object_parameters = template["object_parameters"])
396
+ (num,log) = import_data(db, relationship_parameters = template["relationship_parameters"])
397
+ try:
398
+ db.commit_session("Updated relationship_parameters, object parameters to the Results.sqlite")
399
+ except NothingToCommit:
400
+ print("These parameters have been added before, continuing")
401
+ return 0
402
+
403
+ def initialize_result_database(filename,json_path):
404
+
405
+ with open (json_path) as json_file:
406
+ template = json.load(json_file)
407
+ with DatabaseMapping('sqlite:///' + filename, create = True) as db:
408
+ (num,log) = import_data(db,**template)
409
+ print("Result database initialized")
410
+ db.commit_session("Result database initialized")
411
+
412
+ def migrate_project(old_path, new_path):
413
+ #purpose of this is to update some of the items that users should not need to modify
414
+ #done simply by copying from the git project.json
415
+ #should be replaced if major changes to project.json
416
+
417
+ #items that are copied
418
+ items = [
419
+ "Replace with examples",
420
+ "FlexTool",
421
+ "Import_results",
422
+ "Plot_results",
423
+ "Plot_settings",
424
+ ]
425
+
426
+ with open(old_path) as old_json:
427
+ old_dict = json.load(old_json)
428
+ with open(new_path) as new_json:
429
+ new_dict = json.load(new_json)
430
+
431
+ for item in items:
432
+ if item in old_dict["items"].keys():
433
+ for param in old_dict["items"][item].keys():
434
+ if param != "x" and param != "y":
435
+ old_dict["items"][item][param] = new_dict["items"][item][param]
436
+
437
+ if ("Open_summary" not in old_dict["items"].keys()) and ("Open_summary" in new_dict["items"].keys()):
438
+ old_dict["items"]["Open_summary"] = new_dict["items"]["Open_summary"]
439
+ old_dict["project"]["connections"].append({"name": "from FlexTool3 to Open_summary", "from": ["FlexTool3","bottom"],"to": ["Open_summary","right"]})
440
+ old_dict["project"]["specifications"]["Tool"].append({"type": "path","relative": True,"path": ".spinetoolbox/specifications/Tool/open_summary.json"})
441
+
442
+ if ("Migrate database version" not in old_dict["items"].keys()) and ("Migrate database version" in new_dict["items"].keys()):
443
+ old_dict["items"]["Migrate database version"] = new_dict["items"]["Migrate database version"]
444
+ old_dict["project"]["connections"].append({"name": "from Migrate database version to Input_data",
445
+ "from": ["Migrate database version","right"],
446
+ "to": ["Input_data","bottom"]})
447
+ old_dict["project"]["specifications"]["Tool"].append({ "type": "path",
448
+ "relative": True,
449
+ "path": ".spinetoolbox/specifications/Tool/migrate_database.json"})
450
+
451
+
452
+ with open("./.spinetoolbox/project_temp2.json", "w") as outfile:
453
+ json.dump(old_dict, outfile, indent=4)
454
+
455
+ shutil.copy("./.spinetoolbox/project_temp2.json", new_path)
456
+ os.remove("./.spinetoolbox/project_temp2.json")
457
+
458
+
459
+ def main():
460
+ update_flextool(False)
461
+
462
+
463
+ if __name__ == '__main__':
464
+ main()
@@ -0,0 +1,125 @@
1
+ """Regenerate schemas/spinedb_schema.json from itself + migrations.
2
+
3
+ Workflow:
4
+ 1. Create a temporary DB from the current schemas/spinedb_schema.json
5
+ 2. Run db_migration.py on it (applies any new migration steps)
6
+ 3. Export the migrated DB back to schemas/spinedb_schema.json
7
+
8
+ This ensures the master template always reflects the latest schema
9
+ after db_migration.py has been updated.
10
+
11
+ Usage:
12
+ python -m flextool.update_flextool.sync_master_json_template [--verify]
13
+
14
+ --verify Don't overwrite; just check if the template is up to date.
15
+ Exits with code 1 if it needs regeneration (for CI).
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import argparse
20
+ import json
21
+ import sys
22
+ import tempfile
23
+ from pathlib import Path
24
+
25
+ from spinedb_api import DatabaseMapping, export_data, import_data
26
+
27
+ from flextool.update_flextool.db_migration import migrate_database
28
+
29
+ from flextool._resources import package_data_path
30
+
31
+ SCHEMAS_DIR = package_data_path("schemas")
32
+ SPINEDB_SCHEMA = SCHEMAS_DIR / "spinedb_schema.json"
33
+
34
+
35
+ def _create_db_from_template(template_path: Path, db_path: Path) -> None:
36
+ """Create a fresh SQLite DB from a JSON template."""
37
+ with open(template_path) as f:
38
+ template = json.load(f)
39
+
40
+ url = f"sqlite:///{db_path}"
41
+ with DatabaseMapping(url, create=True) as db:
42
+ for alt in db.find_alternatives():
43
+ try:
44
+ db.remove_alternative(name=alt["name"])
45
+ except Exception:
46
+ pass
47
+ import_data(db, **template)
48
+ db.commit_session("Initialized from master template")
49
+
50
+
51
+ def _export_db_to_json(db_path: Path) -> dict:
52
+ """Export a Spine DB to a JSON-serializable dict."""
53
+ url = f"sqlite:///{db_path}"
54
+ with DatabaseMapping(url) as db:
55
+ data = export_data(db)
56
+ # export_data returns tuples; convert to lists for JSON
57
+ return {k: [list(item) for item in v] for k, v in data.items()}
58
+
59
+
60
+ def sync_master_template(*, verify_only: bool = False) -> bool:
61
+ """Regenerate (or verify) the master JSON template.
62
+
63
+ Returns True if the template is up to date, False if it needed updating.
64
+ """
65
+ if not SPINEDB_SCHEMA.exists():
66
+ print(f"ERROR: Master template not found: {SPINEDB_SCHEMA}", file=sys.stderr)
67
+ sys.exit(2)
68
+
69
+ with tempfile.TemporaryDirectory() as tmp_dir:
70
+ db_path = Path(tmp_dir) / "template.sqlite"
71
+
72
+ # Step 1: Create DB from current master template
73
+ _create_db_from_template(SPINEDB_SCHEMA, db_path)
74
+
75
+ # Step 2: Run all migrations
76
+ migrate_database(str(db_path))
77
+
78
+ # Step 3: Export migrated DB
79
+ new_data = _export_db_to_json(db_path)
80
+
81
+ # Step 4: Compare or overwrite
82
+ new_json = json.dumps(new_data, indent=2, ensure_ascii=False) + "\n"
83
+
84
+ with open(SPINEDB_SCHEMA) as f:
85
+ old_json = f.read()
86
+
87
+ if new_json == old_json:
88
+ print("Master template is up to date.")
89
+ return True
90
+
91
+ if verify_only:
92
+ # Show what changed
93
+ old_data = json.loads(old_json)
94
+ for key in sorted(set(list(old_data.keys()) + list(new_data.keys()))):
95
+ old_count = len(old_data.get(key, []))
96
+ new_count = len(new_data.get(key, []))
97
+ if old_count != new_count:
98
+ print(f" {key}: {old_count} -> {new_count}")
99
+ print("Master template is OUT OF DATE. Run without --verify to regenerate.")
100
+ return False
101
+
102
+ with open(SPINEDB_SCHEMA, "w") as f:
103
+ f.write(new_json)
104
+ print(f"Regenerated {SPINEDB_SCHEMA}")
105
+ return True
106
+
107
+
108
+ def main() -> None:
109
+ parser = argparse.ArgumentParser(
110
+ description="Regenerate schemas/spinedb_schema.json after migration changes."
111
+ )
112
+ parser.add_argument(
113
+ "--verify",
114
+ action="store_true",
115
+ help="Check if template is up to date without overwriting (for CI).",
116
+ )
117
+ args = parser.parse_args()
118
+
119
+ up_to_date = sync_master_template(verify_only=args.verify)
120
+ if not up_to_date:
121
+ sys.exit(1)
122
+
123
+
124
+ if __name__ == "__main__":
125
+ main()