flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1098 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import shutil
5
+ import subprocess
6
+ import sys
7
+ import threading
8
+ import tkinter as tk
9
+ from pathlib import Path
10
+ from tkinter import messagebox, ttk
11
+
12
+ from flextool._resources import package_data_path
13
+ from flextool.gui.dialogs.file_picker import FilePickerDialog
14
+ from flextool.gui.project_utils import get_projects_dir
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ # Default name for the alternative created during old FlexTool import
19
+ _OLD_FLEX_ALTERNATIVE = "base"
20
+
21
+
22
+ def _ask_migration_choice(
23
+ parent: tk.Misc,
24
+ filename: str,
25
+ version_str: str,
26
+ current_version: int,
27
+ ) -> str:
28
+ """Ask how to handle an older Excel file that needs migration.
29
+
30
+ Returns:
31
+ ``"sqlite"`` to convert to Spine DB, ``"excel"`` to migrate and
32
+ keep as Excel, or ``"cancel"`` to do nothing.
33
+ """
34
+ dlg = tk.Toplevel(parent)
35
+ dlg.title("Version migration needed")
36
+ dlg.transient(parent)
37
+ dlg.grab_set()
38
+ dlg.resizable(False, False)
39
+
40
+ result = "cancel"
41
+
42
+ msg = (
43
+ f"'{filename}' is version {version_str} "
44
+ f"(current is {current_version}) and needs to be migrated.\n\n"
45
+ f"Choose how to handle the update:"
46
+ )
47
+ lbl = ttk.Label(dlg, text=msg, wraplength=420, justify="left")
48
+ lbl.pack(padx=16, pady=(16, 8))
49
+
50
+ btn_frame = ttk.Frame(dlg)
51
+ btn_frame.pack(padx=16, pady=(4, 16))
52
+
53
+ def _choose(choice: str) -> None:
54
+ nonlocal result
55
+ result = choice
56
+ dlg.destroy()
57
+
58
+ ttk.Button(
59
+ btn_frame, text="Convert to Spine DB",
60
+ command=lambda: _choose("sqlite"),
61
+ ).pack(side="left", padx=4)
62
+ ttk.Button(
63
+ btn_frame, text="Update Excel",
64
+ command=lambda: _choose("excel"),
65
+ ).pack(side="left", padx=4)
66
+ ttk.Button(
67
+ btn_frame, text="Cancel",
68
+ command=lambda: _choose("cancel"),
69
+ ).pack(side="left", padx=4)
70
+
71
+ dlg.protocol("WM_DELETE_WINDOW", lambda: _choose("cancel"))
72
+
73
+ # Center on parent
74
+ dlg.update_idletasks()
75
+ pw = parent.winfo_width()
76
+ ph = parent.winfo_height()
77
+ px = parent.winfo_rootx()
78
+ py = parent.winfo_rooty()
79
+ dw = dlg.winfo_width()
80
+ dh = dlg.winfo_height()
81
+ dlg.geometry(f"+{px + (pw - dw) // 2}+{py + (ph - dh) // 2}")
82
+
83
+ parent.wait_window(dlg)
84
+ return result
85
+
86
+
87
+ class AddDialog(tk.Toplevel):
88
+ """Modal dialog for adding input source files to a project.
89
+
90
+ After the dialog closes, ``self.result`` is ``True`` if any files were
91
+ added (so the caller knows to refresh the input sources list).
92
+ """
93
+
94
+ def __init__(
95
+ self, parent: tk.Misc, project_path: Path,
96
+ execution_mgr=None, input_source_mgr=None,
97
+ convert_callback=None, update_xlsx_callback=None,
98
+ ) -> None:
99
+ super().__init__(parent)
100
+ self.title("Add input sources")
101
+ self.result: bool = False
102
+ self.old_convert_started: bool = False # True if old FlexTool conversion was started
103
+ self.files_to_convert: list[str] = [] # Excel files converted to sqlite (for the "Done" notice)
104
+ self.files_to_update_xlsx: list[str] = [] # Excel files round-tripped in place (for the "Done" notice)
105
+ self._project_path = project_path
106
+ self._input_dir = project_path / "input_sources"
107
+ self._input_dir.mkdir(parents=True, exist_ok=True)
108
+ self._execution_mgr = execution_mgr
109
+ self._input_source_mgr = input_source_mgr
110
+ # Callbacks that start a migration job immediately (streamed to the
111
+ # Execution window) as soon as the user picks how to handle a file,
112
+ # instead of deferring until the dialog is closed.
113
+ self._convert_callback = convert_callback
114
+ self._update_xlsx_callback = update_xlsx_callback
115
+
116
+ # Root directories for templates
117
+ self._flextool_root = get_projects_dir().parent
118
+
119
+ # ── Modal behaviour ─────────────────────────────────────────
120
+ self.transient(parent)
121
+ self.grab_set()
122
+
123
+ # No fixed size — the dialog takes only the space its widgets need.
124
+ # The old fixed cw*77 x lh*45 box left most of it empty and grew huge
125
+ # once cw/lh scaled up under DPI. resizable(False) keeps the fit.
126
+ self.resizable(False, False)
127
+
128
+ self._build_widgets()
129
+
130
+ # Close via window-manager "X"
131
+ self.protocol("WM_DELETE_WINDOW", self._on_back)
132
+
133
+ # Size to natural content (clamped to the screen) and centre on parent.
134
+ self.update_idletasks()
135
+ w = min(self.winfo_reqwidth(), self.winfo_screenwidth())
136
+ h = min(self.winfo_reqheight(), self.winfo_screenheight())
137
+ px = parent.winfo_rootx()
138
+ py = parent.winfo_rooty()
139
+ pw = parent.winfo_width()
140
+ ph = parent.winfo_height()
141
+ x = px + (pw - w) // 2
142
+ y = py + (ph - h) // 2
143
+ self.geometry(f"{w}x{h}+{x}+{y}")
144
+
145
+ # Block until closed
146
+ parent.wait_window(self)
147
+
148
+ # ── Widget construction ─────────────────────────────────────────
149
+
150
+ def _build_widgets(self) -> None:
151
+ pad = dict(padx=10, pady=5)
152
+ from flextool.gui.ui_metrics import get_metrics
153
+ lh = get_metrics(self).lh
154
+ section_pad = dict(padx=10, pady=(lh, 5))
155
+
156
+ # Use the named-font string so a live UI font size change reaches
157
+ # these LabelFrame titles.
158
+ style = ttk.Style()
159
+ style.configure("AddDialog.TLabelframe.Label", font="TkDefaultFont")
160
+
161
+ # ── Copy to project section ─────────────────────────────────
162
+ copy_frame = ttk.LabelFrame(
163
+ self, text="Copy an existing FlexTool 3.x input file to the project", padding=8, style="AddDialog.TLabelframe",
164
+ )
165
+ copy_frame.pack(fill="x", **pad)
166
+
167
+ choose_copy_btn = ttk.Button(
168
+ copy_frame,
169
+ text="Choose and copy files...",
170
+ command=self._on_choose_and_copy,
171
+ )
172
+ choose_copy_btn.pack(fill="x")
173
+
174
+ # ── Add external reference section ──────────────────────────
175
+ ext_frame = ttk.LabelFrame(
176
+ self,
177
+ text="Add outside input file to the project (reference only, not copied)",
178
+ padding=8, style="AddDialog.TLabelframe",
179
+ )
180
+ ext_frame.pack(fill="x", **pad)
181
+
182
+ choose_ext_btn = ttk.Button(
183
+ ext_frame,
184
+ text="Choose external files...",
185
+ command=self._on_choose_external,
186
+ )
187
+ choose_ext_btn.pack(fill="x")
188
+
189
+ # ── Add empty FlexTool input Excel section ──────────────────
190
+ xlsx_frame = ttk.LabelFrame(
191
+ self, text="Add empty FlexTool input Excel", padding=8,
192
+ style="AddDialog.TLabelframe",
193
+ )
194
+ xlsx_frame.pack(fill="x", **section_pad)
195
+
196
+ xlsx_row = ttk.Frame(xlsx_frame)
197
+ xlsx_row.pack(fill="x")
198
+
199
+ ttk.Label(xlsx_row, text="Name:").pack(side="left")
200
+ self._xlsx_name_var = tk.StringVar(value="input")
201
+ xlsx_entry = ttk.Entry(
202
+ xlsx_row, textvariable=self._xlsx_name_var, width=25
203
+ )
204
+ xlsx_entry.pack(side="left", padx=(5, 5))
205
+ ttk.Label(xlsx_row, text=".xlsx").pack(side="left")
206
+
207
+ xlsx_add_btn = ttk.Button(
208
+ xlsx_row, text="Add", command=self._on_add_xlsx
209
+ )
210
+ xlsx_add_btn.pack(side="right")
211
+
212
+ # ── Add empty FlexTool input database section ───────────────
213
+ sqlite_frame = ttk.LabelFrame(
214
+ self, text="Add empty FlexTool input database", padding=8,
215
+ style="AddDialog.TLabelframe",
216
+ )
217
+ sqlite_frame.pack(fill="x", **section_pad)
218
+
219
+ sqlite_row = ttk.Frame(sqlite_frame)
220
+ sqlite_row.pack(fill="x")
221
+
222
+ ttk.Label(sqlite_row, text="Name:").pack(side="left")
223
+ self._sqlite_name_var = tk.StringVar(value="input")
224
+ sqlite_entry = ttk.Entry(
225
+ sqlite_row, textvariable=self._sqlite_name_var, width=25
226
+ )
227
+ sqlite_entry.pack(side="left", padx=(5, 5))
228
+ ttk.Label(sqlite_row, text=".sqlite").pack(side="left")
229
+
230
+ sqlite_add_btn = ttk.Button(
231
+ sqlite_row, text="Add", command=self._on_add_sqlite
232
+ )
233
+ sqlite_add_btn.pack(side="right")
234
+
235
+ # ── Convert from old FlexTool section ─────────────────────────
236
+ old_frame = ttk.LabelFrame(
237
+ self, text="Convert from FlexTool 2.0 input file", padding=8,
238
+ style="AddDialog.TLabelframe",
239
+ )
240
+ old_frame.pack(fill="x", **section_pad)
241
+
242
+ convert_row = ttk.Frame(old_frame)
243
+ convert_row.pack(fill="x")
244
+ ttk.Label(convert_row, text="1st step:").pack(side="left")
245
+ convert_btn = ttk.Button(
246
+ convert_row,
247
+ text="Convert base input data file...",
248
+ command=self._on_convert_old_flextool,
249
+ )
250
+ convert_btn.pack(side="left", padx=(5, 0))
251
+
252
+ sens_row = ttk.Frame(old_frame)
253
+ sens_row.pack(fill="x", pady=(5, 0))
254
+ ttk.Label(sens_row, text="2nd optional step:").pack(side="left")
255
+ self._import_sens_btn = ttk.Button(
256
+ sens_row,
257
+ text="Import sensitivities from compatible master file...",
258
+ command=self._on_import_sensitivities,
259
+ state="disabled",
260
+ )
261
+ self._import_sens_btn.pack(side="left", padx=(5, 0))
262
+
263
+ # Stored after successful 2.0 base conversion
264
+ self._last_base_xlsm: Path | None = None
265
+ self._last_target_sqlite: Path | None = None
266
+
267
+ # ── Close button (very bottom) ───────────────────────────────
268
+ close_frame = ttk.Frame(self)
269
+ close_frame.pack(fill="x", side="bottom", padx=10, pady=(15, 10))
270
+
271
+ close_btn = ttk.Button(
272
+ close_frame, text="Close", command=self._on_back
273
+ )
274
+ close_btn.pack(side="right")
275
+
276
+ # ── Event handlers ──────────────────────────────────────────────
277
+
278
+ def _on_choose_and_copy(self) -> None:
279
+ """Open a file chooser and immediately copy selected files."""
280
+ initial_dir = self._flextool_root
281
+ if not initial_dir.is_dir():
282
+ initial_dir = Path.home()
283
+
284
+ # Determine dialog size from the main window
285
+ try:
286
+ root = self.winfo_toplevel()
287
+ main_window_width = root.winfo_width()
288
+ screen_height = root.winfo_screenheight()
289
+ except Exception:
290
+ main_window_width = 700
291
+ screen_height = 800
292
+
293
+ picker = FilePickerDialog(
294
+ self,
295
+ title="Select input source files",
296
+ initialdir=str(initial_dir),
297
+ filetypes=[
298
+ ("FlexTool inputs", "*.xlsx *.xlsm *.ods *.sqlite"),
299
+ ("Excel files", "*.xlsx *.xlsm *.ods"),
300
+ ("SQLite databases", "*.sqlite"),
301
+ ("All files", "*"),
302
+ ],
303
+ multiple=True,
304
+ width=main_window_width,
305
+ height=int(screen_height * 0.75),
306
+ )
307
+ filepaths = picker.result
308
+ if not filepaths:
309
+ return
310
+
311
+ # Validate and copy files
312
+ from flextool.process_inputs import (
313
+ detect_excel_format, ExcelFormat, CURRENT_FLEXTOOL_DB_VERSION,
314
+ )
315
+
316
+ errors: list[str] = []
317
+ copied_names: list[str] = []
318
+ for fp in filepaths:
319
+ fp = Path(fp)
320
+
321
+ # Validate Excel files before copying
322
+ needs_migration = False
323
+ migration_choice: str | None = None
324
+ if fp.suffix.lower() in (".xlsx", ".xlsm", ".ods"):
325
+ info = detect_excel_format(fp)
326
+
327
+ if info.format == ExcelFormat.OLD_V2:
328
+ messagebox.showwarning(
329
+ "FlexTool 2.0 file",
330
+ f"'{fp.name}' is a FlexTool 2.0 file.\n\n"
331
+ "Use 'Convert from FlexTool 2.0 input file' instead.",
332
+ parent=self,
333
+ )
334
+ continue
335
+
336
+ # Check whether the file needs migration
337
+ needs_migration = (
338
+ info.format == ExcelFormat.SPECIFICATION
339
+ or (
340
+ info.format == ExcelFormat.SELF_DESCRIBING
341
+ and info.version is not None
342
+ and info.version < CURRENT_FLEXTOOL_DB_VERSION
343
+ )
344
+ )
345
+
346
+ if info.format == ExcelFormat.UNKNOWN:
347
+ messagebox.showwarning(
348
+ "Unrecognised format",
349
+ f"'{fp.name}' does not appear to be a valid FlexTool "
350
+ f"input file (no scenario sheet found).",
351
+ parent=self,
352
+ )
353
+ continue
354
+
355
+ # For older files, ask how to handle migration before copying
356
+ if needs_migration:
357
+ version_str = str(info.version) if info.version is not None else "unknown"
358
+ migration_choice = _ask_migration_choice(
359
+ self,
360
+ fp.name,
361
+ version_str,
362
+ CURRENT_FLEXTOOL_DB_VERSION,
363
+ )
364
+ if migration_choice == "cancel":
365
+ continue
366
+
367
+ dest = self._input_dir / fp.name
368
+ if dest.exists():
369
+ overwrite = messagebox.askyesno(
370
+ "File exists",
371
+ f"'{fp.name}' already exists in input_sources.\n"
372
+ "Do you want to overwrite it?",
373
+ parent=self,
374
+ )
375
+ if not overwrite:
376
+ continue
377
+ try:
378
+ shutil.copy2(str(fp), str(dest))
379
+ copied_names.append(fp.name)
380
+ except OSError as exc:
381
+ errors.append(f"{fp.name}: {exc}")
382
+ logger.error("Failed to copy %s: %s", fp, exc)
383
+ continue
384
+
385
+ # Start the migration job immediately so the user does not have to
386
+ # close the dialog first; progress streams to the Execution window.
387
+ if migration_choice == "sqlite":
388
+ self.files_to_convert.append(fp.name)
389
+ self._run_migration_callback(self._convert_callback, fp.name)
390
+ elif migration_choice == "excel":
391
+ self.files_to_update_xlsx.append(fp.name)
392
+ self._run_migration_callback(self._update_xlsx_callback, fp.name)
393
+
394
+ if errors:
395
+ messagebox.showerror(
396
+ "Copy errors",
397
+ "Some files could not be copied:\n" + "\n".join(errors),
398
+ parent=self,
399
+ )
400
+
401
+ # When no migration callbacks are wired (e.g. in tests), the queued
402
+ # files are migrated by the main window's input-source refresh right
403
+ # after this dialog closes instead.
404
+
405
+ # Files undergoing conversion are excluded from the "Done"
406
+ # notification — they will appear after conversion finishes.
407
+ migrating = set(self.files_to_convert) | set(self.files_to_update_xlsx)
408
+ ready_names = [n for n in copied_names if n not in migrating]
409
+
410
+ if copied_names:
411
+ self.result = True
412
+ if ready_names:
413
+ file_list = "\n".join(f" - {name}" for name in ready_names)
414
+ done_text = (
415
+ f"Done — copied {len(ready_names)} file(s):\n{file_list}"
416
+ )
417
+ messagebox.showinfo(
418
+ "Done",
419
+ done_text,
420
+ parent=self,
421
+ )
422
+
423
+ def _run_migration_callback(self, callback, name: str) -> None:
424
+ """Invoke a migration-start callback with the modal grab released.
425
+
426
+ The callback (``_convert_xlsx_to_sqlite`` / ``_update_xlsx_version``)
427
+ starts a background job and may raise its own message boxes or the
428
+ Execution window. Releasing this dialog's grab first keeps those
429
+ windows reachable; the callback returns promptly (it only kicks off a
430
+ worker thread), after which the dialog re-grabs and stays modal.
431
+ """
432
+ if callback is None:
433
+ return
434
+ had_grab = self.grab_current() is self
435
+ if had_grab:
436
+ self.grab_release()
437
+ try:
438
+ callback(name)
439
+ finally:
440
+ if had_grab and self.winfo_exists():
441
+ self.grab_set()
442
+
443
+ def _on_choose_external(self) -> None:
444
+ """Register external files as input source references (no copy)."""
445
+ if self._input_source_mgr is None:
446
+ messagebox.showerror(
447
+ "Not available",
448
+ "Cannot add external references before a project is loaded.",
449
+ parent=self,
450
+ )
451
+ return
452
+
453
+ initial_dir = self._flextool_root
454
+ if not initial_dir.is_dir():
455
+ initial_dir = Path.home()
456
+
457
+ try:
458
+ root = self.winfo_toplevel()
459
+ main_window_width = root.winfo_width()
460
+ screen_height = root.winfo_screenheight()
461
+ except Exception:
462
+ main_window_width = 700
463
+ screen_height = 800
464
+
465
+ picker = FilePickerDialog(
466
+ self,
467
+ title="Select external input source files",
468
+ initialdir=str(initial_dir),
469
+ filetypes=[
470
+ ("FlexTool inputs", "*.xlsx *.xlsm *.ods *.sqlite"),
471
+ ("Excel files", "*.xlsx *.xlsm *.ods"),
472
+ ("SQLite databases", "*.sqlite"),
473
+ ("All files", "*"),
474
+ ],
475
+ multiple=True,
476
+ width=main_window_width,
477
+ height=int(screen_height * 0.75),
478
+ )
479
+ filepaths = picker.result
480
+ if not filepaths:
481
+ return
482
+
483
+ from flextool.process_inputs import (
484
+ detect_excel_format, ExcelFormat, CURRENT_FLEXTOOL_DB_VERSION,
485
+ )
486
+
487
+ errors: list[str] = []
488
+ added_names: list[str] = []
489
+ for fp in filepaths:
490
+ fp = Path(fp)
491
+
492
+ if fp.suffix.lower() in (".xlsx", ".xlsm", ".ods"):
493
+ info = detect_excel_format(fp)
494
+ if info.format == ExcelFormat.OLD_V2:
495
+ messagebox.showwarning(
496
+ "FlexTool 2.0 file",
497
+ f"'{fp.name}' is a FlexTool 2.0 file and cannot be "
498
+ "referenced externally.\nUse 'Convert from FlexTool "
499
+ "2.0 input file' instead.",
500
+ parent=self,
501
+ )
502
+ continue
503
+ if info.format == ExcelFormat.UNKNOWN:
504
+ messagebox.showwarning(
505
+ "Unrecognised format",
506
+ f"'{fp.name}' does not appear to be a valid FlexTool "
507
+ "input file (no scenario sheet found).",
508
+ parent=self,
509
+ )
510
+ continue
511
+ needs_migration = (
512
+ info.format == ExcelFormat.SPECIFICATION
513
+ or (
514
+ info.format == ExcelFormat.SELF_DESCRIBING
515
+ and info.version is not None
516
+ and info.version < CURRENT_FLEXTOOL_DB_VERSION
517
+ )
518
+ )
519
+ if needs_migration:
520
+ version_str = (
521
+ str(info.version) if info.version is not None else "unknown"
522
+ )
523
+ proceed = messagebox.askyesno(
524
+ "Outdated file version",
525
+ f"'{fp.name}' is version {version_str} "
526
+ f"(current is {CURRENT_FLEXTOOL_DB_VERSION}).\n\n"
527
+ "External references cannot be migrated. Add "
528
+ "anyway? Reading scenarios may still work, but "
529
+ "execution may fail until the file is updated.",
530
+ parent=self,
531
+ )
532
+ if not proceed:
533
+ continue
534
+
535
+ try:
536
+ name, _rel = self._input_source_mgr.add_external_ref(fp)
537
+ added_names.append(name)
538
+ except ValueError as exc:
539
+ errors.append(str(exc))
540
+ except Exception as exc:
541
+ errors.append(f"{fp.name}: {exc}")
542
+ logger.error("Failed to add external ref %s: %s", fp, exc)
543
+
544
+ if errors:
545
+ messagebox.showerror(
546
+ "Errors",
547
+ "Some files could not be added:\n" + "\n".join(errors),
548
+ parent=self,
549
+ )
550
+
551
+ if added_names:
552
+ self.result = True
553
+ file_list = "\n".join(f" - {name}" for name in added_names)
554
+ messagebox.showinfo(
555
+ "Done",
556
+ f"Added {len(added_names)} external reference(s):\n{file_list}",
557
+ parent=self,
558
+ )
559
+
560
+ def _on_add_xlsx(self) -> None:
561
+ """Create an empty FlexTool Excel from the JSON master template."""
562
+ name = self._xlsx_name_var.get().strip()
563
+ if not name:
564
+ messagebox.showwarning(
565
+ "Invalid name", "Please enter a file name.", parent=self
566
+ )
567
+ return
568
+
569
+ filename = f"{name}.xlsx"
570
+ dest = self._input_dir / filename
571
+ if dest.exists():
572
+ messagebox.showwarning(
573
+ "Already exists",
574
+ f"'{filename}' already exists in input_sources.",
575
+ parent=self,
576
+ )
577
+ return
578
+
579
+ json_template = (
580
+ package_data_path("schemas/spinedb_schema.json")
581
+ )
582
+ if not json_template.exists():
583
+ messagebox.showerror(
584
+ "Template missing",
585
+ f"Cannot find template:\n{json_template}",
586
+ parent=self,
587
+ )
588
+ return
589
+
590
+ import tempfile
591
+
592
+ tmp_sqlite = None
593
+ try:
594
+ from flextool.update_flextool.initialize_database import (
595
+ initialize_database,
596
+ )
597
+ from flextool.export_to_tabular.export_to_excel import export_to_excel
598
+ from flextool.export_to_tabular.db_reader import read_database
599
+ from flextool.export_to_tabular.sheet_config import load_settings
600
+ from flextool.gui.dialogs.group_picker import GroupPickerDialog
601
+
602
+ # Create a temporary sqlite from the JSON template
603
+ with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as tmp:
604
+ tmp_sqlite = Path(tmp.name)
605
+ initialize_database(str(json_template), str(tmp_sqlite))
606
+
607
+ # Read parameter groups + required list, then ask the user
608
+ db_url = f"sqlite:///{tmp_sqlite}"
609
+ dc = read_database(db_url)
610
+ settings = load_settings()
611
+ groups = [
612
+ {"name": name, **meta}
613
+ for name, meta in dc.parameter_groups.items()
614
+ ]
615
+ required = settings.get("required_groups", []) or []
616
+
617
+ picker = GroupPickerDialog(self, groups, required)
618
+ if picker.result is None:
619
+ # Cancelled — abort silently, no output created
620
+ return
621
+
622
+ include_groups = (
623
+ None if picker.result == "all" else list(picker.result)
624
+ )
625
+ export_to_excel(db_url, str(dest), include_groups=include_groups)
626
+ except ImportError as exc:
627
+ messagebox.showerror(
628
+ "Missing dependency",
629
+ f"Required dependency not available:\n{exc}",
630
+ parent=self,
631
+ )
632
+ return
633
+ except Exception as exc:
634
+ messagebox.showerror("Error", str(exc), parent=self)
635
+ if dest.exists():
636
+ dest.unlink(missing_ok=True)
637
+ return
638
+ finally:
639
+ if tmp_sqlite is not None and tmp_sqlite.exists():
640
+ tmp_sqlite.unlink(missing_ok=True)
641
+
642
+ self.result = True
643
+ messagebox.showinfo(
644
+ "Done",
645
+ f"Created '{filename}' in input_sources.",
646
+ parent=self,
647
+ )
648
+
649
+ def _on_add_sqlite(self) -> None:
650
+ """Create a new Spine database from the master template."""
651
+ name = self._sqlite_name_var.get().strip()
652
+ if not name:
653
+ messagebox.showwarning(
654
+ "Invalid name", "Please enter a file name.", parent=self
655
+ )
656
+ return
657
+
658
+ filename = f"{name}.sqlite"
659
+ dest = self._input_dir / filename
660
+ if dest.exists():
661
+ messagebox.showwarning(
662
+ "Already exists",
663
+ f"'{filename}' already exists in input_sources.",
664
+ parent=self,
665
+ )
666
+ return
667
+
668
+ json_template = (
669
+ package_data_path("schemas/spinedb_schema.json")
670
+ )
671
+ if not json_template.exists():
672
+ messagebox.showerror(
673
+ "Template missing",
674
+ f"Cannot find template:\n{json_template}",
675
+ parent=self,
676
+ )
677
+ return
678
+
679
+ try:
680
+ from flextool.update_flextool.initialize_database import (
681
+ initialize_database,
682
+ )
683
+
684
+ initialize_database(str(json_template), str(dest))
685
+ except ImportError:
686
+ messagebox.showerror(
687
+ "Missing dependency",
688
+ "spinedb_api is required to create SQLite databases.",
689
+ parent=self,
690
+ )
691
+ return
692
+ except Exception as exc:
693
+ messagebox.showerror("Error", str(exc), parent=self)
694
+ return
695
+
696
+ self.result = True
697
+ messagebox.showinfo(
698
+ "Done",
699
+ f"Created '{filename}' in input_sources.",
700
+ parent=self,
701
+ )
702
+
703
+ def _on_convert_old_flextool(self) -> None:
704
+ """Open a file chooser for an old FlexTool .xlsm, convert to Spine DB.
705
+
706
+ The conversion runs as an auxiliary job in the ExecutionManager.
707
+ The dialog closes immediately and progress is shown in the
708
+ execution window.
709
+ """
710
+ initial_dir = self._flextool_root
711
+
712
+ try:
713
+ root = self.winfo_toplevel()
714
+ main_window_width = root.winfo_width()
715
+ screen_height = root.winfo_screenheight()
716
+ except Exception:
717
+ main_window_width = 700
718
+ screen_height = 800
719
+
720
+ picker = FilePickerDialog(
721
+ self,
722
+ title="Select old FlexTool input file",
723
+ initialdir=str(initial_dir),
724
+ filetypes=[
725
+ ("Old FlexTool Excel", "*.xlsm *.xlsx"),
726
+ ("All files", "*"),
727
+ ],
728
+ multiple=False,
729
+ width=main_window_width,
730
+ height=int(screen_height * 0.75),
731
+ )
732
+ filepath = picker.result
733
+ if not filepath:
734
+ return
735
+
736
+ filepath = Path(filepath)
737
+ dest_name = filepath.stem + ".sqlite"
738
+ dest = self._input_dir / dest_name
739
+
740
+ if dest.exists():
741
+ overwrite = messagebox.askyesno(
742
+ "File exists",
743
+ f"'{dest_name}' already exists in input_sources.\n"
744
+ "Do you want to overwrite it?",
745
+ parent=self,
746
+ )
747
+ if not overwrite:
748
+ return
749
+ try:
750
+ dest.unlink()
751
+ except OSError as exc:
752
+ messagebox.showerror(
753
+ "Error",
754
+ f"Could not remove existing file:\n{exc}",
755
+ parent=self,
756
+ )
757
+ return
758
+
759
+ # The old-FlexTool importer targets a FROZEN schema version (v56) and
760
+ # relies on migration to reach the live schema — so initialise from the
761
+ # frozen import template, not the live schema. The conversion
762
+ # subprocess (or the sync fallback below) migrates to current after
763
+ # writing.
764
+ from flextool.process_inputs.write_old_flextool_to_db import (
765
+ OLD_FLEXTOOL_IMPORT_SCHEMA,
766
+ )
767
+ json_template = package_data_path(OLD_FLEXTOOL_IMPORT_SCHEMA)
768
+ if not json_template.exists():
769
+ messagebox.showerror(
770
+ "Template missing",
771
+ f"Cannot find template:\n{json_template}",
772
+ parent=self,
773
+ )
774
+ return
775
+
776
+ # Create empty Spine DB from template
777
+ try:
778
+ from flextool.update_flextool.initialize_database import (
779
+ initialize_database,
780
+ )
781
+ initialize_database(str(json_template), str(dest))
782
+ except Exception as exc:
783
+ messagebox.showerror(
784
+ "Initialisation failed",
785
+ f"Could not create target database:\n{exc}",
786
+ parent=self,
787
+ )
788
+ if dest.exists():
789
+ dest.unlink(missing_ok=True)
790
+ return
791
+
792
+ # Build the commands. The importer targets a frozen v56 schema and the
793
+ # migration to the current version runs afterwards as its OWN execution
794
+ # job (``--no-migrate`` keeps it out of the conversion subprocess), so
795
+ # the user sees a distinct, visible migration step.
796
+ db_url = f"sqlite:///{dest}"
797
+ cmd = [
798
+ sys.executable, "-m",
799
+ "flextool.cli.cmd_read_old_flextool",
800
+ str(filepath),
801
+ db_url,
802
+ "--no-migrate",
803
+ ]
804
+ migrate_cmd = [
805
+ sys.executable, "-m",
806
+ "flextool.cli.cmd_migrate_database",
807
+ str(dest),
808
+ ]
809
+
810
+ if self._execution_mgr is not None:
811
+ # Run as auxiliary job — close dialog, show in execution window
812
+ from flextool.gui.execution_manager import JobType
813
+
814
+ job = self._execution_mgr.add_auxiliary_job(
815
+ JobType.OLD_CONVERT,
816
+ f"Convert FlexTool 2.0: {filepath.name}",
817
+ f"old_convert:{filepath.name}",
818
+ )
819
+ self._execution_mgr.append_stdout(
820
+ job.job_id, f"Converting '{filepath.name}' \u2192 '{dest_name}'"
821
+ )
822
+ self._execution_mgr.append_stdout(job.job_id, " ".join(cmd))
823
+ self._execution_mgr.append_stdout(job.job_id, "")
824
+
825
+ main_window = self.master
826
+ mgr = self._execution_mgr # capture for threads
827
+
828
+ # Make the output visible even if the execution window was closed.
829
+ main_window._open_or_raise_execution_window()
830
+ if main_window.execution_window is not None:
831
+ main_window.execution_window.select_job(job.job_id)
832
+
833
+ def _stream(stream_cmd: list[str], stream_job) -> int:
834
+ """Run a subprocess, piping its output into ``stream_job``.
835
+
836
+ Returns the process exit code.
837
+ """
838
+ import os as _os
839
+ env = {**_os.environ, "PYTHONUNBUFFERED": "1"}
840
+ proc = subprocess.Popen(
841
+ stream_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
842
+ text=True, bufsize=1, cwd=str(self._flextool_root), env=env,
843
+ )
844
+ with mgr._lock:
845
+ stream_job.process = proc
846
+ for line in proc.stdout: # type: ignore[union-attr]
847
+ mgr.append_stdout(stream_job.job_id, line.rstrip("\n"))
848
+ proc.wait()
849
+ return proc.returncode
850
+
851
+ def _start_migration_job() -> None:
852
+ """Create and run migration as its own job (Tk main thread)."""
853
+ mig_job = mgr.add_auxiliary_job(
854
+ JobType.MIGRATION,
855
+ f"Migrate database: {dest_name}",
856
+ f"old_convert_migrate:{filepath.name}",
857
+ )
858
+ mgr.append_stdout(
859
+ mig_job.job_id,
860
+ f"Migrating '{dest_name}' to the current schema version",
861
+ )
862
+ mgr.append_stdout(mig_job.job_id, " ".join(migrate_cmd))
863
+ mgr.append_stdout(mig_job.job_id, "")
864
+ main_window._open_or_raise_execution_window()
865
+ if main_window.execution_window is not None:
866
+ main_window.execution_window.select_job(mig_job.job_id)
867
+
868
+ def _mig_worker() -> None:
869
+ rc = 1
870
+ try:
871
+ rc = _stream(migrate_cmd, mig_job)
872
+ if rc != 0:
873
+ mgr.append_stdout(
874
+ mig_job.job_id,
875
+ f"\nMigration failed (exit code {rc}).",
876
+ )
877
+ if dest.exists():
878
+ dest.unlink(missing_ok=True)
879
+ except Exception as exc:
880
+ logger.error("Database migration failed: %s", exc, exc_info=True)
881
+ mgr.append_stdout(mig_job.job_id, f"\nError: {exc}")
882
+ if dest.exists():
883
+ dest.unlink(missing_ok=True)
884
+ mgr.finish_job(mig_job.job_id, rc == 0)
885
+ if rc == 0:
886
+ try:
887
+ main_window.after(0, main_window._refresh_input_sources)
888
+ except Exception:
889
+ pass
890
+
891
+ threading.Thread(target=_mig_worker, daemon=True).start()
892
+
893
+ def _worker() -> None:
894
+ rc = 1
895
+ try:
896
+ rc = _stream(cmd, job)
897
+ if rc != 0:
898
+ mgr.append_stdout(
899
+ job.job_id,
900
+ f"\nConversion failed (exit code {rc}).",
901
+ )
902
+ if dest.exists():
903
+ dest.unlink(missing_ok=True)
904
+ except Exception as exc:
905
+ logger.error("Old FlexTool conversion failed: %s", exc, exc_info=True)
906
+ mgr.append_stdout(job.job_id, f"\nError: {exc}")
907
+ if dest.exists():
908
+ dest.unlink(missing_ok=True)
909
+
910
+ mgr.finish_job(job.job_id, rc == 0)
911
+ if rc == 0:
912
+ # Chain migration as its own job, created on the Tk main
913
+ # thread (job/window creation must not run off-thread).
914
+ try:
915
+ main_window.after(0, _start_migration_job)
916
+ except Exception:
917
+ pass
918
+
919
+ threading.Thread(target=_worker, daemon=True).start()
920
+
921
+ self.result = True
922
+ self.old_convert_started = True
923
+ # Store paths so sensitivity import can find them
924
+ self._last_base_xlsm = filepath
925
+ self._last_target_sqlite = dest
926
+ self._import_sens_btn.configure(state="normal")
927
+ else:
928
+ # Fallback: no execution manager — run synchronously with busy cursor
929
+ self.config(cursor="watch")
930
+ self.update()
931
+ try:
932
+ from flextool.process_inputs.write_old_flextool_to_db import (
933
+ import_old_flextool_xlsm,
934
+ )
935
+
936
+ # Full pipeline: the DB was initialised from the frozen
937
+ # template above, so this writes the data and migrates the
938
+ # result up to the current schema version.
939
+ import_old_flextool_xlsm(
940
+ str(filepath), db_url,
941
+ alternative_name=_OLD_FLEX_ALTERNATIVE,
942
+ )
943
+ except Exception as exc:
944
+ logger.error("Old FlexTool conversion failed: %s", exc, exc_info=True)
945
+ messagebox.showerror("Conversion failed", str(exc), parent=self)
946
+ if dest.exists():
947
+ dest.unlink(missing_ok=True)
948
+ return
949
+ finally:
950
+ self.config(cursor="")
951
+
952
+ self.result = True
953
+ self._last_base_xlsm = filepath
954
+ self._last_target_sqlite = dest
955
+ self._import_sens_btn.configure(state="normal")
956
+ messagebox.showinfo(
957
+ "Done",
958
+ f"Converted '{filepath.name}' \u2192 '{dest_name}' in input_sources.\n\n"
959
+ "You can now import sensitivities from a master file, "
960
+ "or close this dialog.",
961
+ parent=self,
962
+ )
963
+
964
+ def _on_import_sensitivities(self) -> None:
965
+ """Import sensitivities from a FlexTool 2.0 master file into the
966
+ database that was just created by the base conversion."""
967
+ if self._last_base_xlsm is None or self._last_target_sqlite is None:
968
+ return
969
+
970
+ try:
971
+ root = self.winfo_toplevel()
972
+ main_window_width = root.winfo_width()
973
+ screen_height = root.winfo_screenheight()
974
+ except Exception:
975
+ main_window_width = 700
976
+ screen_height = 800
977
+
978
+ # Let user pick the master file (start in same directory as base)
979
+ picker = FilePickerDialog(
980
+ self,
981
+ title="Select FlexTool 2.0 master file (with sensitivity definitions)",
982
+ initialdir=str(self._last_base_xlsm.parent),
983
+ filetypes=[
984
+ ("Old FlexTool Excel", "*.xlsm *.xlsx"),
985
+ ("All files", "*"),
986
+ ],
987
+ multiple=False,
988
+ width=main_window_width,
989
+ height=int(screen_height * 0.75),
990
+ )
991
+ master_path = picker.result
992
+ if not master_path:
993
+ return
994
+
995
+ target_db_url = f"sqlite:///{self._last_target_sqlite}"
996
+ cmd = [
997
+ sys.executable, "-m",
998
+ "flextool.cli.cmd_import_sensitivities",
999
+ str(master_path),
1000
+ str(self._last_base_xlsm),
1001
+ target_db_url,
1002
+ ]
1003
+
1004
+ if self._execution_mgr is not None:
1005
+ from flextool.gui.execution_manager import JobType
1006
+
1007
+ job = self._execution_mgr.add_auxiliary_job(
1008
+ JobType.CONVERSION,
1009
+ f"Import sensitivities → '{self._last_target_sqlite.name}'",
1010
+ f"import_sensitivities:{self._last_target_sqlite.name}",
1011
+ )
1012
+ self._execution_mgr.append_stdout(
1013
+ job.job_id,
1014
+ f"Importing sensitivities from '{Path(master_path).name}'\n",
1015
+ )
1016
+ self._execution_mgr.append_stdout(job.job_id, " ".join(cmd))
1017
+ self._execution_mgr.append_stdout(job.job_id, "")
1018
+
1019
+ mgr = self._execution_mgr
1020
+
1021
+ def _worker() -> None:
1022
+ import os as _os
1023
+ success = False
1024
+ try:
1025
+ env = {**_os.environ, "PYTHONUNBUFFERED": "1"}
1026
+ proc = subprocess.Popen(
1027
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
1028
+ text=True, bufsize=1,
1029
+ cwd=str(self._flextool_root), env=env,
1030
+ )
1031
+ with mgr._lock:
1032
+ job.process = proc
1033
+
1034
+ for line in proc.stdout: # type: ignore[union-attr]
1035
+ mgr.append_stdout(job.job_id, line.rstrip("\n"))
1036
+
1037
+ proc.wait()
1038
+ success = proc.returncode == 0
1039
+ if success:
1040
+ mgr.append_stdout(job.job_id, "\nSensitivity import succeeded.")
1041
+ else:
1042
+ mgr.append_stdout(
1043
+ job.job_id,
1044
+ f"\nSensitivity import failed (exit code {proc.returncode}).",
1045
+ )
1046
+ except Exception as exc:
1047
+ logger.error("Sensitivity import failed: %s", exc, exc_info=True)
1048
+ mgr.append_stdout(job.job_id, f"\nError: {exc}")
1049
+
1050
+ mgr.finish_job(job.job_id, success)
1051
+ # Refresh the main window so new scenarios appear checked
1052
+ if success:
1053
+ try:
1054
+ main_window.after(0, main_window._refresh_and_check_new_scenarios)
1055
+ except Exception:
1056
+ pass # Main window may be gone
1057
+
1058
+ main_window = self.master # capture before thread starts
1059
+ threading.Thread(target=_worker, daemon=True).start()
1060
+ self._import_sens_btn.configure(state="disabled")
1061
+ self.result = True
1062
+ else:
1063
+ # Fallback: run synchronously
1064
+ self.config(cursor="watch")
1065
+ self.update()
1066
+ try:
1067
+ from flextool.process_inputs.read_old_flextool import (
1068
+ read_old_flextool, read_old_flextool_sensitivities,
1069
+ )
1070
+ from flextool.process_inputs.write_old_flextool_to_db import (
1071
+ write_sensitivities_to_db,
1072
+ )
1073
+
1074
+ data = read_old_flextool(str(self._last_base_xlsm))
1075
+ sensitivities = read_old_flextool_sensitivities(str(master_path))
1076
+ write_sensitivities_to_db(
1077
+ sensitivities, data, target_db_url,
1078
+ base_alternative=_OLD_FLEX_ALTERNATIVE,
1079
+ )
1080
+ except Exception as exc:
1081
+ logger.error("Sensitivity import failed: %s", exc, exc_info=True)
1082
+ messagebox.showerror("Import failed", str(exc), parent=self)
1083
+ return
1084
+ finally:
1085
+ self.config(cursor="")
1086
+
1087
+ self._import_sens_btn.configure(state="disabled")
1088
+ self.result = True
1089
+ messagebox.showinfo(
1090
+ "Done",
1091
+ f"Sensitivities imported into '{self._last_target_sqlite.name}'.",
1092
+ parent=self,
1093
+ )
1094
+
1095
+ def _on_back(self) -> None:
1096
+ """Close the dialog."""
1097
+ self.grab_release()
1098
+ self.destroy()