flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import subprocess
|
|
5
|
+
import shutil
|
|
6
|
+
try:
|
|
7
|
+
from spinedb_api import import_data, DatabaseMapping
|
|
8
|
+
except ModuleNotFoundError:
|
|
9
|
+
exit("Cannot find the required Spine-Toolbox module. Check that the environment is activated and the toolbox is installed")
|
|
10
|
+
from spinedb_api.exception import NothingToCommit
|
|
11
|
+
from flextool._resources import package_data_path
|
|
12
|
+
from flextool.update_flextool.db_migration import migrate_database
|
|
13
|
+
from flextool.update_flextool.initialize_database import initialize_database
|
|
14
|
+
from flextool.update_flextool import canonical_databases
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _reinstall_if_needed():
|
|
18
|
+
"""Re-install flextool and refresh its declared dependencies.
|
|
19
|
+
|
|
20
|
+
Runs ``pip install --upgrade [-e] <repo_root>`` after every
|
|
21
|
+
``git pull``. Three reasons:
|
|
22
|
+
|
|
23
|
+
1. **Editable installs need this too.** ``pip install -e .`` picks up
|
|
24
|
+
source-tree changes immediately but does NOT pull in new
|
|
25
|
+
dependencies that appeared in ``pyproject.toml``. Without this
|
|
26
|
+
reinstall, an editable user pulling a commit that adds (say)
|
|
27
|
+
``polar-high`` to ``[project.dependencies]`` would see
|
|
28
|
+
``ModuleNotFoundError`` on the next solver invocation.
|
|
29
|
+
2. **``--upgrade``** re-resolves the install target and bumps any
|
|
30
|
+
direct dependency whose pyproject pin moved past the currently
|
|
31
|
+
installed version (e.g. ``highspy>=1.14`` floor bumps land
|
|
32
|
+
automatically). Replaces the previous bespoke
|
|
33
|
+
``pip install --upgrade highspy>=1.14`` workaround.
|
|
34
|
+
3. **NO ``--upgrade-strategy=eager``** — pip's default
|
|
35
|
+
``only-if-needed`` semantics keep transitive dependencies stable
|
|
36
|
+
unless flextool's own pins force a move. This is deliberate:
|
|
37
|
+
eager upgrades churn the user's whole environment (numpy,
|
|
38
|
+
pandas, SQLAlchemy, etc.) and break other editable installs
|
|
39
|
+
(spinedb-api, toolbox extras) that pin to specific versions.
|
|
40
|
+
|
|
41
|
+
The install target is resolved from ``__file__`` rather than ``.``
|
|
42
|
+
so the re-install always targets THIS repo, regardless of the
|
|
43
|
+
caller's CWD (Spine Toolbox typically invokes from the project
|
|
44
|
+
directory, not the flextool repo root).
|
|
45
|
+
"""
|
|
46
|
+
import sys
|
|
47
|
+
import os
|
|
48
|
+
|
|
49
|
+
# flextool/update_flextool/self_update.py → flextool/ → repo root.
|
|
50
|
+
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(
|
|
51
|
+
os.path.abspath(__file__)
|
|
52
|
+
)))
|
|
53
|
+
|
|
54
|
+
result = subprocess.run(
|
|
55
|
+
[sys.executable, "-m", "pip", "show", "flextool"],
|
|
56
|
+
capture_output=True, text=True,
|
|
57
|
+
)
|
|
58
|
+
is_editable = "Editable project location" in result.stdout
|
|
59
|
+
|
|
60
|
+
print(
|
|
61
|
+
f"Refreshing flextool install and dependencies "
|
|
62
|
+
f"({'editable' if is_editable else 'non-editable'}, target={repo_root})..."
|
|
63
|
+
)
|
|
64
|
+
target = ["-e", repo_root] if is_editable else [repo_root]
|
|
65
|
+
completed = subprocess.run(
|
|
66
|
+
[sys.executable, "-m", "pip", "install", "--upgrade", *target],
|
|
67
|
+
)
|
|
68
|
+
if completed.returncode != 0:
|
|
69
|
+
flag = "-e " if is_editable else ""
|
|
70
|
+
print(
|
|
71
|
+
f"Warning: pip install failed. Run "
|
|
72
|
+
f"'pip install --upgrade {flag}{repo_root}' manually."
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _ensure_compatible_solver_stack():
|
|
77
|
+
"""Verify the native solver stack (polars, highspy/HiGHS) runs on this
|
|
78
|
+
machine and, if a component crashes natively, re-install the compatible
|
|
79
|
+
build automatically.
|
|
80
|
+
|
|
81
|
+
Two known cases (see ``flextool.env_check``): the default ``polars``
|
|
82
|
+
wheel on a CPU lacking the SIMD baseline it was built for (cure:
|
|
83
|
+
``polars-lts-cpu``), and ``highspy==1.14.0`` import-crashing on older
|
|
84
|
+
Windows (cure: ``highspy==1.13.1``). Both surface as a native crash —
|
|
85
|
+
a Windows access violation (exit ``3221225477``) or a POSIX
|
|
86
|
+
``SIGILL``/``SIGSEGV``. We probe in a child process so the crash is
|
|
87
|
+
observable and pinpoint which component died, then apply its remedy.
|
|
88
|
+
|
|
89
|
+
Run unattended here (the user already opted into installs by choosing
|
|
90
|
+
"Update FlexTool"), but loudly: the banner names what is happening and
|
|
91
|
+
how to stop it, and everything prints to stdout so it lands in the
|
|
92
|
+
update log window for copy-paste.
|
|
93
|
+
"""
|
|
94
|
+
import sys
|
|
95
|
+
from flextool import env_check
|
|
96
|
+
|
|
97
|
+
print("\nChecking that the solver libraries run on this computer...")
|
|
98
|
+
probe = env_check.probe_solver_stack()
|
|
99
|
+
print(env_check.diagnostics_report(probe))
|
|
100
|
+
if not probe.is_native_fault:
|
|
101
|
+
# OK, or an ordinary error (e.g. a package missing) that a build
|
|
102
|
+
# swap would not fix — leave it for the normal install machinery.
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
steps = env_check.remediation_steps(probe.failed_component, sys.executable)
|
|
106
|
+
if steps is None:
|
|
107
|
+
# A native crash with no package-level remedy (e.g. polar_high).
|
|
108
|
+
print("\n" + env_check.UNFIXABLE_HELP)
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
print("\n" + env_check.remediation_banner(probe.failed_component))
|
|
112
|
+
for step in steps:
|
|
113
|
+
print("\n$ " + " ".join(step))
|
|
114
|
+
completed = subprocess.run(step)
|
|
115
|
+
if completed.returncode != 0:
|
|
116
|
+
print(
|
|
117
|
+
f"Warning: '{' '.join(step)}' exited {completed.returncode}. "
|
|
118
|
+
f"Run it manually to finish the fix."
|
|
119
|
+
)
|
|
120
|
+
return
|
|
121
|
+
|
|
122
|
+
reprobe = env_check.probe_solver_stack()
|
|
123
|
+
print("\n" + reprobe.summary())
|
|
124
|
+
if not reprobe.ok:
|
|
125
|
+
print("\n" + env_check.UNFIXABLE_HELP)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _sync_settings_param_defs(json_template, sqlite_path):
|
|
129
|
+
"""Back-fill parameter DEFINITIONS from a settings template into an
|
|
130
|
+
EXISTING user settings DB.
|
|
131
|
+
|
|
132
|
+
The user-editable settings DBs (``output_settings.sqlite`` etc.) are only
|
|
133
|
+
created-if-absent, so a DB made by an older FlexTool version never gains
|
|
134
|
+
options added later (e.g. ``output-spinedb``). This imports just the
|
|
135
|
+
schema — entity classes + parameter definitions/types — so new options
|
|
136
|
+
appear in the editor, WITHOUT importing ``parameter_values`` (the user's
|
|
137
|
+
existing choices are never overwritten). Idempotent; non-fatal on error.
|
|
138
|
+
"""
|
|
139
|
+
from flextool.update_flextool.export_database import keep_serialized_unparse
|
|
140
|
+
try:
|
|
141
|
+
with open(json_template) as json_file:
|
|
142
|
+
template = json.load(json_file)
|
|
143
|
+
payload = {
|
|
144
|
+
key: template[key]
|
|
145
|
+
for key in ("entity_classes", "parameter_definitions", "parameter_types")
|
|
146
|
+
if key in template
|
|
147
|
+
}
|
|
148
|
+
if not payload:
|
|
149
|
+
return
|
|
150
|
+
with DatabaseMapping('sqlite:///' + sqlite_path, create=False) as db:
|
|
151
|
+
(num, _log) = import_data(db, unparse_value=keep_serialized_unparse, **payload)
|
|
152
|
+
if num:
|
|
153
|
+
db.commit_session(
|
|
154
|
+
"flextool-update: back-fill settings parameter definitions"
|
|
155
|
+
)
|
|
156
|
+
except NothingToCommit:
|
|
157
|
+
pass
|
|
158
|
+
except Exception as exc: # noqa: BLE001
|
|
159
|
+
print(f"Warning: could not sync parameter definitions into "
|
|
160
|
+
f"{sqlite_path}: {exc}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
_PROJECT_FOLDER_TEMPLATE = (
|
|
164
|
+
"# FlexTool output project folder (user-local; not tracked by git).\n"
|
|
165
|
+
"# Write ONE path below — absolute, or relative to the FlexTool root —\n"
|
|
166
|
+
"# to direct this workflow's outputs (output_parquet/, results.sqlite,\n"
|
|
167
|
+
"# plots, and the per-project plot_settings.yaml) into that project folder.\n"
|
|
168
|
+
"# Leave blank to use the FlexTool root.\n"
|
|
169
|
+
"# Use projects sub-folders to allow access by FlexTool GUI as well.\n"
|
|
170
|
+
"# Using the project specific input_sources folder for the input data db\n"
|
|
171
|
+
"# makes it visible in the GUI too.\n"
|
|
172
|
+
"# e.g.: projects/Rivendell\n"
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def ensure_runtime_files(verbose: bool = False) -> None:
|
|
177
|
+
"""Idempotently materialize the per-installation runtime files that
|
|
178
|
+
FlexTool needs but does not track in git.
|
|
179
|
+
|
|
180
|
+
Every step is **create-if-missing**: an existing user-edited file is
|
|
181
|
+
never overwritten, so this is safe to call on every GUI launch *and*
|
|
182
|
+
as the seeding step of :func:`update_flextool`. Each group is wrapped
|
|
183
|
+
so that one failure (e.g. a locked DB) logs a warning and the rest
|
|
184
|
+
still run — a seeding hiccup must never block GUI startup.
|
|
185
|
+
|
|
186
|
+
Paths are interpreted relative to the current working directory (the
|
|
187
|
+
FlexTool workspace root), matching :func:`update_flextool` and the
|
|
188
|
+
GUI's ``get_projects_dir`` convention.
|
|
189
|
+
|
|
190
|
+
Covers: the root input/settings SQLites and their ``templates/``
|
|
191
|
+
copies, the canonical example DBs + XLSX templates, the user-facing
|
|
192
|
+
``example_input.xlsx``, the ``solver_config/<solver>.opt`` files,
|
|
193
|
+
``templates/project_folder.txt``, and ``results.sqlite``.
|
|
194
|
+
|
|
195
|
+
It deliberately does NOT perform the update-only operations (git
|
|
196
|
+
restore/pull, reinstall, the unconditional template refresh that
|
|
197
|
+
keeps artifacts schema-current after a pull, parameter-definition
|
|
198
|
+
back-fill into existing DBs, or DB-version migration) — those remain
|
|
199
|
+
in :func:`update_flextool`.
|
|
200
|
+
"""
|
|
201
|
+
master_json = str(package_data_path("schemas/spinedb_schema.json"))
|
|
202
|
+
output_settings_json = str(package_data_path("schemas/output_settings_template.json"))
|
|
203
|
+
output_info_json = str(package_data_path("schemas/output_info_template.json"))
|
|
204
|
+
comparison_settings_json = str(package_data_path("schemas/comparison_settings_template.json"))
|
|
205
|
+
|
|
206
|
+
os.makedirs("templates", exist_ok=True)
|
|
207
|
+
os.makedirs("solver_config", exist_ok=True)
|
|
208
|
+
|
|
209
|
+
# 1. Root input/settings DBs + their templates/ copies, from the
|
|
210
|
+
# bundled package JSON. Created once; user edits then survive.
|
|
211
|
+
try:
|
|
212
|
+
for sqlite_rel, json_src in (
|
|
213
|
+
("input_data.sqlite", master_json),
|
|
214
|
+
("templates/input_data_template.sqlite", master_json),
|
|
215
|
+
("output_settings.sqlite", output_settings_json),
|
|
216
|
+
("output_info.sqlite", output_info_json),
|
|
217
|
+
("comparison_settings.sqlite", comparison_settings_json),
|
|
218
|
+
("templates/output_settings.sqlite", output_settings_json),
|
|
219
|
+
("templates/output_info.sqlite", output_info_json),
|
|
220
|
+
("templates/comparison_settings.sqlite", comparison_settings_json),
|
|
221
|
+
):
|
|
222
|
+
if not os.path.exists(sqlite_rel):
|
|
223
|
+
if verbose:
|
|
224
|
+
print(f"Seeding {sqlite_rel}")
|
|
225
|
+
initialize_database(json_src, sqlite_rel)
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
logging.warning("ensure_runtime_files: input/settings DB seeding failed: %s", exc)
|
|
228
|
+
|
|
229
|
+
# 2. Canonical example DBs, the XLSX templates derived from them, and
|
|
230
|
+
# the user-facing example_input.xlsx — all create-if-missing.
|
|
231
|
+
try:
|
|
232
|
+
canonical_databases.materialize(overwrite=False)
|
|
233
|
+
from flextool.export_to_tabular import export_to_excel
|
|
234
|
+
for sqlite_rel, xlsx_rel in (
|
|
235
|
+
("templates/examples.sqlite", "templates/example_input_template.xlsx"),
|
|
236
|
+
("templates/input_data_template.sqlite", "templates/empty_input_template.xlsx"),
|
|
237
|
+
):
|
|
238
|
+
if os.path.exists(sqlite_rel) and not os.path.exists(xlsx_rel):
|
|
239
|
+
export_to_excel(f"sqlite:///{sqlite_rel}", xlsx_rel)
|
|
240
|
+
if not os.path.exists("example_input.xlsx") and os.path.exists("templates/example_input_template.xlsx"):
|
|
241
|
+
shutil.copy("templates/example_input_template.xlsx", "example_input.xlsx")
|
|
242
|
+
except Exception as exc:
|
|
243
|
+
logging.warning("ensure_runtime_files: example/template seeding failed: %s", exc)
|
|
244
|
+
|
|
245
|
+
# 3. Live solver options — HiGHS + the four commercial solvers — each
|
|
246
|
+
# seeded from its bundled <solver>.opt.template if missing. The
|
|
247
|
+
# runtime files are gitignored and user-editable (delete one to
|
|
248
|
+
# reset it to the bundled default).
|
|
249
|
+
try:
|
|
250
|
+
for _solver in ("highs", "gurobi", "cplex", "xpress", "copt"):
|
|
251
|
+
_runtime = f"solver_config/{_solver}.opt"
|
|
252
|
+
if not os.path.exists(_runtime):
|
|
253
|
+
shutil.copy(
|
|
254
|
+
str(package_data_path(f"solver_config/{_solver}.opt.template")),
|
|
255
|
+
_runtime,
|
|
256
|
+
)
|
|
257
|
+
except Exception as exc:
|
|
258
|
+
logging.warning("ensure_runtime_files: solver_config seeding failed: %s", exc)
|
|
259
|
+
|
|
260
|
+
# 4. templates/project_folder.txt — a commented guide; its CONTENTS are
|
|
261
|
+
# load-bearing for Spine Toolbox (the run Tool reads the first
|
|
262
|
+
# non-comment line as the output project folder). Gitignored.
|
|
263
|
+
try:
|
|
264
|
+
if not os.path.exists("templates/project_folder.txt"):
|
|
265
|
+
with open("templates/project_folder.txt", "w", encoding="utf-8") as fh:
|
|
266
|
+
fh.write(_PROJECT_FOLDER_TEMPLATE)
|
|
267
|
+
except Exception as exc:
|
|
268
|
+
logging.warning("ensure_runtime_files: project_folder.txt seeding failed: %s", exc)
|
|
269
|
+
|
|
270
|
+
# 5. results.sqlite (and its template), both create-if-missing.
|
|
271
|
+
try:
|
|
272
|
+
result_template_path = str(package_data_path("schemas/pre_v26/flextool_template_results_master.json"))
|
|
273
|
+
if not os.path.exists("templates/results_template.sqlite"):
|
|
274
|
+
initialize_result_database("templates/results_template.sqlite", result_template_path)
|
|
275
|
+
if not os.path.exists("results.sqlite") and os.path.exists("templates/results_template.sqlite"):
|
|
276
|
+
shutil.copy("templates/results_template.sqlite", "results.sqlite")
|
|
277
|
+
except Exception as exc:
|
|
278
|
+
logging.warning("ensure_runtime_files: results DB seeding failed: %s", exc)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def update_flextool(skip_git):
|
|
282
|
+
|
|
283
|
+
shutil.copy("./.spinetoolbox/project.json", "./.spinetoolbox/project_temp.json")
|
|
284
|
+
if not skip_git:
|
|
285
|
+
completed = subprocess.run(["git", "restore", "."])
|
|
286
|
+
if completed.returncode != 0:
|
|
287
|
+
print("Failed to restore version controlled files.")
|
|
288
|
+
exit(-1)
|
|
289
|
+
|
|
290
|
+
completed = subprocess.run(["git", "pull"])
|
|
291
|
+
if completed.returncode != 0:
|
|
292
|
+
print("Failed to get the new version.")
|
|
293
|
+
exit(-1)
|
|
294
|
+
|
|
295
|
+
# Re-install the package so that code changes from git pull take effect.
|
|
296
|
+
# Skip if the package is installed in editable mode (changes are live already).
|
|
297
|
+
_reinstall_if_needed()
|
|
298
|
+
|
|
299
|
+
# After (re)install, make sure the native solver stack actually runs on
|
|
300
|
+
# this machine; auto-fix the known incompatible-build cases (polars CPU
|
|
301
|
+
# baseline, highspy 1.14.0 on older Windows) if a component crashes.
|
|
302
|
+
_ensure_compatible_solver_stack()
|
|
303
|
+
|
|
304
|
+
migrate_project("./.spinetoolbox/project_temp.json","./.spinetoolbox/project.json")
|
|
305
|
+
os.remove("./.spinetoolbox/project_temp.json")
|
|
306
|
+
|
|
307
|
+
# Idempotently materialize all per-installation runtime files
|
|
308
|
+
# (root/settings DBs + templates, canonical examples + XLSX,
|
|
309
|
+
# example_input.xlsx, solver_config/*.opt, project_folder.txt,
|
|
310
|
+
# results.sqlite). Shared with the GUI startup path, so "an update
|
|
311
|
+
# brought a new file to copy" is handled by the same create-if-missing
|
|
312
|
+
# logic in both places — no duplication.
|
|
313
|
+
ensure_runtime_files()
|
|
314
|
+
|
|
315
|
+
# ----- Update-only refresh of EXISTING artifacts (after a git pull) ---
|
|
316
|
+
# These differ from the create-if-missing seeding above: they bring
|
|
317
|
+
# already-present files up to the current schema, so they must NOT run
|
|
318
|
+
# on every GUI launch (that is why they stay here, not in
|
|
319
|
+
# ensure_runtime_files).
|
|
320
|
+
output_settings_json = str(package_data_path("schemas/output_settings_template.json"))
|
|
321
|
+
output_info_json = str(package_data_path("schemas/output_info_template.json"))
|
|
322
|
+
comparison_settings_json = str(package_data_path("schemas/comparison_settings_template.json"))
|
|
323
|
+
|
|
324
|
+
# Back-fill parameter definitions added in newer versions into EXISTING
|
|
325
|
+
# user settings DBs. Options added later (e.g. output-spinedb) must be
|
|
326
|
+
# added to DBs made by older versions. Only schema is synced; user
|
|
327
|
+
# parameter values are untouched.
|
|
328
|
+
for sqlite_rel, json_src in (
|
|
329
|
+
("output_settings.sqlite", output_settings_json),
|
|
330
|
+
("output_info.sqlite", output_info_json),
|
|
331
|
+
("comparison_settings.sqlite", comparison_settings_json),
|
|
332
|
+
):
|
|
333
|
+
if os.path.exists(sqlite_rel):
|
|
334
|
+
_sync_settings_param_defs(json_src, sqlite_rel)
|
|
335
|
+
|
|
336
|
+
# Keep CWD-resident template SQLites up-to-date (Spine Toolbox refs):
|
|
337
|
+
# unconditional regen so a git pull's schema change is reflected.
|
|
338
|
+
for sqlite_rel, json_src in (
|
|
339
|
+
("templates/output_settings.sqlite", output_settings_json),
|
|
340
|
+
("templates/output_info.sqlite", output_info_json),
|
|
341
|
+
("templates/comparison_settings.sqlite", comparison_settings_json),
|
|
342
|
+
):
|
|
343
|
+
if os.path.exists(sqlite_rel):
|
|
344
|
+
os.remove(sqlite_rel)
|
|
345
|
+
initialize_database(json_src, sqlite_rel)
|
|
346
|
+
|
|
347
|
+
# Refresh XLSX templates from the canonical SQLites so they track the
|
|
348
|
+
# current schema (unconditional regen; the binaries are not in git).
|
|
349
|
+
from flextool.export_to_tabular import export_to_excel
|
|
350
|
+
for sqlite_rel, xlsx_rel in (
|
|
351
|
+
("templates/examples.sqlite", "templates/example_input_template.xlsx"),
|
|
352
|
+
("templates/input_data_template.sqlite", "templates/empty_input_template.xlsx"),
|
|
353
|
+
):
|
|
354
|
+
if not os.path.exists(sqlite_rel):
|
|
355
|
+
continue
|
|
356
|
+
try:
|
|
357
|
+
export_to_excel(f"sqlite:///{sqlite_rel}", xlsx_rel)
|
|
358
|
+
except Exception as exc:
|
|
359
|
+
print(f"Warning: failed to generate {xlsx_rel}: {exc}")
|
|
360
|
+
|
|
361
|
+
# Migrate user-owned SQLites (canonical ones are already at current
|
|
362
|
+
# version because they were just materialized from the canonical
|
|
363
|
+
# JSONs, which migrate-all keeps at FLEXTOOL_DB_VERSION).
|
|
364
|
+
db_to_update = [
|
|
365
|
+
"input_data.sqlite",
|
|
366
|
+
"templates/input_data_template.sqlite",
|
|
367
|
+
]
|
|
368
|
+
|
|
369
|
+
# add the database used in the input_data tool
|
|
370
|
+
with open("./.spinetoolbox/project.json") as json_file:
|
|
371
|
+
specifications = json.load(json_file)
|
|
372
|
+
path = specifications["items"]["Input data"]["url"]["database"]["path"]
|
|
373
|
+
db_to_update.append(path)
|
|
374
|
+
|
|
375
|
+
# migrate the databases to new version
|
|
376
|
+
for i in db_to_update:
|
|
377
|
+
migrate_database(i)
|
|
378
|
+
|
|
379
|
+
# Refresh the results template (unconditional) and ensure results.sqlite
|
|
380
|
+
# exists (ensure_runtime_files seeds it; this is a belt-and-braces copy
|
|
381
|
+
# for the case it was removed since).
|
|
382
|
+
result_template_path = str(package_data_path("schemas/pre_v26/flextool_template_results_master.json"))
|
|
383
|
+
if os.path.exists('templates/results_template.sqlite'):
|
|
384
|
+
os.remove('templates/results_template.sqlite')
|
|
385
|
+
initialize_result_database('templates/results_template.sqlite', result_template_path)
|
|
386
|
+
|
|
387
|
+
if not os.path.exists("results.sqlite"):
|
|
388
|
+
shutil.copy("templates/results_template.sqlite", "results.sqlite")
|
|
389
|
+
#update result parameter definitions
|
|
390
|
+
#get template JSON. This can be the master or old template if conflicting migrations in between
|
|
391
|
+
with open (result_template_path) as json_file:
|
|
392
|
+
template = json.load(json_file)
|
|
393
|
+
with DatabaseMapping('sqlite:///' + 'results.sqlite', create = False, upgrade = True) as db:
|
|
394
|
+
#these update the old descriptions, but wont remove them or change names (the new name is created, but old stays)
|
|
395
|
+
(num,log) = import_data(db, object_parameters = template["object_parameters"])
|
|
396
|
+
(num,log) = import_data(db, relationship_parameters = template["relationship_parameters"])
|
|
397
|
+
try:
|
|
398
|
+
db.commit_session("Updated relationship_parameters, object parameters to the Results.sqlite")
|
|
399
|
+
except NothingToCommit:
|
|
400
|
+
print("These parameters have been added before, continuing")
|
|
401
|
+
return 0
|
|
402
|
+
|
|
403
|
+
def initialize_result_database(filename,json_path):
|
|
404
|
+
|
|
405
|
+
with open (json_path) as json_file:
|
|
406
|
+
template = json.load(json_file)
|
|
407
|
+
with DatabaseMapping('sqlite:///' + filename, create = True) as db:
|
|
408
|
+
(num,log) = import_data(db,**template)
|
|
409
|
+
print("Result database initialized")
|
|
410
|
+
db.commit_session("Result database initialized")
|
|
411
|
+
|
|
412
|
+
def migrate_project(old_path, new_path):
|
|
413
|
+
#purpose of this is to update some of the items that users should not need to modify
|
|
414
|
+
#done simply by copying from the git project.json
|
|
415
|
+
#should be replaced if major changes to project.json
|
|
416
|
+
|
|
417
|
+
#items that are copied
|
|
418
|
+
items = [
|
|
419
|
+
"Replace with examples",
|
|
420
|
+
"FlexTool",
|
|
421
|
+
"Import_results",
|
|
422
|
+
"Plot_results",
|
|
423
|
+
"Plot_settings",
|
|
424
|
+
]
|
|
425
|
+
|
|
426
|
+
with open(old_path) as old_json:
|
|
427
|
+
old_dict = json.load(old_json)
|
|
428
|
+
with open(new_path) as new_json:
|
|
429
|
+
new_dict = json.load(new_json)
|
|
430
|
+
|
|
431
|
+
for item in items:
|
|
432
|
+
if item in old_dict["items"].keys():
|
|
433
|
+
for param in old_dict["items"][item].keys():
|
|
434
|
+
if param != "x" and param != "y":
|
|
435
|
+
old_dict["items"][item][param] = new_dict["items"][item][param]
|
|
436
|
+
|
|
437
|
+
if ("Open_summary" not in old_dict["items"].keys()) and ("Open_summary" in new_dict["items"].keys()):
|
|
438
|
+
old_dict["items"]["Open_summary"] = new_dict["items"]["Open_summary"]
|
|
439
|
+
old_dict["project"]["connections"].append({"name": "from FlexTool3 to Open_summary", "from": ["FlexTool3","bottom"],"to": ["Open_summary","right"]})
|
|
440
|
+
old_dict["project"]["specifications"]["Tool"].append({"type": "path","relative": True,"path": ".spinetoolbox/specifications/Tool/open_summary.json"})
|
|
441
|
+
|
|
442
|
+
if ("Migrate database version" not in old_dict["items"].keys()) and ("Migrate database version" in new_dict["items"].keys()):
|
|
443
|
+
old_dict["items"]["Migrate database version"] = new_dict["items"]["Migrate database version"]
|
|
444
|
+
old_dict["project"]["connections"].append({"name": "from Migrate database version to Input_data",
|
|
445
|
+
"from": ["Migrate database version","right"],
|
|
446
|
+
"to": ["Input_data","bottom"]})
|
|
447
|
+
old_dict["project"]["specifications"]["Tool"].append({ "type": "path",
|
|
448
|
+
"relative": True,
|
|
449
|
+
"path": ".spinetoolbox/specifications/Tool/migrate_database.json"})
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
with open("./.spinetoolbox/project_temp2.json", "w") as outfile:
|
|
453
|
+
json.dump(old_dict, outfile, indent=4)
|
|
454
|
+
|
|
455
|
+
shutil.copy("./.spinetoolbox/project_temp2.json", new_path)
|
|
456
|
+
os.remove("./.spinetoolbox/project_temp2.json")
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def main():
|
|
460
|
+
update_flextool(False)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
if __name__ == '__main__':
|
|
464
|
+
main()
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Regenerate schemas/spinedb_schema.json from itself + migrations.
|
|
2
|
+
|
|
3
|
+
Workflow:
|
|
4
|
+
1. Create a temporary DB from the current schemas/spinedb_schema.json
|
|
5
|
+
2. Run db_migration.py on it (applies any new migration steps)
|
|
6
|
+
3. Export the migrated DB back to schemas/spinedb_schema.json
|
|
7
|
+
|
|
8
|
+
This ensures the master template always reflects the latest schema
|
|
9
|
+
after db_migration.py has been updated.
|
|
10
|
+
|
|
11
|
+
Usage:
|
|
12
|
+
python -m flextool.update_flextool.sync_master_json_template [--verify]
|
|
13
|
+
|
|
14
|
+
--verify Don't overwrite; just check if the template is up to date.
|
|
15
|
+
Exits with code 1 if it needs regeneration (for CI).
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import json
|
|
21
|
+
import sys
|
|
22
|
+
import tempfile
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from spinedb_api import DatabaseMapping, export_data, import_data
|
|
26
|
+
|
|
27
|
+
from flextool.update_flextool.db_migration import migrate_database
|
|
28
|
+
|
|
29
|
+
from flextool._resources import package_data_path
|
|
30
|
+
|
|
31
|
+
SCHEMAS_DIR = package_data_path("schemas")
|
|
32
|
+
SPINEDB_SCHEMA = SCHEMAS_DIR / "spinedb_schema.json"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _create_db_from_template(template_path: Path, db_path: Path) -> None:
|
|
36
|
+
"""Create a fresh SQLite DB from a JSON template."""
|
|
37
|
+
with open(template_path) as f:
|
|
38
|
+
template = json.load(f)
|
|
39
|
+
|
|
40
|
+
url = f"sqlite:///{db_path}"
|
|
41
|
+
with DatabaseMapping(url, create=True) as db:
|
|
42
|
+
for alt in db.find_alternatives():
|
|
43
|
+
try:
|
|
44
|
+
db.remove_alternative(name=alt["name"])
|
|
45
|
+
except Exception:
|
|
46
|
+
pass
|
|
47
|
+
import_data(db, **template)
|
|
48
|
+
db.commit_session("Initialized from master template")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _export_db_to_json(db_path: Path) -> dict:
|
|
52
|
+
"""Export a Spine DB to a JSON-serializable dict."""
|
|
53
|
+
url = f"sqlite:///{db_path}"
|
|
54
|
+
with DatabaseMapping(url) as db:
|
|
55
|
+
data = export_data(db)
|
|
56
|
+
# export_data returns tuples; convert to lists for JSON
|
|
57
|
+
return {k: [list(item) for item in v] for k, v in data.items()}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def sync_master_template(*, verify_only: bool = False) -> bool:
|
|
61
|
+
"""Regenerate (or verify) the master JSON template.
|
|
62
|
+
|
|
63
|
+
Returns True if the template is up to date, False if it needed updating.
|
|
64
|
+
"""
|
|
65
|
+
if not SPINEDB_SCHEMA.exists():
|
|
66
|
+
print(f"ERROR: Master template not found: {SPINEDB_SCHEMA}", file=sys.stderr)
|
|
67
|
+
sys.exit(2)
|
|
68
|
+
|
|
69
|
+
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
70
|
+
db_path = Path(tmp_dir) / "template.sqlite"
|
|
71
|
+
|
|
72
|
+
# Step 1: Create DB from current master template
|
|
73
|
+
_create_db_from_template(SPINEDB_SCHEMA, db_path)
|
|
74
|
+
|
|
75
|
+
# Step 2: Run all migrations
|
|
76
|
+
migrate_database(str(db_path))
|
|
77
|
+
|
|
78
|
+
# Step 3: Export migrated DB
|
|
79
|
+
new_data = _export_db_to_json(db_path)
|
|
80
|
+
|
|
81
|
+
# Step 4: Compare or overwrite
|
|
82
|
+
new_json = json.dumps(new_data, indent=2, ensure_ascii=False) + "\n"
|
|
83
|
+
|
|
84
|
+
with open(SPINEDB_SCHEMA) as f:
|
|
85
|
+
old_json = f.read()
|
|
86
|
+
|
|
87
|
+
if new_json == old_json:
|
|
88
|
+
print("Master template is up to date.")
|
|
89
|
+
return True
|
|
90
|
+
|
|
91
|
+
if verify_only:
|
|
92
|
+
# Show what changed
|
|
93
|
+
old_data = json.loads(old_json)
|
|
94
|
+
for key in sorted(set(list(old_data.keys()) + list(new_data.keys()))):
|
|
95
|
+
old_count = len(old_data.get(key, []))
|
|
96
|
+
new_count = len(new_data.get(key, []))
|
|
97
|
+
if old_count != new_count:
|
|
98
|
+
print(f" {key}: {old_count} -> {new_count}")
|
|
99
|
+
print("Master template is OUT OF DATE. Run without --verify to regenerate.")
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
with open(SPINEDB_SCHEMA, "w") as f:
|
|
103
|
+
f.write(new_json)
|
|
104
|
+
print(f"Regenerated {SPINEDB_SCHEMA}")
|
|
105
|
+
return True
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def main() -> None:
|
|
109
|
+
parser = argparse.ArgumentParser(
|
|
110
|
+
description="Regenerate schemas/spinedb_schema.json after migration changes."
|
|
111
|
+
)
|
|
112
|
+
parser.add_argument(
|
|
113
|
+
"--verify",
|
|
114
|
+
action="store_true",
|
|
115
|
+
help="Check if template is up to date without overwriting (for CI).",
|
|
116
|
+
)
|
|
117
|
+
args = parser.parse_args()
|
|
118
|
+
|
|
119
|
+
up_to_date = sync_master_template(verify_only=args.verify)
|
|
120
|
+
if not up_to_date:
|
|
121
|
+
sys.exit(1)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
if __name__ == "__main__":
|
|
125
|
+
main()
|