flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import shutil
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from flextool.gui.data_models import InputSourceInfo, ProjectSettings, ScenarioInfo
|
|
9
|
+
from flextool.gui.settings_io import save_project_settings
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
# File extensions recognised as input sources
|
|
14
|
+
SUPPORTED_EXTENSIONS = {".xlsx", ".ods", ".sqlite"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _relative_posix(target: Path, base: Path) -> str:
|
|
18
|
+
"""Return ``target`` as a POSIX path relative to ``base``.
|
|
19
|
+
|
|
20
|
+
Falls back to ``os.path.relpath`` when the paths live on the same
|
|
21
|
+
drive but ``target`` is not under ``base`` (common for references
|
|
22
|
+
that go up with ``..``). Returns the absolute POSIX path when the
|
|
23
|
+
paths cannot be related (e.g. different Windows drives).
|
|
24
|
+
"""
|
|
25
|
+
import os
|
|
26
|
+
target = target.resolve()
|
|
27
|
+
base = base.resolve()
|
|
28
|
+
try:
|
|
29
|
+
rel = os.path.relpath(str(target), str(base))
|
|
30
|
+
except ValueError:
|
|
31
|
+
return target.as_posix()
|
|
32
|
+
return Path(rel).as_posix()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class InputSourceManager:
|
|
36
|
+
"""Manages input source files and their scenario discovery.
|
|
37
|
+
|
|
38
|
+
Scans the ``input_sources/`` subdirectory of a project, reads scenario
|
|
39
|
+
names from xlsx and sqlite files, and keeps track of persistent source
|
|
40
|
+
numbering via project settings.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(self, project_path: Path, settings: ProjectSettings) -> None:
|
|
44
|
+
self.project_path = project_path
|
|
45
|
+
self.settings = settings
|
|
46
|
+
self.input_dir = project_path / "input_sources"
|
|
47
|
+
self._sources: list[InputSourceInfo] = []
|
|
48
|
+
# Retired "ghost" rows from the last refresh (file gone, results
|
|
49
|
+
# survive). Kept separate from ``_sources`` (live only) so scenario
|
|
50
|
+
# enumeration ignores them while the UI can still render them.
|
|
51
|
+
self._last_ghosts: list[InputSourceInfo] = []
|
|
52
|
+
# Track mtimes for files opened via Edit button (Linux xlsx fallback)
|
|
53
|
+
self._last_known_mtimes: dict[str, float] = {}
|
|
54
|
+
# Track which sources were explicitly opened for editing
|
|
55
|
+
self._editing_sources: set[str] = set()
|
|
56
|
+
|
|
57
|
+
# ── Public API ────────────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
def scan_input_sources(self) -> list[InputSourceInfo]:
|
|
60
|
+
"""List files in the input_sources/ directory plus external references.
|
|
61
|
+
|
|
62
|
+
Returns an :class:`InputSourceInfo` for each ``.xlsx``, ``.ods``, and
|
|
63
|
+
``.sqlite`` file found inside ``input_sources/``, followed by one
|
|
64
|
+
entry per external reference registered in project settings.
|
|
65
|
+
No scenario reading is performed here.
|
|
66
|
+
"""
|
|
67
|
+
sources: list[InputSourceInfo] = []
|
|
68
|
+
if self.input_dir.is_dir():
|
|
69
|
+
for filepath in sorted(self.input_dir.iterdir()):
|
|
70
|
+
if filepath.suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
71
|
+
continue
|
|
72
|
+
if not filepath.is_file():
|
|
73
|
+
continue
|
|
74
|
+
file_type = filepath.suffix.lstrip(".").lower()
|
|
75
|
+
sources.append(
|
|
76
|
+
InputSourceInfo(
|
|
77
|
+
name=filepath.name,
|
|
78
|
+
file_type=file_type,
|
|
79
|
+
number=0, # assigned later in refresh()
|
|
80
|
+
status="error", # updated later
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
for ext_name, rel_path in self.settings.external_refs.items():
|
|
85
|
+
ext = Path(ext_name).suffix.lower()
|
|
86
|
+
if ext not in SUPPORTED_EXTENSIONS:
|
|
87
|
+
continue
|
|
88
|
+
sources.append(
|
|
89
|
+
InputSourceInfo(
|
|
90
|
+
name=ext_name,
|
|
91
|
+
file_type=ext.lstrip("."),
|
|
92
|
+
number=0,
|
|
93
|
+
status="error",
|
|
94
|
+
external_rel_path=rel_path,
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
return sources
|
|
98
|
+
|
|
99
|
+
def resolve_path(self, source: InputSourceInfo) -> Path:
|
|
100
|
+
"""Return the absolute path to a source's underlying file."""
|
|
101
|
+
if source.external_rel_path:
|
|
102
|
+
return (self.project_path / source.external_rel_path).resolve()
|
|
103
|
+
return self.input_dir / source.name
|
|
104
|
+
|
|
105
|
+
def add_external_ref(self, filepath: Path) -> tuple[str, str]:
|
|
106
|
+
"""Register an external file as an input source.
|
|
107
|
+
|
|
108
|
+
Returns ``(name, rel_path)`` where ``rel_path`` is POSIX-style and
|
|
109
|
+
relative to the project root. Raises ``ValueError`` if a reference
|
|
110
|
+
with the same name is already registered.
|
|
111
|
+
"""
|
|
112
|
+
filepath = filepath.resolve()
|
|
113
|
+
rel_path = _relative_posix(filepath, self.project_path)
|
|
114
|
+
name = filepath.name
|
|
115
|
+
if name in self.settings.external_refs:
|
|
116
|
+
raise ValueError(
|
|
117
|
+
f"An external reference named '{name}' is already registered."
|
|
118
|
+
)
|
|
119
|
+
self.settings.external_refs[name] = rel_path
|
|
120
|
+
return name, rel_path
|
|
121
|
+
|
|
122
|
+
def remove_external_ref(self, name: str) -> bool:
|
|
123
|
+
"""Remove an external reference. Returns True if found and removed."""
|
|
124
|
+
if name in self.settings.external_refs:
|
|
125
|
+
del self.settings.external_refs[name]
|
|
126
|
+
return True
|
|
127
|
+
return False
|
|
128
|
+
|
|
129
|
+
def read_scenarios_xlsx(self, filepath: Path) -> list[str] | None:
|
|
130
|
+
"""Read scenario names from the 'scenario' sheet in an xlsx file.
|
|
131
|
+
|
|
132
|
+
Uses zipfile + xml.etree to read the xlsx directly without openpyxl,
|
|
133
|
+
avoiding any file handle leaks that could prevent LibreOffice from
|
|
134
|
+
opening the file afterwards.
|
|
135
|
+
|
|
136
|
+
Reads the 2nd row, skips the first column, and reads cell values
|
|
137
|
+
until an empty cell is encountered.
|
|
138
|
+
"""
|
|
139
|
+
import zipfile
|
|
140
|
+
import xml.etree.ElementTree as ET
|
|
141
|
+
|
|
142
|
+
ns = {"s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"}
|
|
143
|
+
|
|
144
|
+
try:
|
|
145
|
+
with zipfile.ZipFile(str(filepath), "r") as zf:
|
|
146
|
+
# Find the scenario sheet: read workbook.xml for sheet names
|
|
147
|
+
wb_xml = ET.fromstring(zf.read("xl/workbook.xml"))
|
|
148
|
+
sheets = wb_xml.findall(".//s:sheet", ns)
|
|
149
|
+
scenario_sheet_id = None
|
|
150
|
+
for sheet in sheets:
|
|
151
|
+
if sheet.get("name") == "scenario":
|
|
152
|
+
# rId attribute links to the actual sheet file
|
|
153
|
+
scenario_sheet_id = sheet.get(
|
|
154
|
+
"{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id"
|
|
155
|
+
)
|
|
156
|
+
break
|
|
157
|
+
|
|
158
|
+
if scenario_sheet_id is None:
|
|
159
|
+
return None
|
|
160
|
+
|
|
161
|
+
# Resolve rId to a sheet file path via relationships
|
|
162
|
+
rels_xml = ET.fromstring(zf.read("xl/_rels/workbook.xml.rels"))
|
|
163
|
+
sheet_file = None
|
|
164
|
+
for rel in rels_xml:
|
|
165
|
+
if rel.get("Id") == scenario_sheet_id:
|
|
166
|
+
target = rel.get("Target", "")
|
|
167
|
+
# Target may be relative (e.g. "worksheets/sheet3.xml")
|
|
168
|
+
# or absolute (e.g. "/xl/worksheets/sheet3.xml")
|
|
169
|
+
if target.startswith("/"):
|
|
170
|
+
sheet_file = target.lstrip("/")
|
|
171
|
+
else:
|
|
172
|
+
sheet_file = "xl/" + target
|
|
173
|
+
break
|
|
174
|
+
|
|
175
|
+
if sheet_file is None or sheet_file not in zf.namelist():
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
# Read shared strings (cell values may reference these)
|
|
179
|
+
shared_strings: list[str] = []
|
|
180
|
+
if "xl/sharedStrings.xml" in zf.namelist():
|
|
181
|
+
ss_xml = ET.fromstring(zf.read("xl/sharedStrings.xml"))
|
|
182
|
+
for si in ss_xml.findall("s:si", ns):
|
|
183
|
+
t_elem = si.find("s:t", ns)
|
|
184
|
+
shared_strings.append(t_elem.text if t_elem is not None and t_elem.text else "")
|
|
185
|
+
|
|
186
|
+
# Parse the scenario sheet — find row 2 cells
|
|
187
|
+
sheet_xml = ET.fromstring(zf.read(sheet_file))
|
|
188
|
+
scenarios: list[str] = []
|
|
189
|
+
|
|
190
|
+
for row in sheet_xml.findall(".//s:sheetData/s:row", ns):
|
|
191
|
+
if row.get("r") != "2":
|
|
192
|
+
continue
|
|
193
|
+
# Found row 2 — read cells from column B onwards
|
|
194
|
+
for cell in row.findall("s:c", ns):
|
|
195
|
+
ref = cell.get("r", "")
|
|
196
|
+
# Skip column A (ref is exactly "A2", not "AA2", "AB2", etc.)
|
|
197
|
+
if ref == "A2":
|
|
198
|
+
continue
|
|
199
|
+
cell_type = cell.get("t", "")
|
|
200
|
+
value = ""
|
|
201
|
+
if cell_type == "s":
|
|
202
|
+
# Shared string reference
|
|
203
|
+
v_elem = cell.find("s:v", ns)
|
|
204
|
+
if v_elem is not None and v_elem.text is not None:
|
|
205
|
+
idx = int(v_elem.text)
|
|
206
|
+
value = shared_strings[idx] if idx < len(shared_strings) else ""
|
|
207
|
+
elif cell_type == "inlineStr":
|
|
208
|
+
# Inline string: <is><t>text</t></is>
|
|
209
|
+
is_elem = cell.find("s:is/s:t", ns)
|
|
210
|
+
if is_elem is not None and is_elem.text is not None:
|
|
211
|
+
value = is_elem.text
|
|
212
|
+
else:
|
|
213
|
+
v_elem = cell.find("s:v", ns)
|
|
214
|
+
if v_elem is not None and v_elem.text is not None:
|
|
215
|
+
value = v_elem.text
|
|
216
|
+
value = value.strip()
|
|
217
|
+
if value:
|
|
218
|
+
scenarios.append(value)
|
|
219
|
+
break
|
|
220
|
+
|
|
221
|
+
return scenarios
|
|
222
|
+
|
|
223
|
+
except Exception:
|
|
224
|
+
logger.warning("Error reading scenarios from %s", filepath, exc_info=True)
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
def read_scenarios_sqlite(self, filepath: Path) -> list[str] | None:
|
|
228
|
+
"""Read scenario names from a Spine database sqlite file.
|
|
229
|
+
|
|
230
|
+
Uses ``spinedb_api.DatabaseMapping`` to extract scenario names (or
|
|
231
|
+
alternative names as a fallback). Returns ``None`` if the file is
|
|
232
|
+
not a valid Spine database or spinedb_api is not installed.
|
|
233
|
+
"""
|
|
234
|
+
try:
|
|
235
|
+
from spinedb_api import DatabaseMapping
|
|
236
|
+
except ImportError:
|
|
237
|
+
logger.warning("spinedb_api is not installed -- cannot read sqlite scenarios")
|
|
238
|
+
return None
|
|
239
|
+
|
|
240
|
+
db_url = f"sqlite:///{filepath}"
|
|
241
|
+
try:
|
|
242
|
+
with DatabaseMapping(db_url) as db_map:
|
|
243
|
+
# Try scenarios first
|
|
244
|
+
try:
|
|
245
|
+
items = db_map.get_scenario_items()
|
|
246
|
+
scenarios = [x["name"] for x in items]
|
|
247
|
+
if scenarios:
|
|
248
|
+
return scenarios
|
|
249
|
+
except (AttributeError, Exception):
|
|
250
|
+
pass
|
|
251
|
+
|
|
252
|
+
# Fallback to alternatives
|
|
253
|
+
try:
|
|
254
|
+
items = db_map.get_alternative_items()
|
|
255
|
+
alternatives = [x["name"] for x in items]
|
|
256
|
+
if alternatives:
|
|
257
|
+
return alternatives
|
|
258
|
+
except (AttributeError, Exception):
|
|
259
|
+
pass
|
|
260
|
+
|
|
261
|
+
return []
|
|
262
|
+
except Exception:
|
|
263
|
+
logger.warning("Failed to read spine db: %s", filepath, exc_info=True)
|
|
264
|
+
return None
|
|
265
|
+
|
|
266
|
+
def _disk_source_counts(self) -> dict[int, int]:
|
|
267
|
+
"""Map ``source_number -> number of executed-scenario folders`` on disk.
|
|
268
|
+
|
|
269
|
+
Scans ``output_parquet/`` and attributes each result folder to its
|
|
270
|
+
source number (via the ``_N`` suffix, honouring legacy bare-owner
|
|
271
|
+
records). A number present here is "in use" by results even if its
|
|
272
|
+
input file is gone — which is what keeps a ghost row alive and what
|
|
273
|
+
a new source must avoid colliding with.
|
|
274
|
+
"""
|
|
275
|
+
from flextool.gui.scenario_key import resolve_source_number, LEGACY_SOURCE_NUMBER
|
|
276
|
+
|
|
277
|
+
parquet_dir = self.project_path / "output_parquet"
|
|
278
|
+
counts: dict[int, int] = {}
|
|
279
|
+
if not parquet_dir.is_dir():
|
|
280
|
+
return counts
|
|
281
|
+
owners = self.settings.bare_output_owners
|
|
282
|
+
for entry in parquet_dir.iterdir():
|
|
283
|
+
if not entry.is_dir() or entry.name.startswith("_"):
|
|
284
|
+
continue
|
|
285
|
+
n, _ = resolve_source_number(entry.name, owners)
|
|
286
|
+
if n != LEGACY_SOURCE_NUMBER:
|
|
287
|
+
counts[n] = counts.get(n, 0) + 1
|
|
288
|
+
return counts
|
|
289
|
+
|
|
290
|
+
def _rel_path_for(self, source: InputSourceInfo) -> str:
|
|
291
|
+
"""Project-root-relative POSIX path recorded for a source's identity.
|
|
292
|
+
|
|
293
|
+
Internal files live under ``input_sources/``; external references
|
|
294
|
+
already store a project-root-relative path. Used as the primary
|
|
295
|
+
key for reclaiming a source's original number after its settings
|
|
296
|
+
entry was lost (see :meth:`refresh`).
|
|
297
|
+
"""
|
|
298
|
+
if source.external_rel_path:
|
|
299
|
+
return source.external_rel_path
|
|
300
|
+
return f"input_sources/{source.name}"
|
|
301
|
+
|
|
302
|
+
def refresh(self) -> list[InputSourceInfo]:
|
|
303
|
+
"""Re-scan directory, re-read scenarios, and assign persistent numbers.
|
|
304
|
+
|
|
305
|
+
Numbering is reconciled against two persistent records and the
|
|
306
|
+
result folders on disk:
|
|
307
|
+
|
|
308
|
+
* ``input_source_numbers`` (live name→number) gives a stable number
|
|
309
|
+
to a source that already has one.
|
|
310
|
+
* ``source_registry`` (number→identity) lets a source whose
|
|
311
|
+
settings entry was lost **reclaim its original number** by
|
|
312
|
+
matching on path (then name) — this is what stops a re-added or
|
|
313
|
+
settings-reset source from drifting onto a fresh number and
|
|
314
|
+
orphaning its own results.
|
|
315
|
+
* otherwise the source gets the lowest positive integer not used by
|
|
316
|
+
any live source or any executed-scenario result folder.
|
|
317
|
+
|
|
318
|
+
Numbers whose input file is gone but whose result folders survive
|
|
319
|
+
are emitted as **retired "ghost" rows**. The registry is garbage-
|
|
320
|
+
collected so an entry is dropped once its number has neither a live
|
|
321
|
+
file nor any results. Both records are persisted to settings.yaml.
|
|
322
|
+
Returns live sources followed by ghost rows.
|
|
323
|
+
"""
|
|
324
|
+
from flextool.gui.data_models import SourceRecord
|
|
325
|
+
|
|
326
|
+
sources = self.scan_input_sources()
|
|
327
|
+
registry = dict(self.settings.source_registry)
|
|
328
|
+
disk_counts = self._disk_source_counts()
|
|
329
|
+
prior_numbers = dict(self.settings.input_source_numbers)
|
|
330
|
+
|
|
331
|
+
rel_paths = {id(s): self._rel_path_for(s) for s in sources}
|
|
332
|
+
assigned: dict[int, InputSourceInfo] = {} # number -> source
|
|
333
|
+
|
|
334
|
+
# Pass A — keep the number a live source already holds (stability).
|
|
335
|
+
pending: list[InputSourceInfo] = []
|
|
336
|
+
for source in sources:
|
|
337
|
+
num = prior_numbers.get(source.name)
|
|
338
|
+
if num is not None and num not in assigned:
|
|
339
|
+
source.number = num
|
|
340
|
+
assigned[num] = source
|
|
341
|
+
else:
|
|
342
|
+
pending.append(source)
|
|
343
|
+
|
|
344
|
+
# Pass B — reclaim an original number from the registry by matching
|
|
345
|
+
# on path first, then name. Skips numbers already taken this pass.
|
|
346
|
+
def _reclaim(source: InputSourceInfo) -> int | None:
|
|
347
|
+
path = rel_paths[id(source)]
|
|
348
|
+
by_path = sorted(
|
|
349
|
+
int(k) for k, rec in registry.items()
|
|
350
|
+
if rec.path and rec.path == path and int(k) not in assigned
|
|
351
|
+
)
|
|
352
|
+
if by_path:
|
|
353
|
+
return by_path[0]
|
|
354
|
+
by_name = sorted(
|
|
355
|
+
int(k) for k, rec in registry.items()
|
|
356
|
+
if rec.name == source.name and int(k) not in assigned
|
|
357
|
+
)
|
|
358
|
+
if by_name:
|
|
359
|
+
return by_name[0]
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
still_pending: list[InputSourceInfo] = []
|
|
363
|
+
for source in pending:
|
|
364
|
+
num = _reclaim(source)
|
|
365
|
+
if num is not None:
|
|
366
|
+
source.number = num
|
|
367
|
+
assigned[num] = source
|
|
368
|
+
else:
|
|
369
|
+
still_pending.append(source)
|
|
370
|
+
|
|
371
|
+
# Pass C — allocate the lowest free number, avoiding live sources
|
|
372
|
+
# and any result folders on disk (orphaned results of other files).
|
|
373
|
+
reserved = set(assigned) | set(disk_counts)
|
|
374
|
+
|
|
375
|
+
def _next_free() -> int:
|
|
376
|
+
n = 1
|
|
377
|
+
while n in reserved:
|
|
378
|
+
n += 1
|
|
379
|
+
return n
|
|
380
|
+
|
|
381
|
+
for source in still_pending:
|
|
382
|
+
source.number = _next_free()
|
|
383
|
+
assigned[source.number] = source
|
|
384
|
+
reserved.add(source.number)
|
|
385
|
+
|
|
386
|
+
# Read scenarios for each live source and determine status.
|
|
387
|
+
for source in sources:
|
|
388
|
+
filepath = self.resolve_path(source)
|
|
389
|
+
if not filepath.exists():
|
|
390
|
+
source.status = "error"
|
|
391
|
+
continue
|
|
392
|
+
|
|
393
|
+
# Check for lock file (editing indicator)
|
|
394
|
+
if self._check_lock(filepath):
|
|
395
|
+
source.status = "editing"
|
|
396
|
+
continue
|
|
397
|
+
|
|
398
|
+
scenarios = self._read_scenarios(source, filepath)
|
|
399
|
+
if scenarios is None:
|
|
400
|
+
source.status = "error"
|
|
401
|
+
elif len(scenarios) == 0:
|
|
402
|
+
source.status = "empty"
|
|
403
|
+
else:
|
|
404
|
+
source.status = "ok"
|
|
405
|
+
source.scenarios = scenarios
|
|
406
|
+
|
|
407
|
+
# Rebuild the live name→number index from scratch so deleted files
|
|
408
|
+
# drop out automatically, and refresh each live source's registry
|
|
409
|
+
# record with its current name + path.
|
|
410
|
+
numbers = {s.name: s.number for s in sources}
|
|
411
|
+
for source in sources:
|
|
412
|
+
registry[str(source.number)] = SourceRecord(
|
|
413
|
+
name=source.name, path=rel_paths[id(source)]
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
# Emit ghost rows for numbers whose results survive but whose input
|
|
417
|
+
# file is gone, and ensure each such number has a (possibly empty)
|
|
418
|
+
# registry record so it is documented in settings.yaml.
|
|
419
|
+
ghosts: list[InputSourceInfo] = []
|
|
420
|
+
for number in sorted(disk_counts):
|
|
421
|
+
if number in assigned:
|
|
422
|
+
continue # a live source owns this number
|
|
423
|
+
rec = registry.get(str(number))
|
|
424
|
+
if rec is None:
|
|
425
|
+
rec = SourceRecord() # legacy orphan: file identity unknown
|
|
426
|
+
registry[str(number)] = rec
|
|
427
|
+
ext = Path(rec.name).suffix.lstrip(".").lower() if rec.name else ""
|
|
428
|
+
ghosts.append(
|
|
429
|
+
InputSourceInfo(
|
|
430
|
+
name=rec.name or f"(source {number})",
|
|
431
|
+
file_type=ext,
|
|
432
|
+
number=number,
|
|
433
|
+
status="retired",
|
|
434
|
+
external_rel_path=None,
|
|
435
|
+
retired=True,
|
|
436
|
+
result_count=disk_counts[number],
|
|
437
|
+
)
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
# Garbage-collect registry entries no longer backed by a live file
|
|
441
|
+
# or any result folders — so empty ghosts never accumulate.
|
|
442
|
+
keep = set(assigned) | set(disk_counts)
|
|
443
|
+
registry = {k: v for k, v in registry.items() if int(k) in keep}
|
|
444
|
+
|
|
445
|
+
# Persist reconciled records.
|
|
446
|
+
self.settings.input_source_numbers = numbers
|
|
447
|
+
self.settings.source_registry = registry
|
|
448
|
+
save_project_settings(self.project_path, self.settings)
|
|
449
|
+
|
|
450
|
+
self._sources = sources
|
|
451
|
+
self._last_ghosts = ghosts
|
|
452
|
+
return sources + ghosts
|
|
453
|
+
|
|
454
|
+
def result_folders_for_number(self, number: int) -> list[tuple[Path, str]]:
|
|
455
|
+
"""List ``(folder, scenario_name)`` result dirs owned by *number*.
|
|
456
|
+
|
|
457
|
+
Honours legacy bare-owner records, so a bare folder owned by
|
|
458
|
+
*number* is included alongside suffixed folders.
|
|
459
|
+
"""
|
|
460
|
+
from flextool.gui.scenario_key import resolve_source_number
|
|
461
|
+
|
|
462
|
+
parquet = self.project_path / "output_parquet"
|
|
463
|
+
out: list[tuple[Path, str]] = []
|
|
464
|
+
if not parquet.is_dir():
|
|
465
|
+
return out
|
|
466
|
+
owners = self.settings.bare_output_owners
|
|
467
|
+
for entry in sorted(parquet.iterdir()):
|
|
468
|
+
if not entry.is_dir() or entry.name.startswith("_"):
|
|
469
|
+
continue
|
|
470
|
+
n, scen = resolve_source_number(entry.name, owners)
|
|
471
|
+
if n == number:
|
|
472
|
+
out.append((entry, scen))
|
|
473
|
+
return out
|
|
474
|
+
|
|
475
|
+
def _release_bare_owners_for_number(self, number: int) -> None:
|
|
476
|
+
"""Drop any legacy bare-owner records pointing at *number*."""
|
|
477
|
+
owners = self.settings.bare_output_owners
|
|
478
|
+
for scen in [k for k, v in owners.items() if v == number]:
|
|
479
|
+
del owners[scen]
|
|
480
|
+
|
|
481
|
+
def delete_results(self, number: int) -> int:
|
|
482
|
+
"""Delete every result folder attributed to *number*.
|
|
483
|
+
|
|
484
|
+
Releases any legacy bare-owner records for the number and persists
|
|
485
|
+
settings. Returns the count of folders removed. The caller should
|
|
486
|
+
refresh afterwards; the now-unbacked registry entry is GC'd then.
|
|
487
|
+
"""
|
|
488
|
+
folders = self.result_folders_for_number(number)
|
|
489
|
+
for folder, _scen in folders:
|
|
490
|
+
shutil.rmtree(folder, ignore_errors=True)
|
|
491
|
+
self._release_bare_owners_for_number(number)
|
|
492
|
+
save_project_settings(self.project_path, self.settings)
|
|
493
|
+
return len(folders)
|
|
494
|
+
|
|
495
|
+
def relink_results(
|
|
496
|
+
self, old_number: int, new_number: int
|
|
497
|
+
) -> tuple[int, list[str]]:
|
|
498
|
+
"""Re-attribute *old_number*'s result folders to *new_number*.
|
|
499
|
+
|
|
500
|
+
Renames each folder to the self-describing ``<scenario>_<new>``
|
|
501
|
+
form. Returns ``(moved_count, conflicts)`` where *conflicts* lists
|
|
502
|
+
scenarios skipped because a folder already existed under the target
|
|
503
|
+
number (or could not be moved). Releases legacy bare-owner records
|
|
504
|
+
for the old number and persists settings.
|
|
505
|
+
"""
|
|
506
|
+
from flextool.gui.scenario_key import format_subdir
|
|
507
|
+
|
|
508
|
+
parquet = self.project_path / "output_parquet"
|
|
509
|
+
moved = 0
|
|
510
|
+
conflicts: list[str] = []
|
|
511
|
+
for folder, scen in self.result_folders_for_number(old_number):
|
|
512
|
+
target = parquet / format_subdir(new_number, scen)
|
|
513
|
+
if target.exists():
|
|
514
|
+
conflicts.append(scen)
|
|
515
|
+
continue
|
|
516
|
+
try:
|
|
517
|
+
folder.rename(target)
|
|
518
|
+
moved += 1
|
|
519
|
+
except OSError as exc:
|
|
520
|
+
conflicts.append(f"{scen} ({exc})")
|
|
521
|
+
self._release_bare_owners_for_number(old_number)
|
|
522
|
+
save_project_settings(self.project_path, self.settings)
|
|
523
|
+
return moved, conflicts
|
|
524
|
+
|
|
525
|
+
def get_all_scenarios(
|
|
526
|
+
self, selected_sources: list[str] | None = None
|
|
527
|
+
) -> list[ScenarioInfo]:
|
|
528
|
+
"""Get all scenarios, optionally filtered by source names.
|
|
529
|
+
|
|
530
|
+
If *selected_sources* is ``None`` or empty, returns scenarios from
|
|
531
|
+
all sources.
|
|
532
|
+
"""
|
|
533
|
+
result: list[ScenarioInfo] = []
|
|
534
|
+
for source in self._sources:
|
|
535
|
+
if selected_sources and source.name not in selected_sources:
|
|
536
|
+
continue
|
|
537
|
+
if source.status != "ok":
|
|
538
|
+
continue
|
|
539
|
+
for scenario_name in source.scenarios:
|
|
540
|
+
result.append(
|
|
541
|
+
ScenarioInfo(
|
|
542
|
+
name=scenario_name,
|
|
543
|
+
source_number=source.number,
|
|
544
|
+
source_name=source.name,
|
|
545
|
+
)
|
|
546
|
+
)
|
|
547
|
+
return result
|
|
548
|
+
|
|
549
|
+
def plan_db_migrations(
|
|
550
|
+
self,
|
|
551
|
+
) -> tuple[
|
|
552
|
+
list[tuple[str, Path]],
|
|
553
|
+
list[tuple[str, Path, int, int]],
|
|
554
|
+
list[str],
|
|
555
|
+
]:
|
|
556
|
+
"""Probe sqlite input sources for needed FlexTool DB migrations.
|
|
557
|
+
|
|
558
|
+
Returns ``(internal_to_migrate, external_to_migrate, planning_messages)``.
|
|
559
|
+
Internal entries are ``(source_name, abs_path)``; external entries
|
|
560
|
+
include current and target version numbers for the consent dialog.
|
|
561
|
+
"""
|
|
562
|
+
from flextool.gui.db_version_check import (
|
|
563
|
+
_read_flextool_version,
|
|
564
|
+
get_target_flextool_version,
|
|
565
|
+
needs_flextool_migration,
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
internal_to_migrate: list[tuple[str, Path]] = []
|
|
569
|
+
external_to_migrate: list[tuple[str, Path, int, int]] = []
|
|
570
|
+
planning_messages: list[str] = []
|
|
571
|
+
|
|
572
|
+
target_version = get_target_flextool_version()
|
|
573
|
+
|
|
574
|
+
if self.input_dir.is_dir():
|
|
575
|
+
for filepath in sorted(self.input_dir.iterdir()):
|
|
576
|
+
if filepath.suffix.lower() != ".sqlite" or not filepath.is_file():
|
|
577
|
+
continue
|
|
578
|
+
needs = needs_flextool_migration(filepath)
|
|
579
|
+
if needs is True:
|
|
580
|
+
internal_to_migrate.append((filepath.name, filepath))
|
|
581
|
+
elif needs is None:
|
|
582
|
+
planning_messages.append(
|
|
583
|
+
f"{filepath.name}: could not read FlexTool version; "
|
|
584
|
+
f"migration will attempt to surface the error."
|
|
585
|
+
)
|
|
586
|
+
internal_to_migrate.append((filepath.name, filepath))
|
|
587
|
+
|
|
588
|
+
for ext_name, rel_path in self.settings.external_refs.items():
|
|
589
|
+
if Path(ext_name).suffix.lower() != ".sqlite":
|
|
590
|
+
continue
|
|
591
|
+
abs_path = (self.project_path / rel_path).resolve()
|
|
592
|
+
if not abs_path.is_file():
|
|
593
|
+
continue
|
|
594
|
+
needs = needs_flextool_migration(abs_path)
|
|
595
|
+
if needs is True:
|
|
596
|
+
current_version = _read_flextool_version(f"sqlite:///{abs_path}")
|
|
597
|
+
if current_version is None:
|
|
598
|
+
planning_messages.append(
|
|
599
|
+
f"{ext_name}: external file's FlexTool version "
|
|
600
|
+
f"became unreadable between checks; skipped."
|
|
601
|
+
)
|
|
602
|
+
continue
|
|
603
|
+
external_to_migrate.append(
|
|
604
|
+
(ext_name, abs_path, current_version, target_version)
|
|
605
|
+
)
|
|
606
|
+
elif needs is None:
|
|
607
|
+
planning_messages.append(
|
|
608
|
+
f"{ext_name}: could not read external file's FlexTool "
|
|
609
|
+
f"version; skipped (open it in a tool that can read the "
|
|
610
|
+
f"file to investigate)."
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
return internal_to_migrate, external_to_migrate, planning_messages
|
|
614
|
+
|
|
615
|
+
def copy_external_to_project(
|
|
616
|
+
self, source_name: str
|
|
617
|
+
) -> tuple[Path, str | None]:
|
|
618
|
+
"""Copy an external sqlite into ``input_sources/`` and drop the ref.
|
|
619
|
+
|
|
620
|
+
Returns ``(destination_path, error)``. ``error`` is ``None`` on
|
|
621
|
+
success. The caller is responsible for persisting ``self.settings``.
|
|
622
|
+
"""
|
|
623
|
+
rel_path = self.settings.external_refs.get(source_name)
|
|
624
|
+
if rel_path is None:
|
|
625
|
+
return (Path(), f"External reference '{source_name}' not found.")
|
|
626
|
+
|
|
627
|
+
src = (self.project_path / rel_path).resolve()
|
|
628
|
+
dst = self.input_dir / source_name
|
|
629
|
+
|
|
630
|
+
if dst.exists():
|
|
631
|
+
return (
|
|
632
|
+
dst,
|
|
633
|
+
f"Cannot copy {source_name}: a file with that name "
|
|
634
|
+
f"already exists in input_sources.",
|
|
635
|
+
)
|
|
636
|
+
|
|
637
|
+
try:
|
|
638
|
+
shutil.copy2(src, dst)
|
|
639
|
+
except OSError as exc:
|
|
640
|
+
return (dst, f"Failed to copy {source_name}: {exc}")
|
|
641
|
+
|
|
642
|
+
del self.settings.external_refs[source_name]
|
|
643
|
+
return (dst, None)
|
|
644
|
+
|
|
645
|
+
# ── Private helpers ───────────────────────────────────────────────
|
|
646
|
+
|
|
647
|
+
def _read_scenarios(
|
|
648
|
+
self, source: InputSourceInfo, filepath: Path
|
|
649
|
+
) -> list[str] | None:
|
|
650
|
+
"""Dispatch scenario reading based on file type."""
|
|
651
|
+
ext = filepath.suffix.lower()
|
|
652
|
+
if ext in (".xlsx", ".ods"):
|
|
653
|
+
return self.read_scenarios_xlsx(filepath)
|
|
654
|
+
elif ext == ".sqlite":
|
|
655
|
+
return self.read_scenarios_sqlite(filepath)
|
|
656
|
+
return None
|
|
657
|
+
|
|
658
|
+
def mark_as_editing(self, source_name: str) -> None:
|
|
659
|
+
"""Mark a source as being edited (store mtime for comparison).
|
|
660
|
+
|
|
661
|
+
Called when the Edit button opens a file, so that subsequent
|
|
662
|
+
refreshes can detect ongoing editing via mtime comparison
|
|
663
|
+
(Linux xlsx fallback).
|
|
664
|
+
"""
|
|
665
|
+
rel = self.settings.external_refs.get(source_name)
|
|
666
|
+
if rel is not None:
|
|
667
|
+
filepath = (self.project_path / rel).resolve()
|
|
668
|
+
else:
|
|
669
|
+
filepath = self.input_dir / source_name
|
|
670
|
+
self._editing_sources.add(source_name)
|
|
671
|
+
try:
|
|
672
|
+
self._last_known_mtimes[source_name] = filepath.stat().st_mtime
|
|
673
|
+
except OSError:
|
|
674
|
+
pass
|
|
675
|
+
|
|
676
|
+
def _check_lock(self, filepath: Path) -> bool:
|
|
677
|
+
"""Detect if the file is currently being edited.
|
|
678
|
+
|
|
679
|
+
For .xlsx / .ods files:
|
|
680
|
+
- Windows/macOS: check for ``~$filename`` lock file in same directory.
|
|
681
|
+
- Linux: check for ``.~lock.filename#`` (LibreOffice lock file).
|
|
682
|
+
- Linux fallback: if the file was opened via the Edit button,
|
|
683
|
+
compare ``st_mtime`` against the stored value. If mtime
|
|
684
|
+
changed, the file is still being edited.
|
|
685
|
+
|
|
686
|
+
For .sqlite files:
|
|
687
|
+
- Check for ``<filepath>-journal`` or ``<filepath>-wal`` files.
|
|
688
|
+
|
|
689
|
+
Returns ``True`` if the file appears to be in an editing state.
|
|
690
|
+
"""
|
|
691
|
+
ext = filepath.suffix.lower()
|
|
692
|
+
|
|
693
|
+
if ext in (".xlsx", ".ods"):
|
|
694
|
+
parent = filepath.parent
|
|
695
|
+
name = filepath.name
|
|
696
|
+
|
|
697
|
+
# Windows / macOS Excel lock file: ~$<filename>
|
|
698
|
+
excel_lock = parent / f"~${name}"
|
|
699
|
+
if excel_lock.exists():
|
|
700
|
+
return True
|
|
701
|
+
|
|
702
|
+
# LibreOffice lock file (Linux): .~lock.<filename>#
|
|
703
|
+
libreoffice_lock = parent / f".~lock.{name}#"
|
|
704
|
+
if libreoffice_lock.exists():
|
|
705
|
+
return True
|
|
706
|
+
|
|
707
|
+
# Linux mtime-based fallback for files opened via Edit button
|
|
708
|
+
if (
|
|
709
|
+
sys.platform not in ("win32", "darwin")
|
|
710
|
+
and name in self._editing_sources
|
|
711
|
+
):
|
|
712
|
+
try:
|
|
713
|
+
current_mtime = filepath.stat().st_mtime
|
|
714
|
+
except OSError:
|
|
715
|
+
return False
|
|
716
|
+
last_mtime = self._last_known_mtimes.get(name)
|
|
717
|
+
if last_mtime is not None and current_mtime != last_mtime:
|
|
718
|
+
# mtime changed -- file was modified externally
|
|
719
|
+
return True
|
|
720
|
+
|
|
721
|
+
return False
|
|
722
|
+
|
|
723
|
+
if ext == ".sqlite":
|
|
724
|
+
journal = filepath.parent / (filepath.name + "-journal")
|
|
725
|
+
wal = filepath.parent / (filepath.name + "-wal")
|
|
726
|
+
if journal.exists() or wal.exists():
|
|
727
|
+
return True
|
|
728
|
+
return False
|
|
729
|
+
|
|
730
|
+
return False
|