flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Helpers for scenario identity across source and display boundaries.
|
|
2
|
+
|
|
3
|
+
Two scenarios with the same name from different input sources must not
|
|
4
|
+
collide on disk. **New folders are always written with the source-number
|
|
5
|
+
suffix** (``output_parquet/base_3/`` rather than ``output_parquet/base/``)
|
|
6
|
+
so the folder name itself records the source identity — losing the
|
|
7
|
+
project settings.yaml or moving the project elsewhere does not strand
|
|
8
|
+
the data.
|
|
9
|
+
|
|
10
|
+
Encodings
|
|
11
|
+
---------
|
|
12
|
+
|
|
13
|
+
* **Compound key** (settings) — ``"<source_number>|<scenario_name>"``.
|
|
14
|
+
Used in project settings (checked_executed_scenarios, comp_*_scenarios,
|
|
15
|
+
etc.). ``|`` cannot appear in a scenario name.
|
|
16
|
+
|
|
17
|
+
* **Subdirectory name** (on disk) — ``<scenario_name>_<source_number>``
|
|
18
|
+
(e.g. ``base_2``). Always. There is no bare-name shortcut for newly-
|
|
19
|
+
written folders.
|
|
20
|
+
|
|
21
|
+
Legacy bare folders
|
|
22
|
+
-------------------
|
|
23
|
+
|
|
24
|
+
Pre-existing projects may carry folders without the suffix (``base/``)
|
|
25
|
+
from the previous "bare-owner" convention. Those are still recognised
|
|
26
|
+
on the READ side via :attr:`ProjectSettings.bare_output_owners`:
|
|
27
|
+
:func:`resolve_source_number` consults that map first so the old bare
|
|
28
|
+
folder still maps to its original source. New writes never reuse a
|
|
29
|
+
bare folder — if scenario "base" runs again it writes to
|
|
30
|
+
``base_<source_number>/`` regardless of whether ``base/`` exists.
|
|
31
|
+
Users with legacy projects should rename or delete the old bare
|
|
32
|
+
folders manually; the GUI flags them as orphans (foreign source
|
|
33
|
+
number) so they are visible.
|
|
34
|
+
|
|
35
|
+
If the user happens to have a scenario name that itself ends in
|
|
36
|
+
``_<digits>`` (e.g. literally named ``foo_3``), parsing is ambiguous
|
|
37
|
+
when there is no owners record — :func:`parse_subdir` will read
|
|
38
|
+
``foo_3`` as "scenario foo from source 3". The owners map disambiguates
|
|
39
|
+
for legacy data; for new writes the suffix is appended, producing
|
|
40
|
+
``foo_3_<source_number>`` which parses unambiguously.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
|
|
47
|
+
LEGACY_SOURCE_NUMBER = 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def format_subdir(source_number: int, scenario_name: str) -> str:
|
|
51
|
+
"""Return the explicit suffixed subdirectory name.
|
|
52
|
+
|
|
53
|
+
This is the "collision" form; prefer :func:`resolve_subdir_for_read`
|
|
54
|
+
or :func:`choose_output_subdir_for_write` which honour bare-name
|
|
55
|
+
ownership.
|
|
56
|
+
"""
|
|
57
|
+
return f"{scenario_name}_{source_number}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def parse_subdir(subdir: str) -> tuple[int, str]:
|
|
61
|
+
"""Split ``<name>_<N>`` into ``(N, name)``.
|
|
62
|
+
|
|
63
|
+
A subdir without a trailing numeric suffix returns
|
|
64
|
+
``(LEGACY_SOURCE_NUMBER, subdir)`` — callers are expected to consult
|
|
65
|
+
the ownership map to recover the correct source number for such bare
|
|
66
|
+
names.
|
|
67
|
+
"""
|
|
68
|
+
base, sep, suffix = subdir.rpartition("_")
|
|
69
|
+
if sep and suffix.isdigit():
|
|
70
|
+
return int(suffix), base
|
|
71
|
+
return LEGACY_SOURCE_NUMBER, subdir
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def format_key(source_number: int, scenario_name: str) -> str:
|
|
75
|
+
"""Return the settings-level compound key."""
|
|
76
|
+
return f"{source_number}|{scenario_name}"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def parse_key(key: str) -> tuple[int, str]:
|
|
80
|
+
"""Split a compound key into ``(source_number, scenario_name)``."""
|
|
81
|
+
prefix, sep, rest = key.partition("|")
|
|
82
|
+
if sep and prefix.isdigit():
|
|
83
|
+
return int(prefix), rest
|
|
84
|
+
return LEGACY_SOURCE_NUMBER, key
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# ---------------------------------------------------------------------------
|
|
88
|
+
# Ownership-aware resolution
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
def resolve_source_number(
|
|
92
|
+
subdir: str,
|
|
93
|
+
bare_output_owners: dict[str, int],
|
|
94
|
+
) -> tuple[int, str]:
|
|
95
|
+
"""Return ``(source_number, scenario_name)`` for an on-disk subdir.
|
|
96
|
+
|
|
97
|
+
Uses ``bare_output_owners`` (a snapshot of the ownership map) to
|
|
98
|
+
recover the source number for bare names that aren't self-describing.
|
|
99
|
+
Falls back to suffix parsing when the subdir isn't a recorded bare
|
|
100
|
+
name.
|
|
101
|
+
"""
|
|
102
|
+
# An explicit ownership record wins — even if the subdir happens to
|
|
103
|
+
# end in ``_<N>``, if that full string is recorded as a bare owner
|
|
104
|
+
# (e.g. a scenario legitimately named "foo_1"), trust the record.
|
|
105
|
+
if subdir in bare_output_owners:
|
|
106
|
+
return bare_output_owners[subdir], subdir
|
|
107
|
+
src, name = parse_subdir(subdir)
|
|
108
|
+
if src != LEGACY_SOURCE_NUMBER:
|
|
109
|
+
return src, name
|
|
110
|
+
# No suffix, no ownership record → legacy bare.
|
|
111
|
+
return LEGACY_SOURCE_NUMBER, subdir
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def resolve_subdir_for_read(
|
|
115
|
+
bare_output_owners: dict[str, int],
|
|
116
|
+
source_number: int,
|
|
117
|
+
scenario_name: str,
|
|
118
|
+
) -> str:
|
|
119
|
+
"""Return the subdir that holds this scenario's outputs (no mutation).
|
|
120
|
+
|
|
121
|
+
Use this when the folder is expected to already exist: reading plots,
|
|
122
|
+
checking output presence, deleting, etc.
|
|
123
|
+
"""
|
|
124
|
+
if bare_output_owners.get(scenario_name) == source_number:
|
|
125
|
+
return scenario_name
|
|
126
|
+
return format_subdir(source_number, scenario_name)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def choose_output_subdir_for_write(
|
|
130
|
+
project_path: Path,
|
|
131
|
+
bare_output_owners: dict[str, int],
|
|
132
|
+
source_number: int,
|
|
133
|
+
scenario_name: str,
|
|
134
|
+
) -> str:
|
|
135
|
+
"""Pick the subdir to write outputs into.
|
|
136
|
+
|
|
137
|
+
Always returns ``<scenario_name>_<source_number>``. The folder name
|
|
138
|
+
is self-describing — losing or corrupting ``bare_output_owners`` no
|
|
139
|
+
longer makes the folder ambiguous about which source it came from.
|
|
140
|
+
|
|
141
|
+
The ``project_path`` and ``bare_output_owners`` parameters are kept
|
|
142
|
+
in the signature for API stability with callers but are unused; the
|
|
143
|
+
only legitimate purpose of the bare-owner map now is to recognise
|
|
144
|
+
legacy bare folders on the READ side (see :func:`resolve_source_number`).
|
|
145
|
+
"""
|
|
146
|
+
del project_path, bare_output_owners # unused — kept for API stability
|
|
147
|
+
return format_subdir(source_number, scenario_name)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def release_bare_owner(
|
|
151
|
+
bare_output_owners: dict[str, int],
|
|
152
|
+
source_number: int,
|
|
153
|
+
scenario_name: str,
|
|
154
|
+
) -> bool:
|
|
155
|
+
"""Drop the ownership record if *source_number* owns ``scenario_name``.
|
|
156
|
+
|
|
157
|
+
Returns True if a record was removed.
|
|
158
|
+
"""
|
|
159
|
+
if bare_output_owners.get(scenario_name) == source_number:
|
|
160
|
+
del bare_output_owners[scenario_name]
|
|
161
|
+
return True
|
|
162
|
+
return False
|
|
@@ -0,0 +1,516 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import shutil
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from tkinter import ttk
|
|
8
|
+
|
|
9
|
+
from flextool.gui.data_models import ExecutedScenarioInfo, ProjectSettings, ScenarioInfo
|
|
10
|
+
from flextool.gui.scenario_key import (
|
|
11
|
+
format_key,
|
|
12
|
+
parse_key,
|
|
13
|
+
release_bare_owner,
|
|
14
|
+
resolve_source_number,
|
|
15
|
+
resolve_subdir_for_read,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
# Unicode checkbox characters (mirrors main_window.py constants)
|
|
21
|
+
# Using geometric shapes (U+25A1 / U+25A3) which render noticeably larger
|
|
22
|
+
# than ballot box characters at the same font size.
|
|
23
|
+
CHECK_ON = "\u25a0" # ■
|
|
24
|
+
CHECK_OFF = "\u25a1" # □
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AvailableScenarioManager:
|
|
28
|
+
"""Manages the available scenarios list with persistent ordering."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, settings: ProjectSettings) -> None:
|
|
31
|
+
self.settings = settings
|
|
32
|
+
self._scenarios: list[ScenarioInfo] = []
|
|
33
|
+
|
|
34
|
+
def update_scenarios(self, scenarios: list[ScenarioInfo]) -> list[ScenarioInfo]:
|
|
35
|
+
"""Update with new scenario list from InputSourceManager.
|
|
36
|
+
|
|
37
|
+
Apply persistent ordering from settings.scenario_order.
|
|
38
|
+
New scenarios (not in saved order) go to the end.
|
|
39
|
+
Scenarios no longer available are removed from the order.
|
|
40
|
+
Returns the ordered list.
|
|
41
|
+
"""
|
|
42
|
+
saved_order: list[str] = list(self.settings.scenario_order)
|
|
43
|
+
scenario_by_key: dict[str, ScenarioInfo] = {}
|
|
44
|
+
for s in scenarios:
|
|
45
|
+
key = f"{s.source_number}|{s.name}"
|
|
46
|
+
scenario_by_key[key] = s
|
|
47
|
+
|
|
48
|
+
ordered: list[ScenarioInfo] = []
|
|
49
|
+
|
|
50
|
+
# First: add scenarios that are in the saved order and still available
|
|
51
|
+
for key in saved_order:
|
|
52
|
+
if key in scenario_by_key:
|
|
53
|
+
ordered.append(scenario_by_key.pop(key))
|
|
54
|
+
|
|
55
|
+
# Second: append new scenarios (not in saved order) at the end
|
|
56
|
+
for s in scenarios:
|
|
57
|
+
key = f"{s.source_number}|{s.name}"
|
|
58
|
+
if key in scenario_by_key:
|
|
59
|
+
ordered.append(scenario_by_key.pop(key))
|
|
60
|
+
|
|
61
|
+
self._scenarios = ordered
|
|
62
|
+
return ordered
|
|
63
|
+
|
|
64
|
+
def get_order(self) -> list[str]:
|
|
65
|
+
"""Return current order as list of 'source_number|scenario_name' keys for persistence."""
|
|
66
|
+
return [f"{s.source_number}|{s.name}" for s in self._scenarios]
|
|
67
|
+
|
|
68
|
+
def get_checked_scenarios(self, tree: ttk.Treeview) -> list[ScenarioInfo]:
|
|
69
|
+
"""Get scenarios that have checkboxes checked in the treeview."""
|
|
70
|
+
checked: list[ScenarioInfo] = []
|
|
71
|
+
scenario_by_key: dict[str, ScenarioInfo] = {
|
|
72
|
+
f"{s.source_number}|{s.name}": s for s in self._scenarios
|
|
73
|
+
}
|
|
74
|
+
for item in tree.get_children():
|
|
75
|
+
values = tree.item(item, "values")
|
|
76
|
+
if values and values[0] == CHECK_ON:
|
|
77
|
+
key = f"{values[1]}|{values[2]}"
|
|
78
|
+
if key in scenario_by_key:
|
|
79
|
+
checked.append(scenario_by_key[key])
|
|
80
|
+
return checked
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def scenarios(self) -> list[ScenarioInfo]:
|
|
84
|
+
"""The current ordered list of scenarios."""
|
|
85
|
+
return list(self._scenarios)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class ExecutedScenarioManager:
|
|
89
|
+
"""Manages the executed scenarios list with output detection."""
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
project_path: Path,
|
|
94
|
+
settings: ProjectSettings | None = None,
|
|
95
|
+
) -> None:
|
|
96
|
+
self.project_path = project_path
|
|
97
|
+
# The ownership map lets us recover (source_number, scenario_name)
|
|
98
|
+
# for bare-named folders. An empty map is fine — bare folders then
|
|
99
|
+
# parse as ``source_number = LEGACY_SOURCE_NUMBER``.
|
|
100
|
+
self._bare_owners: dict[str, int] = (
|
|
101
|
+
settings.bare_output_owners if settings is not None else {}
|
|
102
|
+
)
|
|
103
|
+
# Retain the full settings reference so mutating writers (delete_results)
|
|
104
|
+
# can update it in place when releasing ownership.
|
|
105
|
+
self._settings = settings
|
|
106
|
+
|
|
107
|
+
def scan_executed(self) -> list[ExecutedScenarioInfo]:
|
|
108
|
+
"""Scan output_parquet/ for subdirectories.
|
|
109
|
+
|
|
110
|
+
Each subdir is either the bare scenario name (preferred when the
|
|
111
|
+
scenario name is uncontested for this source) or
|
|
112
|
+
``<scenario_name>_<source_number>`` when another source already
|
|
113
|
+
owns the bare name. Ownership is recorded in project settings and
|
|
114
|
+
is used here to reverse-map bare folders to their source.
|
|
115
|
+
"""
|
|
116
|
+
parquet_dir = self.project_path / "output_parquet"
|
|
117
|
+
if not parquet_dir.is_dir():
|
|
118
|
+
return []
|
|
119
|
+
|
|
120
|
+
results: list[ExecutedScenarioInfo] = []
|
|
121
|
+
for entry in sorted(parquet_dir.iterdir()):
|
|
122
|
+
if not entry.is_dir():
|
|
123
|
+
continue
|
|
124
|
+
if entry.name.startswith("_"):
|
|
125
|
+
# Skip manifest files/directories (e.g. _axis_bounds.json).
|
|
126
|
+
continue
|
|
127
|
+
try:
|
|
128
|
+
file_mtimes = [
|
|
129
|
+
f.stat().st_mtime
|
|
130
|
+
for f in entry.iterdir()
|
|
131
|
+
if f.is_file()
|
|
132
|
+
]
|
|
133
|
+
mtime = max(file_mtimes) if file_mtimes else entry.stat().st_mtime
|
|
134
|
+
ts = datetime.fromtimestamp(mtime).strftime("%d.%m.%y %H:%M")
|
|
135
|
+
except OSError:
|
|
136
|
+
ts = ""
|
|
137
|
+
|
|
138
|
+
source_number, scenario_name = resolve_source_number(
|
|
139
|
+
entry.name, self._bare_owners
|
|
140
|
+
)
|
|
141
|
+
results.append(
|
|
142
|
+
ExecutedScenarioInfo(
|
|
143
|
+
name=scenario_name,
|
|
144
|
+
source_number=source_number,
|
|
145
|
+
timestamp=ts,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
return results
|
|
150
|
+
|
|
151
|
+
def check_outputs(
|
|
152
|
+
self, scenario_ids: list[tuple[int, str]]
|
|
153
|
+
) -> dict[str, dict[str, bool]]:
|
|
154
|
+
"""For each scenario, check which outputs exist.
|
|
155
|
+
|
|
156
|
+
*scenario_ids* is a list of ``(source_number, scenario_name)`` pairs.
|
|
157
|
+
The result dict is keyed by the compound ``"<src#>|<name>"`` key.
|
|
158
|
+
"""
|
|
159
|
+
result: dict[str, dict[str, bool]] = {}
|
|
160
|
+
for source_number, name in scenario_ids:
|
|
161
|
+
subdir = resolve_subdir_for_read(self._bare_owners, source_number, name)
|
|
162
|
+
|
|
163
|
+
plots_dir = self.project_path / "output_plots" / subdir
|
|
164
|
+
has_plots = plots_dir.is_dir() and any(plots_dir.iterdir())
|
|
165
|
+
|
|
166
|
+
excel_dir = self.project_path / "output_excel"
|
|
167
|
+
has_excel = False
|
|
168
|
+
if excel_dir.is_dir():
|
|
169
|
+
for f in excel_dir.iterdir():
|
|
170
|
+
if f.suffix.lower() == ".xlsx" and subdir in f.stem:
|
|
171
|
+
has_excel = True
|
|
172
|
+
break
|
|
173
|
+
|
|
174
|
+
csv_dir = self.project_path / "output_csv" / subdir
|
|
175
|
+
has_csvs = csv_dir.is_dir() and any(csv_dir.iterdir())
|
|
176
|
+
|
|
177
|
+
result[format_key(source_number, name)] = {
|
|
178
|
+
"has_plots": has_plots,
|
|
179
|
+
"has_excel": has_excel,
|
|
180
|
+
"has_csvs": has_csvs,
|
|
181
|
+
}
|
|
182
|
+
return result
|
|
183
|
+
|
|
184
|
+
def check_comparison_outputs(
|
|
185
|
+
self, scenario_names: list[str]
|
|
186
|
+
) -> dict[str, bool]:
|
|
187
|
+
"""Check if comparison outputs exist for this set of scenarios.
|
|
188
|
+
|
|
189
|
+
Returns: {has_comp_plots: bool, has_comp_excel: bool,
|
|
190
|
+
has_comp_spinedb: bool}
|
|
191
|
+
"""
|
|
192
|
+
comp_dir = self.project_path / "output_plot_comparisons"
|
|
193
|
+
|
|
194
|
+
has_comp_plots = False
|
|
195
|
+
has_comp_excel = False
|
|
196
|
+
|
|
197
|
+
if comp_dir.is_dir():
|
|
198
|
+
for f in comp_dir.iterdir():
|
|
199
|
+
if f.is_file():
|
|
200
|
+
if f.suffix.lower() in (".png", ".svg", ".pdf", ".html"):
|
|
201
|
+
has_comp_plots = True
|
|
202
|
+
break
|
|
203
|
+
|
|
204
|
+
# Check for comparison Excel in the project root directory
|
|
205
|
+
if self.project_path.is_dir():
|
|
206
|
+
for f in self.project_path.iterdir():
|
|
207
|
+
if (
|
|
208
|
+
f.is_file()
|
|
209
|
+
and f.suffix.lower() == ".xlsx"
|
|
210
|
+
and f.stem.startswith("compare_")
|
|
211
|
+
):
|
|
212
|
+
has_comp_excel = True
|
|
213
|
+
break
|
|
214
|
+
|
|
215
|
+
# Single project-wide SpineDB results database written during the
|
|
216
|
+
# solve (see write_spinedb). Existence-only check — it accumulates
|
|
217
|
+
# all executed scenarios as separate alternatives.
|
|
218
|
+
has_comp_spinedb = (self.project_path / "results.sqlite").is_file()
|
|
219
|
+
|
|
220
|
+
return {
|
|
221
|
+
"has_comp_plots": has_comp_plots,
|
|
222
|
+
"has_comp_excel": has_comp_excel,
|
|
223
|
+
"has_comp_spinedb": has_comp_spinedb,
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
def delete_results(self, scenario_ids: list[tuple[int, str]]) -> None:
|
|
227
|
+
"""Delete all output files for given scenarios.
|
|
228
|
+
|
|
229
|
+
Removes ``output_parquet/<subdir>/``, ``output_plots/<subdir>/``,
|
|
230
|
+
``output_csv/<subdir>/`` and any matching ``output_excel/*<subdir>*.xlsx``.
|
|
231
|
+
When the deleted folder is the bare-named one owned by the given
|
|
232
|
+
source, the ownership record is released so another source can
|
|
233
|
+
claim the bare name later.
|
|
234
|
+
|
|
235
|
+
Resolution is robust to drift between ``bare_output_owners`` and
|
|
236
|
+
the actual on-disk layout: we forward-scan ``output_parquet/`` and
|
|
237
|
+
match each entry's ``(source_number, scenario_name)`` against the
|
|
238
|
+
request. The owners map is consulted only as a fallback for the
|
|
239
|
+
idempotent case where the folder is already gone.
|
|
240
|
+
|
|
241
|
+
Also strips the deleted scenario from the shared axis-bounds
|
|
242
|
+
manifest (``output_parquet/_axis_bounds.json``) so it
|
|
243
|
+
doesn't continue to influence y-axis ranges in the viewer.
|
|
244
|
+
"""
|
|
245
|
+
# Forward-scan output_parquet/ once to map (src#, name) -> subdir
|
|
246
|
+
# AND collect a name-keyed fallback (for cases where the ownership
|
|
247
|
+
# record disagrees with the caller-supplied source_number).
|
|
248
|
+
parquet_root = self.project_path / "output_parquet"
|
|
249
|
+
on_disk: dict[tuple[int, str], str] = {}
|
|
250
|
+
on_disk_by_name: dict[str, list[str]] = {}
|
|
251
|
+
if parquet_root.is_dir():
|
|
252
|
+
for entry in parquet_root.iterdir():
|
|
253
|
+
if not entry.is_dir() or entry.name.startswith("_"):
|
|
254
|
+
continue
|
|
255
|
+
src, scen_name = resolve_source_number(
|
|
256
|
+
entry.name, self._bare_owners
|
|
257
|
+
)
|
|
258
|
+
on_disk[(src, scen_name)] = entry.name
|
|
259
|
+
on_disk_by_name.setdefault(scen_name, []).append(entry.name)
|
|
260
|
+
|
|
261
|
+
for source_number, name in scenario_ids:
|
|
262
|
+
key = (source_number, name)
|
|
263
|
+
if key in on_disk:
|
|
264
|
+
subdir = on_disk[key]
|
|
265
|
+
elif name in on_disk_by_name and len(on_disk_by_name[name]) == 1:
|
|
266
|
+
# Stale owners record: only one folder matches the
|
|
267
|
+
# scenario name (so it must be the one the caller meant
|
|
268
|
+
# even though the (src#, name) tuple doesn't match the
|
|
269
|
+
# ownership record). Trust the caller, drop the stale
|
|
270
|
+
# record below if applicable.
|
|
271
|
+
subdir = on_disk_by_name[name][0]
|
|
272
|
+
logger.warning(
|
|
273
|
+
"delete_results: ownership map disagreed for "
|
|
274
|
+
"(source=%d, scenario=%r); using on-disk folder %r",
|
|
275
|
+
source_number, name, subdir,
|
|
276
|
+
)
|
|
277
|
+
else:
|
|
278
|
+
# Folder isn't on disk under any matching encoding — fall back
|
|
279
|
+
# to the owners-map resolution (idempotent no-op when the
|
|
280
|
+
# folder has already been deleted) and log a warning so the
|
|
281
|
+
# caller can still flush dangling settings state.
|
|
282
|
+
subdir = resolve_subdir_for_read(
|
|
283
|
+
self._bare_owners, source_number, name
|
|
284
|
+
)
|
|
285
|
+
logger.warning(
|
|
286
|
+
"delete_results: no on-disk folder found for "
|
|
287
|
+
"(source=%d, scenario=%r); falling back to %r",
|
|
288
|
+
source_number, name, subdir,
|
|
289
|
+
)
|
|
290
|
+
was_bare_owner = subdir == name
|
|
291
|
+
|
|
292
|
+
parquet_dir = self.project_path / "output_parquet" / subdir
|
|
293
|
+
if parquet_dir.is_dir():
|
|
294
|
+
shutil.rmtree(parquet_dir, ignore_errors=True)
|
|
295
|
+
logger.info("Deleted %s", parquet_dir)
|
|
296
|
+
|
|
297
|
+
plots_dir = self.project_path / "output_plots" / subdir
|
|
298
|
+
if plots_dir.is_dir():
|
|
299
|
+
shutil.rmtree(plots_dir, ignore_errors=True)
|
|
300
|
+
logger.info("Deleted %s", plots_dir)
|
|
301
|
+
|
|
302
|
+
excel_dir = self.project_path / "output_excel"
|
|
303
|
+
if excel_dir.is_dir():
|
|
304
|
+
for f in excel_dir.iterdir():
|
|
305
|
+
if f.suffix.lower() == ".xlsx" and subdir in f.stem:
|
|
306
|
+
try:
|
|
307
|
+
f.unlink()
|
|
308
|
+
logger.info("Deleted %s", f)
|
|
309
|
+
except OSError as exc:
|
|
310
|
+
logger.warning("Could not delete %s: %s", f, exc)
|
|
311
|
+
|
|
312
|
+
csv_dir = self.project_path / "output_csv" / subdir
|
|
313
|
+
if csv_dir.is_dir():
|
|
314
|
+
shutil.rmtree(csv_dir, ignore_errors=True)
|
|
315
|
+
logger.info("Deleted %s", csv_dir)
|
|
316
|
+
|
|
317
|
+
# Strip scenario from the shared axis-bounds manifest. The
|
|
318
|
+
# manifest uses the on-disk folder name (``subdir``) as the
|
|
319
|
+
# scenario key, matching how ``ManifestAccumulator`` stores it.
|
|
320
|
+
try:
|
|
321
|
+
from flextool.plot_outputs.shared_manifest import (
|
|
322
|
+
remove_scenario_from_manifest,
|
|
323
|
+
)
|
|
324
|
+
remove_scenario_from_manifest(self.project_path, subdir)
|
|
325
|
+
except Exception as exc: # noqa: BLE001
|
|
326
|
+
logger.warning(
|
|
327
|
+
"Failed to strip %s from shared axis manifest: %s",
|
|
328
|
+
subdir, exc,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
if was_bare_owner:
|
|
332
|
+
# Release the bare-name claim. If the ownership record was
|
|
333
|
+
# stale (different source), drop it anyway — the folder is
|
|
334
|
+
# gone, so no other reader could honour the old record.
|
|
335
|
+
if not release_bare_owner(self._bare_owners, source_number, name):
|
|
336
|
+
if name in self._bare_owners:
|
|
337
|
+
del self._bare_owners[name]
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def prune_dangling_scenario_state(
|
|
341
|
+
settings: ProjectSettings,
|
|
342
|
+
available_keys: set[tuple[int, str]],
|
|
343
|
+
executed_subdirs: set[str],
|
|
344
|
+
) -> bool:
|
|
345
|
+
"""Drop scenario references from settings that no longer have a backing.
|
|
346
|
+
|
|
347
|
+
Rule: a scenario stays in settings.yaml as long as it's EITHER an
|
|
348
|
+
available scenario in some input source OR has on-disk results.
|
|
349
|
+
Once both conditions become false, all references in settings.yaml
|
|
350
|
+
are removed.
|
|
351
|
+
|
|
352
|
+
Parameters
|
|
353
|
+
----------
|
|
354
|
+
settings:
|
|
355
|
+
The project settings to mutate in place.
|
|
356
|
+
available_keys:
|
|
357
|
+
Set of ``(source_number, scenario_name)`` pairs that are present
|
|
358
|
+
in some loaded input source's available-scenarios list.
|
|
359
|
+
executed_subdirs:
|
|
360
|
+
Set of on-disk subdir names under ``output_parquet/`` (already
|
|
361
|
+
filtered to skip ``_``-prefixed entries).
|
|
362
|
+
|
|
363
|
+
Order of operations
|
|
364
|
+
-------------------
|
|
365
|
+
Snapshot ``bare_output_owners`` at the start and use the snapshot
|
|
366
|
+
consistently throughout for resolving compound keys to subdirs. The
|
|
367
|
+
live ``settings.bare_output_owners`` map is then pruned LAST. This
|
|
368
|
+
avoids the "I just removed the ownership record I was about to use
|
|
369
|
+
to resolve another list" footgun.
|
|
370
|
+
|
|
371
|
+
Returns True if any changes were made.
|
|
372
|
+
"""
|
|
373
|
+
changed = False
|
|
374
|
+
|
|
375
|
+
# Snapshot the ownership map so all resolutions use a consistent view,
|
|
376
|
+
# regardless of the prune order below.
|
|
377
|
+
owners_snapshot: dict[str, int] = dict(settings.bare_output_owners)
|
|
378
|
+
|
|
379
|
+
# Set of available scenario names (ignoring source number) for lists
|
|
380
|
+
# that hold bare names rather than compound keys.
|
|
381
|
+
available_names: set[str] = {name for _, name in available_keys}
|
|
382
|
+
|
|
383
|
+
# Helper: does (src, name) correspond to an existing on-disk subdir?
|
|
384
|
+
def _has_executed_subdir(source_number: int, scenario_name: str) -> bool:
|
|
385
|
+
subdir = resolve_subdir_for_read(
|
|
386
|
+
owners_snapshot, source_number, scenario_name
|
|
387
|
+
)
|
|
388
|
+
return subdir in executed_subdirs
|
|
389
|
+
|
|
390
|
+
# 1. scenario_order: drop names not in any input source's scenarios.
|
|
391
|
+
new_scenario_order = [
|
|
392
|
+
n for n in settings.scenario_order if n in available_names
|
|
393
|
+
]
|
|
394
|
+
if new_scenario_order != settings.scenario_order:
|
|
395
|
+
settings.scenario_order = new_scenario_order
|
|
396
|
+
changed = True
|
|
397
|
+
|
|
398
|
+
# 2. comp_plots / comp_excel / comp_viewer scenarios: drop entries
|
|
399
|
+
# whose subdir is not in executed_subdirs. These lists hold either
|
|
400
|
+
# bare scenario names or compound "src#|name" keys depending on
|
|
401
|
+
# when they were written; handle both forms.
|
|
402
|
+
for attr in ("comp_plots_scenarios", "comp_excel_scenarios", "comp_viewer_scenarios"):
|
|
403
|
+
old: list[str] = list(getattr(settings, attr))
|
|
404
|
+
new: list[str] = []
|
|
405
|
+
for entry in old:
|
|
406
|
+
if "|" in entry:
|
|
407
|
+
src, scen_name = parse_key(entry)
|
|
408
|
+
if _has_executed_subdir(src, scen_name):
|
|
409
|
+
new.append(entry)
|
|
410
|
+
else:
|
|
411
|
+
# Bare name — keep if it matches any executed subdir
|
|
412
|
+
# directly or via owners_snapshot.
|
|
413
|
+
if entry in executed_subdirs:
|
|
414
|
+
new.append(entry)
|
|
415
|
+
continue
|
|
416
|
+
# Try to find any (src, entry) with an executed subdir
|
|
417
|
+
kept = False
|
|
418
|
+
for src, scen_name in available_keys:
|
|
419
|
+
if scen_name == entry and _has_executed_subdir(src, scen_name):
|
|
420
|
+
kept = True
|
|
421
|
+
break
|
|
422
|
+
if not kept:
|
|
423
|
+
# Last-ditch: try owner-based resolution with each
|
|
424
|
+
# known source number from owners_snapshot.
|
|
425
|
+
src_for_name = owners_snapshot.get(entry)
|
|
426
|
+
if src_for_name is not None and _has_executed_subdir(
|
|
427
|
+
src_for_name, entry
|
|
428
|
+
):
|
|
429
|
+
kept = True
|
|
430
|
+
if kept:
|
|
431
|
+
new.append(entry)
|
|
432
|
+
if new != old:
|
|
433
|
+
setattr(settings, attr, new)
|
|
434
|
+
changed = True
|
|
435
|
+
|
|
436
|
+
# 3. checked_available_scenarios: compound "src#|name"; drop those
|
|
437
|
+
# whose (src, name) is not in available_keys.
|
|
438
|
+
new_checked_avail = [
|
|
439
|
+
k for k in settings.checked_available_scenarios
|
|
440
|
+
if parse_key(k) in available_keys
|
|
441
|
+
]
|
|
442
|
+
if new_checked_avail != settings.checked_available_scenarios:
|
|
443
|
+
settings.checked_available_scenarios = new_checked_avail
|
|
444
|
+
changed = True
|
|
445
|
+
|
|
446
|
+
# 4. checked_executed_scenarios: compound "src#|name"; drop those
|
|
447
|
+
# whose (src, name) does not correspond to any executed_subdir.
|
|
448
|
+
new_checked_exec = [
|
|
449
|
+
k for k in settings.checked_executed_scenarios
|
|
450
|
+
if _has_executed_subdir(*parse_key(k))
|
|
451
|
+
]
|
|
452
|
+
if new_checked_exec != settings.checked_executed_scenarios:
|
|
453
|
+
settings.checked_executed_scenarios = new_checked_exec
|
|
454
|
+
changed = True
|
|
455
|
+
|
|
456
|
+
# 5. scenario_resource_history: keyed by output subdir; drop missing.
|
|
457
|
+
new_resource_history = {
|
|
458
|
+
k: v for k, v in settings.scenario_resource_history.items()
|
|
459
|
+
if k in executed_subdirs
|
|
460
|
+
}
|
|
461
|
+
if new_resource_history != settings.scenario_resource_history:
|
|
462
|
+
settings.scenario_resource_history = new_resource_history
|
|
463
|
+
changed = True
|
|
464
|
+
|
|
465
|
+
# 6. executed_scenario_order: bare names; keep only those that
|
|
466
|
+
# correspond to some executed_subdir.
|
|
467
|
+
new_executed_order: list[str] = []
|
|
468
|
+
for name in settings.executed_scenario_order:
|
|
469
|
+
# Direct match (subdir == bare name).
|
|
470
|
+
if name in executed_subdirs:
|
|
471
|
+
new_executed_order.append(name)
|
|
472
|
+
continue
|
|
473
|
+
# Any source has an executed subdir for this name?
|
|
474
|
+
kept = False
|
|
475
|
+
# Try owners_snapshot first.
|
|
476
|
+
src_for_name = owners_snapshot.get(name)
|
|
477
|
+
if src_for_name is not None and _has_executed_subdir(src_for_name, name):
|
|
478
|
+
kept = True
|
|
479
|
+
if not kept:
|
|
480
|
+
# Try via suffix-encoded subdirs in executed_subdirs.
|
|
481
|
+
for subdir in executed_subdirs:
|
|
482
|
+
src, scen_name = resolve_source_number(subdir, owners_snapshot)
|
|
483
|
+
if scen_name == name:
|
|
484
|
+
kept = True
|
|
485
|
+
break
|
|
486
|
+
if kept:
|
|
487
|
+
new_executed_order.append(name)
|
|
488
|
+
if new_executed_order != settings.executed_scenario_order:
|
|
489
|
+
settings.executed_scenario_order = new_executed_order
|
|
490
|
+
changed = True
|
|
491
|
+
|
|
492
|
+
# 7. bare_output_owners: drop entries with NO backing in either
|
|
493
|
+
# available_keys OR on-disk executed_subdirs. Pruned LAST so the
|
|
494
|
+
# other lists' resolutions used the original ownership map.
|
|
495
|
+
new_owners: dict[str, int] = {}
|
|
496
|
+
for scen_name, src in settings.bare_output_owners.items():
|
|
497
|
+
# Keep if still an available scenario from this source.
|
|
498
|
+
if (src, scen_name) in available_keys:
|
|
499
|
+
new_owners[scen_name] = src
|
|
500
|
+
continue
|
|
501
|
+
# Keep if there's still an on-disk folder for this (src, name).
|
|
502
|
+
# The folder may be encoded as either bare or suffixed, so check
|
|
503
|
+
# both forms.
|
|
504
|
+
if scen_name in executed_subdirs:
|
|
505
|
+
new_owners[scen_name] = src
|
|
506
|
+
continue
|
|
507
|
+
from flextool.gui.scenario_key import format_subdir
|
|
508
|
+
if format_subdir(src, scen_name) in executed_subdirs:
|
|
509
|
+
new_owners[scen_name] = src
|
|
510
|
+
continue
|
|
511
|
+
# No backing anywhere — drop.
|
|
512
|
+
if new_owners != settings.bare_output_owners:
|
|
513
|
+
settings.bare_output_owners = new_owners
|
|
514
|
+
changed = True
|
|
515
|
+
|
|
516
|
+
return changed
|