flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1423 @@
|
|
|
1
|
+
"""Read self-describing FlexTool Excel files.
|
|
2
|
+
|
|
3
|
+
Each sheet embeds its own metadata via a 2D grid of definitions:
|
|
4
|
+
- A **definition row** labels columns (alternative, entity, parameters, etc.)
|
|
5
|
+
- A **definition column** labels metadata rows (description, data type, etc.)
|
|
6
|
+
- Their intersection is the **crossing point**.
|
|
7
|
+
|
|
8
|
+
The reader auto-discovers the crossing point and interprets the sheet
|
|
9
|
+
layout from the embedded keywords, eliminating the need for an external
|
|
10
|
+
JSON import specification.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
import re
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import openpyxl
|
|
21
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
# Keywords recognised in the definition column (row labels)
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
ROW_KEYWORDS = frozenset({
|
|
30
|
+
"description",
|
|
31
|
+
"data type",
|
|
32
|
+
"parameter",
|
|
33
|
+
"alternative",
|
|
34
|
+
"entity",
|
|
35
|
+
"index",
|
|
36
|
+
"filter",
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
# Keywords that can appear as column definitions in the definition row.
|
|
40
|
+
# Many start with a prefix (e.g. "entity: node"), so we check prefixes.
|
|
41
|
+
COL_DEF_PREFIXES = (
|
|
42
|
+
"alternative",
|
|
43
|
+
"entity",
|
|
44
|
+
"filter",
|
|
45
|
+
"index",
|
|
46
|
+
"parameter",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Special parameter names recognised by the importer.
|
|
50
|
+
#
|
|
51
|
+
# The exporter (``excel_writer.py``) emits exactly ``"entity existence"`` as
|
|
52
|
+
# the header label and ``"TRUE"`` / ``"FALSE"`` as the cell values. The v2
|
|
53
|
+
# reader is a faithful inverse of that exporter — it does NOT accept legacy
|
|
54
|
+
# aliases (the old ``"entity alternative"`` header is rejected so users see
|
|
55
|
+
# a clear error rather than silent renames).
|
|
56
|
+
ENTITY_EXISTENCE = "entity existence"
|
|
57
|
+
# Strings the exporter EVER emits for a boolean-valued cell (case-insensitive
|
|
58
|
+
# match). Anything else is treated as a data error and raised — silent
|
|
59
|
+
# coercion of ``"yes"`` / ``"no"`` / ``"1"`` would mask data corruption.
|
|
60
|
+
_BOOL_TRUE_LITERAL = "true"
|
|
61
|
+
_BOOL_FALSE_LITERAL = "false"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
# Data structures
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class EntityClassDef:
|
|
70
|
+
"""One entity class mapping parsed from the entity definition."""
|
|
71
|
+
class_name: str
|
|
72
|
+
dimensions: list[str]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class SheetMetadata:
|
|
77
|
+
"""Parsed metadata from a single sheet."""
|
|
78
|
+
|
|
79
|
+
sheet_name: str
|
|
80
|
+
|
|
81
|
+
# Crossing point coordinates (0-based)
|
|
82
|
+
def_row: int = 0 # the definition row index
|
|
83
|
+
def_col: int = 0 # the definition column index
|
|
84
|
+
|
|
85
|
+
# Column definitions (from the definition row, left of crossing)
|
|
86
|
+
# Each maps column index -> definition string
|
|
87
|
+
col_defs: dict[int, str] = field(default_factory=dict)
|
|
88
|
+
|
|
89
|
+
# Row definitions (from the definition column, above crossing)
|
|
90
|
+
# Each maps row index -> definition string
|
|
91
|
+
row_defs: dict[int, str] = field(default_factory=dict)
|
|
92
|
+
|
|
93
|
+
# Parameter names (from the definition row, right of crossing)
|
|
94
|
+
# Maps column index -> parameter name
|
|
95
|
+
param_cols: dict[int, str] = field(default_factory=dict)
|
|
96
|
+
|
|
97
|
+
# Default parameter name (from "parameter: xxx" in the label cell)
|
|
98
|
+
default_parameter: str | None = None
|
|
99
|
+
|
|
100
|
+
# Default data type (from "data type: xxx" in the label cell)
|
|
101
|
+
default_data_type: str | None = None
|
|
102
|
+
|
|
103
|
+
# Data types per parameter column index
|
|
104
|
+
data_types: dict[int, str] = field(default_factory=dict)
|
|
105
|
+
|
|
106
|
+
# Descriptions per parameter column index
|
|
107
|
+
descriptions: dict[int, str] = field(default_factory=dict)
|
|
108
|
+
|
|
109
|
+
# Parsed entity class definitions
|
|
110
|
+
entity_classes: list[EntityClassDef] = field(default_factory=list)
|
|
111
|
+
|
|
112
|
+
# Filter column index and regex map {class_name: pattern}
|
|
113
|
+
filter_col: int | None = None
|
|
114
|
+
filter_map: dict[str, str] = field(default_factory=dict)
|
|
115
|
+
|
|
116
|
+
# Index column/row info
|
|
117
|
+
index_col: int | None = None
|
|
118
|
+
index_name: str | None = None
|
|
119
|
+
|
|
120
|
+
# Extra index columns (for ladder-style sheets with multiple ``index:`` cols
|
|
121
|
+
# on the left, e.g. ``index: period | index: tier``). Each entry is
|
|
122
|
+
# ``(column_index, axis_name)``. When non-empty, ``index_col`` /
|
|
123
|
+
# ``index_name`` carries the LAST entry; ``extra_index_cols`` carries the
|
|
124
|
+
# preceding ones in left-to-right order so the data extractor can recover
|
|
125
|
+
# all index levels.
|
|
126
|
+
extra_index_cols: list[tuple[int, str]] = field(default_factory=list)
|
|
127
|
+
|
|
128
|
+
# Alternative column index (or None if alternatives are in a row)
|
|
129
|
+
alt_col: int | None = None
|
|
130
|
+
|
|
131
|
+
# Entity existence column index
|
|
132
|
+
entity_existence_col: int | None = None
|
|
133
|
+
|
|
134
|
+
# Data start row (0-based, first row after all metadata rows)
|
|
135
|
+
data_start_row: int = 0
|
|
136
|
+
|
|
137
|
+
# Whether this is a transposed (timeseries-style) sheet
|
|
138
|
+
is_transposed: bool = False
|
|
139
|
+
|
|
140
|
+
# For transposed: row indices for alternative, parameter, entity
|
|
141
|
+
alt_row: int | None = None
|
|
142
|
+
param_row: int | None = None
|
|
143
|
+
entity_row: int | None = None
|
|
144
|
+
index_col_transposed: int | None = None # column with index values (usually 0)
|
|
145
|
+
|
|
146
|
+
# Whether this is a stochastic (_s) sheet — a transposed sheet whose
|
|
147
|
+
# entity / alternative / parameter live in COLUMN headers (like a
|
|
148
|
+
# timeseries sheet) but whose index is a MULTI-LEVEL nested Map spread
|
|
149
|
+
# across N left-side ``index:`` columns (one column per Map level).
|
|
150
|
+
# The definition column (carrying the ``parameter:`` triplet, the
|
|
151
|
+
# ``entity:`` label and the ``alternative`` label) sits at ``def_col``
|
|
152
|
+
# (= number of index columns). Each data column to its right is one
|
|
153
|
+
# (entity, alternative) series whose cells carry the leaf scalars.
|
|
154
|
+
is_stochastic: bool = False
|
|
155
|
+
|
|
156
|
+
# Ordered ``(column_index, axis_name)`` for the stochastic index
|
|
157
|
+
# columns, left-to-right (outermost Map level first).
|
|
158
|
+
stoch_index_cols: list[tuple[int, str]] = field(default_factory=list)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
# Crossing point detection
|
|
163
|
+
# ---------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _cell_value(ws: Worksheet, row: int, col: int) -> str:
|
|
167
|
+
"""Get cell value as stripped string (0-based row/col)."""
|
|
168
|
+
val = ws.cell(row=row + 1, column=col + 1).value # openpyxl is 1-based
|
|
169
|
+
if val is None:
|
|
170
|
+
return ""
|
|
171
|
+
return str(val).strip()
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _actual_max_row(ws: Worksheet) -> int:
|
|
175
|
+
"""Get the actual last row with data (not the Excel theoretical max)."""
|
|
176
|
+
# ws.max_row in non-read-only mode should be correct, but guard against
|
|
177
|
+
# sheets where it reports 1048576
|
|
178
|
+
max_r = ws.max_row or 0
|
|
179
|
+
if max_r > 100000:
|
|
180
|
+
# Scan backwards from a reasonable limit
|
|
181
|
+
for r in range(min(max_r, 10000), 0, -1):
|
|
182
|
+
for c in range(1, min((ws.max_column or 1) + 1, 50)):
|
|
183
|
+
if ws.cell(row=r, column=c).value is not None:
|
|
184
|
+
return r
|
|
185
|
+
return 0
|
|
186
|
+
return max_r
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _is_info_row(text: str) -> bool:
|
|
190
|
+
"""Check if text is an INFO row (to be skipped by the reader)."""
|
|
191
|
+
return text.upper().startswith("INFO:")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _is_keyword(text: str) -> bool:
|
|
195
|
+
"""Check if text starts with a recognised keyword."""
|
|
196
|
+
t = text.lower()
|
|
197
|
+
for prefix in COL_DEF_PREFIXES:
|
|
198
|
+
if t == prefix or t.startswith(prefix + ":") or t.startswith(prefix + " "):
|
|
199
|
+
return True
|
|
200
|
+
if t in ROW_KEYWORDS:
|
|
201
|
+
return True
|
|
202
|
+
return False
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def find_crossing_point(ws: Worksheet) -> tuple[int, int]:
|
|
206
|
+
"""Find the crossing point of the definition row and definition column.
|
|
207
|
+
|
|
208
|
+
Algorithm:
|
|
209
|
+
1. Scan row 0 (Excel row 1) rightward until a recognised keyword is found.
|
|
210
|
+
That column is the definition column.
|
|
211
|
+
2. Scan that column downward until an empty cell. Go back one row.
|
|
212
|
+
That row is the definition row.
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
(def_row, def_col) — 0-based indices.
|
|
216
|
+
|
|
217
|
+
Raises:
|
|
218
|
+
ValueError: if no crossing point can be found.
|
|
219
|
+
"""
|
|
220
|
+
max_col = ws.max_column or 1
|
|
221
|
+
|
|
222
|
+
# Step 1: scan row 0 rightward for a keyword
|
|
223
|
+
def_col = None
|
|
224
|
+
for c in range(max_col):
|
|
225
|
+
val = _cell_value(ws, 0, c)
|
|
226
|
+
if val and _is_keyword(val):
|
|
227
|
+
def_col = c
|
|
228
|
+
break
|
|
229
|
+
|
|
230
|
+
if def_col is None:
|
|
231
|
+
raise ValueError(
|
|
232
|
+
f"Sheet '{ws.title}': could not find definition column "
|
|
233
|
+
f"(no keyword found in row 1)"
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
# Step 2: scan definition column downward until empty cell
|
|
237
|
+
max_row = _actual_max_row(ws)
|
|
238
|
+
def_row = 0
|
|
239
|
+
for r in range(max_row):
|
|
240
|
+
val = _cell_value(ws, r, def_col)
|
|
241
|
+
if not val:
|
|
242
|
+
# Empty cell found — go back one
|
|
243
|
+
def_row = max(0, r - 1)
|
|
244
|
+
break
|
|
245
|
+
else:
|
|
246
|
+
# No empty cell found — last row with content
|
|
247
|
+
def_row = max_row - 1
|
|
248
|
+
|
|
249
|
+
return (def_row, def_col)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
# ---------------------------------------------------------------------------
|
|
253
|
+
# Definition parsing
|
|
254
|
+
# ---------------------------------------------------------------------------
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _parse_entity_def(text: str) -> list[EntityClassDef]:
|
|
258
|
+
"""Parse an entity definition string.
|
|
259
|
+
|
|
260
|
+
Examples:
|
|
261
|
+
"entity: node"
|
|
262
|
+
→ [EntityClassDef("node", ["node"])]
|
|
263
|
+
"entity: commodity, node"
|
|
264
|
+
→ [EntityClassDef("commodity__node", ["commodity", "node"])]
|
|
265
|
+
"entity: (unit__inputNode: (unit, node), unit__outputNode: (unit, node))"
|
|
266
|
+
→ [EntityClassDef("unit__inputNode", ["unit", "node"]),
|
|
267
|
+
EntityClassDef("unit__outputNode", ["unit", "node"])]
|
|
268
|
+
"""
|
|
269
|
+
# Strip "entity:" prefix
|
|
270
|
+
text = text.strip()
|
|
271
|
+
if text.lower().startswith("entity:"):
|
|
272
|
+
text = text[len("entity:"):].strip()
|
|
273
|
+
elif text.lower().startswith("entity name:"):
|
|
274
|
+
text = text[len("entity name:"):].strip()
|
|
275
|
+
elif text.lower() == "entity":
|
|
276
|
+
return []
|
|
277
|
+
|
|
278
|
+
# Check for multi-class syntax: (class1: (dim1, dim2), class2: (dim3, dim4))
|
|
279
|
+
if text.startswith("(") and text.endswith(")"):
|
|
280
|
+
inner = text[1:-1].strip()
|
|
281
|
+
results = []
|
|
282
|
+
# Split on top-level commas (not inside parens)
|
|
283
|
+
parts = _split_top_level(inner)
|
|
284
|
+
for part in parts:
|
|
285
|
+
part = part.strip()
|
|
286
|
+
# Parse "class_name: (dim1, dim2)"
|
|
287
|
+
match = re.match(r'(\w+)\s*:\s*\(([^)]+)\)', part)
|
|
288
|
+
if match:
|
|
289
|
+
class_name = match.group(1).strip()
|
|
290
|
+
dims = [d.strip() for d in match.group(2).split(",")]
|
|
291
|
+
results.append(EntityClassDef(class_name, dims))
|
|
292
|
+
return results
|
|
293
|
+
|
|
294
|
+
# Simple case: "node" or "commodity, node"
|
|
295
|
+
dims = [d.strip() for d in text.split(",")]
|
|
296
|
+
if len(dims) == 1:
|
|
297
|
+
return [EntityClassDef(dims[0], dims)]
|
|
298
|
+
else:
|
|
299
|
+
class_name = "__".join(dims)
|
|
300
|
+
return [EntityClassDef(class_name, dims)]
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _split_top_level(text: str) -> list[str]:
|
|
304
|
+
"""Split text on commas that are not inside parentheses."""
|
|
305
|
+
parts = []
|
|
306
|
+
depth = 0
|
|
307
|
+
current = []
|
|
308
|
+
for ch in text:
|
|
309
|
+
if ch == "(":
|
|
310
|
+
depth += 1
|
|
311
|
+
current.append(ch)
|
|
312
|
+
elif ch == ")":
|
|
313
|
+
depth -= 1
|
|
314
|
+
current.append(ch)
|
|
315
|
+
elif ch == "," and depth == 0:
|
|
316
|
+
parts.append("".join(current))
|
|
317
|
+
current = []
|
|
318
|
+
else:
|
|
319
|
+
current.append(ch)
|
|
320
|
+
if current:
|
|
321
|
+
parts.append("".join(current))
|
|
322
|
+
return parts
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _parse_filter_def(text: str) -> dict[str, str]:
|
|
326
|
+
"""Parse a filter definition string.
|
|
327
|
+
|
|
328
|
+
Example:
|
|
329
|
+
"filter: {unit__inputNode: ^input$, unit__outputNode: ^output$}"
|
|
330
|
+
→ {"unit__inputNode": "^input$", "unit__outputNode": "^output$"}
|
|
331
|
+
"""
|
|
332
|
+
text = text.strip()
|
|
333
|
+
if text.lower().startswith("filter:"):
|
|
334
|
+
text = text[len("filter:"):].strip()
|
|
335
|
+
|
|
336
|
+
# Remove outer braces or parens
|
|
337
|
+
if (text.startswith("{") and text.endswith("}")) or \
|
|
338
|
+
(text.startswith("(") and text.endswith(")")):
|
|
339
|
+
text = text[1:-1].strip()
|
|
340
|
+
|
|
341
|
+
result = {}
|
|
342
|
+
for part in _split_top_level(text):
|
|
343
|
+
part = part.strip()
|
|
344
|
+
if ":" in part:
|
|
345
|
+
key, val = part.split(":", 1)
|
|
346
|
+
result[key.strip()] = val.strip()
|
|
347
|
+
return result
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _parse_index_def(text: str) -> str:
|
|
351
|
+
"""Parse an index definition: "index: time" → "time"."""
|
|
352
|
+
text = text.strip()
|
|
353
|
+
if text.lower().startswith("index:"):
|
|
354
|
+
return text[len("index:"):].strip()
|
|
355
|
+
return text
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _parse_default_value(text: str) -> tuple[str, str | None]:
|
|
359
|
+
"""Parse a definition that may have a default value.
|
|
360
|
+
|
|
361
|
+
"parameter: profile" → ("parameter", "profile")
|
|
362
|
+
"parameter" → ("parameter", None)
|
|
363
|
+
"data type: float" → ("data type", "float")
|
|
364
|
+
|
|
365
|
+
Also handles the triplet format with ``|`` separator:
|
|
366
|
+
"parameter: profile | data type: float | description: ..."
|
|
367
|
+
→ ("parameter", "profile")
|
|
368
|
+
"""
|
|
369
|
+
# Handle | separator: only parse the first segment
|
|
370
|
+
if "|" in text:
|
|
371
|
+
text = text.split("|")[0].strip()
|
|
372
|
+
|
|
373
|
+
for keyword in sorted(ROW_KEYWORDS, key=len, reverse=True):
|
|
374
|
+
if text.lower().startswith(keyword):
|
|
375
|
+
rest = text[len(keyword):].strip()
|
|
376
|
+
if rest.startswith(":"):
|
|
377
|
+
return keyword, rest[1:].strip()
|
|
378
|
+
elif rest == "":
|
|
379
|
+
return keyword, None
|
|
380
|
+
return text, None
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _extract_triplet_field(text: str, field: str) -> str | None:
|
|
384
|
+
"""Return the value of a labelled segment in a pipe-separated triplet.
|
|
385
|
+
|
|
386
|
+
Looks for ``<field>: <value>`` inside a ``"a | b | c"`` style string
|
|
387
|
+
(case-insensitive on the field name). Returns the trimmed value or
|
|
388
|
+
``None`` when the field isn't present.
|
|
389
|
+
"""
|
|
390
|
+
if not text or "|" not in text:
|
|
391
|
+
return None
|
|
392
|
+
fl = field.lower()
|
|
393
|
+
for seg in text.split("|"):
|
|
394
|
+
s = seg.strip()
|
|
395
|
+
if s.lower().startswith(fl + ":"):
|
|
396
|
+
return s[len(field) + 1:].strip()
|
|
397
|
+
return None
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
# ---------------------------------------------------------------------------
|
|
401
|
+
# Sheet metadata extraction
|
|
402
|
+
# ---------------------------------------------------------------------------
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def parse_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
|
|
406
|
+
"""Parse a sheet's embedded metadata.
|
|
407
|
+
|
|
408
|
+
Returns None if the sheet has no recognisable metadata (e.g. navigate, version).
|
|
409
|
+
"""
|
|
410
|
+
try:
|
|
411
|
+
def_row, def_col = find_crossing_point(ws)
|
|
412
|
+
except ValueError:
|
|
413
|
+
return None
|
|
414
|
+
|
|
415
|
+
meta = SheetMetadata(sheet_name=ws.title, def_row=def_row, def_col=def_col)
|
|
416
|
+
|
|
417
|
+
# --- Parse row definitions (definition column, above crossing point) ---
|
|
418
|
+
for r in range(def_row):
|
|
419
|
+
val = _cell_value(ws, r, def_col)
|
|
420
|
+
if val:
|
|
421
|
+
meta.row_defs[r] = val
|
|
422
|
+
|
|
423
|
+
# The crossing point cell itself is also a row def
|
|
424
|
+
crossing_val = _cell_value(ws, def_row, def_col)
|
|
425
|
+
if crossing_val:
|
|
426
|
+
meta.row_defs[def_row] = crossing_val
|
|
427
|
+
|
|
428
|
+
# --- Parse column definitions (definition row, left of crossing) ---
|
|
429
|
+
for c in range(def_col):
|
|
430
|
+
val = _cell_value(ws, def_row, c)
|
|
431
|
+
if val:
|
|
432
|
+
meta.col_defs[c] = val.strip()
|
|
433
|
+
|
|
434
|
+
# --- Interpret column definitions ---
|
|
435
|
+
# Multiple ``index:`` columns on the left are recorded in
|
|
436
|
+
# ``extra_index_cols`` (in column order); the rightmost wins for
|
|
437
|
+
# ``index_col`` / ``index_name`` to keep backwards compatibility with
|
|
438
|
+
# single-index sheets.
|
|
439
|
+
index_seen: list[tuple[int, str]] = []
|
|
440
|
+
for c, defn in meta.col_defs.items():
|
|
441
|
+
dl = defn.lower()
|
|
442
|
+
if dl == "alternative":
|
|
443
|
+
meta.alt_col = c
|
|
444
|
+
elif dl.startswith("entity"):
|
|
445
|
+
meta.entity_classes = _parse_entity_def(defn)
|
|
446
|
+
elif dl.startswith("filter"):
|
|
447
|
+
meta.filter_col = c
|
|
448
|
+
meta.filter_map = _parse_filter_def(defn)
|
|
449
|
+
elif dl.startswith("index"):
|
|
450
|
+
index_seen.append((c, _parse_index_def(defn)))
|
|
451
|
+
if index_seen:
|
|
452
|
+
index_seen.sort()
|
|
453
|
+
meta.index_col, meta.index_name = index_seen[-1]
|
|
454
|
+
# Preceding (left-of-rightmost) index columns become extras.
|
|
455
|
+
meta.extra_index_cols = list(index_seen[:-1])
|
|
456
|
+
|
|
457
|
+
# --- Parse parameter columns (definition row, right of crossing) ---
|
|
458
|
+
# Stop at the first empty cell (separator column before parameter reference)
|
|
459
|
+
max_col = ws.max_column or 1
|
|
460
|
+
for c in range(def_col + 1, max_col):
|
|
461
|
+
val = _cell_value(ws, def_row, c)
|
|
462
|
+
if not val:
|
|
463
|
+
break # empty separator column = end of parameter area
|
|
464
|
+
vl = val.lower()
|
|
465
|
+
if vl == ENTITY_EXISTENCE:
|
|
466
|
+
meta.entity_existence_col = c
|
|
467
|
+
meta.param_cols[c] = ENTITY_EXISTENCE
|
|
468
|
+
else:
|
|
469
|
+
meta.param_cols[c] = val
|
|
470
|
+
|
|
471
|
+
# --- Parse crossing point for defaults ---
|
|
472
|
+
# NOTE: kept defensively — the current exporter ALWAYS writes a
|
|
473
|
+
# parameter name into every right-of-crossing header cell, so the
|
|
474
|
+
# back-fill loop below is dead code on writer-generated files. It
|
|
475
|
+
# remains in place to keep hand-edited sheets (where the user relies
|
|
476
|
+
# on the crossing's ``parameter: <default>`` to label an empty
|
|
477
|
+
# column) loadable. If the exporter contract changes, revisit.
|
|
478
|
+
keyword, default = _parse_default_value(crossing_val)
|
|
479
|
+
if keyword.lower() == "parameter" and default:
|
|
480
|
+
meta.default_parameter = default
|
|
481
|
+
# Apply default to all param columns that are empty
|
|
482
|
+
for c in range(def_col + 1, max_col):
|
|
483
|
+
if c not in meta.param_cols:
|
|
484
|
+
val = _cell_value(ws, def_row, c)
|
|
485
|
+
if not val:
|
|
486
|
+
meta.param_cols[c] = default
|
|
487
|
+
|
|
488
|
+
# --- Parse row metadata (description, data type) ---
|
|
489
|
+
for r, label in meta.row_defs.items():
|
|
490
|
+
if r == def_row:
|
|
491
|
+
continue # skip the definition row itself
|
|
492
|
+
kw, default_val = _parse_default_value(label)
|
|
493
|
+
kw_lower = kw.lower()
|
|
494
|
+
|
|
495
|
+
if kw_lower == "description":
|
|
496
|
+
for c in meta.param_cols:
|
|
497
|
+
desc = _cell_value(ws, r, c)
|
|
498
|
+
if desc:
|
|
499
|
+
meta.descriptions[c] = desc
|
|
500
|
+
|
|
501
|
+
elif kw_lower == "data type":
|
|
502
|
+
if default_val:
|
|
503
|
+
meta.default_data_type = default_val
|
|
504
|
+
for c in meta.param_cols:
|
|
505
|
+
dt = _cell_value(ws, r, c)
|
|
506
|
+
# NOTE: ``.lower()`` is defensive — the exporter already
|
|
507
|
+
# emits all data-type labels lowercase ("float", "string",
|
|
508
|
+
# "boolean (array)", "string (array)", "*-1d-map", ...).
|
|
509
|
+
# Kept in case a hand-edited sheet uses mixed case so the
|
|
510
|
+
# downstream dtype dispatch (``_convert_value``) keeps
|
|
511
|
+
# matching its lowercase ``elif`` branches.
|
|
512
|
+
if dt:
|
|
513
|
+
meta.data_types[c] = dt.lower()
|
|
514
|
+
elif default_val:
|
|
515
|
+
meta.data_types[c] = default_val.lower()
|
|
516
|
+
|
|
517
|
+
# Data starts after the definition row
|
|
518
|
+
meta.data_start_row = def_row + 1
|
|
519
|
+
|
|
520
|
+
return meta
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def parse_link_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
|
|
524
|
+
"""Parse a link-only sheet (no parameters, just entity dimensions).
|
|
525
|
+
|
|
526
|
+
Link sheets have a simpler format:
|
|
527
|
+
Row 1: navigate link
|
|
528
|
+
Row 2: entity definition in col A, dimension names in cols B+
|
|
529
|
+
Row 3+: data
|
|
530
|
+
"""
|
|
531
|
+
meta = SheetMetadata(sheet_name=ws.title)
|
|
532
|
+
|
|
533
|
+
# Look for entity definition in column A, rows 1-3
|
|
534
|
+
for r in range(min(3, ws.max_row or 1)):
|
|
535
|
+
val = _cell_value(ws, r, 0)
|
|
536
|
+
if val.lower().startswith("entity"):
|
|
537
|
+
meta.entity_classes = _parse_entity_def(val)
|
|
538
|
+
meta.def_row = r
|
|
539
|
+
meta.data_start_row = r + 1
|
|
540
|
+
return meta
|
|
541
|
+
|
|
542
|
+
return None
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def parse_transposed_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
|
|
546
|
+
"""Parse a transposed (timeseries-style) sheet.
|
|
547
|
+
|
|
548
|
+
In transposed sheets, column B serves as the row definition column:
|
|
549
|
+
- Row with "alternative" in col B → alternative values in cols C+
|
|
550
|
+
- Row with "parameter" in col B → parameter names in cols C+
|
|
551
|
+
- Row with "entity: X" in col B → entity names in cols C+
|
|
552
|
+
- Column A has "index: time" and time values below
|
|
553
|
+
"""
|
|
554
|
+
meta = SheetMetadata(sheet_name=ws.title, is_transposed=True)
|
|
555
|
+
max_row = _actual_max_row(ws)
|
|
556
|
+
|
|
557
|
+
# Scan column A for "index:" definition (skip INFO rows)
|
|
558
|
+
for r in range(min(10, max_row)):
|
|
559
|
+
val = _cell_value(ws, r, 0)
|
|
560
|
+
if _is_info_row(val):
|
|
561
|
+
continue
|
|
562
|
+
if val.lower().startswith("index:"):
|
|
563
|
+
meta.index_name = _parse_index_def(val)
|
|
564
|
+
meta.index_col_transposed = 0
|
|
565
|
+
meta.data_start_row = r + 1
|
|
566
|
+
|
|
567
|
+
# Scan column B for row role definitions (skip INFO rows)
|
|
568
|
+
label_col = 1 # column B
|
|
569
|
+
datatype_row: int | None = None
|
|
570
|
+
for r in range(min(10, max_row)):
|
|
571
|
+
val_a = _cell_value(ws, r, 0)
|
|
572
|
+
if _is_info_row(val_a):
|
|
573
|
+
continue
|
|
574
|
+
val = _cell_value(ws, r, label_col)
|
|
575
|
+
if not val:
|
|
576
|
+
continue
|
|
577
|
+
vl = val.lower()
|
|
578
|
+
kw, default_val = _parse_default_value(val)
|
|
579
|
+
kw_lower = kw.lower()
|
|
580
|
+
|
|
581
|
+
if vl == "alternative" or vl.startswith("alternative"):
|
|
582
|
+
meta.alt_row = r
|
|
583
|
+
elif kw_lower == "data type":
|
|
584
|
+
# Multi-param _t sheets carry a per-column data-type row so the
|
|
585
|
+
# leaf scalar type survives the round-trip (single-param sheets
|
|
586
|
+
# fold it into the triplet instead).
|
|
587
|
+
datatype_row = r
|
|
588
|
+
if default_val:
|
|
589
|
+
meta.default_data_type = default_val.lower()
|
|
590
|
+
elif kw_lower == "parameter":
|
|
591
|
+
meta.param_row = r
|
|
592
|
+
if default_val:
|
|
593
|
+
meta.default_parameter = default_val
|
|
594
|
+
elif vl.startswith("entity"):
|
|
595
|
+
meta.entity_row = r
|
|
596
|
+
meta.entity_classes = _parse_entity_def(val)
|
|
597
|
+
|
|
598
|
+
# Read per-column data types (one per transposed data column, C+).
|
|
599
|
+
if datatype_row is not None:
|
|
600
|
+
max_col = ws.max_column or 1
|
|
601
|
+
for c in range(2, max_col):
|
|
602
|
+
dt = _cell_value(ws, datatype_row, c)
|
|
603
|
+
if dt:
|
|
604
|
+
meta.data_types[c] = dt.lower()
|
|
605
|
+
|
|
606
|
+
# Determine data start row (row after last metadata row)
|
|
607
|
+
metadata_rows = [
|
|
608
|
+
r for r in [meta.alt_row, meta.param_row, meta.entity_row, datatype_row]
|
|
609
|
+
if r is not None
|
|
610
|
+
]
|
|
611
|
+
if metadata_rows:
|
|
612
|
+
meta.data_start_row = max(metadata_rows) + 1
|
|
613
|
+
|
|
614
|
+
return meta
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
# ---------------------------------------------------------------------------
|
|
618
|
+
# Scenario sheet parsing
|
|
619
|
+
# ---------------------------------------------------------------------------
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _is_scenario_sheet(ws: Worksheet) -> bool:
|
|
623
|
+
"""Check if the worksheet is a scenario sheet.
|
|
624
|
+
|
|
625
|
+
Scenario sheets have "Scenario names" in row 1, col 2 (0-based: row 0, col 1).
|
|
626
|
+
"""
|
|
627
|
+
val = _cell_value(ws, 0, 1)
|
|
628
|
+
return val.lower() == "scenario names" if val else False
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def parse_scenario_sheet(ws: Worksheet) -> SheetData:
|
|
632
|
+
"""Parse a scenario sheet into SheetData records.
|
|
633
|
+
|
|
634
|
+
Scenario sheet layout (Excel rows, 1-based):
|
|
635
|
+
Row 1: navigate | Scenario names | ...
|
|
636
|
+
Row 2: (empty) | scenario_1 | scenario_2 | ...
|
|
637
|
+
Row 3: base_alternative | alt_for_s1 | alt_for_s2 | ...
|
|
638
|
+
Row 4: alternative_1 | alt_for_s1 | alt_for_s2 | ...
|
|
639
|
+
Row 5: alternative_2 | (empty) | alt_for_s2 | ...
|
|
640
|
+
|
|
641
|
+
Produces records with:
|
|
642
|
+
entity_class = "scenario"
|
|
643
|
+
entity_byname = (scenario_name,)
|
|
644
|
+
param_name = row label (e.g. "base_alternative", "alternative_1")
|
|
645
|
+
alternative = the alternative name in that cell
|
|
646
|
+
value = the rank (row index - 2, i.e. base_alternative=0, alternative_1=1, ...)
|
|
647
|
+
"""
|
|
648
|
+
meta = SheetMetadata(sheet_name=ws.title)
|
|
649
|
+
meta.entity_classes = [EntityClassDef("scenario", ["scenario"])]
|
|
650
|
+
data = SheetData(sheet_name=ws.title, metadata=meta)
|
|
651
|
+
|
|
652
|
+
max_row = _actual_max_row(ws)
|
|
653
|
+
max_col = ws.max_column or 1
|
|
654
|
+
|
|
655
|
+
# Row 2 (0-based row 1): scenario names in cols B+ (0-based col 1+)
|
|
656
|
+
scenario_names: dict[int, str] = {}
|
|
657
|
+
for c in range(1, max_col):
|
|
658
|
+
name = _cell_value(ws, 1, c)
|
|
659
|
+
if name:
|
|
660
|
+
scenario_names[c] = name
|
|
661
|
+
|
|
662
|
+
if not scenario_names:
|
|
663
|
+
return data
|
|
664
|
+
|
|
665
|
+
# Row 3+ (0-based row 2+): row label in col A, alternative names in cols B+
|
|
666
|
+
for r in range(2, max_row):
|
|
667
|
+
row_label = _cell_value(ws, r, 0)
|
|
668
|
+
if not row_label:
|
|
669
|
+
continue
|
|
670
|
+
|
|
671
|
+
rank = r - 2 # base_alternative (row 2, 0-based) = rank 0
|
|
672
|
+
|
|
673
|
+
for c, scenario_name in scenario_names.items():
|
|
674
|
+
alt_name = _cell_value(ws, r, c)
|
|
675
|
+
if not alt_name:
|
|
676
|
+
continue
|
|
677
|
+
|
|
678
|
+
data.records.append({
|
|
679
|
+
"entity_class": "scenario",
|
|
680
|
+
"entity_byname": (scenario_name,),
|
|
681
|
+
"param_name": row_label,
|
|
682
|
+
"alternative": alt_name,
|
|
683
|
+
"value": rank,
|
|
684
|
+
"index_value": None,
|
|
685
|
+
"index_name": None,
|
|
686
|
+
})
|
|
687
|
+
|
|
688
|
+
return data
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
# ---------------------------------------------------------------------------
|
|
692
|
+
# Stochastic (_s) sheet parsing
|
|
693
|
+
# ---------------------------------------------------------------------------
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _is_index_label(text: str) -> bool:
|
|
697
|
+
"""Return True when *text* is an ``index:`` column definition."""
|
|
698
|
+
return text.lower().startswith("index:") or text.lower() == "index"
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _find_stochastic_def_col(ws: Worksheet) -> int | None:
|
|
702
|
+
"""Locate the definition column of a stochastic (_s) sheet, or None.
|
|
703
|
+
|
|
704
|
+
A stochastic sheet is a transposed sheet whose nested-Map index is
|
|
705
|
+
spread across ``N`` left-side ``index:`` columns (one per Map level)
|
|
706
|
+
followed by a definition column that carries the ``alternative`` /
|
|
707
|
+
``parameter`` / ``entity`` row roles (the entity/alt/param VALUES live
|
|
708
|
+
in the column headers to its right, like a timeseries sheet).
|
|
709
|
+
|
|
710
|
+
The signal: there are at least TWO ``index:`` columns on the left, and
|
|
711
|
+
immediately after the last of them comes a column whose header rows
|
|
712
|
+
carry the ``alternative`` (and ``parameter`` / ``entity``) role
|
|
713
|
+
labels. A single-index transposed sheet (timeseries) keeps its lone
|
|
714
|
+
``index:`` in column A and its def labels in column B — that case is
|
|
715
|
+
handled by the existing transposed path and is explicitly NOT matched
|
|
716
|
+
here (it has fewer than two ``index:`` columns).
|
|
717
|
+
"""
|
|
718
|
+
max_row = _actual_max_row(ws)
|
|
719
|
+
max_col = ws.max_column or 1
|
|
720
|
+
scan_rows = min(8, max_row)
|
|
721
|
+
|
|
722
|
+
# The role-definition row is the one carrying the ``alternative`` label
|
|
723
|
+
# (in the definition column). On a stochastic sheet that same row also
|
|
724
|
+
# carries the ``index:`` labels for every Map level in the columns to
|
|
725
|
+
# its left. Anchor on that row, then verify the left columns are all
|
|
726
|
+
# ``index:``.
|
|
727
|
+
for role_row in range(scan_rows):
|
|
728
|
+
def_col: int | None = None
|
|
729
|
+
for c in range(max_col):
|
|
730
|
+
if _cell_value(ws, role_row, c).lower() == "alternative":
|
|
731
|
+
def_col = c
|
|
732
|
+
break
|
|
733
|
+
if def_col is None or def_col < 2:
|
|
734
|
+
# Need at least two ``index:`` columns to the left to be a
|
|
735
|
+
# genuine nested-Map (multi-index) sheet; a lone ``index:`` in
|
|
736
|
+
# column A is an ordinary single-index transposed sheet.
|
|
737
|
+
continue
|
|
738
|
+
if all(
|
|
739
|
+
_is_index_label(_cell_value(ws, role_row, c)) for c in range(def_col)
|
|
740
|
+
):
|
|
741
|
+
return def_col
|
|
742
|
+
return None
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def parse_stochastic_sheet_metadata(ws: Worksheet, def_col: int) -> SheetMetadata:
|
|
746
|
+
"""Parse a stochastic (_s) sheet's embedded metadata.
|
|
747
|
+
|
|
748
|
+
``def_col`` is the 0-based definition column index (= the number of
|
|
749
|
+
left-side ``index:`` columns). The header rows carry the role labels
|
|
750
|
+
(``parameter:``, ``entity:``, ``alternative``) in that column; the
|
|
751
|
+
entity / alternative / leaf-data values live in the columns to its
|
|
752
|
+
right (transposed). The ``index:`` columns 0..def_col-1 give the
|
|
753
|
+
nested-Map level axis names, outermost first.
|
|
754
|
+
"""
|
|
755
|
+
meta = SheetMetadata(
|
|
756
|
+
sheet_name=ws.title, is_transposed=True, is_stochastic=True,
|
|
757
|
+
)
|
|
758
|
+
meta.def_col = def_col
|
|
759
|
+
max_row = _actual_max_row(ws)
|
|
760
|
+
scan_rows = min(8, max_row)
|
|
761
|
+
|
|
762
|
+
# Index columns (left of def_col), in order. Axis name comes from the
|
|
763
|
+
# role-definition row; collect from whichever header row carries it.
|
|
764
|
+
for c in range(def_col):
|
|
765
|
+
axis = ""
|
|
766
|
+
for r in range(scan_rows):
|
|
767
|
+
val = _cell_value(ws, r, c)
|
|
768
|
+
if val and _is_index_label(val):
|
|
769
|
+
axis = _parse_index_def(val)
|
|
770
|
+
break
|
|
771
|
+
meta.stoch_index_cols.append((c, axis))
|
|
772
|
+
if meta.stoch_index_cols:
|
|
773
|
+
# Keep single-index compatibility fields pointing at the LAST level.
|
|
774
|
+
meta.index_col, meta.index_name = meta.stoch_index_cols[-1]
|
|
775
|
+
|
|
776
|
+
# Role rows in the definition column.
|
|
777
|
+
triplet_dtype: str | None = None
|
|
778
|
+
for r in range(scan_rows):
|
|
779
|
+
val = _cell_value(ws, r, def_col)
|
|
780
|
+
if not val:
|
|
781
|
+
continue
|
|
782
|
+
vl = val.lower()
|
|
783
|
+
kw, default_val = _parse_default_value(val)
|
|
784
|
+
kw_lower = kw.lower()
|
|
785
|
+
if vl == "alternative" or vl.startswith("alternative"):
|
|
786
|
+
meta.alt_row = r
|
|
787
|
+
elif kw_lower == "parameter":
|
|
788
|
+
meta.param_row = r
|
|
789
|
+
if default_val:
|
|
790
|
+
meta.default_parameter = default_val
|
|
791
|
+
# The single-param triplet carries the canonical data type.
|
|
792
|
+
triplet_dtype = _extract_triplet_field(val, "data type")
|
|
793
|
+
elif vl.startswith("entity"):
|
|
794
|
+
meta.entity_row = r
|
|
795
|
+
meta.entity_classes = _parse_entity_def(val)
|
|
796
|
+
|
|
797
|
+
if triplet_dtype:
|
|
798
|
+
meta.default_data_type = triplet_dtype.lower()
|
|
799
|
+
|
|
800
|
+
# Data rows start after the last header (role) row.
|
|
801
|
+
role_rows = [
|
|
802
|
+
r for r in (meta.alt_row, meta.param_row, meta.entity_row)
|
|
803
|
+
if r is not None
|
|
804
|
+
]
|
|
805
|
+
meta.data_start_row = (max(role_rows) + 1) if role_rows else def_col + 1
|
|
806
|
+
return meta
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _extract_stochastic(ws: Worksheet, meta: SheetMetadata) -> SheetData:
|
|
810
|
+
"""Extract per-leaf records from a stochastic (_s) sheet.
|
|
811
|
+
|
|
812
|
+
Each data column (def_col+1 onward) is one (entity, alternative)
|
|
813
|
+
series. Each data row carries one full nested-Map index tuple in the
|
|
814
|
+
left ``index:`` columns. We emit one record per (column, row) cell,
|
|
815
|
+
carrying the first N-1 index levels in ``extra_index_values`` and the
|
|
816
|
+
last level in ``index_value`` / ``index_name`` — mirroring the ladder
|
|
817
|
+
reader so the DB-side nested-Map reconstruction can rebuild the
|
|
818
|
+
full ``Map(idx0 -> Map(idx1 -> ... -> leaf))``.
|
|
819
|
+
"""
|
|
820
|
+
data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
|
|
821
|
+
max_row = _actual_max_row(ws)
|
|
822
|
+
max_col = ws.max_column or 1
|
|
823
|
+
def_col = meta.def_col
|
|
824
|
+
|
|
825
|
+
entity_class = meta.entity_classes[0] if meta.entity_classes else None
|
|
826
|
+
if entity_class is None:
|
|
827
|
+
return data
|
|
828
|
+
|
|
829
|
+
n_dims = len(entity_class.dimensions)
|
|
830
|
+
leaf_dtype = meta.default_data_type or "string"
|
|
831
|
+
|
|
832
|
+
for c in range(def_col + 1, max_col):
|
|
833
|
+
alt = _cell_value(ws, meta.alt_row, c) if meta.alt_row is not None else None
|
|
834
|
+
pname = (
|
|
835
|
+
_cell_value(ws, meta.param_row, c) if meta.param_row is not None else None
|
|
836
|
+
)
|
|
837
|
+
ent_name = (
|
|
838
|
+
_cell_value(ws, meta.entity_row, c) if meta.entity_row is not None else None
|
|
839
|
+
)
|
|
840
|
+
if not pname and meta.default_parameter:
|
|
841
|
+
pname = meta.default_parameter
|
|
842
|
+
if not alt or not ent_name or not pname:
|
|
843
|
+
continue
|
|
844
|
+
|
|
845
|
+
# The entity header carries only the FIRST dimension element for a
|
|
846
|
+
# multi-dim class (the writer emits ``byname[0]``). Stochastic
|
|
847
|
+
# params in the schema are single-dimension (profile, node, …); a
|
|
848
|
+
# multi-dim header would be ambiguous, so we only build a byname we
|
|
849
|
+
# can trust.
|
|
850
|
+
if n_dims == 1:
|
|
851
|
+
entity_byname: tuple[str, ...] = (ent_name,)
|
|
852
|
+
else:
|
|
853
|
+
# Reaching here means a multi-dim class was routed onto an _s
|
|
854
|
+
# sheet whose header can't carry the full byname — surface it
|
|
855
|
+
# rather than silently fabricating a wrong entity.
|
|
856
|
+
logger.warning(
|
|
857
|
+
"Sheet '%s' col %d: stochastic sheet for multi-dim class "
|
|
858
|
+
"%r cannot recover the full entity byname from a single "
|
|
859
|
+
"header value %r; skipping column.",
|
|
860
|
+
meta.sheet_name, c + 1, entity_class.class_name, ent_name,
|
|
861
|
+
)
|
|
862
|
+
continue
|
|
863
|
+
|
|
864
|
+
for r in range(meta.data_start_row, max_row):
|
|
865
|
+
val = _cell_value(ws, r, c)
|
|
866
|
+
if not val:
|
|
867
|
+
continue
|
|
868
|
+
# Read all index levels for this row.
|
|
869
|
+
level_vals: list[tuple[str, str | None]] = []
|
|
870
|
+
for col_idx, axis in meta.stoch_index_cols:
|
|
871
|
+
level_vals.append((axis, _cell_value(ws, r, col_idx) or None))
|
|
872
|
+
# A row missing any index level is malformed — skip it.
|
|
873
|
+
if any(v is None for _, v in level_vals):
|
|
874
|
+
continue
|
|
875
|
+
|
|
876
|
+
converted = _convert_value(val, leaf_dtype)
|
|
877
|
+
record: dict[str, Any] = {
|
|
878
|
+
"alternative": alt,
|
|
879
|
+
"entity_class": entity_class.class_name,
|
|
880
|
+
"entity_byname": entity_byname,
|
|
881
|
+
"param_name": pname,
|
|
882
|
+
"value": converted,
|
|
883
|
+
"index_value": level_vals[-1][1],
|
|
884
|
+
"index_name": level_vals[-1][0],
|
|
885
|
+
"data_type": leaf_dtype,
|
|
886
|
+
}
|
|
887
|
+
if len(level_vals) > 1:
|
|
888
|
+
record["extra_index_values"] = level_vals[:-1]
|
|
889
|
+
data.records.append(record)
|
|
890
|
+
|
|
891
|
+
return data
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
# ---------------------------------------------------------------------------
|
|
895
|
+
# Determine sheet type and parse accordingly
|
|
896
|
+
# ---------------------------------------------------------------------------
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def detect_and_parse_sheet(ws: Worksheet) -> tuple[SheetMetadata | None, bool]:
|
|
900
|
+
"""Auto-detect sheet type and parse its metadata.
|
|
901
|
+
|
|
902
|
+
Tries scenario first, then standard (constant/periodic), then link,
|
|
903
|
+
then transposed.
|
|
904
|
+
|
|
905
|
+
Returns:
|
|
906
|
+
(metadata, is_scenario) — metadata is None for unrecognised sheets.
|
|
907
|
+
is_scenario is True when the sheet is a scenario sheet (needs special
|
|
908
|
+
extraction via parse_scenario_sheet).
|
|
909
|
+
"""
|
|
910
|
+
# Skip very small sheets
|
|
911
|
+
if (ws.max_row or 0) < 2 or (ws.max_column or 0) < 2:
|
|
912
|
+
return None, False
|
|
913
|
+
|
|
914
|
+
# Check for scenario sheet: "Scenario names" in row 1, col B
|
|
915
|
+
if _is_scenario_sheet(ws):
|
|
916
|
+
return SheetMetadata(sheet_name=ws.title), True
|
|
917
|
+
|
|
918
|
+
# Check for a stochastic (_s) sheet: N>=2 left ``index:`` columns then a
|
|
919
|
+
# definition column carrying the transposed entity/alt/param roles. Must
|
|
920
|
+
# run BEFORE the standard-sheet probe, which would otherwise misread the
|
|
921
|
+
# transposed ``alternative`` value header as a parameter column.
|
|
922
|
+
stoch_def_col = _find_stochastic_def_col(ws)
|
|
923
|
+
if stoch_def_col is not None:
|
|
924
|
+
return parse_stochastic_sheet_metadata(ws, stoch_def_col), False
|
|
925
|
+
|
|
926
|
+
# Check for transposed sheet first: "alternative" or "parameter" in column B (row 0-4)
|
|
927
|
+
for r in range(min(5, _actual_max_row(ws))):
|
|
928
|
+
val_a = _cell_value(ws, r, 0)
|
|
929
|
+
if val_a and _is_info_row(val_a):
|
|
930
|
+
continue # skip INFO rows
|
|
931
|
+
val_b = _cell_value(ws, r, 1) # column B
|
|
932
|
+
if val_b:
|
|
933
|
+
vl = val_b.lower()
|
|
934
|
+
if vl in ("alternative", "parameter") or vl.startswith("parameter:"):
|
|
935
|
+
return parse_transposed_sheet_metadata(ws), False
|
|
936
|
+
|
|
937
|
+
# Check for standard sheet (has keywords like "description" in row 1)
|
|
938
|
+
for c in range(2, min(20, ws.max_column or 1)): # start at col C (skip A=navigate, B)
|
|
939
|
+
val = _cell_value(ws, 0, c)
|
|
940
|
+
if val and _is_keyword(val) and val.lower() != "navigate":
|
|
941
|
+
return parse_sheet_metadata(ws), False
|
|
942
|
+
|
|
943
|
+
# Check for link sheet (entity: in column A)
|
|
944
|
+
for r in range(min(5, ws.max_row or 1)):
|
|
945
|
+
val = _cell_value(ws, r, 0)
|
|
946
|
+
if val.lower().startswith("entity:") or val.lower().startswith("entity "):
|
|
947
|
+
return parse_link_sheet_metadata(ws), False
|
|
948
|
+
|
|
949
|
+
# Check for transposed sheet (column B has keywords like "alternative", "parameter")
|
|
950
|
+
for r in range(min(10, ws.max_row or 1)):
|
|
951
|
+
val = _cell_value(ws, r, 1)
|
|
952
|
+
if val and val.lower() in ("alternative", "parameter"):
|
|
953
|
+
return parse_transposed_sheet_metadata(ws), False
|
|
954
|
+
|
|
955
|
+
return None, False
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
# ---------------------------------------------------------------------------
|
|
959
|
+
# Data extraction
|
|
960
|
+
# ---------------------------------------------------------------------------
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
@dataclass
|
|
964
|
+
class SheetData:
|
|
965
|
+
"""Extracted data from one sheet, ready for DB import."""
|
|
966
|
+
|
|
967
|
+
sheet_name: str
|
|
968
|
+
metadata: SheetMetadata
|
|
969
|
+
|
|
970
|
+
# For each entity class mapping:
|
|
971
|
+
# list of dicts with keys: alternative, entity_byname, param_name, value,
|
|
972
|
+
# entity_existence (bool or None)
|
|
973
|
+
records: list[dict[str, Any]] = field(default_factory=list)
|
|
974
|
+
|
|
975
|
+
# Entity relationships (for link sheets): list of entity_byname tuples
|
|
976
|
+
link_entities: list[tuple[str, ...]] = field(default_factory=list)
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
def extract_sheet_data(ws: Worksheet, meta: SheetMetadata) -> SheetData:
|
|
980
|
+
"""Extract data records from a parsed sheet."""
|
|
981
|
+
if meta.is_stochastic:
|
|
982
|
+
return _extract_stochastic(ws, meta)
|
|
983
|
+
elif meta.is_transposed:
|
|
984
|
+
return _extract_transposed(ws, meta)
|
|
985
|
+
elif not meta.param_cols and meta.entity_classes:
|
|
986
|
+
return _extract_link(ws, meta)
|
|
987
|
+
else:
|
|
988
|
+
return _extract_standard(ws, meta)
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
def _extract_standard(ws: Worksheet, meta: SheetMetadata) -> SheetData:
|
|
992
|
+
"""Extract data from a standard (constant/periodic) sheet."""
|
|
993
|
+
data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
|
|
994
|
+
max_row = _actual_max_row(ws)
|
|
995
|
+
|
|
996
|
+
# Find entity element columns (columns in definition row between entity def and first
|
|
997
|
+
# non-structural column). These are the dimension name columns.
|
|
998
|
+
for c in sorted(meta.col_defs.keys()):
|
|
999
|
+
defn = meta.col_defs[c].lower()
|
|
1000
|
+
if not (defn == "alternative" or defn.startswith("entity") or
|
|
1001
|
+
defn.startswith("filter") or defn.startswith("index")):
|
|
1002
|
+
continue
|
|
1003
|
+
if defn == "alternative" or defn.startswith("filter") or defn.startswith("index"):
|
|
1004
|
+
continue
|
|
1005
|
+
# This is an entity-related column but not the entity def itself
|
|
1006
|
+
# Actually, the entity def column has the "entity:" prefix.
|
|
1007
|
+
# The dimension columns follow it and have plain dimension names.
|
|
1008
|
+
|
|
1009
|
+
# Simpler approach: entity def column + subsequent dimension columns
|
|
1010
|
+
# are identified from the entity_classes definitions.
|
|
1011
|
+
# The entity column index is the one with "entity:" in col_defs.
|
|
1012
|
+
entity_col: int | None = None
|
|
1013
|
+
for c, defn in meta.col_defs.items():
|
|
1014
|
+
if defn.lower().startswith("entity"):
|
|
1015
|
+
entity_col = c
|
|
1016
|
+
break
|
|
1017
|
+
|
|
1018
|
+
# Dimension element columns follow the entity column
|
|
1019
|
+
# Their count = max dimensions across entity classes
|
|
1020
|
+
if meta.entity_classes:
|
|
1021
|
+
max_dims = max(len(ec.dimensions) for ec in meta.entity_classes)
|
|
1022
|
+
else:
|
|
1023
|
+
max_dims = 0
|
|
1024
|
+
|
|
1025
|
+
if entity_col is not None and max_dims > 1:
|
|
1026
|
+
# For multi-dim entities, dimension element columns follow the entity column.
|
|
1027
|
+
# There are exactly max_dims dimension columns after the entity column.
|
|
1028
|
+
dim_cols = list(range(entity_col + 1, entity_col + 1 + max_dims))
|
|
1029
|
+
elif entity_col is not None:
|
|
1030
|
+
dim_cols = [] # single-dim: entity name is in entity_col itself
|
|
1031
|
+
else:
|
|
1032
|
+
dim_cols = []
|
|
1033
|
+
|
|
1034
|
+
for r in range(meta.data_start_row, max_row):
|
|
1035
|
+
# Read alternative. Normalise empty cells to ``None`` (rather than
|
|
1036
|
+
# ``""``) so downstream entity-only records propagate a real
|
|
1037
|
+
# absence — the writer's alt-creation step skips falsy names and
|
|
1038
|
+
# discards entity-only records before parameter writes, so a
|
|
1039
|
+
# synthesised ``""`` would create an empty-string alternative
|
|
1040
|
+
# in SpineDB.
|
|
1041
|
+
if meta.alt_col is not None:
|
|
1042
|
+
alt = _cell_value(ws, r, meta.alt_col) or None
|
|
1043
|
+
else:
|
|
1044
|
+
alt = None
|
|
1045
|
+
|
|
1046
|
+
# Read entity byname
|
|
1047
|
+
if entity_col is not None:
|
|
1048
|
+
if max_dims <= 1:
|
|
1049
|
+
ent_name = _cell_value(ws, r, entity_col)
|
|
1050
|
+
entity_byname = (ent_name,) if ent_name else None
|
|
1051
|
+
else:
|
|
1052
|
+
elements = []
|
|
1053
|
+
for dc in dim_cols:
|
|
1054
|
+
elements.append(_cell_value(ws, r, dc))
|
|
1055
|
+
if any(elements):
|
|
1056
|
+
entity_byname = tuple(elements)
|
|
1057
|
+
else:
|
|
1058
|
+
entity_byname = None
|
|
1059
|
+
else:
|
|
1060
|
+
entity_byname = None
|
|
1061
|
+
|
|
1062
|
+
if entity_byname is None:
|
|
1063
|
+
continue
|
|
1064
|
+
|
|
1065
|
+
# Determine entity class for this row (using filter if present)
|
|
1066
|
+
if meta.filter_col is not None and meta.filter_map and len(meta.entity_classes) > 1:
|
|
1067
|
+
filter_val = _cell_value(ws, r, meta.filter_col)
|
|
1068
|
+
entity_class = None
|
|
1069
|
+
for ec in meta.entity_classes:
|
|
1070
|
+
pattern = meta.filter_map.get(ec.class_name, "")
|
|
1071
|
+
if pattern and re.match(pattern, filter_val):
|
|
1072
|
+
entity_class = ec
|
|
1073
|
+
break
|
|
1074
|
+
if entity_class is None:
|
|
1075
|
+
logger.warning(
|
|
1076
|
+
"Sheet '%s' row %d: filter value '%s' didn't match any entity class",
|
|
1077
|
+
meta.sheet_name, r + 1, filter_val,
|
|
1078
|
+
)
|
|
1079
|
+
continue
|
|
1080
|
+
elif meta.entity_classes:
|
|
1081
|
+
entity_class = meta.entity_classes[0]
|
|
1082
|
+
else:
|
|
1083
|
+
continue
|
|
1084
|
+
|
|
1085
|
+
# Read index value (for periodic/map sheets)
|
|
1086
|
+
index_val = None
|
|
1087
|
+
if meta.index_col is not None:
|
|
1088
|
+
index_val = _cell_value(ws, r, meta.index_col) or None
|
|
1089
|
+
|
|
1090
|
+
# Read extra index values (e.g. ladder sheets carry index: period
|
|
1091
|
+
# + index: tier). We keep them in the same column order as the
|
|
1092
|
+
# sheet so the DB-write side can reconstruct the nested Map.
|
|
1093
|
+
extra_index_values: list[tuple[str, str | None]] = []
|
|
1094
|
+
for c_extra, name_extra in meta.extra_index_cols:
|
|
1095
|
+
v = _cell_value(ws, r, c_extra) or None
|
|
1096
|
+
extra_index_values.append((name_extra, v))
|
|
1097
|
+
|
|
1098
|
+
# Read entity existence (strict TRUE/FALSE — exporter only ever
|
|
1099
|
+
# writes those two literals; anything else is a data error).
|
|
1100
|
+
entity_existence = None
|
|
1101
|
+
if meta.entity_existence_col is not None:
|
|
1102
|
+
ee_val = _cell_value(ws, r, meta.entity_existence_col)
|
|
1103
|
+
if ee_val:
|
|
1104
|
+
entity_existence = _parse_strict_bool(
|
|
1105
|
+
ee_val,
|
|
1106
|
+
sheet_name=meta.sheet_name,
|
|
1107
|
+
row_1based=r + 1,
|
|
1108
|
+
col_1based=meta.entity_existence_col + 1,
|
|
1109
|
+
field_name="entity existence",
|
|
1110
|
+
)
|
|
1111
|
+
|
|
1112
|
+
# Read parameter values
|
|
1113
|
+
row_has_data = False
|
|
1114
|
+
for c, pname in meta.param_cols.items():
|
|
1115
|
+
if pname == ENTITY_EXISTENCE:
|
|
1116
|
+
continue # already handled
|
|
1117
|
+
val = _cell_value(ws, r, c)
|
|
1118
|
+
if not val:
|
|
1119
|
+
continue
|
|
1120
|
+
|
|
1121
|
+
# Convert based on data type. Fallback order:
|
|
1122
|
+
# 1. explicit per-column data_type
|
|
1123
|
+
# 2. sheet default_data_type
|
|
1124
|
+
# 3. "string" — safe pass-through. The exporter ALWAYS
|
|
1125
|
+
# writes a data type for every parameter column, so
|
|
1126
|
+
# reaching the fallback implies a hand-edited sheet;
|
|
1127
|
+
# "string" preserves the cell verbatim instead of
|
|
1128
|
+
# forcing a ``float()`` that may corrupt string-typed
|
|
1129
|
+
# parameters (a previous "float" fallback turned cell
|
|
1130
|
+
# "123" into 123.0 even for schema-string params).
|
|
1131
|
+
if c in meta.data_types:
|
|
1132
|
+
dtype = meta.data_types[c]
|
|
1133
|
+
elif meta.default_data_type:
|
|
1134
|
+
dtype = meta.default_data_type
|
|
1135
|
+
else:
|
|
1136
|
+
dtype = "string"
|
|
1137
|
+
logger.warning(
|
|
1138
|
+
"Sheet '%s' col %d (parameter %r): no data type declared; "
|
|
1139
|
+
"defaulting to 'string' (raw cell value).",
|
|
1140
|
+
meta.sheet_name, c + 1, pname,
|
|
1141
|
+
)
|
|
1142
|
+
converted = _convert_value(val, dtype)
|
|
1143
|
+
|
|
1144
|
+
record = {
|
|
1145
|
+
"alternative": alt,
|
|
1146
|
+
"entity_class": entity_class.class_name,
|
|
1147
|
+
"entity_byname": entity_byname,
|
|
1148
|
+
"param_name": pname,
|
|
1149
|
+
"value": converted,
|
|
1150
|
+
"index_value": index_val,
|
|
1151
|
+
"index_name": meta.index_name,
|
|
1152
|
+
"data_type": dtype,
|
|
1153
|
+
}
|
|
1154
|
+
if extra_index_values:
|
|
1155
|
+
record["extra_index_values"] = list(extra_index_values)
|
|
1156
|
+
data.records.append(record)
|
|
1157
|
+
row_has_data = True
|
|
1158
|
+
|
|
1159
|
+
# Add entity existence record
|
|
1160
|
+
if entity_existence is not None:
|
|
1161
|
+
data.records.append({
|
|
1162
|
+
"alternative": alt,
|
|
1163
|
+
"entity_class": entity_class.class_name,
|
|
1164
|
+
"entity_byname": entity_byname,
|
|
1165
|
+
"param_name": ENTITY_EXISTENCE,
|
|
1166
|
+
"value": entity_existence,
|
|
1167
|
+
"index_value": None,
|
|
1168
|
+
"index_name": None,
|
|
1169
|
+
})
|
|
1170
|
+
row_has_data = True
|
|
1171
|
+
|
|
1172
|
+
# Create entity-only record if no data was found for this row
|
|
1173
|
+
# (ensures data-less entities are still imported). Propagate the
|
|
1174
|
+
# real ``alt`` even when None — the downstream writer
|
|
1175
|
+
# (``write_self_describing_to_db._import_sheet``) skips
|
|
1176
|
+
# alternative-creation for falsy alt names and discards
|
|
1177
|
+
# entity-only records before parameter writes, so a synthesised
|
|
1178
|
+
# ``""`` alt name would create an empty-string alternative in
|
|
1179
|
+
# SpineDB while a faithful ``None`` is dropped cleanly.
|
|
1180
|
+
if not row_has_data and entity_byname:
|
|
1181
|
+
data.records.append({
|
|
1182
|
+
"alternative": alt,
|
|
1183
|
+
"entity_class": entity_class.class_name,
|
|
1184
|
+
"entity_byname": entity_byname,
|
|
1185
|
+
"param_name": "",
|
|
1186
|
+
"value": None,
|
|
1187
|
+
"index_value": None,
|
|
1188
|
+
"index_name": None,
|
|
1189
|
+
})
|
|
1190
|
+
|
|
1191
|
+
return data
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def _extract_link(ws: Worksheet, meta: SheetMetadata) -> SheetData:
|
|
1195
|
+
"""Extract entity relationships from a link sheet."""
|
|
1196
|
+
data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
|
|
1197
|
+
max_row = _actual_max_row(ws)
|
|
1198
|
+
max_col = ws.max_column or 1
|
|
1199
|
+
|
|
1200
|
+
for r in range(meta.data_start_row, max_row):
|
|
1201
|
+
elements = []
|
|
1202
|
+
for c in range(max_col):
|
|
1203
|
+
val = _cell_value(ws, r, c)
|
|
1204
|
+
if val:
|
|
1205
|
+
elements.append(val)
|
|
1206
|
+
if elements:
|
|
1207
|
+
data.link_entities.append(tuple(elements))
|
|
1208
|
+
|
|
1209
|
+
return data
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
def _extract_transposed(ws: Worksheet, meta: SheetMetadata) -> SheetData:
|
|
1213
|
+
"""Extract data from a transposed (timeseries) sheet."""
|
|
1214
|
+
data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
|
|
1215
|
+
max_row = _actual_max_row(ws)
|
|
1216
|
+
max_col = ws.max_column or 1
|
|
1217
|
+
|
|
1218
|
+
# Each data column (from col 2 onwards) is one data series
|
|
1219
|
+
data_col_start = 2 # columns C+ (0-based index 2)
|
|
1220
|
+
|
|
1221
|
+
# On single-param transposed sheets (timeseries, array-transposed)
|
|
1222
|
+
# the parameter triplet in row 1 / col B carries the canonical data
|
|
1223
|
+
# type — e.g. ``"parameter: solves | data type: boolean (array)"``.
|
|
1224
|
+
# We extract it once and attach it to every emitted record so the
|
|
1225
|
+
# writer can decide whether to reconstruct an ``Array`` or a ``Map``.
|
|
1226
|
+
triplet_dtype: str | None = None
|
|
1227
|
+
if meta.param_row is not None:
|
|
1228
|
+
triplet_text = _cell_value(ws, meta.param_row, 1) # col B
|
|
1229
|
+
triplet_dtype = _extract_triplet_field(triplet_text, "data type")
|
|
1230
|
+
if triplet_dtype is None and meta.default_data_type:
|
|
1231
|
+
triplet_dtype = meta.default_data_type
|
|
1232
|
+
|
|
1233
|
+
for c in range(data_col_start, max_col):
|
|
1234
|
+
# Read column metadata from the header rows
|
|
1235
|
+
alt = _cell_value(ws, meta.alt_row, c) if meta.alt_row is not None else None
|
|
1236
|
+
pname = _cell_value(ws, meta.param_row, c) if meta.param_row is not None else None
|
|
1237
|
+
ent_name = _cell_value(ws, meta.entity_row, c) if meta.entity_row is not None else None
|
|
1238
|
+
|
|
1239
|
+
if not pname and meta.default_parameter:
|
|
1240
|
+
pname = meta.default_parameter
|
|
1241
|
+
if not alt or not ent_name:
|
|
1242
|
+
continue
|
|
1243
|
+
|
|
1244
|
+
entity_class = meta.entity_classes[0] if meta.entity_classes else None
|
|
1245
|
+
if entity_class is None:
|
|
1246
|
+
continue
|
|
1247
|
+
|
|
1248
|
+
entity_byname = (ent_name,)
|
|
1249
|
+
|
|
1250
|
+
# Per-column data type (multi-param _t sheets) takes precedence over
|
|
1251
|
+
# the single-param triplet; fall back to the triplet then "string".
|
|
1252
|
+
col_dtype = meta.data_types.get(c) or triplet_dtype
|
|
1253
|
+
|
|
1254
|
+
# Read time-indexed data
|
|
1255
|
+
for r in range(meta.data_start_row, max_row):
|
|
1256
|
+
index_val = _cell_value(ws, r, 0) if meta.index_col_transposed == 0 else None
|
|
1257
|
+
val = _cell_value(ws, r, c)
|
|
1258
|
+
if not val or not index_val:
|
|
1259
|
+
continue
|
|
1260
|
+
|
|
1261
|
+
# Use the column/triplet data type for value conversion.
|
|
1262
|
+
# The pre-fix code force-cast every cell to float, which only
|
|
1263
|
+
# worked by accident for ``boolean (array)`` (period names
|
|
1264
|
+
# like ``"Y2025"`` raise on ``float()`` and ``_convert_value``
|
|
1265
|
+
# falls back to returning the raw string). A schema-string
|
|
1266
|
+
# series with numeric-looking entries (e.g. ``"01"``) would
|
|
1267
|
+
# silently get coerced to ``1.0`` and round-trip wrong.
|
|
1268
|
+
value_dtype = col_dtype or "string"
|
|
1269
|
+
converted = _convert_value(val, value_dtype)
|
|
1270
|
+
data.records.append({
|
|
1271
|
+
"alternative": alt,
|
|
1272
|
+
"entity_class": entity_class.class_name,
|
|
1273
|
+
"entity_byname": entity_byname,
|
|
1274
|
+
"param_name": pname,
|
|
1275
|
+
"value": converted,
|
|
1276
|
+
"index_value": index_val,
|
|
1277
|
+
"index_name": meta.index_name,
|
|
1278
|
+
"data_type": col_dtype,
|
|
1279
|
+
})
|
|
1280
|
+
|
|
1281
|
+
return data
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _parse_strict_bool(
|
|
1285
|
+
val: str,
|
|
1286
|
+
*,
|
|
1287
|
+
sheet_name: str,
|
|
1288
|
+
row_1based: int,
|
|
1289
|
+
col_1based: int,
|
|
1290
|
+
field_name: str,
|
|
1291
|
+
) -> bool:
|
|
1292
|
+
"""Strict TRUE / FALSE parse, case-insensitive, mirroring the exporter.
|
|
1293
|
+
|
|
1294
|
+
The exporter only ever writes the literal strings ``"TRUE"`` / ``"FALSE"``
|
|
1295
|
+
for entity-existence and boolean-array cells (excel_writer.py:2272 and
|
|
1296
|
+
:2773 — the dropdown validation also enforces those exact tokens). A
|
|
1297
|
+
cell holding anything else (``"yes"``, ``"1"``, ``"Y"``, a typo) is a
|
|
1298
|
+
data error: raise so the user sees the offending sheet / cell rather
|
|
1299
|
+
than silently importing ``False``.
|
|
1300
|
+
"""
|
|
1301
|
+
stripped = val.strip().lower()
|
|
1302
|
+
if stripped == _BOOL_TRUE_LITERAL:
|
|
1303
|
+
return True
|
|
1304
|
+
if stripped == _BOOL_FALSE_LITERAL:
|
|
1305
|
+
return False
|
|
1306
|
+
raise ValueError(
|
|
1307
|
+
f"Sheet '{sheet_name}' row {row_1based} col {col_1based}: "
|
|
1308
|
+
f"{field_name} cell has value {val!r}; expected 'TRUE' or 'FALSE' "
|
|
1309
|
+
f"(case-insensitive). The exporter only emits those literals — "
|
|
1310
|
+
f"fix the cell or restore the exporter-written value."
|
|
1311
|
+
)
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def _convert_value(val: str, dtype: str) -> Any:
|
|
1315
|
+
"""Convert a string cell value to the appropriate Python type.
|
|
1316
|
+
|
|
1317
|
+
``float`` cells accept the IEEE non-finite sentinels ``inf`` /
|
|
1318
|
+
``-inf`` / ``nan`` (case-insensitive) — the export writer emits
|
|
1319
|
+
them as strings because openpyxl drops actual non-finite floats.
|
|
1320
|
+
|
|
1321
|
+
The leaf scalar type is taken from the part of *dtype* BEFORE any
|
|
1322
|
+
``(...)`` container suffix: ``"float (1d-map)"``, ``"float (array)"``,
|
|
1323
|
+
``"float (3d-map)"`` all carry a ``float`` leaf and must convert their
|
|
1324
|
+
per-cell values to ``float`` — otherwise a Map/Array would round-trip
|
|
1325
|
+
with string leaves (``["10", "20"]`` instead of ``[10.0, 20.0]``).
|
|
1326
|
+
Compound leaf types joined with ``/`` (e.g. ``"string/float (1d-map)"``
|
|
1327
|
+
— a param that accepts either) attempt the ``float`` conversion when
|
|
1328
|
+
``float`` is one of the tokens, falling back to the raw string for a
|
|
1329
|
+
genuinely non-numeric cell. Only the exact scalar ``"boolean"``
|
|
1330
|
+
strict-parses TRUE/FALSE: a ``"boolean (array)"`` cell holds a period
|
|
1331
|
+
token (the round-trip preservation form, see _extract_standard) and
|
|
1332
|
+
must stay raw.
|
|
1333
|
+
"""
|
|
1334
|
+
base = dtype.split("(", 1)[0].strip().lower() if dtype else ""
|
|
1335
|
+
base_tokens = base.replace("/", " ").split()
|
|
1336
|
+
if "float" in base_tokens:
|
|
1337
|
+
if isinstance(val, str):
|
|
1338
|
+
lo = val.strip().lower()
|
|
1339
|
+
if lo in ("inf", "+inf", "infinity", "+infinity"):
|
|
1340
|
+
return float("inf")
|
|
1341
|
+
if lo in ("-inf", "-infinity"):
|
|
1342
|
+
return float("-inf")
|
|
1343
|
+
if lo == "nan":
|
|
1344
|
+
return float("nan")
|
|
1345
|
+
try:
|
|
1346
|
+
return float(val)
|
|
1347
|
+
except (ValueError, TypeError):
|
|
1348
|
+
return val # keep as string if conversion fails
|
|
1349
|
+
elif dtype == "boolean":
|
|
1350
|
+
# Strict match — the exporter only writes ``"TRUE"`` / ``"FALSE"``
|
|
1351
|
+
# for boolean cells (see excel_writer.py:2272 / :2773). Without
|
|
1352
|
+
# call-site context we can't name the offending sheet/row, so the
|
|
1353
|
+
# error message asks the user to inspect their workbook; the
|
|
1354
|
+
# per-cell entity-existence path uses ``_parse_strict_bool`` and
|
|
1355
|
+
# gives a much sharper message.
|
|
1356
|
+
stripped = val.strip().lower()
|
|
1357
|
+
if stripped == _BOOL_TRUE_LITERAL:
|
|
1358
|
+
return True
|
|
1359
|
+
if stripped == _BOOL_FALSE_LITERAL:
|
|
1360
|
+
return False
|
|
1361
|
+
raise ValueError(
|
|
1362
|
+
f"Boolean cell has value {val!r}; expected 'TRUE' or 'FALSE' "
|
|
1363
|
+
f"(case-insensitive)."
|
|
1364
|
+
)
|
|
1365
|
+
else:
|
|
1366
|
+
return val
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
# ---------------------------------------------------------------------------
|
|
1370
|
+
# Public API: read an entire workbook
|
|
1371
|
+
# ---------------------------------------------------------------------------
|
|
1372
|
+
|
|
1373
|
+
|
|
1374
|
+
def read_self_describing_excel(
|
|
1375
|
+
file_path: str,
|
|
1376
|
+
skip_sheets: set[str] | None = None,
|
|
1377
|
+
) -> list[SheetData]:
|
|
1378
|
+
"""Read all self-describing sheets from an Excel workbook.
|
|
1379
|
+
|
|
1380
|
+
Args:
|
|
1381
|
+
file_path: Path to the .xlsx file.
|
|
1382
|
+
skip_sheets: Sheet names to skip (e.g. navigate, version).
|
|
1383
|
+
|
|
1384
|
+
Returns:
|
|
1385
|
+
List of SheetData objects, one per successfully parsed sheet.
|
|
1386
|
+
"""
|
|
1387
|
+
if skip_sheets is None:
|
|
1388
|
+
skip_sheets = {"navigate", "version"}
|
|
1389
|
+
|
|
1390
|
+
# Note: do NOT use read_only=True — it reports max_row=1048576 for
|
|
1391
|
+
# sheets without explicit dimensions, which makes iteration hang.
|
|
1392
|
+
wb = openpyxl.load_workbook(file_path, data_only=True)
|
|
1393
|
+
results: list[SheetData] = []
|
|
1394
|
+
|
|
1395
|
+
for sheet_name in wb.sheetnames:
|
|
1396
|
+
if sheet_name in skip_sheets:
|
|
1397
|
+
continue
|
|
1398
|
+
|
|
1399
|
+
ws = wb[sheet_name]
|
|
1400
|
+
meta, is_scenario = detect_and_parse_sheet(ws)
|
|
1401
|
+
|
|
1402
|
+
if is_scenario:
|
|
1403
|
+
sheet_data = parse_scenario_sheet(ws)
|
|
1404
|
+
results.append(sheet_data)
|
|
1405
|
+
logger.info(
|
|
1406
|
+
"Sheet '%s' (scenario): %d records",
|
|
1407
|
+
sheet_name, len(sheet_data.records),
|
|
1408
|
+
)
|
|
1409
|
+
continue
|
|
1410
|
+
|
|
1411
|
+
if meta is None:
|
|
1412
|
+
logger.debug("Skipping sheet '%s': no metadata found", sheet_name)
|
|
1413
|
+
continue
|
|
1414
|
+
|
|
1415
|
+
sheet_data = extract_sheet_data(ws, meta)
|
|
1416
|
+
results.append(sheet_data)
|
|
1417
|
+
logger.info(
|
|
1418
|
+
"Sheet '%s': %d records, %d links",
|
|
1419
|
+
sheet_name, len(sheet_data.records), len(sheet_data.link_entities),
|
|
1420
|
+
)
|
|
1421
|
+
|
|
1422
|
+
wb.close()
|
|
1423
|
+
return results
|