flextool 4.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. flextool/__init__.py +41 -0
  2. flextool/_mem_sampler.py +193 -0
  3. flextool/_resources.py +43 -0
  4. flextool/calibrate/__init__.py +51 -0
  5. flextool/calibrate/__main__.py +11 -0
  6. flextool/calibrate/_cli.py +316 -0
  7. flextool/calibrate/_db_alt.py +166 -0
  8. flextool/calibrate/_final_outputs.py +110 -0
  9. flextool/calibrate/_guard.py +151 -0
  10. flextool/calibrate/_loop.py +558 -0
  11. flextool/calibrate/_readers.py +223 -0
  12. flextool/calibrate/_report.py +263 -0
  13. flextool/calibrate/_sizing.py +699 -0
  14. flextool/calibrate/_solve.py +134 -0
  15. flextool/calibrate/_solve_status.py +495 -0
  16. flextool/cli/__init__.py +9 -0
  17. flextool/cli/_console.py +51 -0
  18. flextool/cli/_timing.py +147 -0
  19. flextool/cli/cmd_execute_flextool_workflow.py +187 -0
  20. flextool/cli/cmd_export_to_tabular.py +56 -0
  21. flextool/cli/cmd_import_sensitivities.py +75 -0
  22. flextool/cli/cmd_migrate_database.py +13 -0
  23. flextool/cli/cmd_open_results_db.py +269 -0
  24. flextool/cli/cmd_read_matpower.py +66 -0
  25. flextool/cli/cmd_read_old_flextool.py +63 -0
  26. flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
  27. flextool/cli/cmd_read_tabular_input.py +81 -0
  28. flextool/cli/cmd_run_flextool.py +1095 -0
  29. flextool/cli/cmd_scenario_results.py +284 -0
  30. flextool/cli/cmd_solve_mps.py +169 -0
  31. flextool/cli/cmd_update_flextool.py +17 -0
  32. flextool/cli/cmd_write_outputs.py +125 -0
  33. flextool/common_utils/__init__.py +1 -0
  34. flextool/common_utils/plot_mem_shape.py +77 -0
  35. flextool/common_utils/precision.py +451 -0
  36. flextool/decomposition/__init__.py +0 -0
  37. flextool/decomposition/region_decomposition.py +128 -0
  38. flextool/decomposition/region_filter.py +1261 -0
  39. flextool/engine_polars/__init__.py +110 -0
  40. flextool/engine_polars/_axis_enums.py +742 -0
  41. flextool/engine_polars/_benders.py +3462 -0
  42. flextool/engine_polars/_block_layout.py +1479 -0
  43. flextool/engine_polars/_blocks.py +1515 -0
  44. flextool/engine_polars/_commodity_ladder.py +660 -0
  45. flextool/engine_polars/_cumulative_invest.py +1165 -0
  46. flextool/engine_polars/_db_loader.py +153 -0
  47. flextool/engine_polars/_db_reader.py +127 -0
  48. flextool/engine_polars/_dc_power_flow.py +445 -0
  49. flextool/engine_polars/_delay.py +442 -0
  50. flextool/engine_polars/_derived_arithmetic.py +432 -0
  51. flextool/engine_polars/_derived_block.py +990 -0
  52. flextool/engine_polars/_derived_branch.py +769 -0
  53. flextool/engine_polars/_derived_existing.py +1353 -0
  54. flextool/engine_polars/_derived_npv.py +1297 -0
  55. flextool/engine_polars/_derived_params.py +9850 -0
  56. flextool/engine_polars/_derived_profile.py +881 -0
  57. flextool/engine_polars/_derived_walks.py +276 -0
  58. flextool/engine_polars/_determinism.py +70 -0
  59. flextool/engine_polars/_direct_params.py +2186 -0
  60. flextool/engine_polars/_dump_csvs.py +1009 -0
  61. flextool/engine_polars/_emit_arc_unions.py +1631 -0
  62. flextool/engine_polars/_emit_calc_params.py +729 -0
  63. flextool/engine_polars/_emit_chain_params.py +709 -0
  64. flextool/engine_polars/_emit_co2_accumulators.py +400 -0
  65. flextool/engine_polars/_emit_dispatchers.py +690 -0
  66. flextool/engine_polars/_emit_energy_margin.py +125 -0
  67. flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
  68. flextool/engine_polars/_emit_entity_annual.py +428 -0
  69. flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
  70. flextool/engine_polars/_emit_leaf_sets.py +550 -0
  71. flextool/engine_polars/_emit_lp_scaling.py +665 -0
  72. flextool/engine_polars/_emit_mid_sets.py +859 -0
  73. flextool/engine_polars/_emit_pdt_params.py +759 -0
  74. flextool/engine_polars/_emit_per_solve.py +774 -0
  75. flextool/engine_polars/_emit_period_calc.py +504 -0
  76. flextool/engine_polars/_emit_period_params.py +2398 -0
  77. flextool/engine_polars/_emit_provider_io.py +141 -0
  78. flextool/engine_polars/_emit_reserve.py +574 -0
  79. flextool/engine_polars/_emit_solve_time.py +311 -0
  80. flextool/engine_polars/_emit_solve_writers.py +1249 -0
  81. flextool/engine_polars/_flex_data_accumulator.py +388 -0
  82. flextool/engine_polars/_flex_data_provider.py +478 -0
  83. flextool/engine_polars/_group_slack.py +1253 -0
  84. flextool/engine_polars/_inmemory_reader.py +140 -0
  85. flextool/engine_polars/_input_source.py +336 -0
  86. flextool/engine_polars/_invest_seeds.py +191 -0
  87. flextool/engine_polars/_native_input_writer.py +100 -0
  88. flextool/engine_polars/_native_run_model.py +1348 -0
  89. flextool/engine_polars/_orchestration.py +4314 -0
  90. flextool/engine_polars/_output_writer.py +439 -0
  91. flextool/engine_polars/_param_shapes.py +1595 -0
  92. flextool/engine_polars/_parquet_bundle.py +723 -0
  93. flextool/engine_polars/_pdt_join.py +167 -0
  94. flextool/engine_polars/_pdt_lookup.py +547 -0
  95. flextool/engine_polars/_per_solve_sets.py +335 -0
  96. flextool/engine_polars/_projection_params.py +2056 -0
  97. flextool/engine_polars/_provider_keys.py +173 -0
  98. flextool/engine_polars/_provider_translators.py +225 -0
  99. flextool/engine_polars/_recursive_solve.py +703 -0
  100. flextool/engine_polars/_region_filter.py +2508 -0
  101. flextool/engine_polars/_reserve.py +649 -0
  102. flextool/engine_polars/_solve_acceptance.py +331 -0
  103. flextool/engine_polars/_solve_config.py +1001 -0
  104. flextool/engine_polars/_solve_context.py +885 -0
  105. flextool/engine_polars/_solve_handoff.py +164 -0
  106. flextool/engine_polars/_solve_state.py +232 -0
  107. flextool/engine_polars/_solver_base.py +36 -0
  108. flextool/engine_polars/_solver_dispatch.py +511 -0
  109. flextool/engine_polars/_spinedb_reader.py +1165 -0
  110. flextool/engine_polars/_stochastic.py +593 -0
  111. flextool/engine_polars/_subprocess_solve.py +1838 -0
  112. flextool/engine_polars/_timeline.py +1416 -0
  113. flextool/engine_polars/_vectorize.py +438 -0
  114. flextool/engine_polars/_warm.py +858 -0
  115. flextool/engine_polars/autoscale/__init__.py +107 -0
  116. flextool/engine_polars/autoscale/_config.py +218 -0
  117. flextool/engine_polars/autoscale/_layer2.py +1253 -0
  118. flextool/engine_polars/autoscale/_layer2_types.py +584 -0
  119. flextool/engine_polars/autoscale/_quantity_types.py +621 -0
  120. flextool/engine_polars/autoscale/_report.py +336 -0
  121. flextool/engine_polars/chain.py +259 -0
  122. flextool/engine_polars/input.py +6638 -0
  123. flextool/engine_polars/model.py +4754 -0
  124. flextool/env_check.py +388 -0
  125. flextool/export_to_tabular/__init__.py +5 -0
  126. flextool/export_to_tabular/db_reader.py +224 -0
  127. flextool/export_to_tabular/excel_writer.py +3559 -0
  128. flextool/export_to_tabular/export_settings.yaml +377 -0
  129. flextool/export_to_tabular/export_to_excel.py +227 -0
  130. flextool/export_to_tabular/formatting.py +543 -0
  131. flextool/export_to_tabular/sheet_config.py +876 -0
  132. flextool/gui/__init__.py +0 -0
  133. flextool/gui/__main__.py +118 -0
  134. flextool/gui/calibrate_commands.py +184 -0
  135. flextool/gui/calibrate_jobs.py +424 -0
  136. flextool/gui/check_tree.py +142 -0
  137. flextool/gui/cli_format.py +83 -0
  138. flextool/gui/config_parser.py +68 -0
  139. flextool/gui/data_models.py +362 -0
  140. flextool/gui/db_editor_integration.py +202 -0
  141. flextool/gui/db_version_check.py +269 -0
  142. flextool/gui/dialogs/__init__.py +0 -0
  143. flextool/gui/dialogs/add_dialog.py +1098 -0
  144. flextool/gui/dialogs/calibrate_dialog.py +1259 -0
  145. flextool/gui/dialogs/file_picker.py +473 -0
  146. flextool/gui/dialogs/group_picker.py +299 -0
  147. flextool/gui/dialogs/migration_consent_dialog.py +106 -0
  148. flextool/gui/dialogs/migration_progress_dialog.py +237 -0
  149. flextool/gui/dialogs/plot_dialog.py +459 -0
  150. flextool/gui/dialogs/plot_settings_picker.py +2184 -0
  151. flextool/gui/dialogs/project_dialog.py +426 -0
  152. flextool/gui/dialogs/update_dialog.py +212 -0
  153. flextool/gui/downsampling.py +88 -0
  154. flextool/gui/error_handling.py +50 -0
  155. flextool/gui/execution_manager.py +1715 -0
  156. flextool/gui/execution_window.py +1377 -0
  157. flextool/gui/hover_tooltip.py +111 -0
  158. flextool/gui/input_sources.py +730 -0
  159. flextool/gui/main_window.py +6181 -0
  160. flextool/gui/network_graph.py +215 -0
  161. flextool/gui/output_actions.py +393 -0
  162. flextool/gui/output_log_window.py +159 -0
  163. flextool/gui/platform_utils.py +421 -0
  164. flextool/gui/plot_cache.py +88 -0
  165. flextool/gui/plot_canvas.py +543 -0
  166. flextool/gui/plot_config_reader.py +272 -0
  167. flextool/gui/project_utils.py +100 -0
  168. flextool/gui/result_viewer.py +4394 -0
  169. flextool/gui/scenario_key.py +162 -0
  170. flextool/gui/scenario_lists.py +516 -0
  171. flextool/gui/settings_io.py +360 -0
  172. flextool/gui/solve_reader.py +103 -0
  173. flextool/gui/tree_reorder.py +88 -0
  174. flextool/gui/ui_metrics.py +420 -0
  175. flextool/input_derivation/__init__.py +281 -0
  176. flextool/input_derivation/_commodity_ladder.py +375 -0
  177. flextool/input_derivation/_commodity_ladder_sets.py +70 -0
  178. flextool/input_derivation/_dc_power_flow.py +377 -0
  179. flextool/input_derivation/_method_constants.py +77 -0
  180. flextool/input_derivation/_process_method.py +258 -0
  181. flextool/input_derivation/_specs.py +1026 -0
  182. flextool/input_derivation/_validators.py +321 -0
  183. flextool/lean_parquet.py +159 -0
  184. flextool/model_builder/__init__.py +5 -0
  185. flextool/model_builder/build_model.py +589 -0
  186. flextool/model_builder/encoding.py +67 -0
  187. flextool/model_builder/names.py +34 -0
  188. flextool/model_builder/profiles.py +129 -0
  189. flextool/plot_outputs/__init__.py +14 -0
  190. flextool/plot_outputs/axis_helpers.py +355 -0
  191. flextool/plot_outputs/color_template.py +888 -0
  192. flextool/plot_outputs/config.py +171 -0
  193. flextool/plot_outputs/format_helpers.py +345 -0
  194. flextool/plot_outputs/legend_helpers.py +143 -0
  195. flextool/plot_outputs/orchestrator.py +1141 -0
  196. flextool/plot_outputs/perf.py +37 -0
  197. flextool/plot_outputs/plan.py +1787 -0
  198. flextool/plot_outputs/plot_bars.py +1510 -0
  199. flextool/plot_outputs/plot_bars_detail.py +753 -0
  200. flextool/plot_outputs/plot_lines.py +951 -0
  201. flextool/plot_outputs/shared_manifest.py +564 -0
  202. flextool/plot_outputs/subplot_helpers.py +137 -0
  203. flextool/process_inputs/__init__.py +188 -0
  204. flextool/process_inputs/import_old_excel_input.json +4159 -0
  205. flextool/process_inputs/read_matpower.py +451 -0
  206. flextool/process_inputs/read_old_flextool.py +1288 -0
  207. flextool/process_inputs/read_self_describing_excel.py +1423 -0
  208. flextool/process_inputs/read_tabular_with_specification.py +1114 -0
  209. flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
  210. flextool/process_inputs/write_self_describing_to_db.py +977 -0
  211. flextool/process_inputs/write_to_input_db.py +269 -0
  212. flextool/process_outputs/__init__.py +7 -0
  213. flextool/process_outputs/_annualize.py +55 -0
  214. flextool/process_outputs/_inmemory_helpers.py +292 -0
  215. flextool/process_outputs/_output_meta.py +672 -0
  216. flextool/process_outputs/calc_capacity_flows.py +107 -0
  217. flextool/process_outputs/calc_connections.py +136 -0
  218. flextool/process_outputs/calc_costs.py +260 -0
  219. flextool/process_outputs/calc_group_flows.py +192 -0
  220. flextool/process_outputs/calc_slacks.py +103 -0
  221. flextool/process_outputs/calc_storage_vre.py +160 -0
  222. flextool/process_outputs/drop_levels.py +208 -0
  223. flextool/process_outputs/handoff_writers.py +1315 -0
  224. flextool/process_outputs/out_ancillary.py +544 -0
  225. flextool/process_outputs/out_capacity.py +179 -0
  226. flextool/process_outputs/out_costs.py +334 -0
  227. flextool/process_outputs/out_flowgroup.py +189 -0
  228. flextool/process_outputs/out_flows.py +301 -0
  229. flextool/process_outputs/out_group.py +475 -0
  230. flextool/process_outputs/out_node.py +190 -0
  231. flextool/process_outputs/persist_realized_slice.py +601 -0
  232. flextool/process_outputs/process_results.py +24 -0
  233. flextool/process_outputs/read_highs_solution.py +2256 -0
  234. flextool/process_outputs/read_parameters.py +1799 -0
  235. flextool/process_outputs/read_sets.py +1095 -0
  236. flextool/process_outputs/read_variables.py +553 -0
  237. flextool/process_outputs/solve_order.py +81 -0
  238. flextool/process_outputs/spinedb_replay.py +412 -0
  239. flextool/process_outputs/union_realized_slice.py +224 -0
  240. flextool/process_outputs/write_outputs.py +1286 -0
  241. flextool/process_outputs/write_spinedb.py +1267 -0
  242. flextool/representative_periods/__init__.py +5 -0
  243. flextool/representative_periods/clustering.py +165 -0
  244. flextool/representative_periods/force_include.py +563 -0
  245. flextool/representative_periods/netload.py +365 -0
  246. flextool/representative_periods/netload_inputs.py +345 -0
  247. flextool/representative_periods/netload_iterate.py +722 -0
  248. flextool/representative_periods/preprocess.py +948 -0
  249. flextool/representative_periods/scenario_stack.py +195 -0
  250. flextool/representative_periods/weights.py +124 -0
  251. flextool/scenario_comparison/__init__.py +13 -0
  252. flextool/scenario_comparison/config_builder.py +158 -0
  253. flextool/scenario_comparison/constants.py +20 -0
  254. flextool/scenario_comparison/data_models.py +222 -0
  255. flextool/scenario_comparison/db_reader.py +399 -0
  256. flextool/scenario_comparison/dispatch_data.py +1002 -0
  257. flextool/scenario_comparison/dispatch_mappings.py +205 -0
  258. flextool/scenario_comparison/dispatch_plots.py +691 -0
  259. flextool/scenario_comparison/input_entity_colors.py +319 -0
  260. flextool/scenario_comparison/orchestrator.py +453 -0
  261. flextool/scenario_comparison/plan_union.py +244 -0
  262. flextool/scenario_comparison/plot_settings_seed.py +205 -0
  263. flextool/schemas/AXIS_CONTRACT.md +71 -0
  264. flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
  265. flextool/schemas/canonical_databases/howto_connections.json +5606 -0
  266. flextool/schemas/canonical_databases/howto_demand.json +5518 -0
  267. flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
  268. flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
  269. flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
  270. flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
  271. flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
  272. flextool/schemas/canonical_databases/templates_examples.json +13532 -0
  273. flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
  274. flextool/schemas/comparison_settings_template.json +197 -0
  275. flextool/schemas/default_plot_settings.yaml +260 -0
  276. flextool/schemas/default_plots.yaml +2293 -0
  277. flextool/schemas/flextool_axis_contract.json +303 -0
  278. flextool/schemas/flextool_axis_contract.schema.json +247 -0
  279. flextool/schemas/old_flextool_import_template.json +4443 -0
  280. flextool/schemas/output_info_template.json +48 -0
  281. flextool/schemas/output_settings_template.json +256 -0
  282. flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
  283. flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
  284. flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
  285. flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
  286. flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
  287. flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
  288. flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
  289. flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
  290. flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
  291. flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
  292. flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
  293. flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
  294. flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
  295. flextool/schemas/spinedb_results_schema.json +581 -0
  296. flextool/schemas/spinedb_schema.json +4636 -0
  297. flextool/solver_config/copt.opt.template +18 -0
  298. flextool/solver_config/cplex.opt.template +25 -0
  299. flextool/solver_config/gurobi.opt.template +18 -0
  300. flextool/solver_config/highs.opt.template +18 -0
  301. flextool/solver_config/xpress.opt.template +26 -0
  302. flextool/spinedb_backend/__init__.py +26 -0
  303. flextool/spinedb_backend/_axis_enums.py +1119 -0
  304. flextool/spinedb_backend/_backend.py +1139 -0
  305. flextool/update_flextool/__init__.py +12 -0
  306. flextool/update_flextool/canonical_databases.py +251 -0
  307. flextool/update_flextool/db_migration.py +7108 -0
  308. flextool/update_flextool/ensure_settings_db.py +138 -0
  309. flextool/update_flextool/export_database.py +103 -0
  310. flextool/update_flextool/extend_tests_fixture.py +772 -0
  311. flextool/update_flextool/generate_canonical.py +274 -0
  312. flextool/update_flextool/initialize_database.py +42 -0
  313. flextool/update_flextool/install_info.py +225 -0
  314. flextool/update_flextool/self_update.py +464 -0
  315. flextool/update_flextool/sync_master_json_template.py +125 -0
  316. flextool/update_flextool/test_fixtures.py +187 -0
  317. flextool-4.0.0.dist-info/METADATA +217 -0
  318. flextool-4.0.0.dist-info/RECORD +322 -0
  319. flextool-4.0.0.dist-info/WHEEL +5 -0
  320. flextool-4.0.0.dist-info/entry_points.txt +17 -0
  321. flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
  322. flextool-4.0.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1423 @@
1
+ """Read self-describing FlexTool Excel files.
2
+
3
+ Each sheet embeds its own metadata via a 2D grid of definitions:
4
+ - A **definition row** labels columns (alternative, entity, parameters, etc.)
5
+ - A **definition column** labels metadata rows (description, data type, etc.)
6
+ - Their intersection is the **crossing point**.
7
+
8
+ The reader auto-discovers the crossing point and interprets the sheet
9
+ layout from the embedded keywords, eliminating the need for an external
10
+ JSON import specification.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import logging
16
+ import re
17
+ from dataclasses import dataclass, field
18
+ from typing import Any
19
+
20
+ import openpyxl
21
+ from openpyxl.worksheet.worksheet import Worksheet
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # ---------------------------------------------------------------------------
26
+ # Keywords recognised in the definition column (row labels)
27
+ # ---------------------------------------------------------------------------
28
+
29
+ ROW_KEYWORDS = frozenset({
30
+ "description",
31
+ "data type",
32
+ "parameter",
33
+ "alternative",
34
+ "entity",
35
+ "index",
36
+ "filter",
37
+ })
38
+
39
+ # Keywords that can appear as column definitions in the definition row.
40
+ # Many start with a prefix (e.g. "entity: node"), so we check prefixes.
41
+ COL_DEF_PREFIXES = (
42
+ "alternative",
43
+ "entity",
44
+ "filter",
45
+ "index",
46
+ "parameter",
47
+ )
48
+
49
+ # Special parameter names recognised by the importer.
50
+ #
51
+ # The exporter (``excel_writer.py``) emits exactly ``"entity existence"`` as
52
+ # the header label and ``"TRUE"`` / ``"FALSE"`` as the cell values. The v2
53
+ # reader is a faithful inverse of that exporter — it does NOT accept legacy
54
+ # aliases (the old ``"entity alternative"`` header is rejected so users see
55
+ # a clear error rather than silent renames).
56
+ ENTITY_EXISTENCE = "entity existence"
57
+ # Strings the exporter EVER emits for a boolean-valued cell (case-insensitive
58
+ # match). Anything else is treated as a data error and raised — silent
59
+ # coercion of ``"yes"`` / ``"no"`` / ``"1"`` would mask data corruption.
60
+ _BOOL_TRUE_LITERAL = "true"
61
+ _BOOL_FALSE_LITERAL = "false"
62
+
63
+
64
+ # ---------------------------------------------------------------------------
65
+ # Data structures
66
+ # ---------------------------------------------------------------------------
67
+
68
+ @dataclass
69
+ class EntityClassDef:
70
+ """One entity class mapping parsed from the entity definition."""
71
+ class_name: str
72
+ dimensions: list[str]
73
+
74
+
75
+ @dataclass
76
+ class SheetMetadata:
77
+ """Parsed metadata from a single sheet."""
78
+
79
+ sheet_name: str
80
+
81
+ # Crossing point coordinates (0-based)
82
+ def_row: int = 0 # the definition row index
83
+ def_col: int = 0 # the definition column index
84
+
85
+ # Column definitions (from the definition row, left of crossing)
86
+ # Each maps column index -> definition string
87
+ col_defs: dict[int, str] = field(default_factory=dict)
88
+
89
+ # Row definitions (from the definition column, above crossing)
90
+ # Each maps row index -> definition string
91
+ row_defs: dict[int, str] = field(default_factory=dict)
92
+
93
+ # Parameter names (from the definition row, right of crossing)
94
+ # Maps column index -> parameter name
95
+ param_cols: dict[int, str] = field(default_factory=dict)
96
+
97
+ # Default parameter name (from "parameter: xxx" in the label cell)
98
+ default_parameter: str | None = None
99
+
100
+ # Default data type (from "data type: xxx" in the label cell)
101
+ default_data_type: str | None = None
102
+
103
+ # Data types per parameter column index
104
+ data_types: dict[int, str] = field(default_factory=dict)
105
+
106
+ # Descriptions per parameter column index
107
+ descriptions: dict[int, str] = field(default_factory=dict)
108
+
109
+ # Parsed entity class definitions
110
+ entity_classes: list[EntityClassDef] = field(default_factory=list)
111
+
112
+ # Filter column index and regex map {class_name: pattern}
113
+ filter_col: int | None = None
114
+ filter_map: dict[str, str] = field(default_factory=dict)
115
+
116
+ # Index column/row info
117
+ index_col: int | None = None
118
+ index_name: str | None = None
119
+
120
+ # Extra index columns (for ladder-style sheets with multiple ``index:`` cols
121
+ # on the left, e.g. ``index: period | index: tier``). Each entry is
122
+ # ``(column_index, axis_name)``. When non-empty, ``index_col`` /
123
+ # ``index_name`` carries the LAST entry; ``extra_index_cols`` carries the
124
+ # preceding ones in left-to-right order so the data extractor can recover
125
+ # all index levels.
126
+ extra_index_cols: list[tuple[int, str]] = field(default_factory=list)
127
+
128
+ # Alternative column index (or None if alternatives are in a row)
129
+ alt_col: int | None = None
130
+
131
+ # Entity existence column index
132
+ entity_existence_col: int | None = None
133
+
134
+ # Data start row (0-based, first row after all metadata rows)
135
+ data_start_row: int = 0
136
+
137
+ # Whether this is a transposed (timeseries-style) sheet
138
+ is_transposed: bool = False
139
+
140
+ # For transposed: row indices for alternative, parameter, entity
141
+ alt_row: int | None = None
142
+ param_row: int | None = None
143
+ entity_row: int | None = None
144
+ index_col_transposed: int | None = None # column with index values (usually 0)
145
+
146
+ # Whether this is a stochastic (_s) sheet — a transposed sheet whose
147
+ # entity / alternative / parameter live in COLUMN headers (like a
148
+ # timeseries sheet) but whose index is a MULTI-LEVEL nested Map spread
149
+ # across N left-side ``index:`` columns (one column per Map level).
150
+ # The definition column (carrying the ``parameter:`` triplet, the
151
+ # ``entity:`` label and the ``alternative`` label) sits at ``def_col``
152
+ # (= number of index columns). Each data column to its right is one
153
+ # (entity, alternative) series whose cells carry the leaf scalars.
154
+ is_stochastic: bool = False
155
+
156
+ # Ordered ``(column_index, axis_name)`` for the stochastic index
157
+ # columns, left-to-right (outermost Map level first).
158
+ stoch_index_cols: list[tuple[int, str]] = field(default_factory=list)
159
+
160
+
161
+ # ---------------------------------------------------------------------------
162
+ # Crossing point detection
163
+ # ---------------------------------------------------------------------------
164
+
165
+
166
+ def _cell_value(ws: Worksheet, row: int, col: int) -> str:
167
+ """Get cell value as stripped string (0-based row/col)."""
168
+ val = ws.cell(row=row + 1, column=col + 1).value # openpyxl is 1-based
169
+ if val is None:
170
+ return ""
171
+ return str(val).strip()
172
+
173
+
174
+ def _actual_max_row(ws: Worksheet) -> int:
175
+ """Get the actual last row with data (not the Excel theoretical max)."""
176
+ # ws.max_row in non-read-only mode should be correct, but guard against
177
+ # sheets where it reports 1048576
178
+ max_r = ws.max_row or 0
179
+ if max_r > 100000:
180
+ # Scan backwards from a reasonable limit
181
+ for r in range(min(max_r, 10000), 0, -1):
182
+ for c in range(1, min((ws.max_column or 1) + 1, 50)):
183
+ if ws.cell(row=r, column=c).value is not None:
184
+ return r
185
+ return 0
186
+ return max_r
187
+
188
+
189
+ def _is_info_row(text: str) -> bool:
190
+ """Check if text is an INFO row (to be skipped by the reader)."""
191
+ return text.upper().startswith("INFO:")
192
+
193
+
194
+ def _is_keyword(text: str) -> bool:
195
+ """Check if text starts with a recognised keyword."""
196
+ t = text.lower()
197
+ for prefix in COL_DEF_PREFIXES:
198
+ if t == prefix or t.startswith(prefix + ":") or t.startswith(prefix + " "):
199
+ return True
200
+ if t in ROW_KEYWORDS:
201
+ return True
202
+ return False
203
+
204
+
205
+ def find_crossing_point(ws: Worksheet) -> tuple[int, int]:
206
+ """Find the crossing point of the definition row and definition column.
207
+
208
+ Algorithm:
209
+ 1. Scan row 0 (Excel row 1) rightward until a recognised keyword is found.
210
+ That column is the definition column.
211
+ 2. Scan that column downward until an empty cell. Go back one row.
212
+ That row is the definition row.
213
+
214
+ Returns:
215
+ (def_row, def_col) — 0-based indices.
216
+
217
+ Raises:
218
+ ValueError: if no crossing point can be found.
219
+ """
220
+ max_col = ws.max_column or 1
221
+
222
+ # Step 1: scan row 0 rightward for a keyword
223
+ def_col = None
224
+ for c in range(max_col):
225
+ val = _cell_value(ws, 0, c)
226
+ if val and _is_keyword(val):
227
+ def_col = c
228
+ break
229
+
230
+ if def_col is None:
231
+ raise ValueError(
232
+ f"Sheet '{ws.title}': could not find definition column "
233
+ f"(no keyword found in row 1)"
234
+ )
235
+
236
+ # Step 2: scan definition column downward until empty cell
237
+ max_row = _actual_max_row(ws)
238
+ def_row = 0
239
+ for r in range(max_row):
240
+ val = _cell_value(ws, r, def_col)
241
+ if not val:
242
+ # Empty cell found — go back one
243
+ def_row = max(0, r - 1)
244
+ break
245
+ else:
246
+ # No empty cell found — last row with content
247
+ def_row = max_row - 1
248
+
249
+ return (def_row, def_col)
250
+
251
+
252
+ # ---------------------------------------------------------------------------
253
+ # Definition parsing
254
+ # ---------------------------------------------------------------------------
255
+
256
+
257
+ def _parse_entity_def(text: str) -> list[EntityClassDef]:
258
+ """Parse an entity definition string.
259
+
260
+ Examples:
261
+ "entity: node"
262
+ → [EntityClassDef("node", ["node"])]
263
+ "entity: commodity, node"
264
+ → [EntityClassDef("commodity__node", ["commodity", "node"])]
265
+ "entity: (unit__inputNode: (unit, node), unit__outputNode: (unit, node))"
266
+ → [EntityClassDef("unit__inputNode", ["unit", "node"]),
267
+ EntityClassDef("unit__outputNode", ["unit", "node"])]
268
+ """
269
+ # Strip "entity:" prefix
270
+ text = text.strip()
271
+ if text.lower().startswith("entity:"):
272
+ text = text[len("entity:"):].strip()
273
+ elif text.lower().startswith("entity name:"):
274
+ text = text[len("entity name:"):].strip()
275
+ elif text.lower() == "entity":
276
+ return []
277
+
278
+ # Check for multi-class syntax: (class1: (dim1, dim2), class2: (dim3, dim4))
279
+ if text.startswith("(") and text.endswith(")"):
280
+ inner = text[1:-1].strip()
281
+ results = []
282
+ # Split on top-level commas (not inside parens)
283
+ parts = _split_top_level(inner)
284
+ for part in parts:
285
+ part = part.strip()
286
+ # Parse "class_name: (dim1, dim2)"
287
+ match = re.match(r'(\w+)\s*:\s*\(([^)]+)\)', part)
288
+ if match:
289
+ class_name = match.group(1).strip()
290
+ dims = [d.strip() for d in match.group(2).split(",")]
291
+ results.append(EntityClassDef(class_name, dims))
292
+ return results
293
+
294
+ # Simple case: "node" or "commodity, node"
295
+ dims = [d.strip() for d in text.split(",")]
296
+ if len(dims) == 1:
297
+ return [EntityClassDef(dims[0], dims)]
298
+ else:
299
+ class_name = "__".join(dims)
300
+ return [EntityClassDef(class_name, dims)]
301
+
302
+
303
+ def _split_top_level(text: str) -> list[str]:
304
+ """Split text on commas that are not inside parentheses."""
305
+ parts = []
306
+ depth = 0
307
+ current = []
308
+ for ch in text:
309
+ if ch == "(":
310
+ depth += 1
311
+ current.append(ch)
312
+ elif ch == ")":
313
+ depth -= 1
314
+ current.append(ch)
315
+ elif ch == "," and depth == 0:
316
+ parts.append("".join(current))
317
+ current = []
318
+ else:
319
+ current.append(ch)
320
+ if current:
321
+ parts.append("".join(current))
322
+ return parts
323
+
324
+
325
+ def _parse_filter_def(text: str) -> dict[str, str]:
326
+ """Parse a filter definition string.
327
+
328
+ Example:
329
+ "filter: {unit__inputNode: ^input$, unit__outputNode: ^output$}"
330
+ → {"unit__inputNode": "^input$", "unit__outputNode": "^output$"}
331
+ """
332
+ text = text.strip()
333
+ if text.lower().startswith("filter:"):
334
+ text = text[len("filter:"):].strip()
335
+
336
+ # Remove outer braces or parens
337
+ if (text.startswith("{") and text.endswith("}")) or \
338
+ (text.startswith("(") and text.endswith(")")):
339
+ text = text[1:-1].strip()
340
+
341
+ result = {}
342
+ for part in _split_top_level(text):
343
+ part = part.strip()
344
+ if ":" in part:
345
+ key, val = part.split(":", 1)
346
+ result[key.strip()] = val.strip()
347
+ return result
348
+
349
+
350
+ def _parse_index_def(text: str) -> str:
351
+ """Parse an index definition: "index: time" → "time"."""
352
+ text = text.strip()
353
+ if text.lower().startswith("index:"):
354
+ return text[len("index:"):].strip()
355
+ return text
356
+
357
+
358
+ def _parse_default_value(text: str) -> tuple[str, str | None]:
359
+ """Parse a definition that may have a default value.
360
+
361
+ "parameter: profile" → ("parameter", "profile")
362
+ "parameter" → ("parameter", None)
363
+ "data type: float" → ("data type", "float")
364
+
365
+ Also handles the triplet format with ``|`` separator:
366
+ "parameter: profile | data type: float | description: ..."
367
+ → ("parameter", "profile")
368
+ """
369
+ # Handle | separator: only parse the first segment
370
+ if "|" in text:
371
+ text = text.split("|")[0].strip()
372
+
373
+ for keyword in sorted(ROW_KEYWORDS, key=len, reverse=True):
374
+ if text.lower().startswith(keyword):
375
+ rest = text[len(keyword):].strip()
376
+ if rest.startswith(":"):
377
+ return keyword, rest[1:].strip()
378
+ elif rest == "":
379
+ return keyword, None
380
+ return text, None
381
+
382
+
383
+ def _extract_triplet_field(text: str, field: str) -> str | None:
384
+ """Return the value of a labelled segment in a pipe-separated triplet.
385
+
386
+ Looks for ``<field>: <value>`` inside a ``"a | b | c"`` style string
387
+ (case-insensitive on the field name). Returns the trimmed value or
388
+ ``None`` when the field isn't present.
389
+ """
390
+ if not text or "|" not in text:
391
+ return None
392
+ fl = field.lower()
393
+ for seg in text.split("|"):
394
+ s = seg.strip()
395
+ if s.lower().startswith(fl + ":"):
396
+ return s[len(field) + 1:].strip()
397
+ return None
398
+
399
+
400
+ # ---------------------------------------------------------------------------
401
+ # Sheet metadata extraction
402
+ # ---------------------------------------------------------------------------
403
+
404
+
405
+ def parse_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
406
+ """Parse a sheet's embedded metadata.
407
+
408
+ Returns None if the sheet has no recognisable metadata (e.g. navigate, version).
409
+ """
410
+ try:
411
+ def_row, def_col = find_crossing_point(ws)
412
+ except ValueError:
413
+ return None
414
+
415
+ meta = SheetMetadata(sheet_name=ws.title, def_row=def_row, def_col=def_col)
416
+
417
+ # --- Parse row definitions (definition column, above crossing point) ---
418
+ for r in range(def_row):
419
+ val = _cell_value(ws, r, def_col)
420
+ if val:
421
+ meta.row_defs[r] = val
422
+
423
+ # The crossing point cell itself is also a row def
424
+ crossing_val = _cell_value(ws, def_row, def_col)
425
+ if crossing_val:
426
+ meta.row_defs[def_row] = crossing_val
427
+
428
+ # --- Parse column definitions (definition row, left of crossing) ---
429
+ for c in range(def_col):
430
+ val = _cell_value(ws, def_row, c)
431
+ if val:
432
+ meta.col_defs[c] = val.strip()
433
+
434
+ # --- Interpret column definitions ---
435
+ # Multiple ``index:`` columns on the left are recorded in
436
+ # ``extra_index_cols`` (in column order); the rightmost wins for
437
+ # ``index_col`` / ``index_name`` to keep backwards compatibility with
438
+ # single-index sheets.
439
+ index_seen: list[tuple[int, str]] = []
440
+ for c, defn in meta.col_defs.items():
441
+ dl = defn.lower()
442
+ if dl == "alternative":
443
+ meta.alt_col = c
444
+ elif dl.startswith("entity"):
445
+ meta.entity_classes = _parse_entity_def(defn)
446
+ elif dl.startswith("filter"):
447
+ meta.filter_col = c
448
+ meta.filter_map = _parse_filter_def(defn)
449
+ elif dl.startswith("index"):
450
+ index_seen.append((c, _parse_index_def(defn)))
451
+ if index_seen:
452
+ index_seen.sort()
453
+ meta.index_col, meta.index_name = index_seen[-1]
454
+ # Preceding (left-of-rightmost) index columns become extras.
455
+ meta.extra_index_cols = list(index_seen[:-1])
456
+
457
+ # --- Parse parameter columns (definition row, right of crossing) ---
458
+ # Stop at the first empty cell (separator column before parameter reference)
459
+ max_col = ws.max_column or 1
460
+ for c in range(def_col + 1, max_col):
461
+ val = _cell_value(ws, def_row, c)
462
+ if not val:
463
+ break # empty separator column = end of parameter area
464
+ vl = val.lower()
465
+ if vl == ENTITY_EXISTENCE:
466
+ meta.entity_existence_col = c
467
+ meta.param_cols[c] = ENTITY_EXISTENCE
468
+ else:
469
+ meta.param_cols[c] = val
470
+
471
+ # --- Parse crossing point for defaults ---
472
+ # NOTE: kept defensively — the current exporter ALWAYS writes a
473
+ # parameter name into every right-of-crossing header cell, so the
474
+ # back-fill loop below is dead code on writer-generated files. It
475
+ # remains in place to keep hand-edited sheets (where the user relies
476
+ # on the crossing's ``parameter: <default>`` to label an empty
477
+ # column) loadable. If the exporter contract changes, revisit.
478
+ keyword, default = _parse_default_value(crossing_val)
479
+ if keyword.lower() == "parameter" and default:
480
+ meta.default_parameter = default
481
+ # Apply default to all param columns that are empty
482
+ for c in range(def_col + 1, max_col):
483
+ if c not in meta.param_cols:
484
+ val = _cell_value(ws, def_row, c)
485
+ if not val:
486
+ meta.param_cols[c] = default
487
+
488
+ # --- Parse row metadata (description, data type) ---
489
+ for r, label in meta.row_defs.items():
490
+ if r == def_row:
491
+ continue # skip the definition row itself
492
+ kw, default_val = _parse_default_value(label)
493
+ kw_lower = kw.lower()
494
+
495
+ if kw_lower == "description":
496
+ for c in meta.param_cols:
497
+ desc = _cell_value(ws, r, c)
498
+ if desc:
499
+ meta.descriptions[c] = desc
500
+
501
+ elif kw_lower == "data type":
502
+ if default_val:
503
+ meta.default_data_type = default_val
504
+ for c in meta.param_cols:
505
+ dt = _cell_value(ws, r, c)
506
+ # NOTE: ``.lower()`` is defensive — the exporter already
507
+ # emits all data-type labels lowercase ("float", "string",
508
+ # "boolean (array)", "string (array)", "*-1d-map", ...).
509
+ # Kept in case a hand-edited sheet uses mixed case so the
510
+ # downstream dtype dispatch (``_convert_value``) keeps
511
+ # matching its lowercase ``elif`` branches.
512
+ if dt:
513
+ meta.data_types[c] = dt.lower()
514
+ elif default_val:
515
+ meta.data_types[c] = default_val.lower()
516
+
517
+ # Data starts after the definition row
518
+ meta.data_start_row = def_row + 1
519
+
520
+ return meta
521
+
522
+
523
+ def parse_link_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
524
+ """Parse a link-only sheet (no parameters, just entity dimensions).
525
+
526
+ Link sheets have a simpler format:
527
+ Row 1: navigate link
528
+ Row 2: entity definition in col A, dimension names in cols B+
529
+ Row 3+: data
530
+ """
531
+ meta = SheetMetadata(sheet_name=ws.title)
532
+
533
+ # Look for entity definition in column A, rows 1-3
534
+ for r in range(min(3, ws.max_row or 1)):
535
+ val = _cell_value(ws, r, 0)
536
+ if val.lower().startswith("entity"):
537
+ meta.entity_classes = _parse_entity_def(val)
538
+ meta.def_row = r
539
+ meta.data_start_row = r + 1
540
+ return meta
541
+
542
+ return None
543
+
544
+
545
+ def parse_transposed_sheet_metadata(ws: Worksheet) -> SheetMetadata | None:
546
+ """Parse a transposed (timeseries-style) sheet.
547
+
548
+ In transposed sheets, column B serves as the row definition column:
549
+ - Row with "alternative" in col B → alternative values in cols C+
550
+ - Row with "parameter" in col B → parameter names in cols C+
551
+ - Row with "entity: X" in col B → entity names in cols C+
552
+ - Column A has "index: time" and time values below
553
+ """
554
+ meta = SheetMetadata(sheet_name=ws.title, is_transposed=True)
555
+ max_row = _actual_max_row(ws)
556
+
557
+ # Scan column A for "index:" definition (skip INFO rows)
558
+ for r in range(min(10, max_row)):
559
+ val = _cell_value(ws, r, 0)
560
+ if _is_info_row(val):
561
+ continue
562
+ if val.lower().startswith("index:"):
563
+ meta.index_name = _parse_index_def(val)
564
+ meta.index_col_transposed = 0
565
+ meta.data_start_row = r + 1
566
+
567
+ # Scan column B for row role definitions (skip INFO rows)
568
+ label_col = 1 # column B
569
+ datatype_row: int | None = None
570
+ for r in range(min(10, max_row)):
571
+ val_a = _cell_value(ws, r, 0)
572
+ if _is_info_row(val_a):
573
+ continue
574
+ val = _cell_value(ws, r, label_col)
575
+ if not val:
576
+ continue
577
+ vl = val.lower()
578
+ kw, default_val = _parse_default_value(val)
579
+ kw_lower = kw.lower()
580
+
581
+ if vl == "alternative" or vl.startswith("alternative"):
582
+ meta.alt_row = r
583
+ elif kw_lower == "data type":
584
+ # Multi-param _t sheets carry a per-column data-type row so the
585
+ # leaf scalar type survives the round-trip (single-param sheets
586
+ # fold it into the triplet instead).
587
+ datatype_row = r
588
+ if default_val:
589
+ meta.default_data_type = default_val.lower()
590
+ elif kw_lower == "parameter":
591
+ meta.param_row = r
592
+ if default_val:
593
+ meta.default_parameter = default_val
594
+ elif vl.startswith("entity"):
595
+ meta.entity_row = r
596
+ meta.entity_classes = _parse_entity_def(val)
597
+
598
+ # Read per-column data types (one per transposed data column, C+).
599
+ if datatype_row is not None:
600
+ max_col = ws.max_column or 1
601
+ for c in range(2, max_col):
602
+ dt = _cell_value(ws, datatype_row, c)
603
+ if dt:
604
+ meta.data_types[c] = dt.lower()
605
+
606
+ # Determine data start row (row after last metadata row)
607
+ metadata_rows = [
608
+ r for r in [meta.alt_row, meta.param_row, meta.entity_row, datatype_row]
609
+ if r is not None
610
+ ]
611
+ if metadata_rows:
612
+ meta.data_start_row = max(metadata_rows) + 1
613
+
614
+ return meta
615
+
616
+
617
+ # ---------------------------------------------------------------------------
618
+ # Scenario sheet parsing
619
+ # ---------------------------------------------------------------------------
620
+
621
+
622
+ def _is_scenario_sheet(ws: Worksheet) -> bool:
623
+ """Check if the worksheet is a scenario sheet.
624
+
625
+ Scenario sheets have "Scenario names" in row 1, col 2 (0-based: row 0, col 1).
626
+ """
627
+ val = _cell_value(ws, 0, 1)
628
+ return val.lower() == "scenario names" if val else False
629
+
630
+
631
+ def parse_scenario_sheet(ws: Worksheet) -> SheetData:
632
+ """Parse a scenario sheet into SheetData records.
633
+
634
+ Scenario sheet layout (Excel rows, 1-based):
635
+ Row 1: navigate | Scenario names | ...
636
+ Row 2: (empty) | scenario_1 | scenario_2 | ...
637
+ Row 3: base_alternative | alt_for_s1 | alt_for_s2 | ...
638
+ Row 4: alternative_1 | alt_for_s1 | alt_for_s2 | ...
639
+ Row 5: alternative_2 | (empty) | alt_for_s2 | ...
640
+
641
+ Produces records with:
642
+ entity_class = "scenario"
643
+ entity_byname = (scenario_name,)
644
+ param_name = row label (e.g. "base_alternative", "alternative_1")
645
+ alternative = the alternative name in that cell
646
+ value = the rank (row index - 2, i.e. base_alternative=0, alternative_1=1, ...)
647
+ """
648
+ meta = SheetMetadata(sheet_name=ws.title)
649
+ meta.entity_classes = [EntityClassDef("scenario", ["scenario"])]
650
+ data = SheetData(sheet_name=ws.title, metadata=meta)
651
+
652
+ max_row = _actual_max_row(ws)
653
+ max_col = ws.max_column or 1
654
+
655
+ # Row 2 (0-based row 1): scenario names in cols B+ (0-based col 1+)
656
+ scenario_names: dict[int, str] = {}
657
+ for c in range(1, max_col):
658
+ name = _cell_value(ws, 1, c)
659
+ if name:
660
+ scenario_names[c] = name
661
+
662
+ if not scenario_names:
663
+ return data
664
+
665
+ # Row 3+ (0-based row 2+): row label in col A, alternative names in cols B+
666
+ for r in range(2, max_row):
667
+ row_label = _cell_value(ws, r, 0)
668
+ if not row_label:
669
+ continue
670
+
671
+ rank = r - 2 # base_alternative (row 2, 0-based) = rank 0
672
+
673
+ for c, scenario_name in scenario_names.items():
674
+ alt_name = _cell_value(ws, r, c)
675
+ if not alt_name:
676
+ continue
677
+
678
+ data.records.append({
679
+ "entity_class": "scenario",
680
+ "entity_byname": (scenario_name,),
681
+ "param_name": row_label,
682
+ "alternative": alt_name,
683
+ "value": rank,
684
+ "index_value": None,
685
+ "index_name": None,
686
+ })
687
+
688
+ return data
689
+
690
+
691
+ # ---------------------------------------------------------------------------
692
+ # Stochastic (_s) sheet parsing
693
+ # ---------------------------------------------------------------------------
694
+
695
+
696
+ def _is_index_label(text: str) -> bool:
697
+ """Return True when *text* is an ``index:`` column definition."""
698
+ return text.lower().startswith("index:") or text.lower() == "index"
699
+
700
+
701
+ def _find_stochastic_def_col(ws: Worksheet) -> int | None:
702
+ """Locate the definition column of a stochastic (_s) sheet, or None.
703
+
704
+ A stochastic sheet is a transposed sheet whose nested-Map index is
705
+ spread across ``N`` left-side ``index:`` columns (one per Map level)
706
+ followed by a definition column that carries the ``alternative`` /
707
+ ``parameter`` / ``entity`` row roles (the entity/alt/param VALUES live
708
+ in the column headers to its right, like a timeseries sheet).
709
+
710
+ The signal: there are at least TWO ``index:`` columns on the left, and
711
+ immediately after the last of them comes a column whose header rows
712
+ carry the ``alternative`` (and ``parameter`` / ``entity``) role
713
+ labels. A single-index transposed sheet (timeseries) keeps its lone
714
+ ``index:`` in column A and its def labels in column B — that case is
715
+ handled by the existing transposed path and is explicitly NOT matched
716
+ here (it has fewer than two ``index:`` columns).
717
+ """
718
+ max_row = _actual_max_row(ws)
719
+ max_col = ws.max_column or 1
720
+ scan_rows = min(8, max_row)
721
+
722
+ # The role-definition row is the one carrying the ``alternative`` label
723
+ # (in the definition column). On a stochastic sheet that same row also
724
+ # carries the ``index:`` labels for every Map level in the columns to
725
+ # its left. Anchor on that row, then verify the left columns are all
726
+ # ``index:``.
727
+ for role_row in range(scan_rows):
728
+ def_col: int | None = None
729
+ for c in range(max_col):
730
+ if _cell_value(ws, role_row, c).lower() == "alternative":
731
+ def_col = c
732
+ break
733
+ if def_col is None or def_col < 2:
734
+ # Need at least two ``index:`` columns to the left to be a
735
+ # genuine nested-Map (multi-index) sheet; a lone ``index:`` in
736
+ # column A is an ordinary single-index transposed sheet.
737
+ continue
738
+ if all(
739
+ _is_index_label(_cell_value(ws, role_row, c)) for c in range(def_col)
740
+ ):
741
+ return def_col
742
+ return None
743
+
744
+
745
+ def parse_stochastic_sheet_metadata(ws: Worksheet, def_col: int) -> SheetMetadata:
746
+ """Parse a stochastic (_s) sheet's embedded metadata.
747
+
748
+ ``def_col`` is the 0-based definition column index (= the number of
749
+ left-side ``index:`` columns). The header rows carry the role labels
750
+ (``parameter:``, ``entity:``, ``alternative``) in that column; the
751
+ entity / alternative / leaf-data values live in the columns to its
752
+ right (transposed). The ``index:`` columns 0..def_col-1 give the
753
+ nested-Map level axis names, outermost first.
754
+ """
755
+ meta = SheetMetadata(
756
+ sheet_name=ws.title, is_transposed=True, is_stochastic=True,
757
+ )
758
+ meta.def_col = def_col
759
+ max_row = _actual_max_row(ws)
760
+ scan_rows = min(8, max_row)
761
+
762
+ # Index columns (left of def_col), in order. Axis name comes from the
763
+ # role-definition row; collect from whichever header row carries it.
764
+ for c in range(def_col):
765
+ axis = ""
766
+ for r in range(scan_rows):
767
+ val = _cell_value(ws, r, c)
768
+ if val and _is_index_label(val):
769
+ axis = _parse_index_def(val)
770
+ break
771
+ meta.stoch_index_cols.append((c, axis))
772
+ if meta.stoch_index_cols:
773
+ # Keep single-index compatibility fields pointing at the LAST level.
774
+ meta.index_col, meta.index_name = meta.stoch_index_cols[-1]
775
+
776
+ # Role rows in the definition column.
777
+ triplet_dtype: str | None = None
778
+ for r in range(scan_rows):
779
+ val = _cell_value(ws, r, def_col)
780
+ if not val:
781
+ continue
782
+ vl = val.lower()
783
+ kw, default_val = _parse_default_value(val)
784
+ kw_lower = kw.lower()
785
+ if vl == "alternative" or vl.startswith("alternative"):
786
+ meta.alt_row = r
787
+ elif kw_lower == "parameter":
788
+ meta.param_row = r
789
+ if default_val:
790
+ meta.default_parameter = default_val
791
+ # The single-param triplet carries the canonical data type.
792
+ triplet_dtype = _extract_triplet_field(val, "data type")
793
+ elif vl.startswith("entity"):
794
+ meta.entity_row = r
795
+ meta.entity_classes = _parse_entity_def(val)
796
+
797
+ if triplet_dtype:
798
+ meta.default_data_type = triplet_dtype.lower()
799
+
800
+ # Data rows start after the last header (role) row.
801
+ role_rows = [
802
+ r for r in (meta.alt_row, meta.param_row, meta.entity_row)
803
+ if r is not None
804
+ ]
805
+ meta.data_start_row = (max(role_rows) + 1) if role_rows else def_col + 1
806
+ return meta
807
+
808
+
809
+ def _extract_stochastic(ws: Worksheet, meta: SheetMetadata) -> SheetData:
810
+ """Extract per-leaf records from a stochastic (_s) sheet.
811
+
812
+ Each data column (def_col+1 onward) is one (entity, alternative)
813
+ series. Each data row carries one full nested-Map index tuple in the
814
+ left ``index:`` columns. We emit one record per (column, row) cell,
815
+ carrying the first N-1 index levels in ``extra_index_values`` and the
816
+ last level in ``index_value`` / ``index_name`` — mirroring the ladder
817
+ reader so the DB-side nested-Map reconstruction can rebuild the
818
+ full ``Map(idx0 -> Map(idx1 -> ... -> leaf))``.
819
+ """
820
+ data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
821
+ max_row = _actual_max_row(ws)
822
+ max_col = ws.max_column or 1
823
+ def_col = meta.def_col
824
+
825
+ entity_class = meta.entity_classes[0] if meta.entity_classes else None
826
+ if entity_class is None:
827
+ return data
828
+
829
+ n_dims = len(entity_class.dimensions)
830
+ leaf_dtype = meta.default_data_type or "string"
831
+
832
+ for c in range(def_col + 1, max_col):
833
+ alt = _cell_value(ws, meta.alt_row, c) if meta.alt_row is not None else None
834
+ pname = (
835
+ _cell_value(ws, meta.param_row, c) if meta.param_row is not None else None
836
+ )
837
+ ent_name = (
838
+ _cell_value(ws, meta.entity_row, c) if meta.entity_row is not None else None
839
+ )
840
+ if not pname and meta.default_parameter:
841
+ pname = meta.default_parameter
842
+ if not alt or not ent_name or not pname:
843
+ continue
844
+
845
+ # The entity header carries only the FIRST dimension element for a
846
+ # multi-dim class (the writer emits ``byname[0]``). Stochastic
847
+ # params in the schema are single-dimension (profile, node, …); a
848
+ # multi-dim header would be ambiguous, so we only build a byname we
849
+ # can trust.
850
+ if n_dims == 1:
851
+ entity_byname: tuple[str, ...] = (ent_name,)
852
+ else:
853
+ # Reaching here means a multi-dim class was routed onto an _s
854
+ # sheet whose header can't carry the full byname — surface it
855
+ # rather than silently fabricating a wrong entity.
856
+ logger.warning(
857
+ "Sheet '%s' col %d: stochastic sheet for multi-dim class "
858
+ "%r cannot recover the full entity byname from a single "
859
+ "header value %r; skipping column.",
860
+ meta.sheet_name, c + 1, entity_class.class_name, ent_name,
861
+ )
862
+ continue
863
+
864
+ for r in range(meta.data_start_row, max_row):
865
+ val = _cell_value(ws, r, c)
866
+ if not val:
867
+ continue
868
+ # Read all index levels for this row.
869
+ level_vals: list[tuple[str, str | None]] = []
870
+ for col_idx, axis in meta.stoch_index_cols:
871
+ level_vals.append((axis, _cell_value(ws, r, col_idx) or None))
872
+ # A row missing any index level is malformed — skip it.
873
+ if any(v is None for _, v in level_vals):
874
+ continue
875
+
876
+ converted = _convert_value(val, leaf_dtype)
877
+ record: dict[str, Any] = {
878
+ "alternative": alt,
879
+ "entity_class": entity_class.class_name,
880
+ "entity_byname": entity_byname,
881
+ "param_name": pname,
882
+ "value": converted,
883
+ "index_value": level_vals[-1][1],
884
+ "index_name": level_vals[-1][0],
885
+ "data_type": leaf_dtype,
886
+ }
887
+ if len(level_vals) > 1:
888
+ record["extra_index_values"] = level_vals[:-1]
889
+ data.records.append(record)
890
+
891
+ return data
892
+
893
+
894
+ # ---------------------------------------------------------------------------
895
+ # Determine sheet type and parse accordingly
896
+ # ---------------------------------------------------------------------------
897
+
898
+
899
+ def detect_and_parse_sheet(ws: Worksheet) -> tuple[SheetMetadata | None, bool]:
900
+ """Auto-detect sheet type and parse its metadata.
901
+
902
+ Tries scenario first, then standard (constant/periodic), then link,
903
+ then transposed.
904
+
905
+ Returns:
906
+ (metadata, is_scenario) — metadata is None for unrecognised sheets.
907
+ is_scenario is True when the sheet is a scenario sheet (needs special
908
+ extraction via parse_scenario_sheet).
909
+ """
910
+ # Skip very small sheets
911
+ if (ws.max_row or 0) < 2 or (ws.max_column or 0) < 2:
912
+ return None, False
913
+
914
+ # Check for scenario sheet: "Scenario names" in row 1, col B
915
+ if _is_scenario_sheet(ws):
916
+ return SheetMetadata(sheet_name=ws.title), True
917
+
918
+ # Check for a stochastic (_s) sheet: N>=2 left ``index:`` columns then a
919
+ # definition column carrying the transposed entity/alt/param roles. Must
920
+ # run BEFORE the standard-sheet probe, which would otherwise misread the
921
+ # transposed ``alternative`` value header as a parameter column.
922
+ stoch_def_col = _find_stochastic_def_col(ws)
923
+ if stoch_def_col is not None:
924
+ return parse_stochastic_sheet_metadata(ws, stoch_def_col), False
925
+
926
+ # Check for transposed sheet first: "alternative" or "parameter" in column B (row 0-4)
927
+ for r in range(min(5, _actual_max_row(ws))):
928
+ val_a = _cell_value(ws, r, 0)
929
+ if val_a and _is_info_row(val_a):
930
+ continue # skip INFO rows
931
+ val_b = _cell_value(ws, r, 1) # column B
932
+ if val_b:
933
+ vl = val_b.lower()
934
+ if vl in ("alternative", "parameter") or vl.startswith("parameter:"):
935
+ return parse_transposed_sheet_metadata(ws), False
936
+
937
+ # Check for standard sheet (has keywords like "description" in row 1)
938
+ for c in range(2, min(20, ws.max_column or 1)): # start at col C (skip A=navigate, B)
939
+ val = _cell_value(ws, 0, c)
940
+ if val and _is_keyword(val) and val.lower() != "navigate":
941
+ return parse_sheet_metadata(ws), False
942
+
943
+ # Check for link sheet (entity: in column A)
944
+ for r in range(min(5, ws.max_row or 1)):
945
+ val = _cell_value(ws, r, 0)
946
+ if val.lower().startswith("entity:") or val.lower().startswith("entity "):
947
+ return parse_link_sheet_metadata(ws), False
948
+
949
+ # Check for transposed sheet (column B has keywords like "alternative", "parameter")
950
+ for r in range(min(10, ws.max_row or 1)):
951
+ val = _cell_value(ws, r, 1)
952
+ if val and val.lower() in ("alternative", "parameter"):
953
+ return parse_transposed_sheet_metadata(ws), False
954
+
955
+ return None, False
956
+
957
+
958
+ # ---------------------------------------------------------------------------
959
+ # Data extraction
960
+ # ---------------------------------------------------------------------------
961
+
962
+
963
+ @dataclass
964
+ class SheetData:
965
+ """Extracted data from one sheet, ready for DB import."""
966
+
967
+ sheet_name: str
968
+ metadata: SheetMetadata
969
+
970
+ # For each entity class mapping:
971
+ # list of dicts with keys: alternative, entity_byname, param_name, value,
972
+ # entity_existence (bool or None)
973
+ records: list[dict[str, Any]] = field(default_factory=list)
974
+
975
+ # Entity relationships (for link sheets): list of entity_byname tuples
976
+ link_entities: list[tuple[str, ...]] = field(default_factory=list)
977
+
978
+
979
+ def extract_sheet_data(ws: Worksheet, meta: SheetMetadata) -> SheetData:
980
+ """Extract data records from a parsed sheet."""
981
+ if meta.is_stochastic:
982
+ return _extract_stochastic(ws, meta)
983
+ elif meta.is_transposed:
984
+ return _extract_transposed(ws, meta)
985
+ elif not meta.param_cols and meta.entity_classes:
986
+ return _extract_link(ws, meta)
987
+ else:
988
+ return _extract_standard(ws, meta)
989
+
990
+
991
+ def _extract_standard(ws: Worksheet, meta: SheetMetadata) -> SheetData:
992
+ """Extract data from a standard (constant/periodic) sheet."""
993
+ data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
994
+ max_row = _actual_max_row(ws)
995
+
996
+ # Find entity element columns (columns in definition row between entity def and first
997
+ # non-structural column). These are the dimension name columns.
998
+ for c in sorted(meta.col_defs.keys()):
999
+ defn = meta.col_defs[c].lower()
1000
+ if not (defn == "alternative" or defn.startswith("entity") or
1001
+ defn.startswith("filter") or defn.startswith("index")):
1002
+ continue
1003
+ if defn == "alternative" or defn.startswith("filter") or defn.startswith("index"):
1004
+ continue
1005
+ # This is an entity-related column but not the entity def itself
1006
+ # Actually, the entity def column has the "entity:" prefix.
1007
+ # The dimension columns follow it and have plain dimension names.
1008
+
1009
+ # Simpler approach: entity def column + subsequent dimension columns
1010
+ # are identified from the entity_classes definitions.
1011
+ # The entity column index is the one with "entity:" in col_defs.
1012
+ entity_col: int | None = None
1013
+ for c, defn in meta.col_defs.items():
1014
+ if defn.lower().startswith("entity"):
1015
+ entity_col = c
1016
+ break
1017
+
1018
+ # Dimension element columns follow the entity column
1019
+ # Their count = max dimensions across entity classes
1020
+ if meta.entity_classes:
1021
+ max_dims = max(len(ec.dimensions) for ec in meta.entity_classes)
1022
+ else:
1023
+ max_dims = 0
1024
+
1025
+ if entity_col is not None and max_dims > 1:
1026
+ # For multi-dim entities, dimension element columns follow the entity column.
1027
+ # There are exactly max_dims dimension columns after the entity column.
1028
+ dim_cols = list(range(entity_col + 1, entity_col + 1 + max_dims))
1029
+ elif entity_col is not None:
1030
+ dim_cols = [] # single-dim: entity name is in entity_col itself
1031
+ else:
1032
+ dim_cols = []
1033
+
1034
+ for r in range(meta.data_start_row, max_row):
1035
+ # Read alternative. Normalise empty cells to ``None`` (rather than
1036
+ # ``""``) so downstream entity-only records propagate a real
1037
+ # absence — the writer's alt-creation step skips falsy names and
1038
+ # discards entity-only records before parameter writes, so a
1039
+ # synthesised ``""`` would create an empty-string alternative
1040
+ # in SpineDB.
1041
+ if meta.alt_col is not None:
1042
+ alt = _cell_value(ws, r, meta.alt_col) or None
1043
+ else:
1044
+ alt = None
1045
+
1046
+ # Read entity byname
1047
+ if entity_col is not None:
1048
+ if max_dims <= 1:
1049
+ ent_name = _cell_value(ws, r, entity_col)
1050
+ entity_byname = (ent_name,) if ent_name else None
1051
+ else:
1052
+ elements = []
1053
+ for dc in dim_cols:
1054
+ elements.append(_cell_value(ws, r, dc))
1055
+ if any(elements):
1056
+ entity_byname = tuple(elements)
1057
+ else:
1058
+ entity_byname = None
1059
+ else:
1060
+ entity_byname = None
1061
+
1062
+ if entity_byname is None:
1063
+ continue
1064
+
1065
+ # Determine entity class for this row (using filter if present)
1066
+ if meta.filter_col is not None and meta.filter_map and len(meta.entity_classes) > 1:
1067
+ filter_val = _cell_value(ws, r, meta.filter_col)
1068
+ entity_class = None
1069
+ for ec in meta.entity_classes:
1070
+ pattern = meta.filter_map.get(ec.class_name, "")
1071
+ if pattern and re.match(pattern, filter_val):
1072
+ entity_class = ec
1073
+ break
1074
+ if entity_class is None:
1075
+ logger.warning(
1076
+ "Sheet '%s' row %d: filter value '%s' didn't match any entity class",
1077
+ meta.sheet_name, r + 1, filter_val,
1078
+ )
1079
+ continue
1080
+ elif meta.entity_classes:
1081
+ entity_class = meta.entity_classes[0]
1082
+ else:
1083
+ continue
1084
+
1085
+ # Read index value (for periodic/map sheets)
1086
+ index_val = None
1087
+ if meta.index_col is not None:
1088
+ index_val = _cell_value(ws, r, meta.index_col) or None
1089
+
1090
+ # Read extra index values (e.g. ladder sheets carry index: period
1091
+ # + index: tier). We keep them in the same column order as the
1092
+ # sheet so the DB-write side can reconstruct the nested Map.
1093
+ extra_index_values: list[tuple[str, str | None]] = []
1094
+ for c_extra, name_extra in meta.extra_index_cols:
1095
+ v = _cell_value(ws, r, c_extra) or None
1096
+ extra_index_values.append((name_extra, v))
1097
+
1098
+ # Read entity existence (strict TRUE/FALSE — exporter only ever
1099
+ # writes those two literals; anything else is a data error).
1100
+ entity_existence = None
1101
+ if meta.entity_existence_col is not None:
1102
+ ee_val = _cell_value(ws, r, meta.entity_existence_col)
1103
+ if ee_val:
1104
+ entity_existence = _parse_strict_bool(
1105
+ ee_val,
1106
+ sheet_name=meta.sheet_name,
1107
+ row_1based=r + 1,
1108
+ col_1based=meta.entity_existence_col + 1,
1109
+ field_name="entity existence",
1110
+ )
1111
+
1112
+ # Read parameter values
1113
+ row_has_data = False
1114
+ for c, pname in meta.param_cols.items():
1115
+ if pname == ENTITY_EXISTENCE:
1116
+ continue # already handled
1117
+ val = _cell_value(ws, r, c)
1118
+ if not val:
1119
+ continue
1120
+
1121
+ # Convert based on data type. Fallback order:
1122
+ # 1. explicit per-column data_type
1123
+ # 2. sheet default_data_type
1124
+ # 3. "string" — safe pass-through. The exporter ALWAYS
1125
+ # writes a data type for every parameter column, so
1126
+ # reaching the fallback implies a hand-edited sheet;
1127
+ # "string" preserves the cell verbatim instead of
1128
+ # forcing a ``float()`` that may corrupt string-typed
1129
+ # parameters (a previous "float" fallback turned cell
1130
+ # "123" into 123.0 even for schema-string params).
1131
+ if c in meta.data_types:
1132
+ dtype = meta.data_types[c]
1133
+ elif meta.default_data_type:
1134
+ dtype = meta.default_data_type
1135
+ else:
1136
+ dtype = "string"
1137
+ logger.warning(
1138
+ "Sheet '%s' col %d (parameter %r): no data type declared; "
1139
+ "defaulting to 'string' (raw cell value).",
1140
+ meta.sheet_name, c + 1, pname,
1141
+ )
1142
+ converted = _convert_value(val, dtype)
1143
+
1144
+ record = {
1145
+ "alternative": alt,
1146
+ "entity_class": entity_class.class_name,
1147
+ "entity_byname": entity_byname,
1148
+ "param_name": pname,
1149
+ "value": converted,
1150
+ "index_value": index_val,
1151
+ "index_name": meta.index_name,
1152
+ "data_type": dtype,
1153
+ }
1154
+ if extra_index_values:
1155
+ record["extra_index_values"] = list(extra_index_values)
1156
+ data.records.append(record)
1157
+ row_has_data = True
1158
+
1159
+ # Add entity existence record
1160
+ if entity_existence is not None:
1161
+ data.records.append({
1162
+ "alternative": alt,
1163
+ "entity_class": entity_class.class_name,
1164
+ "entity_byname": entity_byname,
1165
+ "param_name": ENTITY_EXISTENCE,
1166
+ "value": entity_existence,
1167
+ "index_value": None,
1168
+ "index_name": None,
1169
+ })
1170
+ row_has_data = True
1171
+
1172
+ # Create entity-only record if no data was found for this row
1173
+ # (ensures data-less entities are still imported). Propagate the
1174
+ # real ``alt`` even when None — the downstream writer
1175
+ # (``write_self_describing_to_db._import_sheet``) skips
1176
+ # alternative-creation for falsy alt names and discards
1177
+ # entity-only records before parameter writes, so a synthesised
1178
+ # ``""`` alt name would create an empty-string alternative in
1179
+ # SpineDB while a faithful ``None`` is dropped cleanly.
1180
+ if not row_has_data and entity_byname:
1181
+ data.records.append({
1182
+ "alternative": alt,
1183
+ "entity_class": entity_class.class_name,
1184
+ "entity_byname": entity_byname,
1185
+ "param_name": "",
1186
+ "value": None,
1187
+ "index_value": None,
1188
+ "index_name": None,
1189
+ })
1190
+
1191
+ return data
1192
+
1193
+
1194
+ def _extract_link(ws: Worksheet, meta: SheetMetadata) -> SheetData:
1195
+ """Extract entity relationships from a link sheet."""
1196
+ data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
1197
+ max_row = _actual_max_row(ws)
1198
+ max_col = ws.max_column or 1
1199
+
1200
+ for r in range(meta.data_start_row, max_row):
1201
+ elements = []
1202
+ for c in range(max_col):
1203
+ val = _cell_value(ws, r, c)
1204
+ if val:
1205
+ elements.append(val)
1206
+ if elements:
1207
+ data.link_entities.append(tuple(elements))
1208
+
1209
+ return data
1210
+
1211
+
1212
+ def _extract_transposed(ws: Worksheet, meta: SheetMetadata) -> SheetData:
1213
+ """Extract data from a transposed (timeseries) sheet."""
1214
+ data = SheetData(sheet_name=meta.sheet_name, metadata=meta)
1215
+ max_row = _actual_max_row(ws)
1216
+ max_col = ws.max_column or 1
1217
+
1218
+ # Each data column (from col 2 onwards) is one data series
1219
+ data_col_start = 2 # columns C+ (0-based index 2)
1220
+
1221
+ # On single-param transposed sheets (timeseries, array-transposed)
1222
+ # the parameter triplet in row 1 / col B carries the canonical data
1223
+ # type — e.g. ``"parameter: solves | data type: boolean (array)"``.
1224
+ # We extract it once and attach it to every emitted record so the
1225
+ # writer can decide whether to reconstruct an ``Array`` or a ``Map``.
1226
+ triplet_dtype: str | None = None
1227
+ if meta.param_row is not None:
1228
+ triplet_text = _cell_value(ws, meta.param_row, 1) # col B
1229
+ triplet_dtype = _extract_triplet_field(triplet_text, "data type")
1230
+ if triplet_dtype is None and meta.default_data_type:
1231
+ triplet_dtype = meta.default_data_type
1232
+
1233
+ for c in range(data_col_start, max_col):
1234
+ # Read column metadata from the header rows
1235
+ alt = _cell_value(ws, meta.alt_row, c) if meta.alt_row is not None else None
1236
+ pname = _cell_value(ws, meta.param_row, c) if meta.param_row is not None else None
1237
+ ent_name = _cell_value(ws, meta.entity_row, c) if meta.entity_row is not None else None
1238
+
1239
+ if not pname and meta.default_parameter:
1240
+ pname = meta.default_parameter
1241
+ if not alt or not ent_name:
1242
+ continue
1243
+
1244
+ entity_class = meta.entity_classes[0] if meta.entity_classes else None
1245
+ if entity_class is None:
1246
+ continue
1247
+
1248
+ entity_byname = (ent_name,)
1249
+
1250
+ # Per-column data type (multi-param _t sheets) takes precedence over
1251
+ # the single-param triplet; fall back to the triplet then "string".
1252
+ col_dtype = meta.data_types.get(c) or triplet_dtype
1253
+
1254
+ # Read time-indexed data
1255
+ for r in range(meta.data_start_row, max_row):
1256
+ index_val = _cell_value(ws, r, 0) if meta.index_col_transposed == 0 else None
1257
+ val = _cell_value(ws, r, c)
1258
+ if not val or not index_val:
1259
+ continue
1260
+
1261
+ # Use the column/triplet data type for value conversion.
1262
+ # The pre-fix code force-cast every cell to float, which only
1263
+ # worked by accident for ``boolean (array)`` (period names
1264
+ # like ``"Y2025"`` raise on ``float()`` and ``_convert_value``
1265
+ # falls back to returning the raw string). A schema-string
1266
+ # series with numeric-looking entries (e.g. ``"01"``) would
1267
+ # silently get coerced to ``1.0`` and round-trip wrong.
1268
+ value_dtype = col_dtype or "string"
1269
+ converted = _convert_value(val, value_dtype)
1270
+ data.records.append({
1271
+ "alternative": alt,
1272
+ "entity_class": entity_class.class_name,
1273
+ "entity_byname": entity_byname,
1274
+ "param_name": pname,
1275
+ "value": converted,
1276
+ "index_value": index_val,
1277
+ "index_name": meta.index_name,
1278
+ "data_type": col_dtype,
1279
+ })
1280
+
1281
+ return data
1282
+
1283
+
1284
+ def _parse_strict_bool(
1285
+ val: str,
1286
+ *,
1287
+ sheet_name: str,
1288
+ row_1based: int,
1289
+ col_1based: int,
1290
+ field_name: str,
1291
+ ) -> bool:
1292
+ """Strict TRUE / FALSE parse, case-insensitive, mirroring the exporter.
1293
+
1294
+ The exporter only ever writes the literal strings ``"TRUE"`` / ``"FALSE"``
1295
+ for entity-existence and boolean-array cells (excel_writer.py:2272 and
1296
+ :2773 — the dropdown validation also enforces those exact tokens). A
1297
+ cell holding anything else (``"yes"``, ``"1"``, ``"Y"``, a typo) is a
1298
+ data error: raise so the user sees the offending sheet / cell rather
1299
+ than silently importing ``False``.
1300
+ """
1301
+ stripped = val.strip().lower()
1302
+ if stripped == _BOOL_TRUE_LITERAL:
1303
+ return True
1304
+ if stripped == _BOOL_FALSE_LITERAL:
1305
+ return False
1306
+ raise ValueError(
1307
+ f"Sheet '{sheet_name}' row {row_1based} col {col_1based}: "
1308
+ f"{field_name} cell has value {val!r}; expected 'TRUE' or 'FALSE' "
1309
+ f"(case-insensitive). The exporter only emits those literals — "
1310
+ f"fix the cell or restore the exporter-written value."
1311
+ )
1312
+
1313
+
1314
+ def _convert_value(val: str, dtype: str) -> Any:
1315
+ """Convert a string cell value to the appropriate Python type.
1316
+
1317
+ ``float`` cells accept the IEEE non-finite sentinels ``inf`` /
1318
+ ``-inf`` / ``nan`` (case-insensitive) — the export writer emits
1319
+ them as strings because openpyxl drops actual non-finite floats.
1320
+
1321
+ The leaf scalar type is taken from the part of *dtype* BEFORE any
1322
+ ``(...)`` container suffix: ``"float (1d-map)"``, ``"float (array)"``,
1323
+ ``"float (3d-map)"`` all carry a ``float`` leaf and must convert their
1324
+ per-cell values to ``float`` — otherwise a Map/Array would round-trip
1325
+ with string leaves (``["10", "20"]`` instead of ``[10.0, 20.0]``).
1326
+ Compound leaf types joined with ``/`` (e.g. ``"string/float (1d-map)"``
1327
+ — a param that accepts either) attempt the ``float`` conversion when
1328
+ ``float`` is one of the tokens, falling back to the raw string for a
1329
+ genuinely non-numeric cell. Only the exact scalar ``"boolean"``
1330
+ strict-parses TRUE/FALSE: a ``"boolean (array)"`` cell holds a period
1331
+ token (the round-trip preservation form, see _extract_standard) and
1332
+ must stay raw.
1333
+ """
1334
+ base = dtype.split("(", 1)[0].strip().lower() if dtype else ""
1335
+ base_tokens = base.replace("/", " ").split()
1336
+ if "float" in base_tokens:
1337
+ if isinstance(val, str):
1338
+ lo = val.strip().lower()
1339
+ if lo in ("inf", "+inf", "infinity", "+infinity"):
1340
+ return float("inf")
1341
+ if lo in ("-inf", "-infinity"):
1342
+ return float("-inf")
1343
+ if lo == "nan":
1344
+ return float("nan")
1345
+ try:
1346
+ return float(val)
1347
+ except (ValueError, TypeError):
1348
+ return val # keep as string if conversion fails
1349
+ elif dtype == "boolean":
1350
+ # Strict match — the exporter only writes ``"TRUE"`` / ``"FALSE"``
1351
+ # for boolean cells (see excel_writer.py:2272 / :2773). Without
1352
+ # call-site context we can't name the offending sheet/row, so the
1353
+ # error message asks the user to inspect their workbook; the
1354
+ # per-cell entity-existence path uses ``_parse_strict_bool`` and
1355
+ # gives a much sharper message.
1356
+ stripped = val.strip().lower()
1357
+ if stripped == _BOOL_TRUE_LITERAL:
1358
+ return True
1359
+ if stripped == _BOOL_FALSE_LITERAL:
1360
+ return False
1361
+ raise ValueError(
1362
+ f"Boolean cell has value {val!r}; expected 'TRUE' or 'FALSE' "
1363
+ f"(case-insensitive)."
1364
+ )
1365
+ else:
1366
+ return val
1367
+
1368
+
1369
+ # ---------------------------------------------------------------------------
1370
+ # Public API: read an entire workbook
1371
+ # ---------------------------------------------------------------------------
1372
+
1373
+
1374
+ def read_self_describing_excel(
1375
+ file_path: str,
1376
+ skip_sheets: set[str] | None = None,
1377
+ ) -> list[SheetData]:
1378
+ """Read all self-describing sheets from an Excel workbook.
1379
+
1380
+ Args:
1381
+ file_path: Path to the .xlsx file.
1382
+ skip_sheets: Sheet names to skip (e.g. navigate, version).
1383
+
1384
+ Returns:
1385
+ List of SheetData objects, one per successfully parsed sheet.
1386
+ """
1387
+ if skip_sheets is None:
1388
+ skip_sheets = {"navigate", "version"}
1389
+
1390
+ # Note: do NOT use read_only=True — it reports max_row=1048576 for
1391
+ # sheets without explicit dimensions, which makes iteration hang.
1392
+ wb = openpyxl.load_workbook(file_path, data_only=True)
1393
+ results: list[SheetData] = []
1394
+
1395
+ for sheet_name in wb.sheetnames:
1396
+ if sheet_name in skip_sheets:
1397
+ continue
1398
+
1399
+ ws = wb[sheet_name]
1400
+ meta, is_scenario = detect_and_parse_sheet(ws)
1401
+
1402
+ if is_scenario:
1403
+ sheet_data = parse_scenario_sheet(ws)
1404
+ results.append(sheet_data)
1405
+ logger.info(
1406
+ "Sheet '%s' (scenario): %d records",
1407
+ sheet_name, len(sheet_data.records),
1408
+ )
1409
+ continue
1410
+
1411
+ if meta is None:
1412
+ logger.debug("Skipping sheet '%s': no metadata found", sheet_name)
1413
+ continue
1414
+
1415
+ sheet_data = extract_sheet_data(ws, meta)
1416
+ results.append(sheet_data)
1417
+ logger.info(
1418
+ "Sheet '%s': %d records, %d links",
1419
+ sheet_name, len(sheet_data.records), len(sheet_data.link_entities),
1420
+ )
1421
+
1422
+ wb.close()
1423
+ return results