flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1595 @@
|
|
|
1
|
+
"""Δ.17c — centralized parameter-shape resolver.
|
|
2
|
+
|
|
3
|
+
The user's authoritative directive (Δ.17c dispatch, Gap C):
|
|
4
|
+
|
|
5
|
+
There is no guarantee that e.g. 2d_map is (period, time). There is
|
|
6
|
+
multi-level reliance on different factors. The source of truth
|
|
7
|
+
should be three things:
|
|
8
|
+
1. read from database the dimensionality of the parameter.
|
|
9
|
+
2. list of parameter specific allowed dimensionalities e.g.
|
|
10
|
+
p_efficiency[period,time].
|
|
11
|
+
3. If parameter has multiple choices for e.g. 1d-map like
|
|
12
|
+
p_efficiency[period], p_efficiency[time], then you need to read
|
|
13
|
+
the dimension index label from the database. If it says period,
|
|
14
|
+
it is period; if it says time, it is time. If does not match,
|
|
15
|
+
error to the user.
|
|
16
|
+
|
|
17
|
+
This module implements those three things:
|
|
18
|
+
|
|
19
|
+
* :data:`PARAM_ALLOWED_SHAPES` — per-(entity_class, parameter_name)
|
|
20
|
+
registry of valid shapes. Each entry lists the shapes preprocessing
|
|
21
|
+
accepts (mirrors the ``write_pdtX`` / ``write_pdX`` cascade domains;
|
|
22
|
+
see docstring on each entry).
|
|
23
|
+
* :func:`resolve_param_shape` — DB-driven shape detection: reads the
|
|
24
|
+
parameter's actual nesting depth and per-level index_name labels
|
|
25
|
+
from the DB via :meth:`InputSource.parameter_shape_info`, validates
|
|
26
|
+
against the allow-list, raises :class:`FlexToolConfigError` on
|
|
27
|
+
mismatch.
|
|
28
|
+
* :func:`broadcast_to_period_time` — produces ``[entity, d, t, value]``
|
|
29
|
+
by broadcasting the parameter frame over the active solve's
|
|
30
|
+
``(d, t)`` axis according to the resolved shape.
|
|
31
|
+
* :func:`broadcast_to_period` — produces ``[entity, d, value]`` for
|
|
32
|
+
parameters whose allowed shapes are scalar / 1d_map[period] only
|
|
33
|
+
(e.g. ``co2_max_period``).
|
|
34
|
+
|
|
35
|
+
Why a centralized resolver and not per-helper column checks?
|
|
36
|
+
The previous Δ.17b approach inferred the shape from the column names
|
|
37
|
+
of the source frame. That's flawed when:
|
|
38
|
+
|
|
39
|
+
* A 2d_map column ordering depends on which dim was outermost in the
|
|
40
|
+
Map — *e.g.* ``2d_map[time, period]`` writes a column ``[t, period]``
|
|
41
|
+
but the helper expected ``[period, t]``. Column-shape detection
|
|
42
|
+
silently produced the wrong broadcast.
|
|
43
|
+
* The DB carries an unexpected (non-period, non-time) index_name —
|
|
44
|
+
e.g. ``branch`` — and the column was renamed to the canonical
|
|
45
|
+
default ``period`` by :class:`SpineDbReader._discover_index_cols`.
|
|
46
|
+
The helper saw a "period" column and broadcast over (d, t) when the
|
|
47
|
+
authoring intent was different.
|
|
48
|
+
|
|
49
|
+
The data-driven resolver avoids both: it inspects the DB-reported
|
|
50
|
+
``index_name`` per level and validates against the per-parameter
|
|
51
|
+
allow-list. Mismatches surface as :class:`FlexToolConfigError`.
|
|
52
|
+
|
|
53
|
+
Computational efficiency
|
|
54
|
+
------------------------
|
|
55
|
+
|
|
56
|
+
:func:`resolve_param_shape` is called once per (entity_class,
|
|
57
|
+
parameter_name) per ``apply_direct_params`` invocation — at most ~10
|
|
58
|
+
calls per ``load_flextool``, all cached at the source-plugin level.
|
|
59
|
+
The resolver itself does no per-row work; it only inspects the
|
|
60
|
+
parameter's structural metadata.
|
|
61
|
+
"""
|
|
62
|
+
from __future__ import annotations
|
|
63
|
+
|
|
64
|
+
from dataclasses import dataclass
|
|
65
|
+
from enum import Enum
|
|
66
|
+
from typing import TYPE_CHECKING
|
|
67
|
+
|
|
68
|
+
import polars as pl
|
|
69
|
+
|
|
70
|
+
from polar_high import Param
|
|
71
|
+
|
|
72
|
+
from flextool.engine_polars._axis_enums import (
|
|
73
|
+
cast_frame_axes,
|
|
74
|
+
get_global_axis_enums,
|
|
75
|
+
rename_to_axis,
|
|
76
|
+
)
|
|
77
|
+
from flextool.engine_polars._solve_state import FlexToolConfigError
|
|
78
|
+
|
|
79
|
+
if TYPE_CHECKING:
|
|
80
|
+
from flextool.engine_polars._input_source import InputSource
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Shape enum + per-parameter allow-list
|
|
85
|
+
# ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class AxisName(Enum):
|
|
89
|
+
"""Canonical axis identifiers for parameter-shape declarations.
|
|
90
|
+
|
|
91
|
+
Sourced from ``flextool/schemas/flextool_axis_contract.json`` axes —
|
|
92
|
+
each member's value is the axis's `name` in that contract. Phase A
|
|
93
|
+
introduces only the axes that participate in declared parameter
|
|
94
|
+
shapes today (period ``d``, time ``t``, tier ``i``). Adding
|
|
95
|
+
``block`` / ``branch`` / etc. is a follow-up when the registry
|
|
96
|
+
grows entries that use them.
|
|
97
|
+
|
|
98
|
+
Note: the model-internal ``ROLL`` axis is intentionally NOT included
|
|
99
|
+
— rolling boundaries are an engine concern, never authored as a
|
|
100
|
+
parameter index in input/output data.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
D = "d" # period
|
|
104
|
+
T = "t" # time
|
|
105
|
+
I = "i" # tier (price-ladder) # noqa: E741 — axis name from contract
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class LeafKind(Enum):
|
|
109
|
+
"""Leaf value kind for parameter shape declarations.
|
|
110
|
+
|
|
111
|
+
Phase A introduces three kinds:
|
|
112
|
+
|
|
113
|
+
* ``NUMERIC`` — the historical default; a scalar Float (or Int that
|
|
114
|
+
casts cleanly). Every pre-existing :class:`Shape` member declares
|
|
115
|
+
this leaf kind.
|
|
116
|
+
* ``STR_FROM_LIST`` — a scalar string constrained by a
|
|
117
|
+
``parameter_value_list`` (e.g. ``invest_methods``). The reader
|
|
118
|
+
writes whichever token the DB carries; the registry just records
|
|
119
|
+
that this is a constrained-string leaf.
|
|
120
|
+
* ``FACET_PRICE_QUANTITY`` — a depth-1 ``Map(facet -> value)`` whose
|
|
121
|
+
facet keys are exactly ``{"price", "quantity"}``. Used by the
|
|
122
|
+
commodity price-ladder shapes — Spine encodes the per-tier
|
|
123
|
+
``{price, quantity}`` record as a third Map level.
|
|
124
|
+
|
|
125
|
+
Phase A consumers only need the leaf kind for documentation and
|
|
126
|
+
Phase B reader-routing. ``resolve_param_shape`` accepts the new
|
|
127
|
+
leaf-kind shapes structurally; the reader/writer wiring is Phase B.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
NUMERIC = "numeric"
|
|
131
|
+
STR_FROM_LIST = "str_from_list"
|
|
132
|
+
FACET_PRICE_QUANTITY = "facet[price,quantity]"
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# Frozen facet sets — kept here so ``ParamAxes`` and any Phase B reader
|
|
136
|
+
# can share a single source of truth without re-importing across modules.
|
|
137
|
+
_FACET_PRICE_QUANTITY: frozenset[str] = frozenset(("price", "quantity"))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class Shape(Enum):
|
|
141
|
+
"""Recognised parameter shapes.
|
|
142
|
+
|
|
143
|
+
The string values mirror the user's notation in the dispatch /
|
|
144
|
+
open-issues doc (e.g. ``"1d_map[period]"``). Each member maps to a
|
|
145
|
+
``ParamAxes(map_levels, leaf)`` view via :data:`_SHAPE_AXES` for
|
|
146
|
+
callers that prefer the (axes, leaf) decomposition over the enum tag.
|
|
147
|
+
|
|
148
|
+
The original (period, time)-broadcast family — ``SCALAR``,
|
|
149
|
+
``MAP_PERIOD``, ``MAP_TIME``, ``MAP_PERIOD_TIME``, ``MAP_TIME_PERIOD``
|
|
150
|
+
— covers every entry routed through ``broadcast_to_period[_time]``
|
|
151
|
+
today. Phase A adds three new members:
|
|
152
|
+
|
|
153
|
+
* ``SCALAR_STR`` — terminal scalar string (no map levels), value
|
|
154
|
+
constrained by a ``parameter_value_list``. Used by the
|
|
155
|
+
``{node, unit, connection}.invest_method`` family.
|
|
156
|
+
* ``MAP_TIER_FACET`` — depth-2 Map ``Map(tier -> {price, quantity})``
|
|
157
|
+
via :data:`LeafKind.FACET_PRICE_QUANTITY`. Used by
|
|
158
|
+
``commodity.price_ladder_cumulative`` and the depth-2 variant of
|
|
159
|
+
``commodity.price_ladder_annual``.
|
|
160
|
+
* ``MAP_PERIOD_TIER_FACET`` — depth-3 Map
|
|
161
|
+
``Map(period -> Map(tier -> {price, quantity}))``. Used by the
|
|
162
|
+
depth-3 variant of ``commodity.price_ladder_annual``.
|
|
163
|
+
|
|
164
|
+
The Phase A choice was (a) — extend this enum in-place — because the
|
|
165
|
+
existing ``Shape`` use sites (3 test files + ``_direct_params.py``
|
|
166
|
+
imports only the resolver / broadcasters, not the enum members
|
|
167
|
+
directly) confine the blast radius to this module. A natural
|
|
168
|
+
follow-up (option b in the Phase A design doc) is to make ``Shape``
|
|
169
|
+
a derived view over ``ParamAxes(map_levels, leaf)`` and let
|
|
170
|
+
registry entries quote axis tuples directly — useful when the
|
|
171
|
+
contract grows axes like ``block`` / ``branch``. For now the
|
|
172
|
+
enum-extension keeps the diff minimal.
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
SCALAR = "scalar"
|
|
176
|
+
SCALAR_STR = "scalar[str_from_list]"
|
|
177
|
+
MAP_PERIOD = "1d_map[period]"
|
|
178
|
+
MAP_TIME = "1d_map[time]"
|
|
179
|
+
MAP_PERIOD_TIME = "2d_map[period,time]"
|
|
180
|
+
MAP_TIME_PERIOD = "2d_map[time,period]"
|
|
181
|
+
MAP_TIER_FACET = "2d_map[tier,facet{price,quantity}]"
|
|
182
|
+
MAP_PERIOD_TIER_FACET = "3d_map[period,tier,facet{price,quantity}]"
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@dataclass(frozen=True)
|
|
186
|
+
class ParamAxes:
|
|
187
|
+
"""Decomposed (axes, leaf) view of a :class:`Shape`.
|
|
188
|
+
|
|
189
|
+
``map_levels`` are the outer Map index axes from outer to inner; for
|
|
190
|
+
a ``FACET_PRICE_QUANTITY`` leaf the trailing facet level is NOT
|
|
191
|
+
included in ``map_levels`` — it's carried in ``leaf`` instead, so
|
|
192
|
+
the axis tuple stays a pure list of named axes from
|
|
193
|
+
:class:`AxisName`.
|
|
194
|
+
|
|
195
|
+
Example: ``Shape.MAP_PERIOD_TIER_FACET`` decomposes to
|
|
196
|
+
``ParamAxes(map_levels=(AxisName.D, AxisName.I),
|
|
197
|
+
leaf=LeafKind.FACET_PRICE_QUANTITY)``.
|
|
198
|
+
|
|
199
|
+
Phase A does not consume ``ParamAxes`` from the resolver — it's a
|
|
200
|
+
pure documentation/derived view, accessible via
|
|
201
|
+
:func:`shape_to_axes`. Phase B will use it to route the new
|
|
202
|
+
parameter families through reader-side shape probing.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
map_levels: tuple[AxisName, ...]
|
|
206
|
+
leaf: LeafKind
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# Per-(entity_class, parameter_name) allow-list. Each entry lists the
|
|
210
|
+
# shapes preprocessing accepts for that parameter.
|
|
211
|
+
#
|
|
212
|
+
# The cascade in `write_pdtX` (mod L1227 etc.) is:
|
|
213
|
+
# pbt → pd → pt → p → def1 → 0
|
|
214
|
+
# meaning: 2d_map(period, time) preferred, then 1d_map(period), then
|
|
215
|
+
# 1d_map(time), then scalar. All four shapes are valid authoring options
|
|
216
|
+
# for parameters in the relevant ``X_TIME_PARAM`` taxonomy.
|
|
217
|
+
#
|
|
218
|
+
# `write_pdX` cascade (mod L1115) is:
|
|
219
|
+
# pd → period__branch fold → p → 5000-default-set → 0
|
|
220
|
+
# meaning: 1d_map(period) or scalar, no time variant. Used for
|
|
221
|
+
# parameters in ``X_PERIOD_PARAM`` but NOT ``X_TIME_PARAM``.
|
|
222
|
+
#
|
|
223
|
+
# `write_pdtCommodity` cascade is pt → pd → p → 0 (no pbt).
|
|
224
|
+
PARAM_ALLOWED_SHAPES: dict[tuple[str, str], set[Shape]] = {
|
|
225
|
+
# ─── group: (period, time) Params ────────────────────────────────────
|
|
226
|
+
# GROUP_TIME_PARAM = {co2_price, max_instant_flow, min_instant_flow}.
|
|
227
|
+
# Cascade: pbt → pd → pt → p (entity_period_calc_params:write_pdtGroup).
|
|
228
|
+
("group", "co2_price"): {
|
|
229
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
230
|
+
},
|
|
231
|
+
# v58: the four flow-limit params live on the ``flowGroup`` class.
|
|
232
|
+
("flowGroup", "max_instant_flow"): {
|
|
233
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
234
|
+
},
|
|
235
|
+
("flowGroup", "min_instant_flow"): {
|
|
236
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
237
|
+
},
|
|
238
|
+
# ─── group: period-only Params (no time variant) ─────────────────────
|
|
239
|
+
# GROUP_PERIOD_PARAM \ GROUP_TIME_PARAM ⊇ {co2_max_period}.
|
|
240
|
+
# Cascade: pd → p (entity_period_calc_params:write_pdGroup).
|
|
241
|
+
("group", "co2_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
242
|
+
# Tier-1 silent-default migration (pdGroup_* / p_group_* 1d_map[period]).
|
|
243
|
+
# Each parameter was previously routed through
|
|
244
|
+
# ``_entity_period_scalar`` and so cross-joined silent-default Maps
|
|
245
|
+
# (index_name="x") against the period filter, producing duplicate
|
|
246
|
+
# (g, d) rows. Migrating to ``resolve_param_shape`` +
|
|
247
|
+
# ``broadcast_to_period`` carries the silent-default disambiguation
|
|
248
|
+
# via ``_infer_silent_default_labels`` (allow-list {SCALAR,
|
|
249
|
+
# MAP_PERIOD} ⇒ depth-1 silent default unambiguously resolves to
|
|
250
|
+
# ``period``). Cascade: pd → p (entity_period_calc_params:write_pdGroup
|
|
251
|
+
# — same as ``co2_max_period``).
|
|
252
|
+
("group", "capacity_margin"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
253
|
+
("group", "penalty_capacity_margin"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
254
|
+
("group", "inertia_limit"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
255
|
+
("group", "penalty_inertia"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
256
|
+
("group", "non_synchronous_limit"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
257
|
+
("group", "penalty_non_synchronous"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
258
|
+
("group", "invest_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
259
|
+
("group", "invest_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
260
|
+
("group", "retire_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
261
|
+
("group", "retire_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
262
|
+
("flowGroup", "max_cumulative_flow"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
263
|
+
("flowGroup", "min_cumulative_flow"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
264
|
+
# ─── unit: period-only Params (no time variant) ──────────────────────
|
|
265
|
+
# UC startup cost — Spine schema declares ``unit.startup_cost`` as
|
|
266
|
+
# scalar OR 1d_map(period). Cascade: pd → p (write_pUnit / startup
|
|
267
|
+
# cost wiring in ``_load_online``).
|
|
268
|
+
("unit", "startup_cost"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
269
|
+
# Tier-2 silent-default migration (ed_* multi-class union 1d_map(period)).
|
|
270
|
+
# These six parameters are declared on each of unit/node/connection and
|
|
271
|
+
# are unioned in :func:`_e_period_param_union` into a single
|
|
272
|
+
# ``Param(("e", "d"))`` frame. Pre-migration the helper had a partial
|
|
273
|
+
# ``"x" → "period"`` workaround but silently dropped scalar authoring
|
|
274
|
+
# and didn't value-domain probe. Routing each class through
|
|
275
|
+
# ``resolve_param_shape`` + ``broadcast_to_period`` fixes both: the
|
|
276
|
+
# registry's {SCALAR, MAP_PERIOD} allow-list resolves silent-default
|
|
277
|
+
# ``index_name`` at depth-1 structurally, and SCALAR rows are
|
|
278
|
+
# broadcast across the active solve's periods instead of being
|
|
279
|
+
# dropped. Cascade: pd → p (write_ed* via apply_direct_params_b).
|
|
280
|
+
("unit", "invest_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
281
|
+
("node", "invest_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
282
|
+
("connection", "invest_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
283
|
+
("unit", "invest_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
284
|
+
("node", "invest_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
285
|
+
("connection", "invest_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
286
|
+
("unit", "retire_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
287
|
+
("node", "retire_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
288
|
+
("connection", "retire_max_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
289
|
+
("unit", "retire_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
290
|
+
("node", "retire_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
291
|
+
("connection", "retire_min_period"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
292
|
+
("unit", "cumulative_max_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
293
|
+
("node", "cumulative_max_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
294
|
+
("connection", "cumulative_max_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
295
|
+
("unit", "cumulative_min_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
296
|
+
("node", "cumulative_min_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
297
|
+
("connection", "cumulative_min_capacity"): {Shape.SCALAR, Shape.MAP_PERIOD},
|
|
298
|
+
# ─── node: (period, time) Params ─────────────────────────────────────
|
|
299
|
+
# NODE_TIME_PARAM ⊇ {availability, storage_state_reference_value}.
|
|
300
|
+
# Cascade: pbt → pt → pd → p (write_pdtNode, time_first_priority=True).
|
|
301
|
+
("node", "availability"): {
|
|
302
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
303
|
+
},
|
|
304
|
+
("node", "storage_state_reference_value"): {
|
|
305
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
306
|
+
},
|
|
307
|
+
# ─── unit / connection: (period, time) Params ────────────────────────
|
|
308
|
+
# PROCESS_TIME_PARAM ⊇ {availability}.
|
|
309
|
+
# Cascade: pbt → pd → pt → p (write_pdtProcess). ``processes`` in
|
|
310
|
+
# process_arc_unions.py:write_param_in_use_sets reads the ``process.csv``
|
|
311
|
+
# union of unit + connection, so both classes participate in the LP's
|
|
312
|
+
# ``p_process_availability`` (the Spine schema declares the parameter
|
|
313
|
+
# on each class independently).
|
|
314
|
+
("unit", "availability"): {
|
|
315
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
316
|
+
},
|
|
317
|
+
("connection", "availability"): {
|
|
318
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
319
|
+
},
|
|
320
|
+
# ─── unit / connection: efficiency-curve (period, time) Params ────────
|
|
321
|
+
# ``efficiency``, ``efficiency_at_min_load`` and ``min_load`` are all
|
|
322
|
+
# members of ``_PROCESS_TIME_PARAM``
|
|
323
|
+
# (_emit_arc_unions.py:146-149 — flextool_base.dat:153), so each admits
|
|
324
|
+
# the full (period, time) shape family exactly like ``availability``.
|
|
325
|
+
# Consumed by :func:`._derived_params.p_slope_from_source` /
|
|
326
|
+
# :func:`._derived_params.p_section_from_source` via
|
|
327
|
+
# :func:`._derived_params._eff_param_to_pdt`. ``efficiency`` is
|
|
328
|
+
# authored on both unit and connection; ``efficiency_at_min_load`` /
|
|
329
|
+
# ``min_load`` only feed the unit ``min_load_efficiency`` linearisation.
|
|
330
|
+
("unit", "efficiency"): {
|
|
331
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
332
|
+
},
|
|
333
|
+
("connection", "efficiency"): {
|
|
334
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
335
|
+
},
|
|
336
|
+
("unit", "efficiency_at_min_load"): {
|
|
337
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
338
|
+
},
|
|
339
|
+
("unit", "min_load"): {
|
|
340
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
341
|
+
},
|
|
342
|
+
# ─── commodity: time Params ──────────────────────────────────────────
|
|
343
|
+
# commodityTimeParam = {price}.
|
|
344
|
+
# Cascade: pt → pd → p → 0 (write_pdtCommodity, no pbt).
|
|
345
|
+
("commodity", "price"): {
|
|
346
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME,
|
|
347
|
+
},
|
|
348
|
+
# ─── Tier-3 silent-default migration ─────────────────────────────────
|
|
349
|
+
# Map(period → time) variable-cost / reservation parameters previously
|
|
350
|
+
# routed through inline helpers in ``_direct_params.py`` that gated on
|
|
351
|
+
# ``{"period", "t"}.issubset(cols)`` — silently dropping silent-default
|
|
352
|
+
# Maps whose ``index_name`` carried spinedb_api's ``"x" / "" / None``
|
|
353
|
+
# instead of the canonical ``"period" / "t"``. Routing through
|
|
354
|
+
# ``resolve_param_shape`` + ``broadcast_to_period_time`` carries the
|
|
355
|
+
# silent-default disambiguation structurally (allow-list
|
|
356
|
+
# {SCALAR, MAP_PERIOD, MAP_TIME, MAP_PERIOD_TIME} ⇒ value-domain
|
|
357
|
+
# probing for ambiguous depth-1 cases; depth-2 ambiguity falls back
|
|
358
|
+
# to a Map(period → time) reading).
|
|
359
|
+
#
|
|
360
|
+
# Three are on relationship classes (multi-dim entity columns) — the
|
|
361
|
+
# Tier-3 ``broadcast_to_period_time`` extension accepts a per-source-
|
|
362
|
+
# column rename dict to map those into the LP's canonical
|
|
363
|
+
# ``(p / source / sink / r, ud, g)`` axes.
|
|
364
|
+
("unit__inputNode", "other_operational_cost"): {
|
|
365
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
366
|
+
},
|
|
367
|
+
("unit__outputNode", "other_operational_cost"): {
|
|
368
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
369
|
+
},
|
|
370
|
+
("connection", "other_operational_cost"): {
|
|
371
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
372
|
+
},
|
|
373
|
+
("reserve__upDown__group", "reservation"): {
|
|
374
|
+
Shape.SCALAR, Shape.MAP_PERIOD, Shape.MAP_TIME, Shape.MAP_PERIOD_TIME,
|
|
375
|
+
},
|
|
376
|
+
# ─── Phase A: commodity price-ladder Maps (tier-keyed, facet leaf) ───
|
|
377
|
+
# ``commodity.price_ladder_cumulative`` — depth-2 Map keyed by ``tier``
|
|
378
|
+
# with a facet-dict leaf ``{price, quantity}``. Per the schema
|
|
379
|
+
# description (spinedb_schema.json L874) and the input-derivation
|
|
380
|
+
# contract (_commodity_ladder.py:151-156), this is the only valid
|
|
381
|
+
# authoring shape — period-agnostic, one row per tier. Spine's
|
|
382
|
+
# ``parameter_value_types`` block locks this at ``("commodity",
|
|
383
|
+
# "price_ladder_cumulative", "map", 2)``.
|
|
384
|
+
#
|
|
385
|
+
# ``commodity.price_ladder_annual`` — TWO valid variants per the
|
|
386
|
+
# schema description (L869): a depth-2 form identical to cumulative
|
|
387
|
+
# (the same ``Map(tier -> {price, quantity})`` is expanded across
|
|
388
|
+
# every model period), and a depth-3 form
|
|
389
|
+
# ``Map(period -> Map(tier -> {price, quantity}))`` that varies the
|
|
390
|
+
# cap per period. Spine's ``parameter_value_types`` declares both:
|
|
391
|
+
# ``("commodity", "price_ladder_annual", "map", 2)`` and
|
|
392
|
+
# ``("commodity", "price_ladder_annual", "map", 3)`` — matching the
|
|
393
|
+
# auto-detect branch in :func:`derive_commodity_ladder_annual`
|
|
394
|
+
# (input_derivation/_commodity_ladder.py:237-243).
|
|
395
|
+
#
|
|
396
|
+
# Phase A is contract-only. Phase B will wire these through the
|
|
397
|
+
# readers/writers; today the entries just unblock
|
|
398
|
+
# :func:`resolve_param_shape` from raising on the registry probe.
|
|
399
|
+
("commodity", "price_ladder_cumulative"): {Shape.MAP_TIER_FACET},
|
|
400
|
+
("commodity", "price_ladder_annual"): {
|
|
401
|
+
Shape.MAP_TIER_FACET, Shape.MAP_PERIOD_TIER_FACET,
|
|
402
|
+
},
|
|
403
|
+
# ─── Phase A: scalar string constrained by parameter_value_list ──────
|
|
404
|
+
# ``{node, unit, connection}.invest_method`` — terminal scalar string
|
|
405
|
+
# whose value is constrained by the ``invest_methods`` value list
|
|
406
|
+
# (spinedb_schema.json L358-414 enumerate the allowed tokens).
|
|
407
|
+
# Spine's ``parameter_value_types`` locks each at ``("...",
|
|
408
|
+
# "invest_method", "str", 0)``.
|
|
409
|
+
#
|
|
410
|
+
# Phase A registers these as :class:`Shape.SCALAR_STR` so the
|
|
411
|
+
# registry can express that the leaf is a constrained string, not a
|
|
412
|
+
# numeric. Phase B will wire the readers to honour the leaf-kind
|
|
413
|
+
# tag — today these parameters are read by feature-gate logic that
|
|
414
|
+
# bypasses :func:`resolve_param_shape`.
|
|
415
|
+
("node", "invest_method"): {Shape.SCALAR_STR},
|
|
416
|
+
("unit", "invest_method"): {Shape.SCALAR_STR},
|
|
417
|
+
("connection", "invest_method"): {Shape.SCALAR_STR},
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
# ---------------------------------------------------------------------------
|
|
422
|
+
# ParamAxes — derived (map_levels, leaf) view per :class:`Shape`
|
|
423
|
+
# ---------------------------------------------------------------------------
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
# Per-shape ``ParamAxes`` view. Phase A consumers can read this to
|
|
427
|
+
# decompose a :class:`Shape` into its outer-axis tuple + leaf kind
|
|
428
|
+
# without inspecting the enum tag string. Kept in sync with the
|
|
429
|
+
# :class:`Shape` enum — every member must have an entry here.
|
|
430
|
+
_SHAPE_AXES: "dict[Shape, ParamAxes]" = {
|
|
431
|
+
Shape.SCALAR: ParamAxes((), LeafKind.NUMERIC),
|
|
432
|
+
Shape.SCALAR_STR: ParamAxes((), LeafKind.STR_FROM_LIST),
|
|
433
|
+
Shape.MAP_PERIOD: ParamAxes((AxisName.D,), LeafKind.NUMERIC),
|
|
434
|
+
Shape.MAP_TIME: ParamAxes((AxisName.T,), LeafKind.NUMERIC),
|
|
435
|
+
Shape.MAP_PERIOD_TIME: ParamAxes((AxisName.D, AxisName.T),
|
|
436
|
+
LeafKind.NUMERIC),
|
|
437
|
+
Shape.MAP_TIME_PERIOD: ParamAxes((AxisName.T, AxisName.D),
|
|
438
|
+
LeafKind.NUMERIC),
|
|
439
|
+
Shape.MAP_TIER_FACET: ParamAxes((AxisName.I,),
|
|
440
|
+
LeafKind.FACET_PRICE_QUANTITY),
|
|
441
|
+
Shape.MAP_PERIOD_TIER_FACET: ParamAxes(
|
|
442
|
+
(AxisName.D, AxisName.I), LeafKind.FACET_PRICE_QUANTITY),
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def shape_to_axes(shape: Shape) -> ParamAxes:
|
|
447
|
+
"""Return the :class:`ParamAxes` view of *shape*.
|
|
448
|
+
|
|
449
|
+
Stable read-only accessor — consumers should prefer this over
|
|
450
|
+
indexing :data:`_SHAPE_AXES` directly so the module stays free to
|
|
451
|
+
refactor the storage representation later (e.g. derive Shape from
|
|
452
|
+
ParamAxes if option (b) in the Phase A design is adopted).
|
|
453
|
+
"""
|
|
454
|
+
return _SHAPE_AXES[shape]
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def facet_keys(leaf: LeafKind) -> "frozenset[str]":
|
|
458
|
+
"""Return the facet-key set for a facet-dict :class:`LeafKind`.
|
|
459
|
+
|
|
460
|
+
Raises :class:`KeyError` when *leaf* isn't a facet leaf. Phase B
|
|
461
|
+
readers use this when projecting facet-keyed leaves into columnar
|
|
462
|
+
form (one column per facet) — exposing the contract here keeps the
|
|
463
|
+
facet vocabulary in a single place.
|
|
464
|
+
"""
|
|
465
|
+
if leaf is LeafKind.FACET_PRICE_QUANTITY:
|
|
466
|
+
return _FACET_PRICE_QUANTITY
|
|
467
|
+
raise KeyError(
|
|
468
|
+
f"facet_keys: LeafKind {leaf!r} is not a facet leaf"
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
# Allowed shapes for the (entity, period) family — used by
|
|
473
|
+
# :func:`broadcast_to_period`. These parameters never authorise a time
|
|
474
|
+
# axis. Shapes outside ``{SCALAR, MAP_PERIOD}`` raise on detection.
|
|
475
|
+
_PERIOD_ONLY_SHAPES: frozenset[Shape] = frozenset(
|
|
476
|
+
(Shape.SCALAR, Shape.MAP_PERIOD)
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
# ---------------------------------------------------------------------------
|
|
481
|
+
# ResolvedShape — the resolver's output
|
|
482
|
+
# ---------------------------------------------------------------------------
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
@dataclass(frozen=True)
|
|
486
|
+
class ResolvedShape:
|
|
487
|
+
"""Outcome of :func:`resolve_param_shape`.
|
|
488
|
+
|
|
489
|
+
Carries everything a broadcast helper needs:
|
|
490
|
+
|
|
491
|
+
* :attr:`shape` — the canonical :class:`Shape`.
|
|
492
|
+
* :attr:`frame` — the parameter frame from
|
|
493
|
+
:meth:`InputSource.parameter_explicit` (or :meth:`parameter`),
|
|
494
|
+
or ``None`` when the parameter has no explicit rows.
|
|
495
|
+
* :attr:`entity_dim_columns` — the entity's dim columns (for 0-dim
|
|
496
|
+
classes this is ``["name"]``).
|
|
497
|
+
* :attr:`period_index_column` — the column holding the period index
|
|
498
|
+
(or ``None`` for scalar / time-only shapes). Always ``"period"``
|
|
499
|
+
when present (the SpineDbReader normalises ``index_name`` to the
|
|
500
|
+
Map's authored name; the registry validates it's authored as
|
|
501
|
+
"period").
|
|
502
|
+
* :attr:`time_index_column` — the column holding the time index
|
|
503
|
+
(or ``None`` for scalar / period-only shapes). Always ``"t"``
|
|
504
|
+
when present (per ``SpineDbReader._discover_index_cols`` which
|
|
505
|
+
maps "time" → "t").
|
|
506
|
+
"""
|
|
507
|
+
|
|
508
|
+
shape: Shape
|
|
509
|
+
frame: "pl.DataFrame | None"
|
|
510
|
+
entity_dim_columns: tuple[str, ...]
|
|
511
|
+
period_index_column: "str | None"
|
|
512
|
+
time_index_column: "str | None"
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
# ---------------------------------------------------------------------------
|
|
516
|
+
# resolve_param_shape — DB-driven shape detection + validation
|
|
517
|
+
# ---------------------------------------------------------------------------
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _allowed_shape_names(allowed: "set[Shape]") -> str:
|
|
521
|
+
"""Render a stable, sorted comma-separated name list for error messages."""
|
|
522
|
+
return ", ".join(sorted(s.value for s in allowed))
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
# Labels that the spinedb_api default-fills when the author didn't
|
|
526
|
+
# specify ``index_name`` on a Map. Treated as "silent default" — the
|
|
527
|
+
# resolver returns None (the caller falls through to the seed / leaves
|
|
528
|
+
# the field unset) instead of raising. Per the user advice, the error
|
|
529
|
+
# path applies to *explicit* mismatches, not authoring oversights.
|
|
530
|
+
_SILENT_DEFAULT_LABELS: frozenset[str] = frozenset(("x", ""))
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _normalise_label(n: "str | None") -> "str | None":
|
|
534
|
+
"""Lowercase / strip / collapse silent defaults to ``None``."""
|
|
535
|
+
if n is None:
|
|
536
|
+
return None
|
|
537
|
+
if not isinstance(n, str):
|
|
538
|
+
return None
|
|
539
|
+
s = n.strip().lower()
|
|
540
|
+
if not s or s in _SILENT_DEFAULT_LABELS:
|
|
541
|
+
return None
|
|
542
|
+
return s
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
# Per-shape depth + per-level canonical label, used to disambiguate
|
|
546
|
+
# silent-default ``index_name`` labels by consulting the per-parameter
|
|
547
|
+
# allow-list. Each entry: shape → tuple of canonical labels per depth
|
|
548
|
+
# level (length 0/1/2). Mirrors the enum membership.
|
|
549
|
+
# For facet-leaf shapes every slot is ``None`` — the author chooses the
|
|
550
|
+
# ``index_name`` per parameter value (``"tier"``, ``"i"``, ``"x"``, ...
|
|
551
|
+
# are all valid) so structural detection MUST NOT rely on a canonical
|
|
552
|
+
# label. Phase A routes facet recognition through the registry
|
|
553
|
+
# allow-list in :func:`resolve_param_shape` instead (see
|
|
554
|
+
# :func:`_recognise_facet_shape`). Listing the slots as ``None`` here
|
|
555
|
+
# documents "label-agnostic" semantics — and keeps
|
|
556
|
+
# :func:`_infer_silent_default_labels` consistent for these depths
|
|
557
|
+
# (the helper only fills slots when the candidate set agrees on a
|
|
558
|
+
# canonical label; ``None`` won't promote to anything).
|
|
559
|
+
_SHAPE_LABELS: "dict[Shape, tuple[str | None, ...]]" = {
|
|
560
|
+
Shape.SCALAR: (),
|
|
561
|
+
Shape.SCALAR_STR: (),
|
|
562
|
+
Shape.MAP_PERIOD: ("period",),
|
|
563
|
+
Shape.MAP_TIME: ("time",),
|
|
564
|
+
Shape.MAP_PERIOD_TIME: ("period", "time"),
|
|
565
|
+
Shape.MAP_TIME_PERIOD: ("time", "period"),
|
|
566
|
+
Shape.MAP_TIER_FACET: (None, None),
|
|
567
|
+
Shape.MAP_PERIOD_TIER_FACET: (None, None, None),
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _infer_silent_default_labels(
|
|
572
|
+
raw_labels: "list[str | None]",
|
|
573
|
+
allowed: "set[Shape]",
|
|
574
|
+
) -> "list[str | None]":
|
|
575
|
+
"""Fill silent-default ``index_name`` slots from the allow-list.
|
|
576
|
+
|
|
577
|
+
When the DB authored a Map without ``index_name`` (the spinedb_api
|
|
578
|
+
silent default ``"x"``), the raw label collapses to ``None`` through
|
|
579
|
+
:func:`_normalise_label`. For parameters whose allow-list has a
|
|
580
|
+
*unique* choice at the observed depth + position, the silent default
|
|
581
|
+
is unambiguous: the registry only permits one shape with that
|
|
582
|
+
depth/position, so the author's intent is recoverable without
|
|
583
|
+
rewriting the DB.
|
|
584
|
+
|
|
585
|
+
Example: ``("group", "co2_max_period")`` admits
|
|
586
|
+
``{SCALAR, MAP_PERIOD}``. Depth=1 admits only ``MAP_PERIOD`` →
|
|
587
|
+
position-0 label is unambiguously ``"period"``. Depth=1 for
|
|
588
|
+
``("commodity", "price")`` admits both ``MAP_PERIOD`` and
|
|
589
|
+
``MAP_TIME``: position-0 stays ``None`` (genuinely ambiguous; the
|
|
590
|
+
caller falls back).
|
|
591
|
+
|
|
592
|
+
Returns a fresh list with disambiguated labels filled in; original
|
|
593
|
+
non-silent labels are passed through unchanged. The output has the
|
|
594
|
+
same length as ``raw_labels``.
|
|
595
|
+
"""
|
|
596
|
+
normalised = [_normalise_label(n) for n in raw_labels]
|
|
597
|
+
depth = len(normalised)
|
|
598
|
+
if all(n is not None for n in normalised):
|
|
599
|
+
# No silent defaults to disambiguate.
|
|
600
|
+
return normalised
|
|
601
|
+
# Per the allow-list, which canonical-label tuples have matching
|
|
602
|
+
# depth? Only shapes at the observed depth.
|
|
603
|
+
candidates = [
|
|
604
|
+
_SHAPE_LABELS[s] for s in allowed if len(_SHAPE_LABELS[s]) == depth
|
|
605
|
+
]
|
|
606
|
+
if not candidates:
|
|
607
|
+
# No allowed shape at this depth — leave it for the post-resolve
|
|
608
|
+
# allow-list check / unresolved-shape branch.
|
|
609
|
+
return normalised
|
|
610
|
+
# For each position, the unique label across all candidates fills
|
|
611
|
+
# silent defaults. Mixed positions stay ambiguous.
|
|
612
|
+
filled: "list[str | None]" = list(normalised)
|
|
613
|
+
for pos in range(depth):
|
|
614
|
+
if filled[pos] is not None:
|
|
615
|
+
continue
|
|
616
|
+
labels_at_pos = {cand[pos] for cand in candidates}
|
|
617
|
+
if len(labels_at_pos) == 1:
|
|
618
|
+
filled[pos] = next(iter(labels_at_pos))
|
|
619
|
+
return filled
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _shape_from_indices(index_names: "list[str | None]",
|
|
623
|
+
) -> "Shape | None":
|
|
624
|
+
"""Map a depth-ordered list of raw ``index_name`` labels to a
|
|
625
|
+
:class:`Shape`.
|
|
626
|
+
|
|
627
|
+
The user advice "If it says period, it is period; if it says time,
|
|
628
|
+
it is time. If does not match, error to the user." is enforced
|
|
629
|
+
here for *explicit* mismatches. Empty / ``None`` / spinedb_api's
|
|
630
|
+
``"x"`` silent default propagate as ambiguity → ``return None``,
|
|
631
|
+
letting the caller fall through to whatever non-resolver pathway
|
|
632
|
+
the seed used to fill (see Δ.18+ punch-list to clean up fixtures
|
|
633
|
+
that author Maps without ``index_name``).
|
|
634
|
+
|
|
635
|
+
Returns ``None`` when at least one level is a silent default
|
|
636
|
+
(cannot disambiguate without DB-side authoring fix).
|
|
637
|
+
Returns the resolved :class:`Shape` when every level is explicitly
|
|
638
|
+
"period" or "time" / "t".
|
|
639
|
+
Raises :class:`_UnrecognisedIndex` when at least one level carries
|
|
640
|
+
an explicit label outside that set (e.g. "tier", "branch").
|
|
641
|
+
"""
|
|
642
|
+
# Canonical normalisation: silent defaults collapse to None.
|
|
643
|
+
normalised = [_normalise_label(n) for n in index_names]
|
|
644
|
+
|
|
645
|
+
# Silent defaults at any level → return None (ambiguous; caller
|
|
646
|
+
# falls back). We DON'T raise here because the spinedb_api default
|
|
647
|
+
# is a silent oversight, not a deliberate mismatch.
|
|
648
|
+
#
|
|
649
|
+
# Phase A note: registry-driven recognition of facet-leaf shapes
|
|
650
|
+
# (``MAP_TIER_FACET``, ``MAP_PERIOD_TIER_FACET``) does NOT live
|
|
651
|
+
# here — :func:`resolve_param_shape` handles it via the allow-list
|
|
652
|
+
# because the user can author any ``index_name`` they want for the
|
|
653
|
+
# outer Map levels (``index_name`` is user-set; ``"x"`` is the
|
|
654
|
+
# spinedb_api default when unset). The shape + position is what
|
|
655
|
+
# identifies the parameter family; label inspection only
|
|
656
|
+
# disambiguates the legacy depth-1 / depth-2 period-vs-time case
|
|
657
|
+
# below.
|
|
658
|
+
if any(n is None for n in normalised):
|
|
659
|
+
return None
|
|
660
|
+
|
|
661
|
+
if len(normalised) == 0:
|
|
662
|
+
return Shape.SCALAR
|
|
663
|
+
if len(normalised) == 1:
|
|
664
|
+
n0 = normalised[0]
|
|
665
|
+
if n0 == "period":
|
|
666
|
+
return Shape.MAP_PERIOD
|
|
667
|
+
if n0 in ("time", "t"):
|
|
668
|
+
return Shape.MAP_TIME
|
|
669
|
+
raise _UnrecognisedIndex(n0, depth=0)
|
|
670
|
+
if len(normalised) == 2:
|
|
671
|
+
n0, n1 = normalised[0], normalised[1]
|
|
672
|
+
if n0 == "period" and n1 in ("time", "t"):
|
|
673
|
+
return Shape.MAP_PERIOD_TIME
|
|
674
|
+
if n0 in ("time", "t") and n1 == "period":
|
|
675
|
+
return Shape.MAP_TIME_PERIOD
|
|
676
|
+
raise _UnrecognisedIndex(f"({n0!r}, {n1!r})", depth=0)
|
|
677
|
+
# Higher depth — not supported here.
|
|
678
|
+
raise _UnrecognisedIndex(
|
|
679
|
+
f"depth {len(normalised)} (got {normalised})", depth=0,
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
class _UnrecognisedIndex(Exception):
|
|
684
|
+
"""Internal sentinel for :func:`_shape_from_indices` — caller maps
|
|
685
|
+
to :class:`FlexToolConfigError` with full context."""
|
|
686
|
+
|
|
687
|
+
def __init__(self, label: str, depth: int):
|
|
688
|
+
super().__init__(label)
|
|
689
|
+
self.label = label
|
|
690
|
+
self.depth = depth
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _disambiguate_shape_by_value_domain(
|
|
694
|
+
df: "pl.DataFrame",
|
|
695
|
+
ent_cols: "list[str]",
|
|
696
|
+
resolved_labels: "list[str | None]",
|
|
697
|
+
allowed: "set[Shape]",
|
|
698
|
+
period_filter: "pl.DataFrame | None",
|
|
699
|
+
) -> "Shape | None":
|
|
700
|
+
"""Value-domain probing fallback for silent-default ``index_name`` Maps.
|
|
701
|
+
|
|
702
|
+
When the registry permits multiple shapes at the observed depth
|
|
703
|
+
(e.g. ``("group", "co2_price")`` admits both ``MAP_PERIOD`` and
|
|
704
|
+
``MAP_TIME``) and the DB carried no ``index_name`` label, we
|
|
705
|
+
cannot decide structurally. Mirroring the legacy CSV pipeline
|
|
706
|
+
(``_timeline.separate_period_and_timeseries_data``), we look at
|
|
707
|
+
the Map's actual index values: a Map keyed by ``y2019, y2020, ...``
|
|
708
|
+
is :class:`Shape.MAP_PERIOD`; a Map keyed by ``t0001, t0002, ...``
|
|
709
|
+
is :class:`Shape.MAP_TIME`.
|
|
710
|
+
|
|
711
|
+
Requires *period_filter* with columns ``d`` (periods) and ``t``
|
|
712
|
+
(timesteps). Returns ``None`` when probing is impossible (no
|
|
713
|
+
filter, no candidates, mixed values matching neither set).
|
|
714
|
+
|
|
715
|
+
Currently only handles depth-1 ambiguity (a single Map level). For
|
|
716
|
+
depth-2 Maps we don't yet probe — the registry's allow-list usually
|
|
717
|
+
fixes the ordering by allowing only one 2d shape at a time, so the
|
|
718
|
+
silent-default case there is rare; the standard structural pathway
|
|
719
|
+
handles every fixture observed in the cascade gate as of 2026-05.
|
|
720
|
+
"""
|
|
721
|
+
if period_filter is None:
|
|
722
|
+
return None
|
|
723
|
+
depth = len(resolved_labels)
|
|
724
|
+
if depth != 1:
|
|
725
|
+
return None
|
|
726
|
+
# Identify the silent-default level (the only position with None
|
|
727
|
+
# in resolved_labels — every other depth-1 case yielded a definite
|
|
728
|
+
# shape above and never reached this fallback).
|
|
729
|
+
if resolved_labels[0] is not None:
|
|
730
|
+
return None
|
|
731
|
+
# The candidate set at depth 1 is the intersection of *allowed* with
|
|
732
|
+
# depth-1 shapes: at most MAP_PERIOD and MAP_TIME.
|
|
733
|
+
candidates = {s for s in allowed
|
|
734
|
+
if len(_SHAPE_LABELS[s]) == 1
|
|
735
|
+
and s in (Shape.MAP_PERIOD, Shape.MAP_TIME)}
|
|
736
|
+
if not candidates:
|
|
737
|
+
return None
|
|
738
|
+
# Locate the index column (the first non-entity, non-value column).
|
|
739
|
+
non_ent_cols = [
|
|
740
|
+
c for c in df.columns if c not in ent_cols and c != "value"
|
|
741
|
+
]
|
|
742
|
+
if len(non_ent_cols) != 1:
|
|
743
|
+
# Defensive: depth-1 frame should carry exactly one index column.
|
|
744
|
+
return None
|
|
745
|
+
idx_col = non_ent_cols[0]
|
|
746
|
+
# Get distinct index values from the frame.
|
|
747
|
+
try:
|
|
748
|
+
idx_values = (
|
|
749
|
+
df.lazy()
|
|
750
|
+
.select(pl.col(idx_col).cast(pl.Utf8))
|
|
751
|
+
.unique()
|
|
752
|
+
.collect()
|
|
753
|
+
.get_column(idx_col)
|
|
754
|
+
.to_list()
|
|
755
|
+
)
|
|
756
|
+
except Exception:
|
|
757
|
+
return None
|
|
758
|
+
if not idx_values:
|
|
759
|
+
return None
|
|
760
|
+
idx_set = {str(v) for v in idx_values if v is not None}
|
|
761
|
+
# Build the period / timestep universes from period_filter.
|
|
762
|
+
pf_cols = period_filter.columns
|
|
763
|
+
periods_set: "set[str] | None" = None
|
|
764
|
+
timesteps_set: "set[str] | None" = None
|
|
765
|
+
if "d" in pf_cols:
|
|
766
|
+
periods_set = set(
|
|
767
|
+
period_filter.lazy()
|
|
768
|
+
.select(pl.col("d").cast(pl.Utf8))
|
|
769
|
+
.unique()
|
|
770
|
+
.collect()
|
|
771
|
+
.get_column("d")
|
|
772
|
+
.to_list()
|
|
773
|
+
)
|
|
774
|
+
if "t" in pf_cols:
|
|
775
|
+
timesteps_set = set(
|
|
776
|
+
period_filter.lazy()
|
|
777
|
+
.select(pl.col("t").cast(pl.Utf8))
|
|
778
|
+
.unique()
|
|
779
|
+
.collect()
|
|
780
|
+
.get_column("t")
|
|
781
|
+
.to_list()
|
|
782
|
+
)
|
|
783
|
+
# Intersects-and-not-the-other: the index must overlap exactly one
|
|
784
|
+
# universe and not the other. We use intersection (not subset) so a
|
|
785
|
+
# Map authored with more periods than the active solve realises still
|
|
786
|
+
# resolves cleanly — extra (inactive-period) keys are dropped
|
|
787
|
+
# downstream by ``_filter_param_by_periods``. Mixed (overlaps both)
|
|
788
|
+
# or no-overlap → genuinely ambiguous → don't guess.
|
|
789
|
+
hits_periods = bool(periods_set) and bool(idx_set & periods_set)
|
|
790
|
+
hits_timesteps = bool(timesteps_set) and bool(idx_set & timesteps_set)
|
|
791
|
+
if hits_periods and not hits_timesteps and Shape.MAP_PERIOD in candidates:
|
|
792
|
+
return Shape.MAP_PERIOD
|
|
793
|
+
if hits_timesteps and not hits_periods and Shape.MAP_TIME in candidates:
|
|
794
|
+
return Shape.MAP_TIME
|
|
795
|
+
return None
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _read_index_names_from_source(
|
|
799
|
+
source: "InputSource",
|
|
800
|
+
entity_class: str,
|
|
801
|
+
parameter_name: str,
|
|
802
|
+
) -> "list[str | None]":
|
|
803
|
+
"""Read the raw per-level ``index_name`` labels from the source.
|
|
804
|
+
|
|
805
|
+
Honours the user advice "read from database the dimensionality of
|
|
806
|
+
the parameter" and "read the dimension index label from the
|
|
807
|
+
database".
|
|
808
|
+
|
|
809
|
+
Implementation:
|
|
810
|
+
|
|
811
|
+
* If the source provides :meth:`parameter_shape_info`, use it (the
|
|
812
|
+
canonical path — see :class:`SpineDbReader.parameter_shape_info`).
|
|
813
|
+
* Otherwise (e.g. :class:`InMemoryReader` in unit tests), inspect
|
|
814
|
+
the parameter frame's columns directly. This is the legacy path;
|
|
815
|
+
InMemoryReader test fixtures author frames with column names
|
|
816
|
+
already matching the resolved shape.
|
|
817
|
+
|
|
818
|
+
Returns a list of raw labels per nesting depth (length 0 for
|
|
819
|
+
scalar; length 1/2 for maps). Labels may be ``None`` when the DB
|
|
820
|
+
didn't author one — the caller will raise via the
|
|
821
|
+
:class:`_UnrecognisedIndex` path.
|
|
822
|
+
"""
|
|
823
|
+
fn = getattr(source, "parameter_shape_info", None)
|
|
824
|
+
if fn is not None:
|
|
825
|
+
return list(fn(entity_class, parameter_name))
|
|
826
|
+
# Fallback path (InMemoryReader): infer from the frame's column
|
|
827
|
+
# names, treating the first non-entity column as depth-0 etc.
|
|
828
|
+
df = _try_parameter_frame(source, entity_class, parameter_name)
|
|
829
|
+
if df is None or df.height == 0:
|
|
830
|
+
return []
|
|
831
|
+
ent_cols = _entity_dim_columns_for_frame(df, source, entity_class)
|
|
832
|
+
cols = [c for c in df.columns if c not in ent_cols and c != "value"]
|
|
833
|
+
# The frame's column names are the post-normalisation labels
|
|
834
|
+
# (period stays "period"; time → "t" via SpineDbReader). We translate
|
|
835
|
+
# back to canonical labels for the shape resolver.
|
|
836
|
+
out: list[str | None] = []
|
|
837
|
+
for c in cols:
|
|
838
|
+
if c == "period":
|
|
839
|
+
out.append("period")
|
|
840
|
+
elif c in ("t", "time"):
|
|
841
|
+
out.append("time")
|
|
842
|
+
else:
|
|
843
|
+
out.append(c)
|
|
844
|
+
return out
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def _try_parameter_frame(
|
|
848
|
+
source: "InputSource",
|
|
849
|
+
entity_class: str,
|
|
850
|
+
parameter_name: str,
|
|
851
|
+
) -> "pl.DataFrame | None":
|
|
852
|
+
"""Wrap ``source.parameter_explicit`` / ``source.parameter`` with
|
|
853
|
+
None-on-KeyError semantics. Used by both the InMemory shape-info
|
|
854
|
+
fallback and by :func:`resolve_param_shape` to fetch the data.
|
|
855
|
+
|
|
856
|
+
Δ.28 — when ``parameter_explicit`` returns an empty frame (no
|
|
857
|
+
explicit overrides for any entity), fall through to ``parameter``
|
|
858
|
+
so the schema default (e.g. ``availability = 1.0``) is consulted.
|
|
859
|
+
Without this fall-through, the resolver dropped fields whose
|
|
860
|
+
Spine value is purely the default (``p_process_availability`` on
|
|
861
|
+
fixtures where every unit/connection inherits the 1.0 default,
|
|
862
|
+
e.g. ``work_lh2_three_region``) — the slow path's CSV
|
|
863
|
+
preprocessing always writes the default-broadcast rows so the
|
|
864
|
+
fast path was missing data.
|
|
865
|
+
|
|
866
|
+
Parameters with ``default_value=None`` (the §4.5 None-skip family)
|
|
867
|
+
behave unchanged: ``parameter`` still returns the explicit-only
|
|
868
|
+
frame, so an empty result remains an empty result and the resolver
|
|
869
|
+
correctly drops the field.
|
|
870
|
+
"""
|
|
871
|
+
try:
|
|
872
|
+
explicit = source.parameter_explicit(entity_class, parameter_name)
|
|
873
|
+
except (KeyError, AttributeError):
|
|
874
|
+
explicit = None
|
|
875
|
+
if explicit is not None and explicit.height > 0:
|
|
876
|
+
return explicit
|
|
877
|
+
try:
|
|
878
|
+
return source.parameter(entity_class, parameter_name)
|
|
879
|
+
except (KeyError, AttributeError):
|
|
880
|
+
# AttributeError covers stub ``InputSource`` implementations in
|
|
881
|
+
# unit tests that only override ``parameter_explicit`` (e.g.
|
|
882
|
+
# ``_ZeroStartupSource`` in ``test_a04_a05_online_ramp.py``).
|
|
883
|
+
return explicit
|
|
884
|
+
|
|
885
|
+
|
|
886
|
+
def _recognise_facet_shape(
|
|
887
|
+
raw_labels: "list[str | None]",
|
|
888
|
+
allowed: "set[Shape]",
|
|
889
|
+
) -> "Shape | None":
|
|
890
|
+
"""Registry-driven detection for facet-leaf shapes (Phase A).
|
|
891
|
+
|
|
892
|
+
Spine-side ``index_name`` labels are user-set per parameter value —
|
|
893
|
+
the user may author ``Map(<anything>)`` with any label they like
|
|
894
|
+
(the spinedb_api default is ``"x"``). Phase A therefore identifies
|
|
895
|
+
facet-leaf shapes (``MAP_TIER_FACET``, ``MAP_PERIOD_TIER_FACET``)
|
|
896
|
+
from the **shape + position** of the value, not from the labels:
|
|
897
|
+
|
|
898
|
+
* Depth of the raw shape info matches the candidate facet shape's
|
|
899
|
+
depth in :data:`_SHAPE_AXES` (``len(map_levels) + 1`` because the
|
|
900
|
+
facet itself sits at the innermost level).
|
|
901
|
+
* The allow-list contains exactly one facet shape at that depth —
|
|
902
|
+
so the structural match is unambiguous.
|
|
903
|
+
|
|
904
|
+
When both conditions hold the helper returns the matching facet
|
|
905
|
+
shape. Otherwise ``None`` and the caller falls back to the legacy
|
|
906
|
+
label-based detection via :func:`_shape_from_indices`.
|
|
907
|
+
|
|
908
|
+
The helper does NOT inspect *raw_labels* values — that's the whole
|
|
909
|
+
point: facet recognition is label-agnostic. We only need the depth
|
|
910
|
+
(``len(raw_labels)``).
|
|
911
|
+
"""
|
|
912
|
+
depth = len(raw_labels)
|
|
913
|
+
facet_candidates = [
|
|
914
|
+
s for s in allowed
|
|
915
|
+
if _SHAPE_AXES[s].leaf is LeafKind.FACET_PRICE_QUANTITY
|
|
916
|
+
and len(_SHAPE_AXES[s].map_levels) + 1 == depth
|
|
917
|
+
]
|
|
918
|
+
if len(facet_candidates) == 1:
|
|
919
|
+
return facet_candidates[0]
|
|
920
|
+
# Zero candidates → not a facet shape; >1 candidates → registry has
|
|
921
|
+
# multiple facet shapes at this depth (no current entries do, but
|
|
922
|
+
# be defensive — fall back to the legacy path which will surface
|
|
923
|
+
# the ambiguity through its own checks).
|
|
924
|
+
return None
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def _entity_dim_columns_for_frame(
|
|
928
|
+
df: "pl.DataFrame",
|
|
929
|
+
source: "InputSource",
|
|
930
|
+
entity_class: str,
|
|
931
|
+
) -> list[str]:
|
|
932
|
+
"""Best-effort recovery of the entity dim columns of *df* by
|
|
933
|
+
consulting :meth:`InputSource.entities` for the class.
|
|
934
|
+
|
|
935
|
+
For 0-dim classes the column name is ``"name"``; for n-relationships
|
|
936
|
+
it's the dim-class names with repeats disambiguated. When
|
|
937
|
+
:meth:`entities` raises KeyError (unknown class), fall back to
|
|
938
|
+
``["name"]`` — defensive; the registry only references known
|
|
939
|
+
classes.
|
|
940
|
+
"""
|
|
941
|
+
try:
|
|
942
|
+
ent = source.entities(entity_class)
|
|
943
|
+
cols = ent.columns
|
|
944
|
+
if cols:
|
|
945
|
+
return list(cols)
|
|
946
|
+
except (KeyError, AttributeError):
|
|
947
|
+
# AttributeError covers stub InputSource implementations in
|
|
948
|
+
# unit-tests that only override ``parameter_explicit`` and don't
|
|
949
|
+
# implement ``entities`` (e.g. the ``_ZeroStartupSource`` stub
|
|
950
|
+
# in ``test_a04_a05_online_ramp.py``).
|
|
951
|
+
pass
|
|
952
|
+
return ["name"]
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def resolve_param_shape(
|
|
956
|
+
source: "InputSource",
|
|
957
|
+
entity_class: str,
|
|
958
|
+
parameter_name: str,
|
|
959
|
+
period_filter: "pl.DataFrame | None" = None,
|
|
960
|
+
) -> "ResolvedShape | None":
|
|
961
|
+
"""DB-driven shape detection + per-parameter allow-list validation.
|
|
962
|
+
|
|
963
|
+
Steps:
|
|
964
|
+
|
|
965
|
+
1. Look up :data:`PARAM_ALLOWED_SHAPES` for ``(entity_class,
|
|
966
|
+
parameter_name)``. Missing entry → :class:`FlexToolConfigError`
|
|
967
|
+
(the registry must explicitly list every parameter routed
|
|
968
|
+
through the resolver).
|
|
969
|
+
2. Fetch the parameter frame via
|
|
970
|
+
:meth:`InputSource.parameter_explicit` (falling back to
|
|
971
|
+
:meth:`parameter`). Empty / None → return ``None`` (the caller
|
|
972
|
+
drops this parameter from FlexData).
|
|
973
|
+
3. Read the raw per-level ``index_name`` labels from the DB.
|
|
974
|
+
4. Map the labels to a :class:`Shape`. Unrecognised labels OR a
|
|
975
|
+
shape outside the allow-list → :class:`FlexToolConfigError`
|
|
976
|
+
with full context (parameter name, observed labels, allowed
|
|
977
|
+
shapes).
|
|
978
|
+
|
|
979
|
+
When the index_name labels are silent-default (e.g. spinedb_api's
|
|
980
|
+
``"x"``) AND ``period_filter`` is supplied, the resolver probes the
|
|
981
|
+
index column values against the active solve's known periods /
|
|
982
|
+
timesteps (mirroring the legacy CSV pipeline's value-domain
|
|
983
|
+
discrimination in
|
|
984
|
+
:func:`flextool.engine_polars._timeline.separate_period_and_timeseries_data`).
|
|
985
|
+
This recovers Rivendell-style fixtures where ``group.co2_price`` is
|
|
986
|
+
authored as ``Map(period→value)`` but the author omitted
|
|
987
|
+
``index_name`` so the DB stores the silent ``"x"`` label. Without
|
|
988
|
+
this probing the resolver returns ``None`` and ``p_co2_price`` is
|
|
989
|
+
silently dropped while the feature gate (driven by topology, not
|
|
990
|
+
data) stays active — the LP build then aborts at
|
|
991
|
+
:func:`flextool.engine_polars.model.build_flextool`'s ``CO2_PRICE``
|
|
992
|
+
invariant check.
|
|
993
|
+
|
|
994
|
+
Returns ``None`` only when the parameter has no explicit rows AND
|
|
995
|
+
no scalar default to broadcast (the source plugin's None-skip per
|
|
996
|
+
§4.5). Otherwise returns a :class:`ResolvedShape` carrying
|
|
997
|
+
everything :func:`broadcast_to_period_time` /
|
|
998
|
+
:func:`broadcast_to_period` need.
|
|
999
|
+
"""
|
|
1000
|
+
key = (entity_class, parameter_name)
|
|
1001
|
+
allowed = PARAM_ALLOWED_SHAPES.get(key)
|
|
1002
|
+
if allowed is None:
|
|
1003
|
+
raise FlexToolConfigError(
|
|
1004
|
+
f"resolve_param_shape: parameter {key!r} is not in "
|
|
1005
|
+
f"PARAM_ALLOWED_SHAPES. Add an explicit allow-list entry "
|
|
1006
|
+
f"to flextool/engine_polars/_param_shapes.py.")
|
|
1007
|
+
|
|
1008
|
+
df = _try_parameter_frame(source, entity_class, parameter_name)
|
|
1009
|
+
if df is None or df.height == 0:
|
|
1010
|
+
return None
|
|
1011
|
+
|
|
1012
|
+
ent_cols = _entity_dim_columns_for_frame(df, source, entity_class)
|
|
1013
|
+
raw_labels = _read_index_names_from_source(
|
|
1014
|
+
source, entity_class, parameter_name)
|
|
1015
|
+
|
|
1016
|
+
# Phase A — registry-driven facet recognition. The legacy
|
|
1017
|
+
# label-based ``_shape_from_indices`` can't see the new facet
|
|
1018
|
+
# shapes structurally (and shouldn't: facet outer-axis labels are
|
|
1019
|
+
# user-set per parameter value). Try a label-agnostic depth match
|
|
1020
|
+
# against the allow-list first; on a hit, build the ResolvedShape
|
|
1021
|
+
# straight away — facet shapes share no columns with the legacy
|
|
1022
|
+
# period/time rename block, so the rest of the function (which
|
|
1023
|
+
# exists for legacy period/time disambiguation) does not apply.
|
|
1024
|
+
facet_shape = _recognise_facet_shape(raw_labels, allowed)
|
|
1025
|
+
if facet_shape is not None:
|
|
1026
|
+
# No column rename needed: the frame's outer-axis column names
|
|
1027
|
+
# are user-set and will be projected by the Phase B reader
|
|
1028
|
+
# using :func:`facet_keys` against the inner facet level.
|
|
1029
|
+
period_col = "period" if "period" in df.columns else None
|
|
1030
|
+
time_col = "t" if "t" in df.columns else None
|
|
1031
|
+
return ResolvedShape(
|
|
1032
|
+
shape=facet_shape,
|
|
1033
|
+
frame=df,
|
|
1034
|
+
entity_dim_columns=tuple(ent_cols),
|
|
1035
|
+
period_index_column=period_col,
|
|
1036
|
+
time_index_column=time_col,
|
|
1037
|
+
)
|
|
1038
|
+
|
|
1039
|
+
# Δ.17c follow-up — disambiguate silent-default ``index_name`` labels
|
|
1040
|
+
# against the per-parameter allow-list. For parameters whose
|
|
1041
|
+
# registry entry permits a unique shape at the observed (depth,
|
|
1042
|
+
# position), the silent default is unambiguous and we can recover
|
|
1043
|
+
# the author's intent without rewriting the DB. Required to keep
|
|
1044
|
+
# the Spine path on parity with the legacy CSV pipeline, which
|
|
1045
|
+
# picks index dimensionality from value-domain probing (period
|
|
1046
|
+
# tokens vs. timestep tokens) rather than the index_name label —
|
|
1047
|
+
# see :func:`flextool.engine_polars._timeline.separate_period_and_timeseries_data`.
|
|
1048
|
+
resolved_labels = _infer_silent_default_labels(raw_labels, allowed)
|
|
1049
|
+
try:
|
|
1050
|
+
shape = _shape_from_indices(resolved_labels)
|
|
1051
|
+
except _UnrecognisedIndex as exc:
|
|
1052
|
+
# User advice: "If does not match, error to the user."
|
|
1053
|
+
# We're strict only about *explicit* mismatches (e.g. ``branch`` on
|
|
1054
|
+
# a parameter whose allow-list is {scalar, period, time}). Silent
|
|
1055
|
+
# defaults (empty / None / ``x``) collapse to ``shape=None`` below
|
|
1056
|
+
# via :func:`_shape_from_indices` and trigger the soft-fallback
|
|
1057
|
+
# path instead.
|
|
1058
|
+
raise FlexToolConfigError(
|
|
1059
|
+
f"Parameter ({entity_class!r}, {parameter_name!r}) carries an "
|
|
1060
|
+
f"unrecognised dimension index label {exc.label} (depth "
|
|
1061
|
+
f"{exc.depth}). Allowed shapes: {_allowed_shape_names(allowed)}. "
|
|
1062
|
+
"Edit the source database so the Map's index_name reads "
|
|
1063
|
+
"'period' or 'time' (matching one of the allowed shapes), or "
|
|
1064
|
+
"extend PARAM_ALLOWED_SHAPES if a new shape is intended."
|
|
1065
|
+
) from None
|
|
1066
|
+
|
|
1067
|
+
if shape is None:
|
|
1068
|
+
# Ambiguous shape — at least one Map level carries the spinedb_api
|
|
1069
|
+
# silent default and the per-parameter allow-list permits multiple
|
|
1070
|
+
# interpretations at that depth. Try value-domain probing against
|
|
1071
|
+
# the active solve's known periods / timesteps when *period_filter*
|
|
1072
|
+
# is supplied — this mirrors the legacy CSV pipeline's
|
|
1073
|
+
# ``separate_period_and_timeseries_data`` discriminator (a Map's
|
|
1074
|
+
# index values reveal whether it indexes by period or timestep,
|
|
1075
|
+
# regardless of the silent ``"x"`` index_name).
|
|
1076
|
+
shape = _disambiguate_shape_by_value_domain(
|
|
1077
|
+
df, ent_cols, resolved_labels, allowed, period_filter,
|
|
1078
|
+
)
|
|
1079
|
+
if shape is None:
|
|
1080
|
+
# Still ambiguous (no period_filter, or values match neither
|
|
1081
|
+
# the period nor the timestep set). Per Δ.17c policy: don't
|
|
1082
|
+
# raise; return None so the caller falls back to its
|
|
1083
|
+
# non-resolver pathway (typically the seed CSV). Δ.18+
|
|
1084
|
+
# punch-list: re-author fixtures so every Map level carries
|
|
1085
|
+
# an explicit ``index_name``.
|
|
1086
|
+
return None
|
|
1087
|
+
# Update the resolved labels so the downstream rename block sees
|
|
1088
|
+
# the disambiguated label and renames the silent-default column
|
|
1089
|
+
# (typically ``"x"``) to ``"period"`` or ``"t"``.
|
|
1090
|
+
resolved_labels = list(_SHAPE_LABELS[shape])
|
|
1091
|
+
|
|
1092
|
+
# Phase A: depth-0 string-leaf promotion. ``_shape_from_indices``
|
|
1093
|
+
# only sees the Map-nesting structure — every scalar (numeric or
|
|
1094
|
+
# string) collapses to :class:`Shape.SCALAR`. When the registry
|
|
1095
|
+
# entry permits :class:`Shape.SCALAR_STR` but not :class:`Shape.SCALAR`,
|
|
1096
|
+
# the author's intent is a constrained-string leaf (per the
|
|
1097
|
+
# ``parameter_value_list``); promote so the allow-list check below
|
|
1098
|
+
# accepts it. Leaf-type confirmation against the DB column dtype is
|
|
1099
|
+
# Phase B — today the structural depth-0 match plus the registry's
|
|
1100
|
+
# exclusivity unambiguously identifies these params.
|
|
1101
|
+
if (shape is Shape.SCALAR
|
|
1102
|
+
and Shape.SCALAR_STR in allowed
|
|
1103
|
+
and Shape.SCALAR not in allowed):
|
|
1104
|
+
shape = Shape.SCALAR_STR
|
|
1105
|
+
|
|
1106
|
+
if shape not in allowed:
|
|
1107
|
+
# Render observed shape for the message — labels can be empty
|
|
1108
|
+
# so reconstruct from the labels list.
|
|
1109
|
+
observed = (shape.value if shape is not None
|
|
1110
|
+
else f"index_names={resolved_labels}")
|
|
1111
|
+
raise FlexToolConfigError(
|
|
1112
|
+
f"Parameter ({entity_class!r}, {parameter_name!r}) was authored "
|
|
1113
|
+
f"as {observed} but allowed shapes are: "
|
|
1114
|
+
f"{_allowed_shape_names(allowed)}. Edit the source database "
|
|
1115
|
+
"or extend PARAM_ALLOWED_SHAPES."
|
|
1116
|
+
)
|
|
1117
|
+
|
|
1118
|
+
# Δ.17c follow-up — when the resolver filled silent-default
|
|
1119
|
+
# ``index_name`` labels from the allow-list, the source frame's
|
|
1120
|
+
# columns still carry the silent-default names (e.g. ``"x"``) used
|
|
1121
|
+
# by :meth:`SpineDbReader._discover_index_cols`. Rename them to the
|
|
1122
|
+
# canonical broadcast keys (``"period"`` / ``"t"``) so downstream
|
|
1123
|
+
# broadcasters can locate the index columns.
|
|
1124
|
+
df_for_broadcast = df
|
|
1125
|
+
if any(_normalise_label(r) is None for r in raw_labels):
|
|
1126
|
+
rename_map: "dict[str, str]" = {}
|
|
1127
|
+
# raw_labels and resolved_labels are aligned per depth. The
|
|
1128
|
+
# SpineDbReader put non-entity, non-value columns in depth order;
|
|
1129
|
+
# walk them in parallel.
|
|
1130
|
+
non_ent_cols = [
|
|
1131
|
+
c for c in df.columns if c not in ent_cols and c != "value"
|
|
1132
|
+
]
|
|
1133
|
+
for col, raw, resolved_lbl in zip(
|
|
1134
|
+
non_ent_cols, raw_labels, resolved_labels,
|
|
1135
|
+
):
|
|
1136
|
+
if _normalise_label(raw) is not None:
|
|
1137
|
+
continue
|
|
1138
|
+
if resolved_lbl == "period" and col != "period":
|
|
1139
|
+
rename_map[col] = "period"
|
|
1140
|
+
elif resolved_lbl in ("time", "t") and col != "t":
|
|
1141
|
+
rename_map[col] = "t"
|
|
1142
|
+
if rename_map:
|
|
1143
|
+
df_for_broadcast = df.rename(rename_map)
|
|
1144
|
+
|
|
1145
|
+
# Resolve the column names per the (possibly renamed) frame.
|
|
1146
|
+
period_col = "period" if "period" in df_for_broadcast.columns else None
|
|
1147
|
+
time_col = "t" if "t" in df_for_broadcast.columns else None
|
|
1148
|
+
return ResolvedShape(
|
|
1149
|
+
shape=shape,
|
|
1150
|
+
frame=df_for_broadcast,
|
|
1151
|
+
entity_dim_columns=tuple(ent_cols),
|
|
1152
|
+
period_index_column=period_col,
|
|
1153
|
+
time_index_column=time_col,
|
|
1154
|
+
)
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
# ---------------------------------------------------------------------------
|
|
1158
|
+
# Broadcast helpers — produce per-(entity, d, t) and per-(entity, d) frames
|
|
1159
|
+
# ---------------------------------------------------------------------------
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def _resolve_entity_dim_aliases(
|
|
1163
|
+
resolved: "ResolvedShape",
|
|
1164
|
+
entity_dim_aliases: "str | dict[str, str]",
|
|
1165
|
+
helper_name: str,
|
|
1166
|
+
) -> "tuple[dict[str, str], tuple[str, ...]]":
|
|
1167
|
+
"""Normalise the ``entity_dim_aliases`` argument into a
|
|
1168
|
+
``(rename_map, target_keys)`` pair.
|
|
1169
|
+
|
|
1170
|
+
Δ.17c-Tier3 — both :func:`broadcast_to_period_time` and
|
|
1171
|
+
:func:`broadcast_to_period` accept either a single string (the
|
|
1172
|
+
0-dim 1-source-column case — historical signature) or a per-source
|
|
1173
|
+
column rename dict (multi-dim relationship classes such as
|
|
1174
|
+
``unit__inputNode``, ``reserve__upDown__group``).
|
|
1175
|
+
|
|
1176
|
+
* ``str`` → renames the entity's sole dim column to that string.
|
|
1177
|
+
Defensive when ``resolved.entity_dim_columns`` is multi-dim.
|
|
1178
|
+
* ``dict[str, str]`` → per-source-column rename; the helper
|
|
1179
|
+
validates every entity dim column has an entry, then orders the
|
|
1180
|
+
target keys to match :attr:`ResolvedShape.entity_dim_columns` so
|
|
1181
|
+
the output Param's dim tuple is stable across runs (which is
|
|
1182
|
+
important for the deterministic LP-column assignment in
|
|
1183
|
+
:mod:`polar_high`).
|
|
1184
|
+
|
|
1185
|
+
The returned ``target_keys`` are the post-rename column names in
|
|
1186
|
+
source-column order — the helper uses them as the first
|
|
1187
|
+
``len(entity_dim_columns)`` columns of every ``select(...)`` and as
|
|
1188
|
+
the leading Param dims.
|
|
1189
|
+
"""
|
|
1190
|
+
ent_cols = resolved.entity_dim_columns
|
|
1191
|
+
if isinstance(entity_dim_aliases, str):
|
|
1192
|
+
# 0-dim entity class — single "name" column → single alias.
|
|
1193
|
+
if ent_cols != ("name",):
|
|
1194
|
+
raise FlexToolConfigError(
|
|
1195
|
+
f"{helper_name}: entity_dim_aliases was passed as a single "
|
|
1196
|
+
f"string {entity_dim_aliases!r} but the entity class has "
|
|
1197
|
+
f"multi-dim columns {ent_cols}. Pass a per-source-column "
|
|
1198
|
+
"rename dict instead."
|
|
1199
|
+
)
|
|
1200
|
+
return ({"name": entity_dim_aliases}, (entity_dim_aliases,))
|
|
1201
|
+
if not isinstance(entity_dim_aliases, dict):
|
|
1202
|
+
raise FlexToolConfigError(
|
|
1203
|
+
f"{helper_name}: entity_dim_aliases must be str or dict; got "
|
|
1204
|
+
f"{type(entity_dim_aliases).__name__}.")
|
|
1205
|
+
# Multi-dim: every source dim column must have a target name.
|
|
1206
|
+
missing = [c for c in ent_cols if c not in entity_dim_aliases]
|
|
1207
|
+
if missing:
|
|
1208
|
+
raise FlexToolConfigError(
|
|
1209
|
+
f"{helper_name}: entity_dim_aliases dict is missing rename "
|
|
1210
|
+
f"target(s) for source column(s) {missing}; entity dim "
|
|
1211
|
+
f"columns are {ent_cols}.")
|
|
1212
|
+
extra = [k for k in entity_dim_aliases if k not in ent_cols]
|
|
1213
|
+
if extra:
|
|
1214
|
+
raise FlexToolConfigError(
|
|
1215
|
+
f"{helper_name}: entity_dim_aliases dict has rename target(s) "
|
|
1216
|
+
f"for unknown source column(s) {extra}; entity dim columns "
|
|
1217
|
+
f"are {ent_cols}.")
|
|
1218
|
+
# Order target keys by source-column order for stable Param dims.
|
|
1219
|
+
target_keys = tuple(entity_dim_aliases[c] for c in ent_cols)
|
|
1220
|
+
return (dict(entity_dim_aliases), target_keys)
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def broadcast_to_period_time(
|
|
1224
|
+
resolved: "ResolvedShape | None",
|
|
1225
|
+
entity_dim_aliases: "str | dict[str, str]",
|
|
1226
|
+
period_filter: "pl.DataFrame | None",
|
|
1227
|
+
*,
|
|
1228
|
+
filter_zero: bool = False,
|
|
1229
|
+
) -> "Param | None":
|
|
1230
|
+
"""Resolve a source frame into a ``Param`` whose dims match the
|
|
1231
|
+
authored shape — **without** materialising the (entity, d, t)
|
|
1232
|
+
cross-product for sources that don't carry that axis natively.
|
|
1233
|
+
|
|
1234
|
+
Phase E.1: the returned Param's dims depend on the resolved shape:
|
|
1235
|
+
|
|
1236
|
+
============================ ====================================
|
|
1237
|
+
Source shape Returned Param dims
|
|
1238
|
+
============================ ====================================
|
|
1239
|
+
``Shape.SCALAR`` ``(*entity_keys,)``
|
|
1240
|
+
``Shape.MAP_PERIOD`` ``(*entity_keys, "d")``
|
|
1241
|
+
``Shape.MAP_TIME`` ``(*entity_keys, "t")``
|
|
1242
|
+
``Shape.MAP_PERIOD_TIME`` ``(*entity_keys, "d", "t")``
|
|
1243
|
+
``Shape.MAP_TIME_PERIOD`` ``(*entity_keys, "d", "t")``
|
|
1244
|
+
============================ ====================================
|
|
1245
|
+
|
|
1246
|
+
``entity_keys`` is derived from *entity_dim_aliases*:
|
|
1247
|
+
|
|
1248
|
+
* ``str`` — the entity class is 0-dim (sole column ``"name"``); the
|
|
1249
|
+
string becomes the only entity key (backwards-compatible).
|
|
1250
|
+
* ``dict[str, str]`` — per-source-column rename for n-dim
|
|
1251
|
+
relationship classes; ``entity_keys`` is ordered by
|
|
1252
|
+
:attr:`ResolvedShape.entity_dim_columns` so the Param's dim tuple
|
|
1253
|
+
is deterministic across runs.
|
|
1254
|
+
|
|
1255
|
+
The returned Param is fully lazy — no ``.collect()`` happens inside
|
|
1256
|
+
this helper. ``polar_high.Param`` broadcasts the smaller-dim Params
|
|
1257
|
+
against ``(entity, d, t)``-keyed Vars at constraint-emission time
|
|
1258
|
+
via shared-dim inner joins on LazyFrames, so the dense cross-product
|
|
1259
|
+
only ever lives in the per-term collect that polar_high already
|
|
1260
|
+
streams to HiGHS one row at a time.
|
|
1261
|
+
|
|
1262
|
+
*period_filter* must carry ``[d, t]`` columns (typically the active
|
|
1263
|
+
solve's ``flex_data.dt`` frame). For the two-axis shapes
|
|
1264
|
+
(``MAP_PERIOD_TIME`` / ``MAP_TIME_PERIOD``) it restricts to active
|
|
1265
|
+
``(d, t)`` pairs. For ``MAP_PERIOD`` / ``MAP_TIME`` it restricts to
|
|
1266
|
+
the active periods / times respectively. For ``SCALAR`` the filter
|
|
1267
|
+
is consulted only as the *gating* signal: when no period is active
|
|
1268
|
+
we return ``None`` (no Params produced for an empty solve), but no
|
|
1269
|
+
join is needed since SCALAR doesn't carry a d/t axis.
|
|
1270
|
+
|
|
1271
|
+
*filter_zero* mirrors the CSV cascade's "drop rows where the
|
|
1272
|
+
explicit value is 0" semantic for fields like ``co2_price``
|
|
1273
|
+
(preprocessing emits zero-defaults but downstream gates filter
|
|
1274
|
+
them out).
|
|
1275
|
+
|
|
1276
|
+
Returns ``None`` when:
|
|
1277
|
+
|
|
1278
|
+
* *resolved* is ``None`` or its frame is empty,
|
|
1279
|
+
* a required ``period_filter`` is missing or empty.
|
|
1280
|
+
"""
|
|
1281
|
+
if resolved is None or resolved.frame is None:
|
|
1282
|
+
return None
|
|
1283
|
+
df = resolved.frame
|
|
1284
|
+
if df.height == 0:
|
|
1285
|
+
return None
|
|
1286
|
+
if period_filter is None or period_filter.height == 0:
|
|
1287
|
+
return None
|
|
1288
|
+
pf_cols = set(period_filter.columns)
|
|
1289
|
+
if not {"d", "t"}.issubset(pf_cols):
|
|
1290
|
+
return None
|
|
1291
|
+
|
|
1292
|
+
# Δ.17c-Tier3 — multi-dim relationship classes (unit__inputNode,
|
|
1293
|
+
# reserve__upDown__group, …) supply a per-source-column rename dict;
|
|
1294
|
+
# 0-dim object classes keep the historical single-string signature.
|
|
1295
|
+
rename_map, entity_keys = _resolve_entity_dim_aliases(
|
|
1296
|
+
resolved, entity_dim_aliases, "broadcast_to_period_time")
|
|
1297
|
+
lf = (df.lazy()
|
|
1298
|
+
.pipe(rename_to_axis, rename_map)
|
|
1299
|
+
.filter(pl.col("value").is_not_null()))
|
|
1300
|
+
if filter_zero:
|
|
1301
|
+
lf = lf.filter(pl.col("value") != 0.0)
|
|
1302
|
+
# Phase 4 — align producer-side dim dtypes to the canonical Enum
|
|
1303
|
+
# vocabulary so the downstream join against ``period_filter`` (which
|
|
1304
|
+
# carries Enum d/t after :func:`apply_derived_a` runs) doesn't trip
|
|
1305
|
+
# over Utf8-vs-Enum on the ``t`` (or entity) axis. The cascade
|
|
1306
|
+
# contract is "Enum when global vocabulary is active"; the resolver
|
|
1307
|
+
# reads frames in String land via ``InputSource.parameter*``, so the
|
|
1308
|
+
# cast happens here at the broadcast boundary.
|
|
1309
|
+
_enums = get_global_axis_enums()
|
|
1310
|
+
if _enums is not None:
|
|
1311
|
+
lf = cast_frame_axes(lf, _enums)
|
|
1312
|
+
|
|
1313
|
+
shape = resolved.shape
|
|
1314
|
+
if shape in (Shape.MAP_PERIOD_TIME, Shape.MAP_TIME_PERIOD):
|
|
1315
|
+
# (d, t)-keyed fill. Both authoring orders unroll to the same
|
|
1316
|
+
# flat frame carrying ``period`` + ``t`` columns (the
|
|
1317
|
+
# SpineDbReader flattens a 2d_map regardless of index order), so
|
|
1318
|
+
# the two shapes share one code path. Inner-join on dt to
|
|
1319
|
+
# restrict to the active solve's (d, t) grid.
|
|
1320
|
+
#
|
|
1321
|
+
# Mixed-authoring guard (mirrors the MAP_PERIOD / MAP_TIME
|
|
1322
|
+
# branches below): when SOME flowGroups author a full 2d
|
|
1323
|
+
# Map(period, time) while OTHERS author only a 1d Map(period)
|
|
1324
|
+
# (or a scalar) for the same parameter, the whole frame's
|
|
1325
|
+
# resolved shape becomes MAP_PERIOD_TIME — SpineDbReader's
|
|
1326
|
+
# ``parameter_shape_info`` reports the DEEPEST row's nesting and
|
|
1327
|
+
# ``_unroll_rows`` discovers index columns from the widest row,
|
|
1328
|
+
# so the shallower rows arrive with a NULL ``t`` (period-only
|
|
1329
|
+
# default) or NULL ``d`` (time-only default). A plain inner-join
|
|
1330
|
+
# on (d, t) SILENTLY DROPS every such row — annihilating those
|
|
1331
|
+
# entities' floors/caps entirely (the reported ``min_instant_flow``
|
|
1332
|
+
# correctness bug: adding one period-time floor on a new
|
|
1333
|
+
# flowGroup made the pre-existing period-scalar floors on other
|
|
1334
|
+
# flowGroups vanish, LOWERING the optimum below the un-floored
|
|
1335
|
+
# baseline). Split by which index axes are populated and
|
|
1336
|
+
# broadcast the missing axis across the active grid instead of
|
|
1337
|
+
# dropping. For a non-mixed (pure 2d) frame every row is fully
|
|
1338
|
+
# keyed, so only the ``lf_full`` branch is non-empty and the
|
|
1339
|
+
# result is byte-identical to the old inner-join.
|
|
1340
|
+
dt_lf = period_filter.lazy().select("d", "t").unique()
|
|
1341
|
+
lf_dt = (lf.pipe(rename_to_axis, {"period": "d"})
|
|
1342
|
+
.select(*entity_keys, "d", "t", "value"))
|
|
1343
|
+
# Fully-keyed rows: exact (d, t) match against the active grid.
|
|
1344
|
+
lf_full = (lf_dt.filter(pl.col("d").is_not_null()
|
|
1345
|
+
& pl.col("t").is_not_null())
|
|
1346
|
+
.join(dt_lf, on=["d", "t"], how="inner")
|
|
1347
|
+
.select(*entity_keys, "d", "t", "value"))
|
|
1348
|
+
# Period-only default (null t): broadcast across every active
|
|
1349
|
+
# timestep of the matching active period.
|
|
1350
|
+
lf_period = (lf_dt.filter(pl.col("d").is_not_null()
|
|
1351
|
+
& pl.col("t").is_null())
|
|
1352
|
+
.select(*entity_keys, "d", "value")
|
|
1353
|
+
.join(dt_lf, on="d", how="inner")
|
|
1354
|
+
.select(*entity_keys, "d", "t", "value"))
|
|
1355
|
+
# Time-only default (null d): broadcast across every active
|
|
1356
|
+
# period of the matching active timestep.
|
|
1357
|
+
lf_time = (lf_dt.filter(pl.col("d").is_null()
|
|
1358
|
+
& pl.col("t").is_not_null())
|
|
1359
|
+
.select(*entity_keys, "t", "value")
|
|
1360
|
+
.join(dt_lf, on="t", how="inner")
|
|
1361
|
+
.select(*entity_keys, "d", "t", "value"))
|
|
1362
|
+
# Scalar default (both null): broadcast across the whole grid.
|
|
1363
|
+
lf_scalar = (lf_dt.filter(pl.col("d").is_null()
|
|
1364
|
+
& pl.col("t").is_null())
|
|
1365
|
+
.select(*entity_keys, "value")
|
|
1366
|
+
.join(dt_lf, how="cross")
|
|
1367
|
+
.select(*entity_keys, "d", "t", "value"))
|
|
1368
|
+
out_lf = pl.concat([lf_full, lf_period, lf_time, lf_scalar])
|
|
1369
|
+
return Param((*entity_keys, "d", "t"), out_lf)
|
|
1370
|
+
elif shape == Shape.MAP_PERIOD:
|
|
1371
|
+
# 1d_map(period) → (entity, d) Param. Phase E.1: do NOT
|
|
1372
|
+
# broadcast across ``t``; polar_high handles the (entity, d) →
|
|
1373
|
+
# (entity, d, t) broadcast lazily at constraint emission.
|
|
1374
|
+
#
|
|
1375
|
+
# Mixed authoring: some entities may carry a scalar default
|
|
1376
|
+
# (period column null) while others carry an explicit map.
|
|
1377
|
+
# SpineDbReader unifies these into a single frame with a
|
|
1378
|
+
# nullable index column; rows with null index represent the
|
|
1379
|
+
# entity's scalar default and must be broadcast across all
|
|
1380
|
+
# active periods — INNER JOIN on a null index would drop them
|
|
1381
|
+
# silently. Split into scalar-default and explicit branches,
|
|
1382
|
+
# broadcast independently across the active-period universe,
|
|
1383
|
+
# then concatenate.
|
|
1384
|
+
d_lf = period_filter.lazy().select("d").unique()
|
|
1385
|
+
lf_p = lf.pipe(rename_to_axis, {"period": "d"})
|
|
1386
|
+
lf_scalar = (lf_p.filter(pl.col("d").is_null())
|
|
1387
|
+
.select(*entity_keys, "value")
|
|
1388
|
+
.join(d_lf, how="cross")
|
|
1389
|
+
.select(*entity_keys, "d", "value"))
|
|
1390
|
+
lf_explicit = (lf_p.filter(pl.col("d").is_not_null())
|
|
1391
|
+
.select(*entity_keys, "d", "value")
|
|
1392
|
+
.join(d_lf, on="d", how="inner")
|
|
1393
|
+
.select(*entity_keys, "d", "value"))
|
|
1394
|
+
out_lf = pl.concat([lf_explicit, lf_scalar])
|
|
1395
|
+
return Param((*entity_keys, "d"), out_lf)
|
|
1396
|
+
elif shape == Shape.MAP_TIME:
|
|
1397
|
+
# 1d_map(time) → (entity, t) Param. Phase E.1: do NOT
|
|
1398
|
+
# broadcast across ``d``; polar_high handles the (entity, t) →
|
|
1399
|
+
# (entity, d, t) broadcast lazily at constraint emission.
|
|
1400
|
+
#
|
|
1401
|
+
# Same mixed-authoring guard as MAP_PERIOD above: rows whose
|
|
1402
|
+
# ``t`` index is null are scalar defaults for that entity and
|
|
1403
|
+
# must broadcast across all active times instead of being
|
|
1404
|
+
# dropped by the inner-join on ``t``. Covers fixtures like
|
|
1405
|
+
# ``network_coal_wind_battery_co2_fullYear_availability`` where
|
|
1406
|
+
# ``coal_plant`` is MAP_TIME but ``wind_plant`` is scalar 0.7.
|
|
1407
|
+
t_lf = period_filter.lazy().select("t").unique()
|
|
1408
|
+
lf_scalar = (lf.filter(pl.col("t").is_null())
|
|
1409
|
+
.select(*entity_keys, "value")
|
|
1410
|
+
.join(t_lf, how="cross")
|
|
1411
|
+
.select(*entity_keys, "t", "value"))
|
|
1412
|
+
lf_explicit = (lf.filter(pl.col("t").is_not_null())
|
|
1413
|
+
.select(*entity_keys, "t", "value")
|
|
1414
|
+
.join(t_lf, on="t", how="inner")
|
|
1415
|
+
.select(*entity_keys, "t", "value"))
|
|
1416
|
+
out_lf = pl.concat([lf_explicit, lf_scalar])
|
|
1417
|
+
return Param((*entity_keys, "t"), out_lf)
|
|
1418
|
+
elif shape == Shape.SCALAR:
|
|
1419
|
+
# Phase E.1: scalar stays scalar — one row per entity, no
|
|
1420
|
+
# cross-join with (d, t). polar_high broadcasts the (entity,)
|
|
1421
|
+
# Param against (entity, d, t)-keyed Vars at constraint
|
|
1422
|
+
# emission via a shared-dim inner-join on ``entity``.
|
|
1423
|
+
# ``period_filter`` already gated us above as the active-solve
|
|
1424
|
+
# signal; no further join needed here.
|
|
1425
|
+
out_lf = lf.select(*entity_keys, "value")
|
|
1426
|
+
return Param(tuple(entity_keys), out_lf)
|
|
1427
|
+
else: # pragma: no cover — guarded by allow-list check.
|
|
1428
|
+
raise FlexToolConfigError(
|
|
1429
|
+
f"broadcast_to_period_time: unhandled shape {shape!r}")
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def broadcast_to_period(
|
|
1433
|
+
resolved: "ResolvedShape | None",
|
|
1434
|
+
entity_dim_aliases: "str | dict[str, str]",
|
|
1435
|
+
period_filter: "pl.DataFrame | None",
|
|
1436
|
+
*,
|
|
1437
|
+
filter_zero: bool = False,
|
|
1438
|
+
filter_null: bool = True,
|
|
1439
|
+
) -> "Param | None":
|
|
1440
|
+
"""Resolve a source frame into a ``Param`` for parameters whose
|
|
1441
|
+
allow-list excludes time — keeping dims at the authored level.
|
|
1442
|
+
|
|
1443
|
+
Phase E.1 (Δ.17c-Tier3 multi-dim extension):
|
|
1444
|
+
|
|
1445
|
+
==================== ====================================
|
|
1446
|
+
Source shape Returned Param dims
|
|
1447
|
+
==================== ====================================
|
|
1448
|
+
``Shape.SCALAR`` ``(*entity_keys,)`` / ``(*entity_keys, "d")``
|
|
1449
|
+
(Δ.17c-Tier1 cross-join — see below)
|
|
1450
|
+
``Shape.MAP_PERIOD`` ``(*entity_keys, "d")``
|
|
1451
|
+
==================== ====================================
|
|
1452
|
+
|
|
1453
|
+
``entity_keys`` is derived from *entity_dim_aliases* (see
|
|
1454
|
+
:func:`_resolve_entity_dim_aliases`):
|
|
1455
|
+
|
|
1456
|
+
* ``str`` — the entity class is 0-dim (sole column ``"name"``); the
|
|
1457
|
+
string becomes the only entity key (backwards-compatible with all
|
|
1458
|
+
pre-Tier3 callers).
|
|
1459
|
+
* ``dict[str, str]`` — per-source-column rename for n-dim
|
|
1460
|
+
relationship classes; ``entity_keys`` is ordered by
|
|
1461
|
+
:attr:`ResolvedShape.entity_dim_columns` so the Param's dim tuple
|
|
1462
|
+
is deterministic across runs.
|
|
1463
|
+
|
|
1464
|
+
The returned Param is fully lazy. For ``MAP_PERIOD`` we inner-join
|
|
1465
|
+
on ``period_filter['d']`` to restrict to the active solve's periods.
|
|
1466
|
+
For ``SCALAR`` the filter is only consulted as the gating signal —
|
|
1467
|
+
no join is needed since a scalar doesn't carry a ``d`` axis.
|
|
1468
|
+
|
|
1469
|
+
Other shapes raise :class:`FlexToolConfigError` (the resolver should
|
|
1470
|
+
have rejected them already; this is a defensive guard).
|
|
1471
|
+
"""
|
|
1472
|
+
if resolved is None or resolved.frame is None:
|
|
1473
|
+
return None
|
|
1474
|
+
df = resolved.frame
|
|
1475
|
+
if df.height == 0:
|
|
1476
|
+
return None
|
|
1477
|
+
|
|
1478
|
+
# Δ.17c-Tier3 — multi-dim relationship classes supply a per-source-
|
|
1479
|
+
# column rename dict; 0-dim object classes keep the historical
|
|
1480
|
+
# single-string signature. See :func:`_resolve_entity_dim_aliases`.
|
|
1481
|
+
rename_map, entity_keys = _resolve_entity_dim_aliases(
|
|
1482
|
+
resolved, entity_dim_aliases, "broadcast_to_period")
|
|
1483
|
+
lf = df.lazy().pipe(rename_to_axis, rename_map)
|
|
1484
|
+
if filter_null:
|
|
1485
|
+
lf = lf.filter(pl.col("value").is_not_null())
|
|
1486
|
+
if filter_zero:
|
|
1487
|
+
lf = lf.filter(pl.col("value") != 0.0)
|
|
1488
|
+
# Phase 4 — align producer-side dim dtypes to the canonical Enum
|
|
1489
|
+
# vocabulary so the downstream join against ``period_filter`` (which
|
|
1490
|
+
# carries Enum d after :func:`apply_derived_a` runs) doesn't trip
|
|
1491
|
+
# over Utf8-vs-Enum on the ``d`` (or entity) axis. See the matching
|
|
1492
|
+
# comment in :func:`broadcast_to_period_time`.
|
|
1493
|
+
_enums = get_global_axis_enums()
|
|
1494
|
+
if _enums is not None:
|
|
1495
|
+
lf = cast_frame_axes(lf, _enums)
|
|
1496
|
+
|
|
1497
|
+
shape = resolved.shape
|
|
1498
|
+
if shape == Shape.SCALAR:
|
|
1499
|
+
# Δ.17c-Tier1 — emit ``(entity, d)`` even for scalar sources by
|
|
1500
|
+
# cross-joining with the active solve's period axis. Mirrors
|
|
1501
|
+
# the legacy ``_entity_period_scalar`` semantic (one
|
|
1502
|
+
# (entity, d, value) row per active period) so eager consumers
|
|
1503
|
+
# like :mod:`flextool.engine_polars._cumulative_invest` (which
|
|
1504
|
+
# does ``cap_param.frame.select("g", "d")``) keep working.
|
|
1505
|
+
#
|
|
1506
|
+
# Phase E.1's "scalar stays (entity,)" optimisation was designed
|
|
1507
|
+
# for Vars/Params consumed exclusively through polar_high's
|
|
1508
|
+
# lazy broadcast. The (entity, d) parameters declared on
|
|
1509
|
+
# :class:`FlexData` (e.g. ``p_co2_max_period``,
|
|
1510
|
+
# ``pd_max_cumulative_flow``) are documented with a ``(g, d)``
|
|
1511
|
+
# contract — emitting ``(entity, d)`` keeps the contract
|
|
1512
|
+
# consistent across SCALAR / MAP_PERIOD source shapes.
|
|
1513
|
+
if period_filter is None or period_filter.height == 0:
|
|
1514
|
+
return None
|
|
1515
|
+
d_lf = period_filter.lazy().select("d").unique()
|
|
1516
|
+
out_lf = (lf.select(*entity_keys, "value")
|
|
1517
|
+
.join(d_lf, how="cross")
|
|
1518
|
+
.select(*entity_keys, "d", "value"))
|
|
1519
|
+
return Param((*entity_keys, "d"), out_lf)
|
|
1520
|
+
elif shape == Shape.MAP_PERIOD:
|
|
1521
|
+
# Mixed authoring: some entities may carry a scalar default
|
|
1522
|
+
# (period column null) while others carry an explicit map.
|
|
1523
|
+
# SpineDbReader unifies these into a single frame with a
|
|
1524
|
+
# nullable index column; rows with null index represent the
|
|
1525
|
+
# entity's scalar default and must be broadcast across all
|
|
1526
|
+
# active periods — an INNER JOIN on a null index would drop
|
|
1527
|
+
# them silently. Mirrors the matching split in
|
|
1528
|
+
# :func:`broadcast_to_period_time`'s MAP_PERIOD branch.
|
|
1529
|
+
#
|
|
1530
|
+
# Observed in practice: within a single group parameter one
|
|
1531
|
+
# entity authors its value as Map(period→…) while a sibling
|
|
1532
|
+
# carries a scalar (Map index null) — without this split the
|
|
1533
|
+
# scalar entity drops from the LP.
|
|
1534
|
+
lf_p = lf.pipe(rename_to_axis, {"period": "d"})
|
|
1535
|
+
if period_filter is not None and period_filter.height > 0:
|
|
1536
|
+
d_lf = period_filter.lazy().select("d").unique()
|
|
1537
|
+
lf_scalar = (lf_p.filter(pl.col("d").is_null())
|
|
1538
|
+
.select(*entity_keys, "value")
|
|
1539
|
+
.join(d_lf, how="cross")
|
|
1540
|
+
.select(*entity_keys, "d", "value"))
|
|
1541
|
+
lf_explicit = (lf_p.filter(pl.col("d").is_not_null())
|
|
1542
|
+
.select(*entity_keys, "d", "value")
|
|
1543
|
+
.join(d_lf, on="d", how="inner")
|
|
1544
|
+
.select(*entity_keys, "d", "value"))
|
|
1545
|
+
out_lf = pl.concat([lf_explicit, lf_scalar])
|
|
1546
|
+
else:
|
|
1547
|
+
# No period_filter — only explicit-period rows can be
|
|
1548
|
+
# carried; null-index scalars have no axis to broadcast on.
|
|
1549
|
+
out_lf = (lf_p.filter(pl.col("d").is_not_null())
|
|
1550
|
+
.select(*entity_keys, "d", "value"))
|
|
1551
|
+
return Param((*entity_keys, "d"), out_lf)
|
|
1552
|
+
else:
|
|
1553
|
+
raise FlexToolConfigError(
|
|
1554
|
+
f"broadcast_to_period: shape {shape.value} is not supported "
|
|
1555
|
+
f"for (entity, period) parameters. Allowed: "
|
|
1556
|
+
f"{_allowed_shape_names(_PERIOD_ONLY_SHAPES)}")
|
|
1557
|
+
|
|
1558
|
+
|
|
1559
|
+
def promote_param_to_dt(
|
|
1560
|
+
param: "Param",
|
|
1561
|
+
dt: "pl.DataFrame | pl.LazyFrame",
|
|
1562
|
+
) -> "pl.LazyFrame":
|
|
1563
|
+
"""Return a LazyFrame view of ``param`` whose columns include both
|
|
1564
|
+
``"d"`` and ``"t"`` — promoting via lazy joins on ``dt`` when the
|
|
1565
|
+
Param's authored shape is narrower.
|
|
1566
|
+
|
|
1567
|
+
Phase E.1 makes flex_data Params keep their authored dims
|
|
1568
|
+
(``(entity,)`` / ``(entity, d)`` / ``(entity, t)`` / ``(entity, d,
|
|
1569
|
+
t)``). Polar_high algebra handles the broadcast lazily at
|
|
1570
|
+
constraint emission, but a small number of consumers in the cascade
|
|
1571
|
+
do eager ``.frame.join(..., on=["x", "d", "t"], how="left")`` style
|
|
1572
|
+
densification (e.g. ``model.py`` flow_upper_rhs availability fold,
|
|
1573
|
+
``_region_filter.py`` half-flow injection). Those consumers need a
|
|
1574
|
+
(entity, d, t)-shaped LazyFrame on the right-hand side; this helper
|
|
1575
|
+
gives it to them without forcing the source Param to materialise
|
|
1576
|
+
eagerly.
|
|
1577
|
+
|
|
1578
|
+
* ``(entity, d, t)`` Params — returned unchanged.
|
|
1579
|
+
* ``(entity, d)`` — inner-join on ``d`` against ``dt[d, t].unique()``.
|
|
1580
|
+
* ``(entity, t)`` — inner-join on ``t`` against ``dt[d, t].unique()``.
|
|
1581
|
+
* ``(entity,)`` — cross-join against ``dt[d, t].unique()``.
|
|
1582
|
+
"""
|
|
1583
|
+
lf = param.lazy
|
|
1584
|
+
has_d = "d" in param.dims
|
|
1585
|
+
has_t = "t" in param.dims
|
|
1586
|
+
dt_lf = dt.lazy() if isinstance(dt, pl.DataFrame) else dt
|
|
1587
|
+
if has_d and has_t:
|
|
1588
|
+
return lf
|
|
1589
|
+
dt_sel = dt_lf.select("d", "t").unique()
|
|
1590
|
+
if has_d: # missing t
|
|
1591
|
+
return lf.join(dt_sel, on="d", how="inner")
|
|
1592
|
+
if has_t: # missing d
|
|
1593
|
+
return lf.join(dt_sel, on="t", how="inner")
|
|
1594
|
+
# missing both — scalar-per-entity broadcast over (d, t).
|
|
1595
|
+
return lf.join(dt_sel, how="cross")
|