flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,742 @@
|
|
|
1
|
+
"""Canonical axis :class:`pl.Enum` dtypes for FlexData dimension columns.
|
|
2
|
+
|
|
3
|
+
Phase 2 of the ``pl.Enum`` dtype refactor (see
|
|
4
|
+
``specs/enum_dtype_refactor_handoff.md``). This module is pure — no
|
|
5
|
+
side effects on import, no globals mutated, no string-cache enablement.
|
|
6
|
+
|
|
7
|
+
The vocabulary builder (`build_axis_enums`) was Path-A of the original
|
|
8
|
+
refactor plan and never landed any live callers; it has been removed
|
|
9
|
+
along with the workdir-CSV vocabulary seed helpers. The remaining
|
|
10
|
+
public surface — :func:`cast_frame_axes` / :func:`cast_value_axes` /
|
|
11
|
+
:func:`cast_flexdata_axes` / :func:`schema_dtype` / :func:`cast_dim` /
|
|
12
|
+
:func:`empty_like` / :func:`align_join_dtypes` — operates on an
|
|
13
|
+
``enums`` dict supplied by the caller (when it is `{}` the casters
|
|
14
|
+
short-circuit to no-ops).
|
|
15
|
+
|
|
16
|
+
Public API
|
|
17
|
+
----------
|
|
18
|
+
|
|
19
|
+
* :func:`cast_frame_axes` — cast a frame's dim columns in-place.
|
|
20
|
+
* :func:`cast_value_axes` — same for arbitrary Param/Value containers.
|
|
21
|
+
* :func:`cast_flexdata_axes` — same for the FlexData container.
|
|
22
|
+
* :func:`schema_dtype` — column-name → canonical dtype lookup.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import contextvars
|
|
27
|
+
import dataclasses
|
|
28
|
+
from typing import TYPE_CHECKING
|
|
29
|
+
|
|
30
|
+
import polars as pl
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
# Avoid a circular import at module-load time — FlexData lives in
|
|
34
|
+
# ``input.py`` which itself transitively imports this module.
|
|
35
|
+
from flextool.engine_polars.input import FlexData
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# Axis families
|
|
40
|
+
#
|
|
41
|
+
# Every column name on the LHS gets the Enum dtype of the column name on
|
|
42
|
+
# the RHS. Keeps the public mapping concise: callers see one dtype per
|
|
43
|
+
# logical axis, and synonym columns (``d_invest``, ``t_previous``, …)
|
|
44
|
+
# pick up the matching dtype automatically.
|
|
45
|
+
|
|
46
|
+
_AXIS_SYNONYMS: dict[str, str] = {
|
|
47
|
+
# Period synonyms
|
|
48
|
+
"period": "d",
|
|
49
|
+
"d_invest": "d",
|
|
50
|
+
"d_divest": "d",
|
|
51
|
+
"d_previous": "d",
|
|
52
|
+
"d_upper": "d",
|
|
53
|
+
"d_back": "d",
|
|
54
|
+
# ``anchor`` is the cascade column name used in _derived_params.py
|
|
55
|
+
# during the period__branch overlay (period→anchor rename). The
|
|
56
|
+
# values are period labels, so the column belongs to the ``d`` axis.
|
|
57
|
+
"anchor": "d",
|
|
58
|
+
# Time-step synonyms
|
|
59
|
+
"step": "t",
|
|
60
|
+
"t_previous": "t",
|
|
61
|
+
"t_previous_within_block": "t",
|
|
62
|
+
"t_previous_within_timeset": "t",
|
|
63
|
+
"t_previous_within_solve": "t",
|
|
64
|
+
"t_upper": "t",
|
|
65
|
+
"t_back": "t",
|
|
66
|
+
"t_source": "t",
|
|
67
|
+
"t_sink": "t",
|
|
68
|
+
# Friendly long names that occasionally leak from helper code.
|
|
69
|
+
"node": "n",
|
|
70
|
+
"process": "p",
|
|
71
|
+
"commodity": "c",
|
|
72
|
+
"group": "g",
|
|
73
|
+
"entity": "e",
|
|
74
|
+
# Multi-dim entity-class element columns (Spine emits ``node_1`` /
|
|
75
|
+
# ``node_2`` for the two ends of a connection, ``unit`` / ``connection``
|
|
76
|
+
# for the process role). Each is a single-axis token.
|
|
77
|
+
"unit": "p",
|
|
78
|
+
"connection": "p",
|
|
79
|
+
"node_1": "n",
|
|
80
|
+
"node_2": "n",
|
|
81
|
+
# Mixed-vocab columns (per axis contract). source/sink columns
|
|
82
|
+
# carry a union of node + process names and must be cast against
|
|
83
|
+
# the 'e' (entity) union axis, not against a same-named single-class
|
|
84
|
+
# axis. See schemas/flextool_axis_contract.json mixed_vocab_columns.
|
|
85
|
+
"source": "e",
|
|
86
|
+
"sink": "e",
|
|
87
|
+
# Renamed-axis short forms surfaced during Phase 1's column-letter
|
|
88
|
+
# de-collision work (see schemas/flextool_axis_contract.json
|
|
89
|
+
# _review_notes.c_collision / b_collision).
|
|
90
|
+
"cn": "constraint",
|
|
91
|
+
"bk": "block",
|
|
92
|
+
# Branch column synonym (per contract: branch.column_synonyms = ["b"]).
|
|
93
|
+
"b": "branch",
|
|
94
|
+
# ----------------------------------------------------------------
|
|
95
|
+
# WARNING — easily confused: ``b_f`` is the BLOCK-AXIS short form
|
|
96
|
+
# (process-side block column in BlockBundle, renamed from "block")
|
|
97
|
+
# while ``b_first`` / ``b_next`` are TIMESTEP-AXIS short forms
|
|
98
|
+
# (cardinal timestep labels in coarse-block successor frames, NOT
|
|
99
|
+
# block names — see _derived_block.py period_block_succ where bsd
|
|
100
|
+
# "step" column is renamed to "b_first" / "b_next" via successor
|
|
101
|
+
# walk). Don't conflate them.
|
|
102
|
+
# ----------------------------------------------------------------
|
|
103
|
+
"b_first": "t",
|
|
104
|
+
"b_next": "t",
|
|
105
|
+
"b_f": "block",
|
|
106
|
+
# Period history column — holds period tokens (subset of d).
|
|
107
|
+
"d_h": "d",
|
|
108
|
+
# ``d_first`` — column holding period tokens of "first" periods
|
|
109
|
+
# (e.g. in ``_lifetime_expired_pairs_lf`` cascade). Subset of d.
|
|
110
|
+
"d_first": "d",
|
|
111
|
+
# Output-side timestep column — same vocabulary as cascade "t".
|
|
112
|
+
"time": "t",
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Canonical axis short names — the set of values that ``_resolve_axis``
|
|
117
|
+
# may return. Kept as a separate constant (rather than self-references
|
|
118
|
+
# in :data:`_AXIS_SYNONYMS`) so cluster-conversion tooling and ad-hoc
|
|
119
|
+
# membership tests can ask "is this name axis-aware?" via
|
|
120
|
+
# ``is_axis_aware(name)`` below.
|
|
121
|
+
_CANONICAL_AXES: frozenset[str] = frozenset({
|
|
122
|
+
"n", "p", "c", "g", "f", "d", "e", "t", "i",
|
|
123
|
+
"branch", "block", "constraint", "side", "klass", "ud", "r",
|
|
124
|
+
"d_anchor",
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def is_axis_aware(name: str) -> bool:
|
|
129
|
+
"""Return ``True`` iff ``name`` resolves to a canonical axis.
|
|
130
|
+
|
|
131
|
+
Used by cluster-conversion tooling and any "should I cast this
|
|
132
|
+
column?" callsite to decide whether ``name`` references an
|
|
133
|
+
axis-typed dim. A name is axis-aware iff it appears in
|
|
134
|
+
:data:`_AXIS_SYNONYMS` or matches a member of
|
|
135
|
+
:data:`_CANONICAL_AXES` directly.
|
|
136
|
+
"""
|
|
137
|
+
return name in _AXIS_SYNONYMS or name in _CANONICAL_AXES
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _resolve_axis(name: str) -> str:
|
|
141
|
+
"""Resolve a column name to its canonical axis name.
|
|
142
|
+
|
|
143
|
+
Looks up ``name`` in :data:`_AXIS_SYNONYMS`; if absent, returns the
|
|
144
|
+
name unchanged (assumes it is already canonical). Used by
|
|
145
|
+
:func:`schema_dtype` and :func:`cast_dim` so callers can pass either
|
|
146
|
+
short canonical names ("n", "p", "e") or column-name-style synonyms
|
|
147
|
+
("node", "source", "entity") interchangeably.
|
|
148
|
+
"""
|
|
149
|
+
return _AXIS_SYNONYMS.get(name, name)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# ---------------------------------------------------------------------------
|
|
153
|
+
# Cascade-wide axis enum vocabulary.
|
|
154
|
+
#
|
|
155
|
+
# Populated by ``load_flextool`` after ``build_axis_enums`` runs against
|
|
156
|
+
# the SpineDBBackend. Read by the substrate (``schema_dtype`` /
|
|
157
|
+
# ``cast_dim`` / ``rename_to_axis`` / ``lit_axis``) at every scratch-
|
|
158
|
+
# frame / rename / literal site across the cascade.
|
|
159
|
+
#
|
|
160
|
+
# Default ``None`` means "no activation" — substrate helpers fall back
|
|
161
|
+
# to ``pl.Utf8`` and downstream joins compose in Utf8 as before. Setting
|
|
162
|
+
# to a populated dict flips activation on for the duration of the
|
|
163
|
+
# cascade. ``load_flextool`` uses a try/finally to reset on exit.
|
|
164
|
+
#
|
|
165
|
+
# Stored in a :class:`contextvars.ContextVar` so concurrent cascades
|
|
166
|
+
# (current code is single-threaded but tests may set/reset around
|
|
167
|
+
# fixtures) do not see each other's state.
|
|
168
|
+
|
|
169
|
+
_LIVE_AXIS_ENUMS_CTX: "contextvars.ContextVar[dict[str, pl.Enum] | None]" = (
|
|
170
|
+
contextvars.ContextVar("_LIVE_AXIS_ENUMS", default=None)
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def set_global_axis_enums(
|
|
175
|
+
enums: "dict[str, pl.Enum] | None",
|
|
176
|
+
) -> "contextvars.Token":
|
|
177
|
+
"""Set the cascade-wide axis enum vocabulary.
|
|
178
|
+
|
|
179
|
+
Called from ``load_flextool`` immediately after
|
|
180
|
+
:func:`flextool.spinedb_backend._axis_enums.build_axis_enums`. The
|
|
181
|
+
cascade's substrate (``schema_dtype`` / ``cast_dim`` /
|
|
182
|
+
``rename_to_axis`` / ``lit_axis``) reads this global at every
|
|
183
|
+
invocation so scratch frames, renames, and literals all pick up
|
|
184
|
+
Enum dtypes uniformly.
|
|
185
|
+
|
|
186
|
+
Returns the ``contextvars.Token`` the caller can pass to
|
|
187
|
+
:func:`reset_global_axis_enums` to undo this set. Callers that
|
|
188
|
+
don't care about token-based reset can ignore the return value
|
|
189
|
+
and call ``set_global_axis_enums(None)`` to clear.
|
|
190
|
+
|
|
191
|
+
Pass ``None`` to reset to the no-activation default.
|
|
192
|
+
"""
|
|
193
|
+
return _LIVE_AXIS_ENUMS_CTX.set(enums)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def reset_global_axis_enums(token: "contextvars.Token") -> None:
|
|
197
|
+
"""Restore the axis enum vocabulary to whatever it was before the
|
|
198
|
+
matching :func:`set_global_axis_enums` call. Use the token
|
|
199
|
+
returned by ``set_global_axis_enums``.
|
|
200
|
+
"""
|
|
201
|
+
_LIVE_AXIS_ENUMS_CTX.reset(token)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def get_global_axis_enums() -> "dict[str, pl.Enum] | None":
|
|
205
|
+
"""Return the current cascade-wide axis enum vocabulary, or ``None``
|
|
206
|
+
when activation is off.
|
|
207
|
+
|
|
208
|
+
Reads the ``contextvars.ContextVar`` storage so concurrent /
|
|
209
|
+
nested calls observe their own state.
|
|
210
|
+
"""
|
|
211
|
+
return _LIVE_AXIS_ENUMS_CTX.get()
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def cast_frame_axes(
|
|
215
|
+
frame: "pl.DataFrame | pl.LazyFrame",
|
|
216
|
+
enums: dict[str, pl.Enum],
|
|
217
|
+
*,
|
|
218
|
+
strict: bool = False,
|
|
219
|
+
skip_columns: "set[str] | frozenset[str] | None" = None,
|
|
220
|
+
) -> "pl.DataFrame | pl.LazyFrame":
|
|
221
|
+
"""Cast every dim column in ``frame`` whose name is in ``enums`` to
|
|
222
|
+
the matching Enum dtype. Columns absent from ``enums`` (value
|
|
223
|
+
columns, unmapped axes, etc.) are left untouched.
|
|
224
|
+
|
|
225
|
+
Parameters
|
|
226
|
+
----------
|
|
227
|
+
frame
|
|
228
|
+
Eager ``pl.DataFrame`` or lazy ``pl.LazyFrame``. The return
|
|
229
|
+
type matches the input type — no eager materialisation of
|
|
230
|
+
LazyFrames.
|
|
231
|
+
enums
|
|
232
|
+
The mapping returned by :func:`build_axis_enums`.
|
|
233
|
+
strict
|
|
234
|
+
If ``False`` (default), unknown values silently become null
|
|
235
|
+
on cast — useful during the Phase 3 rollout so a missing
|
|
236
|
+
vocabulary entry surfaces as nulls rather than an exception.
|
|
237
|
+
Flip to ``True`` once the loader cascade is clean.
|
|
238
|
+
skip_columns
|
|
239
|
+
Column names to leave completely untouched even when their
|
|
240
|
+
(synonym-resolved) axis is present in ``enums``. Used for
|
|
241
|
+
columns whose *name* collides with a canonical axis but whose
|
|
242
|
+
*values* belong to a different vocabulary — e.g. the RP
|
|
243
|
+
base-period / rep-start label columns are named ``"b"`` / ``"r"``
|
|
244
|
+
(so the synonym table would cast them against the BRANCH / rep
|
|
245
|
+
enums) but actually hold timestep labels (``t0001`` …); casting
|
|
246
|
+
them to the branch enum would silently null every value.
|
|
247
|
+
|
|
248
|
+
Notes
|
|
249
|
+
-----
|
|
250
|
+
Skips columns that are already in the target Enum dtype, so the
|
|
251
|
+
cast is idempotent.
|
|
252
|
+
"""
|
|
253
|
+
if isinstance(frame, pl.LazyFrame):
|
|
254
|
+
schema = frame.collect_schema()
|
|
255
|
+
columns = list(schema.keys())
|
|
256
|
+
else:
|
|
257
|
+
schema = frame.schema
|
|
258
|
+
columns = frame.columns
|
|
259
|
+
|
|
260
|
+
exprs = []
|
|
261
|
+
for col in columns:
|
|
262
|
+
if skip_columns is not None and col in skip_columns:
|
|
263
|
+
continue
|
|
264
|
+
# Resolve synonyms before enum lookup so ``source``/``sink`` /
|
|
265
|
+
# ``node``/``period`` etc. all find their canonical axis enum.
|
|
266
|
+
canonical = _resolve_axis(col)
|
|
267
|
+
target = enums.get(canonical)
|
|
268
|
+
if target is None:
|
|
269
|
+
continue
|
|
270
|
+
if schema[col] == target:
|
|
271
|
+
continue
|
|
272
|
+
exprs.append(pl.col(col).cast(target, strict=strict))
|
|
273
|
+
if not exprs:
|
|
274
|
+
return frame
|
|
275
|
+
return frame.with_columns(exprs)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def align_join_dtypes(left: "pl.LazyFrame | pl.DataFrame",
|
|
279
|
+
right: "pl.LazyFrame | pl.DataFrame",
|
|
280
|
+
cols: list[str] | tuple[str, ...]):
|
|
281
|
+
"""Make the dtypes of ``cols`` on ``left`` and ``right`` match.
|
|
282
|
+
|
|
283
|
+
Strategy: prefer the Enum dtype if either side has one (Enum joins
|
|
284
|
+
are zero-copy when both sides match exactly); fall back to
|
|
285
|
+
``pl.Utf8`` otherwise. Returns the (possibly modified) (left,
|
|
286
|
+
right) pair — no schema rebuild when the dtypes already agree.
|
|
287
|
+
|
|
288
|
+
Used to bridge the cascade-helper boundary where one side comes
|
|
289
|
+
from a fresh CSV read (String) and the other from an Enum-cast
|
|
290
|
+
FlexData field.
|
|
291
|
+
"""
|
|
292
|
+
if isinstance(left, pl.LazyFrame):
|
|
293
|
+
lschema = left.collect_schema()
|
|
294
|
+
else:
|
|
295
|
+
lschema = left.schema
|
|
296
|
+
if isinstance(right, pl.LazyFrame):
|
|
297
|
+
rschema = right.collect_schema()
|
|
298
|
+
else:
|
|
299
|
+
rschema = right.schema
|
|
300
|
+
|
|
301
|
+
left_casts = []
|
|
302
|
+
right_casts = []
|
|
303
|
+
for c in cols:
|
|
304
|
+
ld = lschema.get(c)
|
|
305
|
+
rd = rschema.get(c)
|
|
306
|
+
if ld is None or rd is None or ld == rd:
|
|
307
|
+
continue
|
|
308
|
+
# Prefer Enum side; otherwise fall back to String coercion.
|
|
309
|
+
if isinstance(ld, pl.Enum):
|
|
310
|
+
target = ld
|
|
311
|
+
right_casts.append(pl.col(c).cast(target, strict=False))
|
|
312
|
+
elif isinstance(rd, pl.Enum):
|
|
313
|
+
target = rd
|
|
314
|
+
left_casts.append(pl.col(c).cast(target, strict=False))
|
|
315
|
+
else:
|
|
316
|
+
# Two non-Enum non-matching dtypes — cast both to String.
|
|
317
|
+
left_casts.append(pl.col(c).cast(pl.Utf8, strict=False))
|
|
318
|
+
right_casts.append(pl.col(c).cast(pl.Utf8, strict=False))
|
|
319
|
+
if left_casts:
|
|
320
|
+
left = left.with_columns(left_casts)
|
|
321
|
+
if right_casts:
|
|
322
|
+
right = right.with_columns(right_casts)
|
|
323
|
+
return left, right
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def empty_like(frame: "pl.DataFrame | pl.LazyFrame",
|
|
327
|
+
columns: list[str] | tuple[str, ...],
|
|
328
|
+
extra: dict[str, "pl.DataType"] | None = None,
|
|
329
|
+
*,
|
|
330
|
+
lazy: bool = False) -> "pl.DataFrame | pl.LazyFrame":
|
|
331
|
+
"""Build an empty frame whose dim-column dtypes match ``frame``.
|
|
332
|
+
|
|
333
|
+
Designed for cascade scratch frames that were previously declared
|
|
334
|
+
as ``pl.LazyFrame(schema={"e": pl.Utf8, "d": pl.Utf8, ...})`` —
|
|
335
|
+
those break the moment ``frame`` is cast to Enum. This helper
|
|
336
|
+
inspects ``frame``'s schema for each requested column and reuses
|
|
337
|
+
its dtype, falling back to ``pl.Utf8`` only when the column is
|
|
338
|
+
absent from ``frame``. ``extra`` lets the caller add value-typed
|
|
339
|
+
columns (typically ``"value": pl.Float64``) that aren't in
|
|
340
|
+
``frame``.
|
|
341
|
+
|
|
342
|
+
Usage::
|
|
343
|
+
|
|
344
|
+
bounded_walk = empty_like(anchor_lf, ["e", "d", "d_all"], lazy=True)
|
|
345
|
+
|
|
346
|
+
``frame`` may itself be eager or lazy.
|
|
347
|
+
"""
|
|
348
|
+
if isinstance(frame, pl.LazyFrame):
|
|
349
|
+
schema = frame.collect_schema()
|
|
350
|
+
else:
|
|
351
|
+
schema = frame.schema
|
|
352
|
+
out_schema: dict[str, "pl.DataType"] = {}
|
|
353
|
+
for c in columns:
|
|
354
|
+
out_schema[c] = schema.get(c, pl.Utf8)
|
|
355
|
+
if extra:
|
|
356
|
+
out_schema.update(extra)
|
|
357
|
+
if lazy:
|
|
358
|
+
return pl.LazyFrame(schema=out_schema)
|
|
359
|
+
return pl.DataFrame(schema=out_schema)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def cast_dim(col_expr: "pl.Expr",
|
|
363
|
+
enums: "dict[str, pl.Enum] | None",
|
|
364
|
+
axis: str) -> "pl.Expr":
|
|
365
|
+
"""Align a populated-frame ``pl.Expr`` with its empty-frame
|
|
366
|
+
:func:`schema_dtype` counterpart.
|
|
367
|
+
|
|
368
|
+
Used at every cascade-helper site that pairs an empty-frame branch
|
|
369
|
+
declared with ``schema_dtype(_enums, axis)`` against a populated
|
|
370
|
+
branch that emits dim columns from raw String CSV reads. When
|
|
371
|
+
``enums`` is populated, casts ``col_expr`` to the canonical Enum
|
|
372
|
+
dtype for ``axis``; otherwise (and for axes absent from the
|
|
373
|
+
mapping) returns the expression unchanged — preserving current
|
|
374
|
+
behaviour when ``_enums`` is ``None``.
|
|
375
|
+
|
|
376
|
+
Synonym-aware: ``axis`` may be a canonical short name (``"n"``,
|
|
377
|
+
``"e"``, ``"d"``) or a column-name-style synonym (``"node"``,
|
|
378
|
+
``"source"``, ``"period"``) — the latter resolves via
|
|
379
|
+
:data:`_AXIS_SYNONYMS` before lookup.
|
|
380
|
+
|
|
381
|
+
The non-strict cast nulls out values not in the Enum vocabulary
|
|
382
|
+
(consistent with :func:`cast_frame_axes` defaults).
|
|
383
|
+
|
|
384
|
+
When ``enums`` is ``None``, the helper falls back to the live
|
|
385
|
+
cascade-wide vocabulary (see :func:`get_global_axis_enums`). This
|
|
386
|
+
lets substrate modules pass ``_enums = None`` at the call site and
|
|
387
|
+
automatically pick up activation when ``load_flextool`` flips the
|
|
388
|
+
global on.
|
|
389
|
+
"""
|
|
390
|
+
if enums is None:
|
|
391
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
392
|
+
if enums is None:
|
|
393
|
+
return col_expr
|
|
394
|
+
canonical = _resolve_axis(axis)
|
|
395
|
+
dt = enums.get(canonical)
|
|
396
|
+
if dt is None:
|
|
397
|
+
return col_expr
|
|
398
|
+
return col_expr.cast(dt, strict=False)
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def schema_dtype(enums: "dict[str, pl.Enum] | None",
|
|
402
|
+
axis: str) -> "pl.DataType":
|
|
403
|
+
"""Return the Enum dtype for ``axis`` if ``enums`` is populated.
|
|
404
|
+
|
|
405
|
+
Otherwise fall back to :class:`pl.Utf8` (the current default).
|
|
406
|
+
|
|
407
|
+
Synonym-aware: ``axis`` may be a canonical short name (``"n"``,
|
|
408
|
+
``"e"``, ``"d"``) or a column-name-style synonym (``"node"``,
|
|
409
|
+
``"source"``, ``"period"``) — the latter resolves via
|
|
410
|
+
:data:`_AXIS_SYNONYMS` before lookup.
|
|
411
|
+
|
|
412
|
+
Designed for scratch-frame schema declarations in the broadcast
|
|
413
|
+
cascade and adjacent helpers. Each site that previously hard-coded
|
|
414
|
+
``schema={"e": pl.Utf8, "d": pl.Utf8}`` becomes::
|
|
415
|
+
|
|
416
|
+
return pl.DataFrame(schema={
|
|
417
|
+
"e": schema_dtype(_enums, "e"),
|
|
418
|
+
"d": schema_dtype(_enums, "d"),
|
|
419
|
+
})
|
|
420
|
+
|
|
421
|
+
With ``_enums is None`` (default during the cascade pre-activation)
|
|
422
|
+
the lookup falls back to the live cascade-wide vocabulary; if that
|
|
423
|
+
is also unset, returns ``pl.Utf8`` (the hard-coded form). When
|
|
424
|
+
``load_flextool`` calls :func:`set_global_axis_enums` with a
|
|
425
|
+
populated dict, scratch frames pick up the canonical Enum dtype
|
|
426
|
+
automatically without each substrate module needing to plumb the
|
|
427
|
+
dict through.
|
|
428
|
+
"""
|
|
429
|
+
if enums is None:
|
|
430
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
431
|
+
if enums is None:
|
|
432
|
+
return pl.Utf8
|
|
433
|
+
canonical = _resolve_axis(axis)
|
|
434
|
+
return enums.get(canonical, pl.Utf8)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def rename_to_axis(
|
|
438
|
+
frame: "pl.DataFrame | pl.LazyFrame",
|
|
439
|
+
mapping: "dict[str, str]",
|
|
440
|
+
) -> "pl.DataFrame | pl.LazyFrame":
|
|
441
|
+
"""Rename columns AND cast them to the matching axis Enum in one
|
|
442
|
+
call.
|
|
443
|
+
|
|
444
|
+
The FlexTool convention is that any rename which introduces a
|
|
445
|
+
canonical axis column name (``"n"``, ``"p"``, ``"e"``, ``"source"``,
|
|
446
|
+
``"sink"``, ``"period"``, …) takes ownership of the dim-column
|
|
447
|
+
semantics at the rename target. Without this helper, every such
|
|
448
|
+
rename has to be followed by an explicit ``.with_columns(cast_dim(
|
|
449
|
+
pl.col(new_name), _enums, new_name))`` — easy to forget, easy to
|
|
450
|
+
omit at a new site.
|
|
451
|
+
|
|
452
|
+
This helper folds the cast into the rename so the call is a single
|
|
453
|
+
one-liner:
|
|
454
|
+
|
|
455
|
+
.. code-block:: python
|
|
456
|
+
|
|
457
|
+
df.pipe(rename_to_axis, {"node": "source", "period": "d"})
|
|
458
|
+
|
|
459
|
+
Behavior:
|
|
460
|
+
|
|
461
|
+
* Applies ``mapping`` via :meth:`pl.DataFrame.rename` (or the
|
|
462
|
+
:class:`pl.LazyFrame` equivalent).
|
|
463
|
+
* Reads :data:`_LIVE_AXIS_ENUMS` (set by ``load_flextool``). If
|
|
464
|
+
unset (``None``), the cast is skipped — behavior identical to a
|
|
465
|
+
plain ``.rename`` so loader-level tests bypassing
|
|
466
|
+
``load_flextool`` keep their pre-activation Utf8 semantics.
|
|
467
|
+
* For each renamed-to column name, resolves it via
|
|
468
|
+
:data:`_AXIS_SYNONYMS` to its canonical axis. If the canonical
|
|
469
|
+
axis exists in ``_LIVE_AXIS_ENUMS``, the column is cast to that
|
|
470
|
+
Enum. Non-axis renames (data columns like ``"method"``,
|
|
471
|
+
``"value"``) pass through unchanged.
|
|
472
|
+
|
|
473
|
+
Mixed-vocab columns (``"source"``, ``"sink"``) resolve through the
|
|
474
|
+
synonym table to ``"e"`` (the entity-union axis), so the helper
|
|
475
|
+
casts them against the union enum without callers needing to know
|
|
476
|
+
that fact.
|
|
477
|
+
"""
|
|
478
|
+
out = frame.rename(mapping)
|
|
479
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
480
|
+
if enums is None:
|
|
481
|
+
return out
|
|
482
|
+
casts: list[pl.Expr] = []
|
|
483
|
+
for new_name in mapping.values():
|
|
484
|
+
canonical = _resolve_axis(new_name)
|
|
485
|
+
dt = enums.get(canonical)
|
|
486
|
+
if dt is None:
|
|
487
|
+
continue
|
|
488
|
+
casts.append(pl.col(new_name).cast(dt, strict=False))
|
|
489
|
+
if not casts:
|
|
490
|
+
return out
|
|
491
|
+
return out.with_columns(*casts)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def alias_to_axis(
|
|
495
|
+
source: "str | pl.Expr",
|
|
496
|
+
target_axis: str,
|
|
497
|
+
) -> "pl.Expr":
|
|
498
|
+
"""``pl.col(source).alias(target_axis)`` with axis-aware cast.
|
|
499
|
+
|
|
500
|
+
The sibling of :func:`rename_to_axis` for ``select()`` /
|
|
501
|
+
``with_columns()`` projections that rename via alias rather than via
|
|
502
|
+
``.rename(...)``. Two equivalent call forms:
|
|
503
|
+
|
|
504
|
+
.. code-block:: python
|
|
505
|
+
|
|
506
|
+
# Column-name source
|
|
507
|
+
df.select(alias_to_axis("name", "p"), ...)
|
|
508
|
+
|
|
509
|
+
# Expression source (arithmetic, str ops, conditionals, etc.)
|
|
510
|
+
df.select(alias_to_axis(pl.col("a") + pl.col("b"), "d"), ...)
|
|
511
|
+
|
|
512
|
+
Behavior:
|
|
513
|
+
|
|
514
|
+
* If ``source`` is a string, treated as ``pl.col(source)``.
|
|
515
|
+
* Reads :data:`_LIVE_AXIS_ENUMS`. When unset, reduces to a plain
|
|
516
|
+
``expr.alias(target_axis)`` — pre-activation passthrough.
|
|
517
|
+
* Resolves ``target_axis`` via :data:`_AXIS_SYNONYMS` (so
|
|
518
|
+
``"period"`` → ``"d"``, ``"source"`` → ``"e"`` etc).
|
|
519
|
+
* When the resolved canonical axis is in the live enum dict, casts
|
|
520
|
+
the source expression to that Enum before aliasing. Otherwise
|
|
521
|
+
(non-axis target like ``"value"``, ``"method"``) just aliases.
|
|
522
|
+
|
|
523
|
+
The non-strict cast nulls out values not in the Enum vocabulary,
|
|
524
|
+
consistent with :func:`rename_to_axis` / :func:`cast_dim`
|
|
525
|
+
/:func:`cast_frame_axes` defaults.
|
|
526
|
+
"""
|
|
527
|
+
expr = pl.col(source) if isinstance(source, str) else source
|
|
528
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
529
|
+
if enums is None:
|
|
530
|
+
return expr.alias(target_axis)
|
|
531
|
+
canonical = _resolve_axis(target_axis)
|
|
532
|
+
dt = enums.get(canonical)
|
|
533
|
+
if dt is None:
|
|
534
|
+
return expr.alias(target_axis)
|
|
535
|
+
return expr.cast(dt, strict=False).alias(target_axis)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def lit_axis(value: object, axis: str) -> "pl.Expr":
|
|
539
|
+
""":func:`pl.lit` that emits the canonical axis Enum dtype.
|
|
540
|
+
|
|
541
|
+
Use at any site that injects a literal token into an axis-aware
|
|
542
|
+
column. Without this helper, ``pl.lit("source")`` returns a
|
|
543
|
+
``pl.Utf8`` literal which will SchemaError against a frame whose
|
|
544
|
+
``"side"`` column is ``pl.Enum``.
|
|
545
|
+
|
|
546
|
+
.. code-block:: python
|
|
547
|
+
|
|
548
|
+
df.with_columns(side=lit_axis("source", "side"))
|
|
549
|
+
df.with_columns(klass=lit_axis("unit", "klass"))
|
|
550
|
+
|
|
551
|
+
Synonym-aware: ``axis`` may be a canonical short name or a
|
|
552
|
+
column-name-style synonym (``"node"``, ``"source"``, ``"period"``).
|
|
553
|
+
|
|
554
|
+
When :data:`_LIVE_AXIS_ENUMS` is ``None`` (no activation), returns
|
|
555
|
+
a plain ``pl.lit(value)`` — pre-activation behavior.
|
|
556
|
+
"""
|
|
557
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
558
|
+
if enums is None:
|
|
559
|
+
return pl.lit(value)
|
|
560
|
+
canonical = _resolve_axis(axis)
|
|
561
|
+
dt = enums.get(canonical)
|
|
562
|
+
if dt is None:
|
|
563
|
+
return pl.lit(value)
|
|
564
|
+
return pl.lit(value).cast(dt, strict=False)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def cast_value_axes(value, enums: dict[str, pl.Enum], *, strict: bool = False,
|
|
568
|
+
skip_columns: "set[str] | frozenset[str] | None" = None):
|
|
569
|
+
"""Recursively cast a value's dim columns to the canonical Enums.
|
|
570
|
+
|
|
571
|
+
Handles:
|
|
572
|
+
* :class:`polar_high.Param` — rebuilds with cast frame.
|
|
573
|
+
* :class:`pl.DataFrame` / :class:`pl.LazyFrame` — applies
|
|
574
|
+
:func:`cast_frame_axes`.
|
|
575
|
+
* ``dict`` / ``tuple`` / ``list`` — recursively walks and rebuilds
|
|
576
|
+
the same container shape with cast children.
|
|
577
|
+
* Everything else — returned unchanged.
|
|
578
|
+
|
|
579
|
+
Used by :func:`load_flextool` to wrap the return values of each
|
|
580
|
+
``_load_*`` function in a single shot at the call site, so the
|
|
581
|
+
interior of every loader doesn't need an individual cast injection.
|
|
582
|
+
|
|
583
|
+
``skip_columns`` is forwarded to :func:`cast_frame_axes` (see there)
|
|
584
|
+
so callers can protect name-collides-axis-but-different-vocabulary
|
|
585
|
+
columns from being nulled on cast.
|
|
586
|
+
"""
|
|
587
|
+
# Late import to avoid circular dependency.
|
|
588
|
+
from polar_high import Param
|
|
589
|
+
|
|
590
|
+
if value is None:
|
|
591
|
+
return value
|
|
592
|
+
if isinstance(value, Param):
|
|
593
|
+
# Operate on the lazy form to avoid forcing a collect() — every
|
|
594
|
+
# Param keeps an internal ``lazy`` LazyFrame regardless of whether
|
|
595
|
+
# ``.frame`` has been materialised yet.
|
|
596
|
+
cast_lazy = cast_frame_axes(value.lazy, enums, strict=strict,
|
|
597
|
+
skip_columns=skip_columns)
|
|
598
|
+
if cast_lazy is value.lazy:
|
|
599
|
+
return value
|
|
600
|
+
return Param(value.dims, cast_lazy,
|
|
601
|
+
name=getattr(value, "name", None),
|
|
602
|
+
_sources=getattr(value, "_sources", None))
|
|
603
|
+
if isinstance(value, (pl.DataFrame, pl.LazyFrame)):
|
|
604
|
+
return cast_frame_axes(value, enums, strict=strict,
|
|
605
|
+
skip_columns=skip_columns)
|
|
606
|
+
if isinstance(value, dict):
|
|
607
|
+
return {k: cast_value_axes(v, enums, strict=strict,
|
|
608
|
+
skip_columns=skip_columns)
|
|
609
|
+
for k, v in value.items()}
|
|
610
|
+
if isinstance(value, list):
|
|
611
|
+
return [cast_value_axes(v, enums, strict=strict,
|
|
612
|
+
skip_columns=skip_columns) for v in value]
|
|
613
|
+
if isinstance(value, tuple):
|
|
614
|
+
return tuple(cast_value_axes(v, enums, strict=strict,
|
|
615
|
+
skip_columns=skip_columns)
|
|
616
|
+
for v in value)
|
|
617
|
+
return value
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
# Per-field column protection for the end-of-load FlexData → Enum
|
|
621
|
+
# sweep. The representative-period base-period / rep-start relations
|
|
622
|
+
# name their label columns ``"b"`` (base-period start) and ``"r"``
|
|
623
|
+
# (rep-block start) so ``model.py``'s inter-period-closure wiring can
|
|
624
|
+
# join on the ``.mod`` symbols — but those labels are TIMESTEP tokens
|
|
625
|
+
# (``t0001`` …), NOT branch / rep-axis tokens. The synonym table maps
|
|
626
|
+
# ``"b" → branch`` and treats ``"r"`` as the (rep) axis, so a blind
|
|
627
|
+
# sweep casts these columns to the wrong enum and silently nulls every
|
|
628
|
+
# value (branch vocab is e.g. ``{eff, noEff}``; the rep axis enum is
|
|
629
|
+
# empty for a non-stochastic solve). That nulls ``rp_base_period_set``
|
|
630
|
+
# → ``v_state_inter``'s index collapses → the seasonal-closure
|
|
631
|
+
# constraints (``rp_inter_period_cyclic`` / ``_balance``) are dropped
|
|
632
|
+
# and the LP solves an under-constrained, too-cheap problem. These
|
|
633
|
+
# columns are left as the reader produced them (plain string labels);
|
|
634
|
+
# ``model.py`` casts them explicitly against the correct timestep /
|
|
635
|
+
# base-period vocabulary at each use site (model.py:1234-1291). Mirrors
|
|
636
|
+
# how coarse-block frames dodge the identical ``b``/branch collision by
|
|
637
|
+
# using ``"bk"`` for the block axis.
|
|
638
|
+
_FLEXDATA_SWEEP_SKIP_COLUMNS: "dict[str, frozenset[str]]" = {
|
|
639
|
+
"rp_base_period_set": frozenset({"b"}),
|
|
640
|
+
"rp_base_first": frozenset({"b"}),
|
|
641
|
+
"rp_base_last": frozenset({"b"}),
|
|
642
|
+
"rp_base_chain": frozenset({"b", "b_prev"}),
|
|
643
|
+
"rp_base__rep": frozenset({"b", "r"}),
|
|
644
|
+
"p_rp_last_step": frozenset({"r"}),
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def cast_flexdata_axes(flex_data: "FlexData",
|
|
649
|
+
enums: dict[str, pl.Enum],
|
|
650
|
+
*, strict: bool = False) -> "FlexData":
|
|
651
|
+
"""Walk every Param / DataFrame / LazyFrame field on ``flex_data``
|
|
652
|
+
and cast dim columns to the canonical Enums in place. Returns the
|
|
653
|
+
same FlexData (mutated) for fluent use.
|
|
654
|
+
|
|
655
|
+
Skips ``block_layout`` and other non-frame fields. Skips fields
|
|
656
|
+
whose value is ``None``. For the representative-period label
|
|
657
|
+
relations listed in :data:`_FLEXDATA_SWEEP_SKIP_COLUMNS` the
|
|
658
|
+
name-collides-but-different-vocabulary label columns (``b`` / ``r``)
|
|
659
|
+
are left untouched — see that constant for the full rationale.
|
|
660
|
+
"""
|
|
661
|
+
from polar_high import Param
|
|
662
|
+
|
|
663
|
+
for f in dataclasses.fields(flex_data):
|
|
664
|
+
val = getattr(flex_data, f.name, None)
|
|
665
|
+
if val is None:
|
|
666
|
+
continue
|
|
667
|
+
if isinstance(val, (Param, pl.DataFrame, pl.LazyFrame)):
|
|
668
|
+
skip = _FLEXDATA_SWEEP_SKIP_COLUMNS.get(f.name)
|
|
669
|
+
new_val = cast_value_axes(val, enums, strict=strict,
|
|
670
|
+
skip_columns=skip)
|
|
671
|
+
if new_val is not val:
|
|
672
|
+
setattr(flex_data, f.name, new_val)
|
|
673
|
+
return flex_data
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def axis_lazyframe(
|
|
677
|
+
data: "dict[str, list]",
|
|
678
|
+
axis_for_column: "dict[str, str] | None" = None,
|
|
679
|
+
) -> "pl.LazyFrame":
|
|
680
|
+
"""Build a :class:`pl.LazyFrame` from a ``{column: list}`` mapping with
|
|
681
|
+
axis-aware dtype casts.
|
|
682
|
+
|
|
683
|
+
Each ``column`` whose name resolves (via :data:`_AXIS_SYNONYMS`) to an
|
|
684
|
+
axis present in the live cascade-wide enum dict is cast to that
|
|
685
|
+
canonical Enum. When :func:`get_global_axis_enums` returns ``None``
|
|
686
|
+
(pre-activation) the cast is a no-op and the columns stay as polars'
|
|
687
|
+
default-inferred dtypes — same behaviour as a raw
|
|
688
|
+
:class:`pl.LazyFrame` constructor.
|
|
689
|
+
|
|
690
|
+
Parameters
|
|
691
|
+
----------
|
|
692
|
+
data
|
|
693
|
+
``{column_name: list_of_values}``. Lists may be Python strings
|
|
694
|
+
(typical for period / entity tokens) or any polars-coercible
|
|
695
|
+
Python scalar; the cast is non-strict so unknown values become
|
|
696
|
+
null.
|
|
697
|
+
axis_for_column
|
|
698
|
+
Optional override map ``{column_name: axis_name}`` for columns
|
|
699
|
+
whose name is not itself an axis synonym (e.g. ``"d_first"`` may
|
|
700
|
+
be omitted from :data:`_AXIS_SYNONYMS` but here you want it cast
|
|
701
|
+
against ``"d"``). Per-column entry overrides the default
|
|
702
|
+
synonym-table lookup.
|
|
703
|
+
|
|
704
|
+
Returns
|
|
705
|
+
-------
|
|
706
|
+
pl.LazyFrame
|
|
707
|
+
Lazy frame with the requested data and axis-cast schema.
|
|
708
|
+
"""
|
|
709
|
+
lf = pl.LazyFrame(data)
|
|
710
|
+
enums = _LIVE_AXIS_ENUMS_CTX.get()
|
|
711
|
+
if enums is None:
|
|
712
|
+
return lf
|
|
713
|
+
casts: list[pl.Expr] = []
|
|
714
|
+
for col in data.keys():
|
|
715
|
+
ax = (axis_for_column or {}).get(col, col)
|
|
716
|
+
canonical = _resolve_axis(ax)
|
|
717
|
+
dt = enums.get(canonical)
|
|
718
|
+
if dt is None:
|
|
719
|
+
continue
|
|
720
|
+
casts.append(pl.col(col).cast(dt, strict=False))
|
|
721
|
+
if not casts:
|
|
722
|
+
return lf
|
|
723
|
+
return lf.with_columns(*casts)
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
__all__ = [
|
|
727
|
+
"axis_lazyframe",
|
|
728
|
+
"cast_dim",
|
|
729
|
+
"cast_frame_axes",
|
|
730
|
+
"cast_value_axes",
|
|
731
|
+
"cast_flexdata_axes",
|
|
732
|
+
"empty_like",
|
|
733
|
+
"align_join_dtypes",
|
|
734
|
+
"schema_dtype",
|
|
735
|
+
"rename_to_axis",
|
|
736
|
+
"alias_to_axis",
|
|
737
|
+
"lit_axis",
|
|
738
|
+
"set_global_axis_enums",
|
|
739
|
+
"reset_global_axis_enums",
|
|
740
|
+
"get_global_axis_enums",
|
|
741
|
+
"is_axis_aware",
|
|
742
|
+
]
|