flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from spinedb_api import DatabaseMapping, from_database, to_database, SpineDBAPIError, Asterisk
|
|
3
|
+
from spinedb_api.exception import NothingToCommit
|
|
4
|
+
from flextool.update_flextool import FLEXTOOL_DB_VERSION
|
|
5
|
+
|
|
6
|
+
flextool_db_version = FLEXTOOL_DB_VERSION
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _import_legacy_timeset_timeline(db, input_path: str) -> None:
|
|
10
|
+
"""Set the ``timeset.timeline`` parameter from a legacy Excel sheet.
|
|
11
|
+
|
|
12
|
+
Pre-v25 specification-format Excels store the timeset->timeline link as a
|
|
13
|
+
two-column ``timeblockSet_timeline`` sheet (column 0 = timeset name,
|
|
14
|
+
column 1 = timeline name; a header row, no alternative column). The
|
|
15
|
+
modern ``timeset_timeline`` sheet has a different layout, so this link is
|
|
16
|
+
not picked up by the generic spec mappings.
|
|
17
|
+
|
|
18
|
+
The v25 migration (``update_timestructure``) normally collapses the old
|
|
19
|
+
``timeblockSet__timeline`` relationship into the ``timeset.timeline``
|
|
20
|
+
parameter, attaching each value to the alternative of that timeset's
|
|
21
|
+
duration. Mirror that here so a pre-v25 Excel imports directly into the
|
|
22
|
+
v25+ schema: for every ``(timeset, alternative)`` that has a
|
|
23
|
+
``timeset_duration`` value, set ``timeline`` in the same alternative.
|
|
24
|
+
Without it the solve fails with "Failed to map timeset to timeline".
|
|
25
|
+
"""
|
|
26
|
+
import openpyxl
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True)
|
|
30
|
+
except Exception:
|
|
31
|
+
return
|
|
32
|
+
try:
|
|
33
|
+
if "timeblockSet_timeline" not in wb.sheetnames:
|
|
34
|
+
return
|
|
35
|
+
timeset_to_timeline: dict[str, str] = {}
|
|
36
|
+
for i, row in enumerate(wb["timeblockSet_timeline"].iter_rows(values_only=True)):
|
|
37
|
+
if i == 0: # header row: (timeblockSet, timeline)
|
|
38
|
+
continue
|
|
39
|
+
if not row or row[0] is None or row[1] is None:
|
|
40
|
+
continue
|
|
41
|
+
timeset_to_timeline[str(row[0])] = str(row[1])
|
|
42
|
+
finally:
|
|
43
|
+
wb.close()
|
|
44
|
+
|
|
45
|
+
if not timeset_to_timeline:
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
# Already-present (entity, alternative) pairs — stay idempotent.
|
|
49
|
+
existing = {
|
|
50
|
+
(pv["entity_byname"], pv["alternative_name"])
|
|
51
|
+
for pv in db.find_parameter_values(
|
|
52
|
+
entity_class_name="timeset", parameter_definition_name="timeline"
|
|
53
|
+
)
|
|
54
|
+
}
|
|
55
|
+
added = 0
|
|
56
|
+
for pv in db.find_parameter_values(
|
|
57
|
+
entity_class_name="timeset", parameter_definition_name="timeset_duration"
|
|
58
|
+
):
|
|
59
|
+
byname = pv["entity_byname"]
|
|
60
|
+
timeline_name = timeset_to_timeline.get(byname[0])
|
|
61
|
+
if timeline_name is None:
|
|
62
|
+
continue
|
|
63
|
+
key = (byname, pv["alternative_name"])
|
|
64
|
+
if key in existing:
|
|
65
|
+
continue
|
|
66
|
+
value, type_ = to_database(timeline_name)
|
|
67
|
+
try:
|
|
68
|
+
db.add_parameter_value(
|
|
69
|
+
entity_class_name="timeset",
|
|
70
|
+
parameter_definition_name="timeline",
|
|
71
|
+
entity_byname=byname,
|
|
72
|
+
alternative_name=pv["alternative_name"],
|
|
73
|
+
value=value,
|
|
74
|
+
type=type_,
|
|
75
|
+
)
|
|
76
|
+
existing.add(key)
|
|
77
|
+
added += 1
|
|
78
|
+
except Exception as e:
|
|
79
|
+
print(f" Could not set timeset.timeline for {byname[0]}: {e}")
|
|
80
|
+
if added:
|
|
81
|
+
print(
|
|
82
|
+
f"\nImported legacy timeset->timeline links "
|
|
83
|
+
f"({added} (timeset, alternative) pairs) from 'timeblockSet_timeline'."
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
def write_to_flextool_input_db(input_path, tabular_reader, target_db_url, input_type='excel',
|
|
87
|
+
migration_follows: bool = False):
|
|
88
|
+
"""Write tabular data to FlexTool input database, reading one sheet/file at a time.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
input_path: Path to Excel/ODS file or directory containing CSV files
|
|
92
|
+
tabular_reader: TabularReader instance with loaded specification
|
|
93
|
+
target_db_url: URL to target SQLite database
|
|
94
|
+
input_type: Either 'excel' or 'csv' to determine reading method
|
|
95
|
+
migration_follows: If True, accept a version mismatch because
|
|
96
|
+
the caller will migrate the database after import
|
|
97
|
+
|
|
98
|
+
This function reads sheets/files one at a time to minimize memory usage.
|
|
99
|
+
"""
|
|
100
|
+
# Refuse specification-format Excels whose template layout predates the
|
|
101
|
+
# importer. Deferred import avoids a circular dependency with the package.
|
|
102
|
+
if input_type == 'excel':
|
|
103
|
+
from flextool.process_inputs import (
|
|
104
|
+
MIN_SUPPORTED_SPECIFICATION_VERSION,
|
|
105
|
+
read_specification_changelog_version,
|
|
106
|
+
unsupported_specification_message,
|
|
107
|
+
)
|
|
108
|
+
changelog_version = read_specification_changelog_version(input_path)
|
|
109
|
+
if (
|
|
110
|
+
changelog_version is not None
|
|
111
|
+
and changelog_version < MIN_SUPPORTED_SPECIFICATION_VERSION
|
|
112
|
+
):
|
|
113
|
+
print(unsupported_specification_message(changelog_version))
|
|
114
|
+
exit(1)
|
|
115
|
+
|
|
116
|
+
if target_db_url.startswith('sqlite://') or target_db_url.startswith('http://'):
|
|
117
|
+
db_url = target_db_url
|
|
118
|
+
elif os.path.exists(target_db_url) and target_db_url.endswith(".sqlite"):
|
|
119
|
+
db_url = 'sqlite:///' + target_db_url
|
|
120
|
+
else:
|
|
121
|
+
print("No sqlite file at " + target_db_url)
|
|
122
|
+
exit(-1)
|
|
123
|
+
|
|
124
|
+
with DatabaseMapping(db_url, create = False, upgrade = True) as db:
|
|
125
|
+
sq = db.object_parameter_definition_sq
|
|
126
|
+
version_parameter = db.query(sq).filter(sq.c.object_class_name == "model").filter(sq.c.parameter_name == "version").one_or_none()
|
|
127
|
+
if version_parameter is None:
|
|
128
|
+
#if no version assume version 0
|
|
129
|
+
print(f"No FlexTool database version found. Needs to be proper FlexTool database at version: {flextool_db_version}")
|
|
130
|
+
exit(-1)
|
|
131
|
+
else:
|
|
132
|
+
version = from_database(version_parameter.default_value, version_parameter.default_type)
|
|
133
|
+
if version == flextool_db_version:
|
|
134
|
+
print(f"Valid FlexTool input database with correct version: {flextool_db_version}")
|
|
135
|
+
elif migration_follows:
|
|
136
|
+
print(f"Importing into a fresh FlexTool schema-v{int(version)} database "
|
|
137
|
+
"(a migration to the current version follows this import).")
|
|
138
|
+
else:
|
|
139
|
+
print(f"Wrong FlexTool input database version: {version}. Should be: {flextool_db_version}")
|
|
140
|
+
exit(-1)
|
|
141
|
+
|
|
142
|
+
# Get list of tables to process
|
|
143
|
+
selected_tables = tabular_reader.get_selected_table_names()
|
|
144
|
+
print(f"\nProcessing {len(selected_tables)} selected tables")
|
|
145
|
+
|
|
146
|
+
# Purge the database empty of data times
|
|
147
|
+
try:
|
|
148
|
+
db.remove_entity(id=Asterisk)
|
|
149
|
+
db.remove_alternative(id=Asterisk)
|
|
150
|
+
db.remove_scenario(id=Asterisk)
|
|
151
|
+
db.remove_parameter_value(id=Asterisk)
|
|
152
|
+
db.commit_session("Purged the db")
|
|
153
|
+
except Exception as e:
|
|
154
|
+
raise RuntimeError(f"Failed to purge the input database before writing. {e}")
|
|
155
|
+
|
|
156
|
+
# Process each table one at a time
|
|
157
|
+
for table_name in selected_tables:
|
|
158
|
+
print(f"\nProcessing table {table_name + ': ':<30}", end="")
|
|
159
|
+
|
|
160
|
+
# Read the sheet/file that contains the data
|
|
161
|
+
try:
|
|
162
|
+
if input_type == 'excel':
|
|
163
|
+
raw_df, column_types = tabular_reader.read_excel_sheet(input_path, table_name)
|
|
164
|
+
elif input_type == 'csv':
|
|
165
|
+
csv_file = os.path.join(input_path, f"{table_name}.csv")
|
|
166
|
+
if not os.path.exists(csv_file):
|
|
167
|
+
print(f" Warning: CSV file not found: {csv_file}")
|
|
168
|
+
continue
|
|
169
|
+
raw_df, column_types = tabular_reader.read_csv_file(csv_file)
|
|
170
|
+
else:
|
|
171
|
+
raise ValueError(f"Unknown input_type: {input_type}")
|
|
172
|
+
|
|
173
|
+
# Skip if sheet not found in the file
|
|
174
|
+
if raw_df is None:
|
|
175
|
+
print(f"Skipping {table_name} - no sheet found")
|
|
176
|
+
continue
|
|
177
|
+
|
|
178
|
+
# Skip if no data (e.g., sheet not selected)
|
|
179
|
+
if raw_df.empty:
|
|
180
|
+
print(f" Skipping {table_name} - no data")
|
|
181
|
+
continue
|
|
182
|
+
|
|
183
|
+
except Exception as e:
|
|
184
|
+
print(f" Error reading table '{table_name}': {e}")
|
|
185
|
+
continue # Continue to next table on read error
|
|
186
|
+
|
|
187
|
+
# Get mappings for this sheet
|
|
188
|
+
mappings = tabular_reader.get_table_mappings(table_name)
|
|
189
|
+
if not mappings:
|
|
190
|
+
raise ValueError(f"No specification found for sheet {table_name}")
|
|
191
|
+
|
|
192
|
+
table_options = tabular_reader.get_table_options(table_name)
|
|
193
|
+
if 'row' in table_options:
|
|
194
|
+
raw_df = raw_df.iloc[table_options['row']:]
|
|
195
|
+
if 'column' in table_options:
|
|
196
|
+
raw_df = raw_df.iloc[table_options['column']:]
|
|
197
|
+
|
|
198
|
+
# Process each mapping in the table
|
|
199
|
+
for mapping_name in mappings.keys():
|
|
200
|
+
print(f"{mapping_name:>25}", end="")
|
|
201
|
+
|
|
202
|
+
mapping_info = tabular_reader._parse_mapping(mappings[mapping_name])
|
|
203
|
+
if not mapping_info:
|
|
204
|
+
continue
|
|
205
|
+
|
|
206
|
+
# Read data and process the dataframe
|
|
207
|
+
try:
|
|
208
|
+
(data_df, ent_zip_list, ent_act_zip, scen_array, scen_alt_df) = tabular_reader._extract_data(raw_df, mapping_info, table_options, column_types, table_name, mapping_name)
|
|
209
|
+
except Exception as e:
|
|
210
|
+
print(
|
|
211
|
+
f"\n Error processing sheet '{table_name}', mapping "
|
|
212
|
+
f"'{mapping_name}': {type(e).__name__}: {e}"
|
|
213
|
+
)
|
|
214
|
+
continue # Continue to next mapping on processing error
|
|
215
|
+
|
|
216
|
+
if ent_zip_list:
|
|
217
|
+
tabular_reader._add_entities(ent_zip_list, db, table_name, mapping_name)
|
|
218
|
+
|
|
219
|
+
# Add alternatives to the database
|
|
220
|
+
if data_df is not None:
|
|
221
|
+
alt_array = data_df.columns.get_level_values('Alternative').unique().to_list()
|
|
222
|
+
tabular_reader._add_alternatives(alt_array, db, table_name, mapping_name)
|
|
223
|
+
|
|
224
|
+
# Write re-organised dataframe to database
|
|
225
|
+
if data_df is not None:
|
|
226
|
+
#try:
|
|
227
|
+
# check_type:
|
|
228
|
+
value_type = mapping_info['rules'].get('ParameterValueType')
|
|
229
|
+
if value_type:
|
|
230
|
+
value_type = value_type['value']
|
|
231
|
+
else:
|
|
232
|
+
value_type = 'constant'
|
|
233
|
+
tabular_reader._add_parameters(data_df, db, table_name, mapping_name, value_type)
|
|
234
|
+
|
|
235
|
+
# Ensure alternatives referenced by entity_alternatives exist
|
|
236
|
+
if ent_act_zip is not None:
|
|
237
|
+
ent_act_list = list(ent_act_zip)
|
|
238
|
+
ea_alts = list({alt for _cls, _ent, alt, _act in ent_act_list})
|
|
239
|
+
if ea_alts:
|
|
240
|
+
tabular_reader._add_alternatives(ea_alts, db, table_name, mapping_name)
|
|
241
|
+
tabular_reader._add_entity_alternatives(ent_act_list, db, table_name, mapping_name)
|
|
242
|
+
|
|
243
|
+
# Add scenarios to the database
|
|
244
|
+
if scen_array is not None:
|
|
245
|
+
for scen in scen_array:
|
|
246
|
+
try:
|
|
247
|
+
db.add_scenario(name=scen)
|
|
248
|
+
except Exception as e:
|
|
249
|
+
raise SpineDBAPIError(f'Could not add scenario {scen} to the database: {e}')
|
|
250
|
+
|
|
251
|
+
# Add scenario_alternatives to the database
|
|
252
|
+
if scen_alt_df is not None:
|
|
253
|
+
tabular_reader._add_scenario_alternatives(scen_alt_df, db, table_name, mapping_name)
|
|
254
|
+
|
|
255
|
+
# Backfill the timeset->timeline link from the legacy two-column sheet
|
|
256
|
+
# if present (pre-v25 Excels); modern files set it via the spec.
|
|
257
|
+
if input_type == 'excel':
|
|
258
|
+
_import_legacy_timeset_timeline(db, input_path)
|
|
259
|
+
|
|
260
|
+
# Commit the changes
|
|
261
|
+
try:
|
|
262
|
+
db.commit_session(f"Imported data from mapping '{mapping_name}'")
|
|
263
|
+
tabular_reader.logger.info(f"Successfully committed data and items from mapping {mapping_name}")
|
|
264
|
+
except NothingToCommit:
|
|
265
|
+
pass
|
|
266
|
+
except Exception as e:
|
|
267
|
+
raise SpineDBAPIError(f"Could not commit data and items based on {mapping_name}: {e}")
|
|
268
|
+
|
|
269
|
+
print("\nAll data processed successfully! (* = No data found for the mapping)")
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Output data processing: reads solver CSV results, post-processes, and writes outputs."""
|
|
2
|
+
from flextool.process_outputs.read_variables import read_variables
|
|
3
|
+
from flextool.process_outputs.read_parameters import read_parameters
|
|
4
|
+
from flextool.process_outputs.read_sets import read_sets
|
|
5
|
+
from flextool.process_outputs.process_results import post_process_results
|
|
6
|
+
from flextool.process_outputs.write_outputs import write_outputs
|
|
7
|
+
__all__ = ['read_variables', 'read_parameters', 'read_sets', 'post_process_results', 'write_outputs']
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Shared dt→d annualisation helper for the output-processing stage.
|
|
2
|
+
|
|
3
|
+
A single place that performs the *extensive* per-step → per-period→annual
|
|
4
|
+
aggregation so that no individual output site drifts from the cost-weighting
|
|
5
|
+
convention.
|
|
6
|
+
|
|
7
|
+
The LP objective annualises every per-timestep cost term with the
|
|
8
|
+
representative weight ``p_timestep_weight[d, t]`` (1.0 for a full / evenly
|
|
9
|
+
sampled timeline, a per-step share of the year for a representative timeset
|
|
10
|
+
with ``timeset_weights``). Annual *energy / extensive* outputs must use the
|
|
11
|
+
**same** weight so that the energy the model is scaled to serve, reports, and
|
|
12
|
+
costs all coincide. When ``w ≡ 1`` (the common full-timeline case) the result
|
|
13
|
+
is byte-identical to the unweighted formula.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def annualize_dt_to_d(
|
|
22
|
+
frame: pd.DataFrame,
|
|
23
|
+
timestep_weight: pd.Series,
|
|
24
|
+
complete_period_share_of_year: pd.Series,
|
|
25
|
+
step_duration: "pd.Series | None" = None,
|
|
26
|
+
*,
|
|
27
|
+
div_level: "int | None" = None,
|
|
28
|
+
) -> pd.DataFrame:
|
|
29
|
+
"""Weight a per-``(d, t)`` extensive frame and annualise it to per-period.
|
|
30
|
+
|
|
31
|
+
Steps (matching the objective's annualisation):
|
|
32
|
+
|
|
33
|
+
1. If ``step_duration`` is given (the frame holds power, MW), multiply by
|
|
34
|
+
it to obtain energy per step (MWh). Inflow-type frames are already
|
|
35
|
+
MWh/step — pass ``step_duration=None``.
|
|
36
|
+
2. Multiply by ``timestep_weight`` (the per-``(d, t)`` representative weight,
|
|
37
|
+
1.0 when no/uniform ``timeset_weights``).
|
|
38
|
+
3. Sum over the ``period`` level.
|
|
39
|
+
4. Divide by ``complete_period_share_of_year`` (the uniform annualiser
|
|
40
|
+
``Σ_t step_duration / 8760``).
|
|
41
|
+
|
|
42
|
+
``timestep_weight`` and ``step_duration`` share the identical
|
|
43
|
+
``(period, time)`` row index (both stripped of the ``solve`` level in
|
|
44
|
+
``drop_levels``), so ``.mul(..., axis=0)`` broadcasts the same way.
|
|
45
|
+
|
|
46
|
+
``div_level`` mirrors the per-site ``.div(..., axis=0, level=...)``
|
|
47
|
+
convention: pass ``level=1`` where the caller's frame has a multi-level
|
|
48
|
+
row index whose period sits at level 1 (e.g. ``out_node`` slacks).
|
|
49
|
+
"""
|
|
50
|
+
f = frame if step_duration is None else frame.mul(step_duration, axis=0)
|
|
51
|
+
f = f.mul(timestep_weight, axis=0)
|
|
52
|
+
summed = f.groupby(level="period").sum()
|
|
53
|
+
if div_level is None:
|
|
54
|
+
return summed.div(complete_period_share_of_year, axis=0)
|
|
55
|
+
return summed.div(complete_period_share_of_year, axis=0, level=div_level)
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"""Δ.31 — In-memory FlexData → pandas helpers for read_parameters / read_sets.
|
|
2
|
+
|
|
3
|
+
These helpers translate the polars long-form FlexData fields into the
|
|
4
|
+
pandas wide-format shapes that the legacy CSV-based ``read_parameters``
|
|
5
|
+
and ``read_sets`` produced. They are stateless utilities; the
|
|
6
|
+
per-attribute mapping lives in
|
|
7
|
+
:mod:`flextool.process_outputs.read_parameters` and
|
|
8
|
+
:mod:`flextool.process_outputs.read_sets`.
|
|
9
|
+
|
|
10
|
+
Naming convention: dim columns in FlexData are short (``n``, ``p``,
|
|
11
|
+
``d``, ``t``, ``g``, ``c``, ``e``, …). The legacy pandas namespace
|
|
12
|
+
spells them out in full (``node``, ``process``, ``period``, ``time``,
|
|
13
|
+
``group``, ``commodity``, ``entity``, …). :data:`DIM_NAMES` is the
|
|
14
|
+
canonical translation table.
|
|
15
|
+
|
|
16
|
+
Failure mode: every helper raises (``KeyError``, ``ValueError``)
|
|
17
|
+
loudly when a FlexData field is absent or has an unexpected schema.
|
|
18
|
+
The legacy CSV path was tolerant of empty / missing files — the
|
|
19
|
+
in-memory replacement is strict so authoring bugs surface at the
|
|
20
|
+
call-site instead of producing silently-empty outputs.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Sequence
|
|
25
|
+
|
|
26
|
+
import pandas as pd
|
|
27
|
+
import polars as pl
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
# Canonical dim-name translation table.
|
|
32
|
+
#
|
|
33
|
+
# FlexData uses short, model-internal names (``n``, ``p``, ``d``, ``t``).
|
|
34
|
+
# Legacy pandas output namespace spells them out so plot / write_outputs
|
|
35
|
+
# / out_*.py modules read them as ``period`` / ``time`` / ``node`` /
|
|
36
|
+
# ``process`` etc. Keep this map in sync with the per-field comments in
|
|
37
|
+
# :class:`flextool.engine_polars.input.FlexData`.
|
|
38
|
+
DIM_NAMES: dict[str, str] = {
|
|
39
|
+
"n": "node",
|
|
40
|
+
"p": "process",
|
|
41
|
+
"g": "group",
|
|
42
|
+
"c": "commodity",
|
|
43
|
+
"f": "profile",
|
|
44
|
+
"d": "period",
|
|
45
|
+
"t": "time",
|
|
46
|
+
"e": "entity",
|
|
47
|
+
"i": "tier",
|
|
48
|
+
"ud": "upDown",
|
|
49
|
+
"r": "reserve",
|
|
50
|
+
"source": "source",
|
|
51
|
+
"sink": "sink",
|
|
52
|
+
"method": "method",
|
|
53
|
+
"param": "param",
|
|
54
|
+
# Storage / time-helper extensions
|
|
55
|
+
"td": "td",
|
|
56
|
+
# Block / branch / lookback names that already match
|
|
57
|
+
"b_first": "b_first",
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def long_dim(d: str) -> str:
|
|
62
|
+
"""Return the long pandas dim-name for a FlexData short dim."""
|
|
63
|
+
if d in DIM_NAMES:
|
|
64
|
+
return DIM_NAMES[d]
|
|
65
|
+
return d
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
# Frame → pandas helpers
|
|
70
|
+
# ---------------------------------------------------------------------------
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _to_pandas(frame_pl: "pl.DataFrame") -> pd.DataFrame:
|
|
74
|
+
"""``polars.DataFrame.to_pandas()`` with a small empty-frame guard."""
|
|
75
|
+
if frame_pl is None:
|
|
76
|
+
raise ValueError("expected a polars DataFrame, got None")
|
|
77
|
+
return frame_pl.to_pandas()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def wide_per_entity(
|
|
81
|
+
frame_pl: "pl.DataFrame",
|
|
82
|
+
*,
|
|
83
|
+
row_dims: Sequence[str],
|
|
84
|
+
col_dim: str,
|
|
85
|
+
value: str = "value",
|
|
86
|
+
row_names: Sequence[str] | None = None,
|
|
87
|
+
col_name: str | None = None,
|
|
88
|
+
) -> pd.DataFrame:
|
|
89
|
+
"""Pivot a polars long-form frame to a single-level-column pandas
|
|
90
|
+
DataFrame.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
frame_pl : polars.DataFrame
|
|
95
|
+
Source frame; must carry ``row_dims + (col_dim, value)``.
|
|
96
|
+
row_dims : sequence of str
|
|
97
|
+
Dim columns that become the (multi-)row-index.
|
|
98
|
+
col_dim : str
|
|
99
|
+
Dim column that becomes the column header.
|
|
100
|
+
value : str, default "value"
|
|
101
|
+
Column carrying the numeric values.
|
|
102
|
+
row_names : sequence of str, optional
|
|
103
|
+
If supplied, the row index level names are set to these.
|
|
104
|
+
Otherwise the source dim names are used (translated via
|
|
105
|
+
:data:`DIM_NAMES`).
|
|
106
|
+
col_name : str, optional
|
|
107
|
+
Column index name (e.g. ``"node"``, ``"entity"``). Defaults
|
|
108
|
+
to the translated ``col_dim``.
|
|
109
|
+
"""
|
|
110
|
+
pdf = _to_pandas(frame_pl)
|
|
111
|
+
pivoted = pdf.pivot(index=list(row_dims), columns=col_dim, values=value)
|
|
112
|
+
pivoted = pivoted.astype(float)
|
|
113
|
+
if row_names is not None:
|
|
114
|
+
pivoted.index.names = list(row_names)
|
|
115
|
+
else:
|
|
116
|
+
pivoted.index.names = [long_dim(d) for d in row_dims]
|
|
117
|
+
pivoted.columns.name = col_name if col_name is not None else long_dim(col_dim)
|
|
118
|
+
return pivoted
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def wide_multi_col(
|
|
122
|
+
frame_pl: "pl.DataFrame",
|
|
123
|
+
*,
|
|
124
|
+
row_dims: Sequence[str],
|
|
125
|
+
col_dims: Sequence[str],
|
|
126
|
+
value: str = "value",
|
|
127
|
+
row_names: Sequence[str] | None = None,
|
|
128
|
+
col_names: Sequence[str] | None = None,
|
|
129
|
+
) -> pd.DataFrame:
|
|
130
|
+
"""Pivot a polars long-form frame to a wide-column pandas DataFrame
|
|
131
|
+
where the columns are a MultiIndex of ``col_dims``.
|
|
132
|
+
|
|
133
|
+
Used for ``flow_min`` / ``flow_max`` / ``process_source_sink_varCost`` /
|
|
134
|
+
``reserve_upDown_group_reservation`` (column MultiIndex of 3 levels).
|
|
135
|
+
"""
|
|
136
|
+
pdf = _to_pandas(frame_pl)
|
|
137
|
+
pivoted = pdf.pivot(index=list(row_dims), columns=list(col_dims), values=value)
|
|
138
|
+
pivoted = pivoted.astype(float)
|
|
139
|
+
if row_names is not None:
|
|
140
|
+
pivoted.index.names = list(row_names)
|
|
141
|
+
else:
|
|
142
|
+
pivoted.index.names = [long_dim(d) for d in row_dims]
|
|
143
|
+
if col_names is not None:
|
|
144
|
+
pivoted.columns.names = list(col_names)
|
|
145
|
+
else:
|
|
146
|
+
pivoted.columns.names = [long_dim(d) for d in col_dims]
|
|
147
|
+
return pivoted
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def series_with_index(
|
|
151
|
+
frame_pl: "pl.DataFrame",
|
|
152
|
+
*,
|
|
153
|
+
dim: str,
|
|
154
|
+
value: str = "value",
|
|
155
|
+
name: str | None = None,
|
|
156
|
+
) -> pd.Series:
|
|
157
|
+
"""Polars long-form (single-dim) → pandas Series with a named index."""
|
|
158
|
+
pdf = _to_pandas(frame_pl)
|
|
159
|
+
s = pdf.set_index(dim)[value].astype(float)
|
|
160
|
+
s.index.name = name if name is not None else long_dim(dim)
|
|
161
|
+
return s
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def series_from_two_dim(
|
|
165
|
+
frame_pl: "pl.DataFrame",
|
|
166
|
+
*,
|
|
167
|
+
dims: Sequence[str],
|
|
168
|
+
value: str = "value",
|
|
169
|
+
names: Sequence[str] | None = None,
|
|
170
|
+
) -> pd.Series:
|
|
171
|
+
"""Polars long-form (multi-dim) → pandas Series with MultiIndex."""
|
|
172
|
+
pdf = _to_pandas(frame_pl)
|
|
173
|
+
s = pdf.set_index(list(dims))[value].astype(float)
|
|
174
|
+
if names is not None:
|
|
175
|
+
s.index.names = list(names)
|
|
176
|
+
else:
|
|
177
|
+
s.index.names = [long_dim(d) for d in dims]
|
|
178
|
+
return s
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def series_with_multi_index(
|
|
182
|
+
frame_pl: "pl.DataFrame",
|
|
183
|
+
*,
|
|
184
|
+
dims: Sequence[str],
|
|
185
|
+
value: str = "value",
|
|
186
|
+
names: Sequence[str] | None = None,
|
|
187
|
+
) -> pd.Series:
|
|
188
|
+
"""polars long-form → pandas Series whose index is a MultiIndex.
|
|
189
|
+
|
|
190
|
+
The legacy CSV path produced these for parameters with
|
|
191
|
+
``header=[0, 1, 2], index_col=0`` and a single ``value`` row that
|
|
192
|
+
pandas exposed as a Series — e.g. ``process_sink_conversion_flow_coeff``,
|
|
193
|
+
``reserve_upDown_group_penalty``. The Series's index carries the
|
|
194
|
+
multi-key tuple per cell; the in-memory equivalent is a Series with
|
|
195
|
+
a MultiIndex.
|
|
196
|
+
"""
|
|
197
|
+
return series_from_two_dim(frame_pl, dims=dims, value=value, names=names)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# ---------------------------------------------------------------------------
|
|
201
|
+
# Set helpers (read_sets)
|
|
202
|
+
# ---------------------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def to_index(frame_pl: "pl.DataFrame", *, dim: str, name: str | None = None) -> pd.Index:
|
|
206
|
+
"""Polars (single-dim) → pandas Index with a named axis."""
|
|
207
|
+
pdf = _to_pandas(frame_pl.select(dim))
|
|
208
|
+
return pd.Index(pdf[dim].tolist(), name=name if name is not None else long_dim(dim))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def to_multi_index(
|
|
212
|
+
frame_pl: "pl.DataFrame",
|
|
213
|
+
*,
|
|
214
|
+
dims: Sequence[str],
|
|
215
|
+
names: Sequence[str] | None = None,
|
|
216
|
+
) -> pd.MultiIndex:
|
|
217
|
+
"""Polars long-form → pandas MultiIndex over the supplied dims.
|
|
218
|
+
|
|
219
|
+
The frame is restricted to the ``dims`` columns (de-duplicated /
|
|
220
|
+
not — left to the caller; we don't introduce extra ordering).
|
|
221
|
+
"""
|
|
222
|
+
pdf = _to_pandas(frame_pl.select(list(dims)))
|
|
223
|
+
if names is None:
|
|
224
|
+
names = [long_dim(d) for d in dims]
|
|
225
|
+
return pd.MultiIndex.from_frame(pdf, names=list(names))
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def empty_index(name: str | None = None, dtype: str = "object") -> pd.Index:
|
|
229
|
+
"""Empty :class:`pd.Index` with a stable ``name`` / ``dtype``."""
|
|
230
|
+
return pd.Index([], dtype=dtype, name=name)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def empty_multi_index(names: Sequence[str]) -> pd.MultiIndex:
|
|
234
|
+
"""Empty :class:`pd.MultiIndex` with ``len(names)`` levels."""
|
|
235
|
+
return pd.MultiIndex.from_arrays([[]] * len(names), names=list(names))
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# ---------------------------------------------------------------------------
|
|
239
|
+
# Solve-name injection
|
|
240
|
+
# ---------------------------------------------------------------------------
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def with_solve_column(frame_pl: "pl.DataFrame", solve_name: str) -> "pl.DataFrame":
|
|
244
|
+
"""Return ``frame_pl`` with a leading ``solve`` column = ``solve_name``.
|
|
245
|
+
|
|
246
|
+
FlexData fields drop the ``solve`` column at load time (see
|
|
247
|
+
``input.py:_read_long`` family). The legacy pandas namespace
|
|
248
|
+
keeps it as the leftmost index level. Inject it back in so the
|
|
249
|
+
pivoted DataFrame's row MultiIndex starts with ``solve``.
|
|
250
|
+
"""
|
|
251
|
+
if frame_pl is None:
|
|
252
|
+
return None
|
|
253
|
+
return frame_pl.with_columns(pl.lit(solve_name).alias("solve"))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def add_solve_to_pandas(df: "pd.DataFrame | pd.Series", solve_name: str) -> "pd.DataFrame | pd.Series":
|
|
257
|
+
"""Prepend a constant ``solve`` level to the row index.
|
|
258
|
+
|
|
259
|
+
Used when the frame already carries other dims and we want to
|
|
260
|
+
inject the solve level *after* pivoting (so wide-column pivots
|
|
261
|
+
don't wear the constant column unnecessarily).
|
|
262
|
+
"""
|
|
263
|
+
if isinstance(df.index, pd.MultiIndex):
|
|
264
|
+
new = pd.MultiIndex.from_arrays(
|
|
265
|
+
[[solve_name] * len(df.index)] + [df.index.get_level_values(i) for i in range(df.index.nlevels)],
|
|
266
|
+
names=["solve"] + list(df.index.names),
|
|
267
|
+
)
|
|
268
|
+
else:
|
|
269
|
+
new = pd.MultiIndex.from_arrays(
|
|
270
|
+
[[solve_name] * len(df.index), list(df.index)],
|
|
271
|
+
names=["solve", df.index.name],
|
|
272
|
+
)
|
|
273
|
+
out = df.copy()
|
|
274
|
+
out.index = new
|
|
275
|
+
return out
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
__all__ = [
|
|
279
|
+
"DIM_NAMES",
|
|
280
|
+
"long_dim",
|
|
281
|
+
"wide_per_entity",
|
|
282
|
+
"wide_multi_col",
|
|
283
|
+
"series_with_index",
|
|
284
|
+
"series_from_two_dim",
|
|
285
|
+
"series_with_multi_index",
|
|
286
|
+
"to_index",
|
|
287
|
+
"to_multi_index",
|
|
288
|
+
"empty_index",
|
|
289
|
+
"empty_multi_index",
|
|
290
|
+
"with_solve_column",
|
|
291
|
+
"add_solve_to_pandas",
|
|
292
|
+
]
|