flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""In-memory implementation of the :class:`InputSource` Protocol.
|
|
2
|
+
|
|
3
|
+
Used by unit tests for processing-layer Param helpers. The caller
|
|
4
|
+
supplies hand-crafted entity / parameter frames already in the
|
|
5
|
+
post-resolution shape — no SpineDB, no scenario filtering, no default
|
|
6
|
+
fill is applied by the reader (the caller is responsible for shaping
|
|
7
|
+
the data exactly as a real source would have produced it).
|
|
8
|
+
|
|
9
|
+
This is the migration-velocity unlock for Γ.1/Γ.2/Γ.3: every Direct /
|
|
10
|
+
Projection / Derived helper takes an :class:`InputSource`, so test
|
|
11
|
+
coverage no longer requires standing up sqlite or generating fixtures.
|
|
12
|
+
See ``audit/db_direct_param_map.md §4.4`` and ``§8.3``.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Any, Mapping
|
|
17
|
+
|
|
18
|
+
import polars as pl
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class InMemoryReader:
|
|
22
|
+
"""Trivial dict-backed :class:`InputSource`.
|
|
23
|
+
|
|
24
|
+
Parameters
|
|
25
|
+
----------
|
|
26
|
+
entities : Mapping[str, pl.DataFrame]
|
|
27
|
+
``{entity_class_name: frame}``. Frame schema follows
|
|
28
|
+
:meth:`InputSource.entities`: one ``[name]`` column for 0-dim
|
|
29
|
+
classes; one column per dim (named after the dim class) for
|
|
30
|
+
n-relationship classes.
|
|
31
|
+
parameters : Mapping[tuple[str, str], pl.DataFrame]
|
|
32
|
+
``{(entity_class, parameter_name): frame}``. Frame schema
|
|
33
|
+
follows :meth:`InputSource.parameter`.
|
|
34
|
+
defaults : Mapping[tuple[str, str], Any] | None
|
|
35
|
+
Optional ``{(entity_class, parameter_name): default_value}``.
|
|
36
|
+
Absent keys imply ``None`` default (§4.5 None-skip branch).
|
|
37
|
+
|
|
38
|
+
Lookups raise :class:`KeyError` on unknown classes / parameters.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
entities: Mapping[str, pl.DataFrame],
|
|
44
|
+
parameters: Mapping[tuple[str, str], pl.DataFrame],
|
|
45
|
+
defaults: Mapping[tuple[str, str], Any] | None = None,
|
|
46
|
+
):
|
|
47
|
+
# Defensive copy: the caller may mutate their inputs after
|
|
48
|
+
# constructing us. Polars frames are cheap to wrap.
|
|
49
|
+
self._entities: dict[str, pl.DataFrame] = dict(entities)
|
|
50
|
+
self._parameters: dict[tuple[str, str], pl.DataFrame] = dict(parameters)
|
|
51
|
+
self._defaults: dict[tuple[str, str], Any] = (
|
|
52
|
+
dict(defaults) if defaults is not None else {}
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# ------------------------------------------------------------------
|
|
56
|
+
# InputSource Protocol
|
|
57
|
+
|
|
58
|
+
def entities(self, entity_class: str) -> pl.DataFrame:
|
|
59
|
+
try:
|
|
60
|
+
return self._entities[entity_class]
|
|
61
|
+
except KeyError:
|
|
62
|
+
raise KeyError(
|
|
63
|
+
f"InMemoryReader: unknown entity_class {entity_class!r}"
|
|
64
|
+
) from None
|
|
65
|
+
|
|
66
|
+
def parameter(self, entity_class: str, parameter_name: str) -> pl.DataFrame:
|
|
67
|
+
key = (entity_class, parameter_name)
|
|
68
|
+
try:
|
|
69
|
+
return self._parameters[key]
|
|
70
|
+
except KeyError:
|
|
71
|
+
raise KeyError(
|
|
72
|
+
f"InMemoryReader: unknown parameter "
|
|
73
|
+
f"({entity_class!r}, {parameter_name!r})"
|
|
74
|
+
) from None
|
|
75
|
+
|
|
76
|
+
def parameter_default(self, entity_class: str, parameter_name: str) -> Any:
|
|
77
|
+
return self._defaults.get((entity_class, parameter_name))
|
|
78
|
+
|
|
79
|
+
def parameter_explicit(self, entity_class: str,
|
|
80
|
+
parameter_name: str) -> pl.DataFrame:
|
|
81
|
+
"""Mirror of :meth:`SpineDbReader.parameter_explicit`.
|
|
82
|
+
|
|
83
|
+
InMemoryReader holds frames the caller passed in directly — the
|
|
84
|
+
Protocol treats those frames as already containing only
|
|
85
|
+
explicit values (no default broadcast). So this is identical
|
|
86
|
+
to :meth:`parameter` for the in-memory case.
|
|
87
|
+
"""
|
|
88
|
+
return self.parameter(entity_class, parameter_name)
|
|
89
|
+
|
|
90
|
+
def parameter_shape_info(self, entity_class: str,
|
|
91
|
+
parameter_name: str) -> "list[str | None]":
|
|
92
|
+
"""Δ.17c — raw per-level ``index_name`` labels for the
|
|
93
|
+
parameter.
|
|
94
|
+
|
|
95
|
+
InMemoryReader callers author frames already in the post-
|
|
96
|
+
resolution shape (column names like ``period`` / ``t`` / etc.)
|
|
97
|
+
— no DB-level metadata is held. We infer the labels from the
|
|
98
|
+
frame's column names: any column named ``period`` →
|
|
99
|
+
``"period"``; any column named ``t`` / ``time`` → ``"time"``;
|
|
100
|
+
anything else is propagated as-is so the resolver can flag it.
|
|
101
|
+
|
|
102
|
+
The frame's entity-dim columns are excluded by walking the
|
|
103
|
+
registered entities frame for the same class.
|
|
104
|
+
"""
|
|
105
|
+
df = self.parameter(entity_class, parameter_name)
|
|
106
|
+
try:
|
|
107
|
+
ent_df = self.entities(entity_class)
|
|
108
|
+
ent_cols = set(ent_df.columns)
|
|
109
|
+
except KeyError:
|
|
110
|
+
ent_cols = {"name"}
|
|
111
|
+
out: list[str | None] = []
|
|
112
|
+
for c in df.columns:
|
|
113
|
+
if c in ent_cols or c == "value":
|
|
114
|
+
continue
|
|
115
|
+
if c == "period":
|
|
116
|
+
out.append("period")
|
|
117
|
+
elif c in ("t", "time"):
|
|
118
|
+
out.append("time")
|
|
119
|
+
else:
|
|
120
|
+
# Unknown column → caller raises; pass through raw.
|
|
121
|
+
out.append(c)
|
|
122
|
+
return out
|
|
123
|
+
|
|
124
|
+
# ------------------------------------------------------------------
|
|
125
|
+
# Diagnostics
|
|
126
|
+
|
|
127
|
+
def __repr__(self) -> str:
|
|
128
|
+
return (
|
|
129
|
+
f"InMemoryReader(classes={len(self._entities)}, "
|
|
130
|
+
f"params={len(self._parameters)}, "
|
|
131
|
+
f"defaults={len(self._defaults)})"
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def known_classes(self) -> list[str]:
|
|
136
|
+
return sorted(self._entities)
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def known_parameters(self) -> list[tuple[str, str]]:
|
|
140
|
+
return sorted(self._parameters)
|
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
"""Source abstractions for flextool's input data.
|
|
2
|
+
|
|
3
|
+
This module hosts **two** Protocols, used by separate phases of the
|
|
4
|
+
DB-direct migration:
|
|
5
|
+
|
|
6
|
+
* :class:`FlexInputSource` — the **CSV-shaped** source used for the
|
|
7
|
+
fixture / pre-built-workdir path (:class:`CsvSource`). Materialises
|
|
8
|
+
flextool's ``input/`` + ``solve_data/`` CSV layout on disk;
|
|
9
|
+
``load_flextool`` walks them with ``polars.read_csv``.
|
|
10
|
+
* :class:`InputSource` — the **per-(entity_class, parameter_name)
|
|
11
|
+
frame** Protocol introduced in Γ.1 of the deeper DB-direct migration
|
|
12
|
+
(audit/db_direct_param_map.md §4.3). Implementations
|
|
13
|
+
(:class:`flextool._spinedb_reader.SpineDbReader`,
|
|
14
|
+
:class:`flextool._inmemory_reader.InMemoryReader`) return individual
|
|
15
|
+
parameter frames in their natural shape, scenario-resolved, with
|
|
16
|
+
defaults applied per §4.5. This is the abstraction Γ.1/Γ.2/Γ.3
|
|
17
|
+
helpers compose against.
|
|
18
|
+
|
|
19
|
+
The two Protocols coexist: ``FlexInputSource`` keeps the existing
|
|
20
|
+
CSV-shaped loader for fixture workdirs, while ``InputSource`` is the
|
|
21
|
+
DB-direct abstraction used by the live cascade
|
|
22
|
+
(:func:`flextool.engine_polars.run_chain_from_db`).
|
|
23
|
+
|
|
24
|
+
CSV-shaped source notes:
|
|
25
|
+
|
|
26
|
+
Today's downstream consumer (:func:`flextool.input.load_flextool`) reads
|
|
27
|
+
CSVs via ``polars.read_csv`` directly off the directory tree, so the
|
|
28
|
+
Protocol exposes both:
|
|
29
|
+
|
|
30
|
+
* :pyattr:`FlexInputSource.input_dir` and
|
|
31
|
+
:pyattr:`FlexInputSource.solve_data_dir` — Paths to the materialised
|
|
32
|
+
CSV directories (the existing reader walks these as before).
|
|
33
|
+
* :meth:`FlexInputSource.get` — convenience accessor for callers that
|
|
34
|
+
want a frame by ``(kind, name)`` without dealing with paths.
|
|
35
|
+
|
|
36
|
+
For :class:`CsvSource` the directories are just ``workdir/input`` and
|
|
37
|
+
``workdir/solve_data`` with no materialisation work.
|
|
38
|
+
"""
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import logging
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Any, Literal, Protocol, runtime_checkable
|
|
44
|
+
|
|
45
|
+
import polars as pl
|
|
46
|
+
import polars.exceptions as pl_exc
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
_LOGGER = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
Kind = Literal["input", "solve_data"]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
_active_cache: dict[Path, pl.DataFrame] | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _read_csv_file(path: "Path | str") -> pl.DataFrame:
|
|
59
|
+
"""Single residual ``polars.read_csv`` site for the engine_polars
|
|
60
|
+
package.
|
|
61
|
+
|
|
62
|
+
CSV-retirement (Γ.8.F) gates every workdir CSV read in the loader
|
|
63
|
+
path through this helper so the package-wide grep for
|
|
64
|
+
``pl.read_csv`` returns only the ``CsvSource``-internal sites
|
|
65
|
+
(``CsvSource.get`` plus this helper).
|
|
66
|
+
|
|
67
|
+
Δ.12a — when a per-solve cache is active (set via
|
|
68
|
+
:func:`_install_csv_cache` from the ``SolveContext`` constructor),
|
|
69
|
+
repeated reads of the same absolute path hit memory.
|
|
70
|
+
"""
|
|
71
|
+
if _active_cache is not None:
|
|
72
|
+
# Use the str form as cache key — avoids the per-call ``Path.resolve``
|
|
73
|
+
# syscall (which adds ~50µs each and dominates the cache miss path
|
|
74
|
+
# for small fixtures with few duplicate reads). Different string
|
|
75
|
+
# forms of the same file (e.g. ``./x.csv`` vs ``x.csv``) miss the
|
|
76
|
+
# cache but the loader path always constructs paths from the same
|
|
77
|
+
# workdir prefix so collisions are negligible in practice.
|
|
78
|
+
key = str(path)
|
|
79
|
+
cached = _active_cache.get(key)
|
|
80
|
+
if cached is not None:
|
|
81
|
+
return cached
|
|
82
|
+
df = pl.read_csv(path)
|
|
83
|
+
_active_cache[key] = df
|
|
84
|
+
return df
|
|
85
|
+
return pl.read_csv(path)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def read_csv_fallback(path: "Path | str") -> pl.DataFrame:
|
|
89
|
+
"""Off-cascade disk read of a single CSV.
|
|
90
|
+
|
|
91
|
+
Reserved for callers in :pyfile:`flextool/engine_polars/input.py`
|
|
92
|
+
that still serve workdir-only loader-unit tests. Cascade code MUST
|
|
93
|
+
go through :class:`FlexDataProvider`; this is the single sanctioned
|
|
94
|
+
entry point for the residual disk-fallback path so the Rule 1
|
|
95
|
+
invariant scan can confirm input.py never calls ``_read_csv_file``
|
|
96
|
+
or ``pl.read_csv`` directly.
|
|
97
|
+
"""
|
|
98
|
+
return _read_csv_file(path)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def seed_provider_from_dir(
|
|
102
|
+
provider,
|
|
103
|
+
directory: "Path | str",
|
|
104
|
+
kind: str,
|
|
105
|
+
*,
|
|
106
|
+
names: "tuple[str, ...] | None" = None,
|
|
107
|
+
) -> int:
|
|
108
|
+
"""Off-cascade test/bridge helper: populate *provider* by reading
|
|
109
|
+
CSV files under *directory* and keying them under both
|
|
110
|
+
``"<stem>"`` and ``"{kind}/<stem>"``.
|
|
111
|
+
|
|
112
|
+
Mirrors the dual-key convention used by :func:`capture_frames`.
|
|
113
|
+
Returns the count of files seeded. Missing directories are a
|
|
114
|
+
no-op (return 0). Callers in cascade code must NOT reach for
|
|
115
|
+
this helper: it exists for test fixtures, region-decomposition
|
|
116
|
+
seeding, and the off-cascade workdir bridge.
|
|
117
|
+
|
|
118
|
+
Parameters
|
|
119
|
+
----------
|
|
120
|
+
provider
|
|
121
|
+
:class:`FlexDataProvider` to populate.
|
|
122
|
+
directory
|
|
123
|
+
Source directory.
|
|
124
|
+
kind
|
|
125
|
+
Prefix for the parent-qualified key (``"input"`` or
|
|
126
|
+
``"solve_data"``).
|
|
127
|
+
names
|
|
128
|
+
Optional explicit allow-list of stems (without ``.csv``) to
|
|
129
|
+
consume. When ``None`` every ``*.csv`` is read. Use the
|
|
130
|
+
explicit form to skip non-canonical files in directories that
|
|
131
|
+
also carry ragged or human-readable artefacts (e.g.
|
|
132
|
+
``solve_progress.csv``).
|
|
133
|
+
"""
|
|
134
|
+
d = Path(directory)
|
|
135
|
+
if not d.exists() or not d.is_dir():
|
|
136
|
+
return 0
|
|
137
|
+
if names is not None:
|
|
138
|
+
targets = [d / f"{n}.csv" for n in names if (d / f"{n}.csv").exists()]
|
|
139
|
+
else:
|
|
140
|
+
targets = sorted(d.glob("*.csv"))
|
|
141
|
+
seeded = 0
|
|
142
|
+
for p in targets:
|
|
143
|
+
try:
|
|
144
|
+
df = _read_csv_file(p)
|
|
145
|
+
provider.put(f"{kind}/{p.stem}", df)
|
|
146
|
+
except (pl_exc.ComputeError, pl_exc.NoDataError) as exc:
|
|
147
|
+
_LOGGER.warning(
|
|
148
|
+
"seed_provider_from_dir: skipping malformed CSV %s "
|
|
149
|
+
"(%s: %s)",
|
|
150
|
+
p,
|
|
151
|
+
type(exc).__name__,
|
|
152
|
+
exc,
|
|
153
|
+
)
|
|
154
|
+
continue
|
|
155
|
+
except Exception as exc: # noqa: BLE001 — log + continue for stray files
|
|
156
|
+
_LOGGER.warning(
|
|
157
|
+
"seed_provider_from_dir: skipping unreadable CSV %s "
|
|
158
|
+
"(%s: %s)",
|
|
159
|
+
p,
|
|
160
|
+
type(exc).__name__,
|
|
161
|
+
exc,
|
|
162
|
+
)
|
|
163
|
+
continue
|
|
164
|
+
seeded += 1
|
|
165
|
+
return seeded
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _install_csv_cache(cache: "dict[Path, pl.DataFrame] | None") -> None:
|
|
169
|
+
"""Δ.12a — install / clear the process-level CSV-read cache.
|
|
170
|
+
|
|
171
|
+
Called by ``SolveContext.__enter__`` / ``__exit__`` (or the
|
|
172
|
+
explicit ``activate_cache`` / ``deactivate_cache`` helpers) to
|
|
173
|
+
install the per-solve cache so :func:`_read_csv_file` calls in any
|
|
174
|
+
helper hit memory on repeats.
|
|
175
|
+
|
|
176
|
+
Pass ``None`` to disable caching (default). Multiple
|
|
177
|
+
activate/deactivate cycles within a single process are supported;
|
|
178
|
+
nesting is the caller's responsibility (typically via the SolveContext
|
|
179
|
+
context-manager boundary which is one-deep per solve).
|
|
180
|
+
"""
|
|
181
|
+
global _active_cache
|
|
182
|
+
_active_cache = cache
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------------------
|
|
186
|
+
# Γ.1 — per-(entity_class, parameter_name) Protocol
|
|
187
|
+
# ---------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@runtime_checkable
|
|
191
|
+
class InputSource(Protocol):
|
|
192
|
+
"""Source-agnostic per-(entity_class, parameter_name) read API.
|
|
193
|
+
|
|
194
|
+
Implementations are bound to a single scenario at construction;
|
|
195
|
+
:meth:`entities` and :meth:`parameter` return scenario-resolved
|
|
196
|
+
frames with defaults applied per §4.5 of the audit spec.
|
|
197
|
+
|
|
198
|
+
Frames are deterministic in row order (sorted by entity dim columns
|
|
199
|
+
first, then index columns) so per-Param parity assertions are
|
|
200
|
+
stable across runs.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def entities(self, entity_class: str) -> pl.DataFrame:
|
|
204
|
+
"""Return the entity universe for *entity_class*.
|
|
205
|
+
|
|
206
|
+
Schema:
|
|
207
|
+
* 0-dim object class (e.g. ``"node"``): one column ``[name]``.
|
|
208
|
+
* n-relationship class (e.g.
|
|
209
|
+
``"commodity__node"``, ``"connection__node__node"``):
|
|
210
|
+
one column per dim, named after the dim's class. Repeated
|
|
211
|
+
dim classes are disambiguated by appending a 1-based
|
|
212
|
+
suffix (e.g. ``connection__node__node`` →
|
|
213
|
+
``[connection, node_1, node_2]``).
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
def parameter(self,
|
|
217
|
+
entity_class: str,
|
|
218
|
+
parameter_name: str,
|
|
219
|
+
) -> pl.DataFrame:
|
|
220
|
+
"""Return the parameter frame for ``(entity_class, parameter_name)``.
|
|
221
|
+
|
|
222
|
+
Schema:
|
|
223
|
+
* Entity dim columns from :meth:`entities`,
|
|
224
|
+
* Followed by index columns implied by the parameter's
|
|
225
|
+
value type (period / tier / t / branch / sub_index, in
|
|
226
|
+
the parameter's natural index order),
|
|
227
|
+
* Followed by a single ``value`` column (typed:
|
|
228
|
+
``pl.Float64`` for numerics, ``pl.Boolean`` /
|
|
229
|
+
``pl.Utf8`` for the occasional non-numeric).
|
|
230
|
+
|
|
231
|
+
Default policy (§4.5):
|
|
232
|
+
* ``parameter_definition.default_value is None`` → return
|
|
233
|
+
only entities with explicit overrides; no fill-in rows.
|
|
234
|
+
* Scalar default + scalar parameter → broadcast: one row
|
|
235
|
+
per entity with the default for entities that have no
|
|
236
|
+
override.
|
|
237
|
+
* Scalar default + indexed parameter → return only entities
|
|
238
|
+
with overrides; the default is exposed via
|
|
239
|
+
:meth:`parameter_default` so helpers can ``fill_null``
|
|
240
|
+
against their own index frames.
|
|
241
|
+
"""
|
|
242
|
+
|
|
243
|
+
def parameter_default(self,
|
|
244
|
+
entity_class: str,
|
|
245
|
+
parameter_name: str,
|
|
246
|
+
) -> Any:
|
|
247
|
+
"""Return the parameter's scalar default, or ``None``.
|
|
248
|
+
|
|
249
|
+
Used by helpers to ``fill_null`` against their own index
|
|
250
|
+
frames in the scalar-default-on-indexed case (§4.5).
|
|
251
|
+
"""
|
|
252
|
+
|
|
253
|
+
def parameter_shape_info(self,
|
|
254
|
+
entity_class: str,
|
|
255
|
+
parameter_name: str,
|
|
256
|
+
) -> "list[str | None]":
|
|
257
|
+
"""Return the raw per-level ``Map.index_name`` labels for the
|
|
258
|
+
parameter (Δ.17c).
|
|
259
|
+
|
|
260
|
+
Schema:
|
|
261
|
+
|
|
262
|
+
* Empty list (``[]``) — scalar parameter (no Map nesting).
|
|
263
|
+
* One entry per Map nesting level, in order from outermost to
|
|
264
|
+
innermost. Entries are the raw labels exactly as authored
|
|
265
|
+
in the source database (``None`` when unset / empty).
|
|
266
|
+
|
|
267
|
+
Used by :func:`flextool.engine_polars._param_shapes.resolve_param_shape`
|
|
268
|
+
to validate a parameter's actual shape against an explicit
|
|
269
|
+
per-parameter allow-list. See the Δ.17c dispatch / open-issues
|
|
270
|
+
doc for the user advice that mandated this.
|
|
271
|
+
|
|
272
|
+
Implementations that lack explicit DB metadata (e.g.
|
|
273
|
+
:class:`InMemoryReader` in unit tests) infer labels from the
|
|
274
|
+
parameter frame's column names — see the per-implementation
|
|
275
|
+
docstring for details.
|
|
276
|
+
"""
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@runtime_checkable
|
|
280
|
+
class FlexInputSource(Protocol):
|
|
281
|
+
"""Protocol every input-source must satisfy.
|
|
282
|
+
|
|
283
|
+
Once :pyattr:`input_dir` / :pyattr:`solve_data_dir` return, the
|
|
284
|
+
directories must be populated and ready for the existing CSV
|
|
285
|
+
reader to walk.
|
|
286
|
+
"""
|
|
287
|
+
|
|
288
|
+
@property
|
|
289
|
+
def input_dir(self) -> Path: ...
|
|
290
|
+
@property
|
|
291
|
+
def solve_data_dir(self) -> Path: ...
|
|
292
|
+
|
|
293
|
+
def get(self, kind: Kind, name: str) -> pl.DataFrame | None:
|
|
294
|
+
"""Return the named frame from ``input/`` or ``solve_data/``.
|
|
295
|
+
|
|
296
|
+
``name`` may be given with or without the ``.csv`` suffix.
|
|
297
|
+
Returns ``None`` when the file is absent. Empty (header-only)
|
|
298
|
+
files yield an empty DataFrame, consistent with
|
|
299
|
+
``polars.read_csv`` behaviour.
|
|
300
|
+
"""
|
|
301
|
+
...
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
class CsvSource:
|
|
305
|
+
"""Wraps a flextool workdir on disk (the pre-DB-migration layout).
|
|
306
|
+
|
|
307
|
+
Construction is trivial and read-only; both directories must exist
|
|
308
|
+
or be missing in the same way they would be when calling
|
|
309
|
+
``load_flextool(workdir)`` directly.
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
def __init__(self, workdir: Path | str):
|
|
313
|
+
self._workdir = Path(workdir)
|
|
314
|
+
|
|
315
|
+
@property
|
|
316
|
+
def workdir(self) -> Path:
|
|
317
|
+
return self._workdir
|
|
318
|
+
|
|
319
|
+
@property
|
|
320
|
+
def input_dir(self) -> Path:
|
|
321
|
+
return self._workdir / "input"
|
|
322
|
+
|
|
323
|
+
@property
|
|
324
|
+
def solve_data_dir(self) -> Path:
|
|
325
|
+
return self._workdir / "solve_data"
|
|
326
|
+
|
|
327
|
+
def get(self, kind: Kind, name: str) -> pl.DataFrame | None:
|
|
328
|
+
d = self.input_dir if kind == "input" else self.solve_data_dir
|
|
329
|
+
fname = name if name.endswith(".csv") else f"{name}.csv"
|
|
330
|
+
path = d / fname
|
|
331
|
+
if not path.exists():
|
|
332
|
+
return None
|
|
333
|
+
return _read_csv_file(path)
|
|
334
|
+
|
|
335
|
+
def __repr__(self) -> str:
|
|
336
|
+
return f"CsvSource(workdir={self._workdir!s})"
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Workdir-CSV seed readers for the invest/divest cascade.
|
|
2
|
+
|
|
3
|
+
These helpers exist for one reason: the **synthetic per-sub-solve**
|
|
4
|
+
case. When ``_apply_db_overrides`` detects an active solve whose name
|
|
5
|
+
does not appear in Spine (per-period sub-solves like
|
|
6
|
+
``invest_5weeks_p2020`` synthesised at runtime by the orchestrator),
|
|
7
|
+
the per-solve override chain ``apply_derived_a..g`` is skipped — its
|
|
8
|
+
``_solve_periods(source, active_solve, ...)`` lookups would return
|
|
9
|
+
empty and wipe out the legitimate invest activity captured in the
|
|
10
|
+
workdir snapshot.
|
|
11
|
+
|
|
12
|
+
Post-Step-2.5 these helpers consume the canonical
|
|
13
|
+
``solve_data/*.csv`` frames exclusively through the
|
|
14
|
+
:class:`FlexDataProvider`. The disk-fallback arms that previously
|
|
15
|
+
re-read ``<workdir>/solve_data/<name>.csv`` from disk are gone — the
|
|
16
|
+
writer cascade (``_emit_per_solve.write_invest_csvs`` and friends)
|
|
17
|
+
seeds every required key in the Provider before this loader runs.
|
|
18
|
+
|
|
19
|
+
When the active solve **is** in Spine, the override chain
|
|
20
|
+
(``apply_derived_c``) overlays its own values on top of these seeds,
|
|
21
|
+
so the helpers are functionally seeds-only on the non-synthetic path.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
import polars as pl
|
|
29
|
+
|
|
30
|
+
from ._axis_enums import (
|
|
31
|
+
cast_dim,
|
|
32
|
+
rename_to_axis,
|
|
33
|
+
schema_dtype,
|
|
34
|
+
)
|
|
35
|
+
from ._emit_provider_io import _provider_key
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Substrate handle for the cascade-wide axis enum vocabulary.
|
|
39
|
+
# Bare ``None`` here; ``cast_dim`` / ``schema_dtype`` in
|
|
40
|
+
# ``_axis_enums`` fall back to ``_LIVE_AXIS_ENUMS_CTX`` (the live
|
|
41
|
+
# ContextVar) when this is ``None``, so substrate sites pick up
|
|
42
|
+
# activation set by ``load_flextool`` automatically.
|
|
43
|
+
_enums: "dict | None" = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _provider_get(provider, path: "Path") -> "pl.DataFrame | None":
|
|
47
|
+
"""Provider-only fetch. Returns ``None`` when the Provider is
|
|
48
|
+
missing or doesn't carry *path*'s canonical key.
|
|
49
|
+
"""
|
|
50
|
+
if provider is None:
|
|
51
|
+
return None
|
|
52
|
+
key = _provider_key(path)
|
|
53
|
+
if not provider.has(key):
|
|
54
|
+
return None
|
|
55
|
+
return provider.get(key)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
# (e, d) / (p, d) / (n, d) set frames
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def read_invest_set(workdir_solve_data: Path, name: str,
|
|
64
|
+
kind_col: str, *, provider=None) -> pl.DataFrame:
|
|
65
|
+
"""Read ``ed_invest.csv`` / ``ed_divest.csv`` and rename the
|
|
66
|
+
entity-axis column to *kind_col* (``e``).
|
|
67
|
+
|
|
68
|
+
``ed_invest.csv`` etc. are the canonical Python-preprocessing
|
|
69
|
+
outputs that ``flextool.mod`` reads via ``table data IN``
|
|
70
|
+
(flextool.mod:1428). The ``solve__``-prefixed twins are .mod
|
|
71
|
+
printf debug-exports of the *current solve's* subset and must NOT
|
|
72
|
+
be used as inputs — using them silently drops invest variables for
|
|
73
|
+
non-realized periods.
|
|
74
|
+
"""
|
|
75
|
+
empty = pl.DataFrame(schema={kind_col: schema_dtype(_enums, kind_col),
|
|
76
|
+
"d": schema_dtype(_enums, "d")})
|
|
77
|
+
path = workdir_solve_data / f"{name}.csv"
|
|
78
|
+
df = _provider_get(provider, path)
|
|
79
|
+
if df is None or df.height == 0:
|
|
80
|
+
return empty
|
|
81
|
+
rename_src = ("entity" if "entity" in df.columns
|
|
82
|
+
else "node" if "node" in df.columns
|
|
83
|
+
else "process")
|
|
84
|
+
return df.pipe(rename_to_axis,
|
|
85
|
+
{rename_src: kind_col, "period": "d"}).select(
|
|
86
|
+
cast_dim(pl.col(kind_col), _enums, kind_col),
|
|
87
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def read_forbidden_no_investment(workdir_solve_data: Path,
|
|
92
|
+
*, provider=None) -> pl.DataFrame:
|
|
93
|
+
"""Read ``ed_invest_forbidden_no_investment.csv``.
|
|
94
|
+
|
|
95
|
+
Entities that may NOT invest in specified periods
|
|
96
|
+
(lifetime_method=no_investment combined with
|
|
97
|
+
invest_method=invest_no_limit at periods where the lifetime window
|
|
98
|
+
disallows new build). flextool encodes this as
|
|
99
|
+
``fix_v_invest_no_investment_eq`` pinning the variable to 0; we
|
|
100
|
+
achieve the same effect by removing the (entity, period) tuple
|
|
101
|
+
from every invest set so the variable is never created.
|
|
102
|
+
|
|
103
|
+
Returns an empty (e, d) frame when the Provider doesn't carry the
|
|
104
|
+
key or it's empty.
|
|
105
|
+
"""
|
|
106
|
+
empty = pl.DataFrame(schema={"e": schema_dtype(_enums, "e"),
|
|
107
|
+
"d": schema_dtype(_enums, "d")})
|
|
108
|
+
path = workdir_solve_data / "ed_invest_forbidden_no_investment.csv"
|
|
109
|
+
df = _provider_get(provider, path)
|
|
110
|
+
if df is None or df.height == 0:
|
|
111
|
+
return empty
|
|
112
|
+
return df.pipe(rename_to_axis, {"entity": "e", "period": "d"}).select(
|
|
113
|
+
cast_dim(pl.col("e"), _enums, "e"),
|
|
114
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def read_set_seed(workdir_solve_data: Path, name: str,
|
|
119
|
+
kind_col: str, *, provider=None) -> pl.DataFrame:
|
|
120
|
+
"""Read ``pd_invest.csv`` / ``pd_divest.csv`` / ``nd_invest.csv``
|
|
121
|
+
/ ``nd_divest.csv``. Each is a per-(entity, period) seed frame.
|
|
122
|
+
"""
|
|
123
|
+
empty = pl.DataFrame(schema={kind_col: schema_dtype(_enums, kind_col),
|
|
124
|
+
"d": schema_dtype(_enums, "d")})
|
|
125
|
+
path = workdir_solve_data / f"{name}.csv"
|
|
126
|
+
df = _provider_get(provider, path)
|
|
127
|
+
if df is None or df.height == 0:
|
|
128
|
+
return empty
|
|
129
|
+
rename_src = ("entity" if "entity" in df.columns
|
|
130
|
+
else "node" if "node" in df.columns
|
|
131
|
+
else "process" if "process" in df.columns
|
|
132
|
+
else None)
|
|
133
|
+
if rename_src is None or "period" not in df.columns:
|
|
134
|
+
return empty
|
|
135
|
+
return df.pipe(rename_to_axis,
|
|
136
|
+
{rename_src: kind_col, "period": "d"}).select(
|
|
137
|
+
cast_dim(pl.col(kind_col), _enums, kind_col),
|
|
138
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def read_edd_invest(workdir_solve_data: Path,
|
|
143
|
+
*, provider=None) -> pl.DataFrame:
|
|
144
|
+
"""Read ``edd_invest.csv`` — (entity, d_invest, period) triple set.
|
|
145
|
+
|
|
146
|
+
Canonical CSV uses ``period_history`` for d_invest; tolerate both
|
|
147
|
+
column names.
|
|
148
|
+
"""
|
|
149
|
+
empty = pl.DataFrame(schema={
|
|
150
|
+
"e": schema_dtype(_enums, "e"),
|
|
151
|
+
"d_invest": schema_dtype(_enums, "d_invest"),
|
|
152
|
+
"d": schema_dtype(_enums, "d")})
|
|
153
|
+
path = workdir_solve_data / "edd_invest.csv"
|
|
154
|
+
df = _provider_get(provider, path)
|
|
155
|
+
if df is None or df.height == 0:
|
|
156
|
+
return empty
|
|
157
|
+
ren = {}
|
|
158
|
+
if "entity" in df.columns:
|
|
159
|
+
ren["entity"] = "e"
|
|
160
|
+
if "period_history" in df.columns:
|
|
161
|
+
ren["period_history"] = "d_invest"
|
|
162
|
+
if "period" in df.columns:
|
|
163
|
+
ren["period"] = "d"
|
|
164
|
+
df = df.pipe(rename_to_axis, ren)
|
|
165
|
+
if not {"e", "d_invest", "d"}.issubset(df.columns):
|
|
166
|
+
return empty
|
|
167
|
+
return df.select(
|
|
168
|
+
cast_dim(pl.col("e"), _enums, "e"),
|
|
169
|
+
cast_dim(pl.col("d_invest"), _enums, "d_invest"),
|
|
170
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def read_period_set(workdir_solve_data: Path, name: str,
|
|
175
|
+
*, provider=None) -> pl.DataFrame | None:
|
|
176
|
+
"""Read ``ed_invest_period.csv`` / ``ed_divest_period.csv`` — the
|
|
177
|
+
(entity, period) tuples with per-period invest / divest caps.
|
|
178
|
+
|
|
179
|
+
Returns None (not empty) when the Provider doesn't carry the key
|
|
180
|
+
or it's empty so the seed assignment in ``_load_invest`` mirrors
|
|
181
|
+
the original ``None``-or-non-empty contract that downstream
|
|
182
|
+
consumers (``model.py:1517``) gate on.
|
|
183
|
+
"""
|
|
184
|
+
path = workdir_solve_data / f"{name}.csv"
|
|
185
|
+
df = _provider_get(provider, path)
|
|
186
|
+
if df is None or df.height == 0:
|
|
187
|
+
return None
|
|
188
|
+
return df.pipe(rename_to_axis, {"entity": "e", "period": "d"}).select(
|
|
189
|
+
cast_dim(pl.col("e"), _enums, "e"),
|
|
190
|
+
cast_dim(pl.col("d"), _enums, "d"),
|
|
191
|
+
)
|