flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,665 @@
|
|
|
1
|
+
"""LP row-scaling capacity proxies — per-solve emitters.
|
|
2
|
+
|
|
3
|
+
Called per-solve from ``_emit_solve_time.run`` via
|
|
4
|
+
:func:`emit_lp_scaling_params`.
|
|
5
|
+
|
|
6
|
+
The 9 CSVs compute a power-of-10 capacity proxy per (node, period) and
|
|
7
|
+
(group, period) so the LP solver sees coefficient ranges compressed
|
|
8
|
+
near O(1). When ``p_use_row_scaling[solve] < 0.5`` the scalers collapse
|
|
9
|
+
to 1 (Mode A, pre-Agent-5 behaviour).
|
|
10
|
+
|
|
11
|
+
All output values are floats, pre-stringified with ``repr(float(v))``
|
|
12
|
+
for byte-identical parity with the legacy emitter.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import math
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import polars as pl
|
|
20
|
+
|
|
21
|
+
from flextool.engine_polars._emit_provider_io import _emit
|
|
22
|
+
from flextool.engine_polars._vectorize import _render_value_column
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
# CSV I/O helpers — same conventions as ``_emit_per_solve``.
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _read_csv(path: Path, columns: list[str],
|
|
31
|
+
*, provider: "object | None" = None) -> pl.DataFrame:
|
|
32
|
+
"""Provider-only — Step 2.5 Phase C dropped the disk-fallback arm."""
|
|
33
|
+
from flextool.engine_polars._emit_provider_io import (
|
|
34
|
+
_provider_key,
|
|
35
|
+
_provider_lookup_positional,
|
|
36
|
+
)
|
|
37
|
+
seeded = _provider_lookup_positional(
|
|
38
|
+
provider, _provider_key(path), path, columns,
|
|
39
|
+
)
|
|
40
|
+
if seeded is not None:
|
|
41
|
+
return seeded
|
|
42
|
+
return pl.DataFrame(
|
|
43
|
+
{c: [] for c in columns},
|
|
44
|
+
schema={c: pl.Utf8 for c in columns},
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _read_singles(path: Path,
|
|
49
|
+
*, provider: "object | None" = None) -> list[str]:
|
|
50
|
+
df = _read_csv(path, ["v"], provider=provider)
|
|
51
|
+
return [v for v in df["v"].to_list() if v]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _read_pairs(path: Path,
|
|
55
|
+
*, provider: "object | None" = None) -> list[tuple[str, str]]:
|
|
56
|
+
df = _read_csv(path, ["a", "b"], provider=provider)
|
|
57
|
+
return [(a, b) for a, b in zip(df["a"].to_list(), df["b"].to_list())
|
|
58
|
+
if a and b]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _read_keyed_value(path: Path,
|
|
62
|
+
*, provider: "object | None" = None) -> dict[str, float]:
|
|
63
|
+
df = _read_csv(path, ["key", "value"], provider=provider)
|
|
64
|
+
out: dict[str, float] = {}
|
|
65
|
+
for k, v in zip(df["key"].to_list(), df["value"].to_list()):
|
|
66
|
+
if not k or v is None or v == "":
|
|
67
|
+
continue
|
|
68
|
+
try:
|
|
69
|
+
out[k] = float(v)
|
|
70
|
+
except ValueError:
|
|
71
|
+
continue
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _read_triples(path: Path,
|
|
76
|
+
*, provider: "object | None" = None) -> list[tuple[str, str, str]]:
|
|
77
|
+
df = _read_csv(path, ["a", "b", "c"], provider=provider)
|
|
78
|
+
return [(a, b, c) for a, b, c in zip(df["a"].to_list(),
|
|
79
|
+
df["b"].to_list(),
|
|
80
|
+
df["c"].to_list())
|
|
81
|
+
if a and b and c]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _read_node_period_value(path: Path,
|
|
85
|
+
*, provider: "object | None" = None,
|
|
86
|
+
) -> dict[tuple[str, str], float]:
|
|
87
|
+
df = _read_csv(path, ["node", "period", "value"], provider=provider)
|
|
88
|
+
out: dict[tuple[str, str], float] = {}
|
|
89
|
+
for n, d, v in zip(df["node"].to_list(),
|
|
90
|
+
df["period"].to_list(),
|
|
91
|
+
df["value"].to_list()):
|
|
92
|
+
if not n or not d:
|
|
93
|
+
continue
|
|
94
|
+
try:
|
|
95
|
+
out[(n, d)] = float(v)
|
|
96
|
+
except (ValueError, TypeError):
|
|
97
|
+
continue
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _rows_to_frame(
|
|
102
|
+
header: tuple[str, str, str],
|
|
103
|
+
rows: list[tuple[str, str, object]],
|
|
104
|
+
) -> pl.DataFrame:
|
|
105
|
+
"""Materialise (k1, k2, repr(value)) rows as an all-Utf8 frame.
|
|
106
|
+
|
|
107
|
+
Pre-stringifies values via ``repr(v)`` (NOT ``repr(float(v))``) so
|
|
108
|
+
that an empty ``sum(())`` -> int 0 round-trips as ``"0"`` and a
|
|
109
|
+
float arithmetic result round-trips as ``"1.0"`` — matching the
|
|
110
|
+
legacy 3-col emitter's byte shape exactly.
|
|
111
|
+
"""
|
|
112
|
+
return pl.DataFrame(
|
|
113
|
+
{
|
|
114
|
+
header[0]: [r[0] for r in rows],
|
|
115
|
+
header[1]: [r[1] for r in rows],
|
|
116
|
+
header[2]: [repr(r[2]) for r in rows],
|
|
117
|
+
},
|
|
118
|
+
schema={h: pl.Utf8 for h in header},
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------------------------------------------------------------------
|
|
123
|
+
# Numeric primitive.
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _pow10_round_clamped(v: float) -> float:
|
|
128
|
+
"""``max(1e-6, min(1e9, 10 ** round(log10(v))))`` with v ≤ 0 → 1."""
|
|
129
|
+
if v <= 0:
|
|
130
|
+
return 1.0
|
|
131
|
+
return max(1e-6, min(1e9, 10.0 ** round(math.log10(v))))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Top-level writer.
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _compute_lp_scaling_frames(
|
|
140
|
+
input_dir: Path, solve_data_dir: Path,
|
|
141
|
+
*, provider: "object | None" = None,
|
|
142
|
+
) -> dict[str, pl.DataFrame]:
|
|
143
|
+
"""Compute every LP-scaling CSV in one pass, returning a dict keyed
|
|
144
|
+
by output basename.
|
|
145
|
+
|
|
146
|
+
Retained as the parity ORACLE for
|
|
147
|
+
:func:`_compute_lp_scaling_frames_vectorized` (see
|
|
148
|
+
``tests/engine_polars/test_vectorize_lp_scaling_parity.py``). The 9
|
|
149
|
+
CSVs share heavy cross-CSV state (``raw_dict`` -> ``pow10_dict`` ->
|
|
150
|
+
``ncfs_dict`` -> ``grp_raw`` -> ``grp_pow10`` -> group_capacity);
|
|
151
|
+
the dict-of-frames adapter pattern from the audit doc is the
|
|
152
|
+
appropriate refactor here.
|
|
153
|
+
"""
|
|
154
|
+
out: dict[str, pl.DataFrame] = {}
|
|
155
|
+
|
|
156
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
157
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
158
|
+
period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
|
|
159
|
+
provider=provider)
|
|
160
|
+
|
|
161
|
+
# solve_current → scaling_active. Mod-faithful test:
|
|
162
|
+
# sum_{c in solve_current} p_use_row_scaling[c] >= 0.5
|
|
163
|
+
p_use_row_scaling = _read_keyed_value(
|
|
164
|
+
solve_data_dir / "p_use_row_scaling.csv",
|
|
165
|
+
provider=provider,
|
|
166
|
+
)
|
|
167
|
+
solve_current = _read_singles(solve_data_dir / "solve_current.csv",
|
|
168
|
+
provider=provider)
|
|
169
|
+
scaling_active = sum(
|
|
170
|
+
p_use_row_scaling.get(c, 0.0) for c in solve_current
|
|
171
|
+
) >= 0.5
|
|
172
|
+
|
|
173
|
+
# process_source_sink + p_entity_unitsize → _node_cap_unitsize_sum.
|
|
174
|
+
pss = _read_triples(solve_data_dir / "process_source_sink.csv",
|
|
175
|
+
provider=provider)
|
|
176
|
+
p_entity_unitsize = _read_keyed_value(
|
|
177
|
+
solve_data_dir / "p_entity_unitsize.csv",
|
|
178
|
+
provider=provider,
|
|
179
|
+
)
|
|
180
|
+
inflow_fallback = _read_node_period_value(
|
|
181
|
+
solve_data_dir / "_node_cap_inflow_fallback.csv",
|
|
182
|
+
provider=provider,
|
|
183
|
+
)
|
|
184
|
+
group_node = _read_pairs(input_dir / "group__node.csv",
|
|
185
|
+
provider=provider)
|
|
186
|
+
nodes_for_group: dict[str, list[str]] = {}
|
|
187
|
+
for g, n in group_node:
|
|
188
|
+
nodes_for_group.setdefault(g, []).append(n)
|
|
189
|
+
|
|
190
|
+
node_set = frozenset(nodes)
|
|
191
|
+
|
|
192
|
+
# ── _node_cap_unitsize_sum ───────────────────────────────────────────
|
|
193
|
+
# Σ p_entity_unitsize[p] for each arc end ∈ nodes.
|
|
194
|
+
cap_unitsize: dict[str, float] = {n: 0.0 for n in nodes}
|
|
195
|
+
for p, source, sink in pss:
|
|
196
|
+
usz = p_entity_unitsize.get(p, 0.0)
|
|
197
|
+
if sink in node_set:
|
|
198
|
+
cap_unitsize[sink] += usz
|
|
199
|
+
if source in node_set:
|
|
200
|
+
cap_unitsize[source] += usz
|
|
201
|
+
|
|
202
|
+
rows_unitsize: list[tuple[str, str, float]] = [
|
|
203
|
+
(n, d, cap_unitsize.get(n, 0.0)) for n in nodes for d in period_in_use
|
|
204
|
+
]
|
|
205
|
+
out["_node_cap_unitsize_sum.csv"] = _rows_to_frame(
|
|
206
|
+
("node", "period", "value"), rows_unitsize,
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
# ── _node_cap_raw ────────────────────────────────────────────────────
|
|
210
|
+
# unitsize_sum if > 0 else inflow_fallback if > 0 else 1.
|
|
211
|
+
raw_dict: dict[tuple[str, str], float] = {}
|
|
212
|
+
rows_raw: list[tuple[str, str, float]] = []
|
|
213
|
+
for n in nodes:
|
|
214
|
+
usz = cap_unitsize.get(n, 0.0)
|
|
215
|
+
for d in period_in_use:
|
|
216
|
+
if usz > 0:
|
|
217
|
+
v = usz
|
|
218
|
+
else:
|
|
219
|
+
fb = inflow_fallback.get((n, d), 0.0)
|
|
220
|
+
v = fb if fb > 0 else 1.0
|
|
221
|
+
raw_dict[(n, d)] = v
|
|
222
|
+
rows_raw.append((n, d, v))
|
|
223
|
+
out["_node_cap_raw.csv"] = _rows_to_frame(
|
|
224
|
+
("node", "period", "value"), rows_raw,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# ── _node_cap_pow10 ──────────────────────────────────────────────────
|
|
228
|
+
pow10_dict: dict[tuple[str, str], float] = {}
|
|
229
|
+
rows_pow10: list[tuple[str, str, float]] = []
|
|
230
|
+
for (n, d), v in raw_dict.items():
|
|
231
|
+
p10 = _pow10_round_clamped(v)
|
|
232
|
+
pow10_dict[(n, d)] = p10
|
|
233
|
+
rows_pow10.append((n, d, p10))
|
|
234
|
+
out["_node_cap_pow10.csv"] = _rows_to_frame(
|
|
235
|
+
("node", "period", "value"), rows_pow10,
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
# ── node_capacity_for_scaling + inv_node_cap ─────────────────────────
|
|
239
|
+
ncfs_dict: dict[tuple[str, str], float] = {}
|
|
240
|
+
rows_ncfs: list[tuple[str, str, float]] = []
|
|
241
|
+
rows_inc: list[tuple[str, str, float]] = []
|
|
242
|
+
for n in nodes:
|
|
243
|
+
for d in period_in_use:
|
|
244
|
+
v = pow10_dict.get((n, d), 1.0) if scaling_active else 1.0
|
|
245
|
+
ncfs_dict[(n, d)] = v
|
|
246
|
+
rows_ncfs.append((n, d, v))
|
|
247
|
+
rows_inc.append((n, d, 1.0 / v if v != 0 else 0.0))
|
|
248
|
+
out["node_capacity_for_scaling.csv"] = _rows_to_frame(
|
|
249
|
+
("node", "period", "value"), rows_ncfs,
|
|
250
|
+
)
|
|
251
|
+
out["inv_node_cap.csv"] = _rows_to_frame(
|
|
252
|
+
("node", "period", "value"), rows_inc,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
# ── _group_cap_raw ───────────────────────────────────────────────────
|
|
256
|
+
# NOTE: Python's ``sum(())`` returns int ``0`` (not ``0.0``) for empty
|
|
257
|
+
# groups; the legacy emitter preserves this int/float inconsistency
|
|
258
|
+
# via ``repr(v)``. The byte-parity gate requires we preserve it too.
|
|
259
|
+
# The Phase C accumulator parity test (which round-trips through
|
|
260
|
+
# ``pl.read_csv`` -> Float64 cast) skips this basename for the same
|
|
261
|
+
# reason — see ``_flex_data_accumulator.expected_basenames()`` for
|
|
262
|
+
# the exclusion list.
|
|
263
|
+
grp_raw: dict[tuple[str, str], float] = {}
|
|
264
|
+
rows_graw: list[tuple[str, str, float]] = []
|
|
265
|
+
for g in groups:
|
|
266
|
+
for d in period_in_use:
|
|
267
|
+
v = sum(ncfs_dict.get((n, d), 1.0)
|
|
268
|
+
for n in nodes_for_group.get(g, ()))
|
|
269
|
+
grp_raw[(g, d)] = v
|
|
270
|
+
rows_graw.append((g, d, v))
|
|
271
|
+
out["_group_cap_raw.csv"] = _rows_to_frame(
|
|
272
|
+
("group", "period", "value"), rows_graw,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
# ── _group_cap_pow10 ─────────────────────────────────────────────────
|
|
276
|
+
grp_pow10: dict[tuple[str, str], float] = {}
|
|
277
|
+
rows_gpow10: list[tuple[str, str, float]] = []
|
|
278
|
+
for (g, d), v in grp_raw.items():
|
|
279
|
+
p10 = _pow10_round_clamped(v) if v > 0 else 1.0
|
|
280
|
+
grp_pow10[(g, d)] = p10
|
|
281
|
+
rows_gpow10.append((g, d, p10))
|
|
282
|
+
out["_group_cap_pow10.csv"] = _rows_to_frame(
|
|
283
|
+
("group", "period", "value"), rows_gpow10,
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# ── group_capacity_for_scaling + inv_group_cap ───────────────────────
|
|
287
|
+
rows_gcfs: list[tuple[str, str, float]] = []
|
|
288
|
+
rows_igc: list[tuple[str, str, float]] = []
|
|
289
|
+
for g in groups:
|
|
290
|
+
for d in period_in_use:
|
|
291
|
+
v = grp_pow10.get((g, d), 1.0) if scaling_active else 1.0
|
|
292
|
+
rows_gcfs.append((g, d, v))
|
|
293
|
+
rows_igc.append((g, d, 1.0 / v if v != 0 else 0.0))
|
|
294
|
+
out["group_capacity_for_scaling.csv"] = _rows_to_frame(
|
|
295
|
+
("group", "period", "value"), rows_gcfs,
|
|
296
|
+
)
|
|
297
|
+
out["inv_group_cap.csv"] = _rows_to_frame(
|
|
298
|
+
("group", "period", "value"), rows_igc,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
return out
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
# ---------------------------------------------------------------------------
|
|
305
|
+
# Vectorized twin of _compute_lp_scaling_frames (vectorize-per-roll).
|
|
306
|
+
#
|
|
307
|
+
# Built as a FULL COPY of the legacy body, with each stage's COMPUTE
|
|
308
|
+
# replaced by vectorized polars incrementally (node chain → node
|
|
309
|
+
# capacity-for-scaling → group chain). The in-memory dicts the legacy
|
|
310
|
+
# stages consume (``cap_unitsize``, ``raw_dict``, ``pow10_dict``,
|
|
311
|
+
# ``ncfs_dict``, ``grp_raw``, ``grp_pow10``) are reconstructed from each
|
|
312
|
+
# vectorized frame right after it is built, so any not-yet-vectorized
|
|
313
|
+
# downstream stage still works at every commit (full-9-key parity-gateable
|
|
314
|
+
# per commit). The legacy :func:`_compute_lp_scaling_frames` is KEPT as
|
|
315
|
+
# the parity oracle.
|
|
316
|
+
#
|
|
317
|
+
# Tier policy (critique Defect X): NO lp key is hard-asserted Tier A. On
|
|
318
|
+
# real fixtures the unitsize / group sums are exact-integer so no ULP drift
|
|
319
|
+
# occurs and the LP-coefficient keys land Tier A byte-exact in practice,
|
|
320
|
+
# but a pathological sum landing on a half-decade ``10^(k+0.5)`` could flip
|
|
321
|
+
# a ``_pow10_round_clamped`` decade bucket → factor-10 gap → the parity
|
|
322
|
+
# gate RAISES (loud, correct — never masked). The pow10 stages reuse the
|
|
323
|
+
# SAME scalar :func:`_pow10_round_clamped` UDF for byte-fidelity.
|
|
324
|
+
# ---------------------------------------------------------------------------
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _empty_value_frame(header: tuple[str, str, str]) -> pl.DataFrame:
|
|
328
|
+
"""An explicit all-Utf8 empty 3-col frame (key1, key2, value)."""
|
|
329
|
+
return pl.DataFrame(
|
|
330
|
+
{h: [] for h in header},
|
|
331
|
+
schema={h: pl.Utf8 for h in header},
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _ordered_value_frame(
|
|
336
|
+
df: pl.DataFrame,
|
|
337
|
+
header: tuple[str, str, str],
|
|
338
|
+
order_cols: list[str],
|
|
339
|
+
) -> pl.DataFrame:
|
|
340
|
+
"""Sort *df* by *order_cols*, render its ``value_f`` Float64 column via
|
|
341
|
+
``repr`` and project to the all-Utf8 ``(key1, key2, value)`` shape.
|
|
342
|
+
|
|
343
|
+
*df* must carry the two key columns named ``header[0]``/``header[1]``,
|
|
344
|
+
a Float64 ``value_f`` column, and the integer *order_cols*. An empty
|
|
345
|
+
*df* yields the explicit empty schema.
|
|
346
|
+
"""
|
|
347
|
+
if df.height == 0:
|
|
348
|
+
return _empty_value_frame(header)
|
|
349
|
+
df = df.sort(order_cols)
|
|
350
|
+
value = _render_value_column(df["value_f"])
|
|
351
|
+
return df.select([header[0], header[1]]).with_columns(
|
|
352
|
+
value.alias(header[2]),
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _pow10_value_column(df: pl.DataFrame) -> pl.DataFrame:
|
|
357
|
+
"""Add a ``value_f`` column = ``_pow10_round_clamped(raw)`` per row.
|
|
358
|
+
|
|
359
|
+
Applies the EXACT legacy scalar UDF per cell (critique-corrected: the
|
|
360
|
+
same Python fn guarantees byte-identical bits — it re-derives the
|
|
361
|
+
clamp / ``v<=0`` / NaN edge cases identically; a re-implemented polars
|
|
362
|
+
``log10().round()`` would risk diverging on those edges). The Float64
|
|
363
|
+
``raw`` lifted from the dict is bit-identical to the legacy Python
|
|
364
|
+
float, so ``round(math.log10(v))`` matches.
|
|
365
|
+
"""
|
|
366
|
+
raw = df["raw"].to_list()
|
|
367
|
+
return df.with_columns(
|
|
368
|
+
pl.Series("value_f", [_pow10_round_clamped(v) for v in raw],
|
|
369
|
+
dtype=pl.Float64),
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _compute_lp_scaling_frames_vectorized(
|
|
374
|
+
input_dir: Path, solve_data_dir: Path,
|
|
375
|
+
*, provider: "object | None" = None,
|
|
376
|
+
) -> dict[str, pl.DataFrame]:
|
|
377
|
+
"""Vectorized twin of :func:`_compute_lp_scaling_frames`.
|
|
378
|
+
|
|
379
|
+
Same reader block, same in-memory dicts, same stage order. Of the 9
|
|
380
|
+
stage frames the legacy twin computes, only the 3 CONSUMED outputs are
|
|
381
|
+
emitted: ``node_capacity_for_scaling``, ``group_capacity_for_scaling``,
|
|
382
|
+
``inv_group_cap``. The 6 stage-to-stage middle-products
|
|
383
|
+
(``_node_cap_unitsize_sum``, ``_node_cap_raw``, ``_node_cap_pow10``,
|
|
384
|
+
``inv_node_cap``, ``_group_cap_raw``, ``_group_cap_pow10``) have no
|
|
385
|
+
functional reader and are no longer written; every in-memory
|
|
386
|
+
frame/dict that feeds a survivor is retained, so the surviving outputs
|
|
387
|
+
are byte-identical. (The legacy twin still computes all 9 frames
|
|
388
|
+
in-memory — the parity test's oracle for the dropped stages.)
|
|
389
|
+
"""
|
|
390
|
+
out: dict[str, pl.DataFrame] = {}
|
|
391
|
+
|
|
392
|
+
# ── Sources (copied verbatim from the legacy reader block) ─────────
|
|
393
|
+
nodes = _read_singles(input_dir / "node.csv", provider=provider)
|
|
394
|
+
groups = _read_singles(input_dir / "group.csv", provider=provider)
|
|
395
|
+
period_in_use = _read_singles(solve_data_dir / "period_in_use_set.csv",
|
|
396
|
+
provider=provider)
|
|
397
|
+
|
|
398
|
+
p_use_row_scaling = _read_keyed_value(
|
|
399
|
+
solve_data_dir / "p_use_row_scaling.csv",
|
|
400
|
+
provider=provider,
|
|
401
|
+
)
|
|
402
|
+
solve_current = _read_singles(solve_data_dir / "solve_current.csv",
|
|
403
|
+
provider=provider)
|
|
404
|
+
scaling_active = sum(
|
|
405
|
+
p_use_row_scaling.get(c, 0.0) for c in solve_current
|
|
406
|
+
) >= 0.5
|
|
407
|
+
|
|
408
|
+
pss = _read_triples(solve_data_dir / "process_source_sink.csv",
|
|
409
|
+
provider=provider)
|
|
410
|
+
p_entity_unitsize = _read_keyed_value(
|
|
411
|
+
solve_data_dir / "p_entity_unitsize.csv",
|
|
412
|
+
provider=provider,
|
|
413
|
+
)
|
|
414
|
+
inflow_fallback = _read_node_period_value(
|
|
415
|
+
solve_data_dir / "_node_cap_inflow_fallback.csv",
|
|
416
|
+
provider=provider,
|
|
417
|
+
)
|
|
418
|
+
group_node = _read_pairs(input_dir / "group__node.csv",
|
|
419
|
+
provider=provider)
|
|
420
|
+
# (the legacy ``nodes_for_group`` per-group node list is NOT built: the
|
|
421
|
+
# vectorized group chain joins the ``group_node`` pairs directly.)
|
|
422
|
+
|
|
423
|
+
node_set = frozenset(nodes)
|
|
424
|
+
|
|
425
|
+
# ── Shared order frames (node list order, group order, period order) ─
|
|
426
|
+
node_eo = pl.DataFrame(
|
|
427
|
+
{"node": list(nodes), "__eo": list(range(len(nodes)))},
|
|
428
|
+
schema={"node": pl.Utf8, "__eo": pl.Int64},
|
|
429
|
+
)
|
|
430
|
+
period_po = pl.DataFrame(
|
|
431
|
+
{"period": list(period_in_use),
|
|
432
|
+
"__po": list(range(len(period_in_use)))},
|
|
433
|
+
schema={"period": pl.Utf8, "__po": pl.Int64},
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
# ── Stage U: _node_cap_unitsize_sum (Tier B) ───────────────────────
|
|
437
|
+
# cap_unitsize pre-seeded {n: 0.0}; for each pss (p, source, sink) with
|
|
438
|
+
# usz = p_entity_unitsize.get(p, 0.0): TWO INDEPENDENT ifs —
|
|
439
|
+
# if sink in node_set: cap_unitsize[sink] += usz
|
|
440
|
+
# if source in node_set: cap_unitsize[source] += usz
|
|
441
|
+
# (a self-loop source==sink adds TWICE). Vectorize as TWO concatenated
|
|
442
|
+
# contributions so the double-count survives (do NOT struct-union /
|
|
443
|
+
# .unique()), group-by-sum, then densify over node × period.
|
|
444
|
+
pss_df = pl.DataFrame(
|
|
445
|
+
{"process": [p for p, _s, _k in pss],
|
|
446
|
+
"source": [s for _p, s, _k in pss],
|
|
447
|
+
"sink": [k for _p, _s, k in pss]},
|
|
448
|
+
schema={"process": pl.Utf8, "source": pl.Utf8, "sink": pl.Utf8},
|
|
449
|
+
)
|
|
450
|
+
usz_lk = pl.DataFrame(
|
|
451
|
+
{"process": list(p_entity_unitsize.keys()),
|
|
452
|
+
"v_usz": list(p_entity_unitsize.values())},
|
|
453
|
+
schema={"process": pl.Utf8, "v_usz": pl.Float64},
|
|
454
|
+
)
|
|
455
|
+
node_list = list(node_set)
|
|
456
|
+
pss_usz = pss_df.join(usz_lk, on="process", how="left").with_columns(
|
|
457
|
+
pl.col("v_usz").fill_null(0.0),
|
|
458
|
+
)
|
|
459
|
+
sink_contrib = (
|
|
460
|
+
pss_usz.filter(pl.col("sink").is_in(node_list))
|
|
461
|
+
.select(pl.col("sink").alias("node"), pl.col("v_usz"))
|
|
462
|
+
)
|
|
463
|
+
source_contrib = (
|
|
464
|
+
pss_usz.filter(pl.col("source").is_in(node_list))
|
|
465
|
+
.select(pl.col("source").alias("node"), pl.col("v_usz"))
|
|
466
|
+
)
|
|
467
|
+
contrib = pl.concat([sink_contrib, source_contrib], how="vertical")
|
|
468
|
+
if contrib.height:
|
|
469
|
+
unitsize_sum_df = (
|
|
470
|
+
contrib.group_by("node")
|
|
471
|
+
.agg(pl.col("v_usz").sum().alias("v_unitsize"))
|
|
472
|
+
)
|
|
473
|
+
else:
|
|
474
|
+
unitsize_sum_df = pl.DataFrame(
|
|
475
|
+
{"node": [], "v_unitsize": []},
|
|
476
|
+
schema={"node": pl.Utf8, "v_unitsize": pl.Float64},
|
|
477
|
+
)
|
|
478
|
+
# (``_node_cap_unitsize_sum.csv`` is a dead stage-to-stage middle-product
|
|
479
|
+
# — no functional reader — so it is no longer emitted. ``unitsize_sum_df``
|
|
480
|
+
# is retained: it feeds the ``cap_unitsize`` dict consumed by Stage R.)
|
|
481
|
+
# Reconstruct cap_unitsize {n: float} (period-independent) for legacy
|
|
482
|
+
# downstream stages still in scalar form.
|
|
483
|
+
cap_unitsize: dict[str, float] = {n: 0.0 for n in nodes}
|
|
484
|
+
for n, v in unitsize_sum_df.select(["node", "v_unitsize"]).iter_rows():
|
|
485
|
+
cap_unitsize[n] = v
|
|
486
|
+
|
|
487
|
+
# ── Stage R: _node_cap_raw (Tier A on real fixtures) ───────────────
|
|
488
|
+
# usz = cap_unitsize.get(n, 0.0); per (n, d):
|
|
489
|
+
# v = usz if usz > 0 else (fb if (fb := inflow_fallback[(n,d)]) > 0
|
|
490
|
+
# else 1.0)
|
|
491
|
+
# Dense over node × period, no drop.
|
|
492
|
+
usz_lk2 = pl.DataFrame(
|
|
493
|
+
{"node": list(nodes),
|
|
494
|
+
"v_usz": [cap_unitsize.get(n, 0.0) for n in nodes]},
|
|
495
|
+
schema={"node": pl.Utf8, "v_usz": pl.Float64},
|
|
496
|
+
)
|
|
497
|
+
fb_lk = pl.DataFrame(
|
|
498
|
+
{"node": [k[0] for k in inflow_fallback],
|
|
499
|
+
"period": [k[1] for k in inflow_fallback],
|
|
500
|
+
"v_fb": list(inflow_fallback.values())},
|
|
501
|
+
schema={"node": pl.Utf8, "period": pl.Utf8, "v_fb": pl.Float64},
|
|
502
|
+
)
|
|
503
|
+
raw_grid = (
|
|
504
|
+
node_eo.join(period_po, how="cross")
|
|
505
|
+
.join(usz_lk2, on="node", how="left")
|
|
506
|
+
.join(fb_lk, on=["node", "period"], how="left")
|
|
507
|
+
.with_columns(
|
|
508
|
+
pl.col("v_usz").fill_null(0.0),
|
|
509
|
+
pl.col("v_fb").fill_null(0.0),
|
|
510
|
+
)
|
|
511
|
+
.with_columns(
|
|
512
|
+
pl.when(pl.col("v_usz") > 0.0)
|
|
513
|
+
.then(pl.col("v_usz"))
|
|
514
|
+
.otherwise(
|
|
515
|
+
pl.when(pl.col("v_fb") > 0.0)
|
|
516
|
+
.then(pl.col("v_fb"))
|
|
517
|
+
.otherwise(pl.lit(1.0)))
|
|
518
|
+
.alias("value_f"),
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
# (``_node_cap_raw.csv`` is a dead middle-product — no functional reader —
|
|
522
|
+
# so it is no longer emitted. ``raw_grid`` is retained: it feeds P10.
|
|
523
|
+
# raw_dict is not reconstructed: the vectorized P10 stage reads
|
|
524
|
+
# ``raw_grid`` directly rather than iterating a scalar dict.)
|
|
525
|
+
|
|
526
|
+
# ── Stage P10: _node_cap_pow10 ─────────────────────────────────────
|
|
527
|
+
# _pow10_round_clamped(raw) per cell — reuse the SAME scalar UDF for
|
|
528
|
+
# byte-fidelity (the Float64 raw lifted above is bit-identical to the
|
|
529
|
+
# legacy Python float).
|
|
530
|
+
pow10_src = raw_grid.select(
|
|
531
|
+
["node", "period", "__eo", "__po",
|
|
532
|
+
pl.col("value_f").alias("raw")])
|
|
533
|
+
pow10_grid = _pow10_value_column(pow10_src)
|
|
534
|
+
# (``_node_cap_pow10.csv`` is a dead middle-product — no functional reader —
|
|
535
|
+
# so it is no longer emitted. ``pow10_grid`` is retained: it feeds NCFS.
|
|
536
|
+
# pow10_dict is not reconstructed: the vectorized NCFS stage reads
|
|
537
|
+
# ``pow10_grid`` directly rather than iterating a scalar dict.)
|
|
538
|
+
|
|
539
|
+
# ── node_capacity_for_scaling + inv_node_cap (Tier B) ──────────────
|
|
540
|
+
# ncfs = pow10 if scaling_active else 1.0 (scaling_active is a Python
|
|
541
|
+
# bool — branch in Python); inc = 1/ncfs if ncfs != 0 else 0.0. Dense
|
|
542
|
+
# over node × period. When scaling_active the value frame is the
|
|
543
|
+
# already-computed pow10_grid (every (n, d) is present, no .get default
|
|
544
|
+
# needed); otherwise a constant-1.0 dense grid.
|
|
545
|
+
if scaling_active:
|
|
546
|
+
ncfs_df = pow10_grid.select(
|
|
547
|
+
["node", "period", "__eo", "__po",
|
|
548
|
+
pl.col("value_f")])
|
|
549
|
+
else:
|
|
550
|
+
ncfs_df = node_eo.join(period_po, how="cross").with_columns(
|
|
551
|
+
pl.lit(1.0, dtype=pl.Float64).alias("value_f"),
|
|
552
|
+
)
|
|
553
|
+
out["node_capacity_for_scaling.csv"] = _ordered_value_frame(
|
|
554
|
+
ncfs_df, ("node", "period", "value"), ["__eo", "__po"],
|
|
555
|
+
)
|
|
556
|
+
# (``inv_node_cap.csv`` and its ``inc_df`` build are a dead leaf — no
|
|
557
|
+
# functional reader, and absent from the output meta-files copy — so
|
|
558
|
+
# neither the 1/ncfs frame nor its emit is produced.)
|
|
559
|
+
# (ncfs_dict is not reconstructed: the vectorized group chain reads the
|
|
560
|
+
# ncfs FRAME directly when summing ncfs over a group's member nodes.)
|
|
561
|
+
|
|
562
|
+
# Group order frame (group list position → __eo).
|
|
563
|
+
group_eo = pl.DataFrame(
|
|
564
|
+
{"group": list(groups), "__eo": list(range(len(groups)))},
|
|
565
|
+
schema={"group": pl.Utf8, "__eo": pl.Int64},
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
# ── _group_cap_raw (Tier B + S5 int-0) — UNCONDITIONAL (Defect Y) ───
|
|
569
|
+
# v = sum(ncfs_dict.get((n, d), 1.0) for n in nodes_for_group.get(g,()))
|
|
570
|
+
# Join group_node→ncfs on node (left-join + fill_null(1.0) reproduces
|
|
571
|
+
# the .get((n, d), 1.0) default for a member node absent from ncfs),
|
|
572
|
+
# group_by([group, period]).sum, densify over group × period.
|
|
573
|
+
# S5 int-0: a group with NO member nodes → sum(()) == int 0 → emit the
|
|
574
|
+
# literal "0" (NOT "0.0"). GR is NOT gated by scaling_active.
|
|
575
|
+
ncfs_lk = ncfs_df.select(
|
|
576
|
+
["node", "period", pl.col("value_f").alias("v_ncfs")])
|
|
577
|
+
gn_df = pl.DataFrame(
|
|
578
|
+
{"group": [g for g, _n in group_node],
|
|
579
|
+
"node": [n for _g, n in group_node]},
|
|
580
|
+
schema={"group": pl.Utf8, "node": pl.Utf8},
|
|
581
|
+
)
|
|
582
|
+
# group × member-node × period, then sum the ncfs default-1.0 value.
|
|
583
|
+
gn_member_sum = (
|
|
584
|
+
gn_df
|
|
585
|
+
.join(period_po, how="cross")
|
|
586
|
+
.join(ncfs_lk, on=["node", "period"], how="left")
|
|
587
|
+
.with_columns(pl.col("v_ncfs").fill_null(1.0))
|
|
588
|
+
.group_by(["group", "period"])
|
|
589
|
+
.agg(pl.col("v_ncfs").sum().alias("v_graw"))
|
|
590
|
+
)
|
|
591
|
+
graw_grid = (
|
|
592
|
+
group_eo.join(period_po, how="cross")
|
|
593
|
+
.join(gn_member_sum, on=["group", "period"], how="left")
|
|
594
|
+
)
|
|
595
|
+
# (``_group_cap_raw.csv`` is a dead middle-product — no functional reader,
|
|
596
|
+
# and never in the manifest — so it is no longer rendered/emitted.
|
|
597
|
+
# ``graw_grid`` is retained: it feeds the ``grp_raw`` dict consumed by
|
|
598
|
+
# GP10.)
|
|
599
|
+
# Reconstruct grp_raw {(g, d): float} (member-less → 0.0; the int/float
|
|
600
|
+
# distinction is irrelevant to the downstream ``v > 0`` GP10 test).
|
|
601
|
+
grp_raw: dict[tuple[str, str], float] = {
|
|
602
|
+
(r[0], r[1]): (r[2] if r[2] is not None else 0.0)
|
|
603
|
+
for r in graw_grid.select(["group", "period", "v_graw"]).iter_rows()
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
# ── _group_cap_pow10 — UNCONDITIONAL (Defect Y), scalar UDF ─────────
|
|
607
|
+
# _pow10_round_clamped(graw) if graw > 0 else 1.0. Reuse the SAME
|
|
608
|
+
# scalar UDF (note: it already returns 1.0 for v <= 0, so the explicit
|
|
609
|
+
# ``if graw > 0 else 1.0`` is subsumed by the UDF — apply it directly).
|
|
610
|
+
gpow10_grid = (
|
|
611
|
+
group_eo.join(period_po, how="cross")
|
|
612
|
+
.join(
|
|
613
|
+
pl.DataFrame(
|
|
614
|
+
{"group": [k[0] for k in grp_raw],
|
|
615
|
+
"period": [k[1] for k in grp_raw],
|
|
616
|
+
"raw": list(grp_raw.values())},
|
|
617
|
+
schema={"group": pl.Utf8, "period": pl.Utf8,
|
|
618
|
+
"raw": pl.Float64}),
|
|
619
|
+
on=["group", "period"], how="left")
|
|
620
|
+
)
|
|
621
|
+
gpow10_grid = _pow10_value_column(gpow10_grid)
|
|
622
|
+
# (``_group_cap_pow10.csv`` is a dead middle-product — no functional
|
|
623
|
+
# reader — so it is no longer emitted. ``gpow10_grid`` is retained: it
|
|
624
|
+
# feeds GCFS. grp_pow10 is not reconstructed: the vectorized GCFS stage
|
|
625
|
+
# reads the gpow10 FRAME directly.)
|
|
626
|
+
|
|
627
|
+
# ── group_capacity_for_scaling + inv_group_cap (Tier B) ────────────
|
|
628
|
+
# gcfs = grp_pow10 if scaling_active else 1.0 (scaling_active is a
|
|
629
|
+
# Python bool); igc = 1/gcfs if gcfs != 0 else 0.0. GCFS IS gated.
|
|
630
|
+
if scaling_active:
|
|
631
|
+
gcfs_df = gpow10_grid.select(
|
|
632
|
+
["group", "period", "__eo", "__po",
|
|
633
|
+
pl.col("value_f")])
|
|
634
|
+
else:
|
|
635
|
+
gcfs_df = group_eo.join(period_po, how="cross").with_columns(
|
|
636
|
+
pl.lit(1.0, dtype=pl.Float64).alias("value_f"),
|
|
637
|
+
)
|
|
638
|
+
out["group_capacity_for_scaling.csv"] = _ordered_value_frame(
|
|
639
|
+
gcfs_df, ("group", "period", "value"), ["__eo", "__po"],
|
|
640
|
+
)
|
|
641
|
+
igc_df = gcfs_df.with_columns(
|
|
642
|
+
pl.when(pl.col("value_f") != 0.0)
|
|
643
|
+
.then(1.0 / pl.col("value_f"))
|
|
644
|
+
.otherwise(pl.lit(0.0))
|
|
645
|
+
.alias("value_f"),
|
|
646
|
+
)
|
|
647
|
+
out["inv_group_cap.csv"] = _ordered_value_frame(
|
|
648
|
+
igc_df, ("group", "period", "value"), ["__eo", "__po"],
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
return out
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def emit_lp_scaling_params(
|
|
655
|
+
input_dir: Path, solve_data_dir: Path,
|
|
656
|
+
*, provider,
|
|
657
|
+
) -> None:
|
|
658
|
+
"""Emit ``lp_scaling_params`` to the Provider.
|
|
659
|
+
Emits the 3 CONSUMED frames under ``solve_data/<basename>`` keys via
|
|
660
|
+
:func:`_emit` (dual-key registration).
|
|
661
|
+
"""
|
|
662
|
+
frames = _compute_lp_scaling_frames_vectorized(
|
|
663
|
+
input_dir, solve_data_dir, provider=provider)
|
|
664
|
+
for basename, df in frames.items():
|
|
665
|
+
_emit(provider, f"solve_data/{basename}", df)
|