flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1515 @@
|
|
|
1
|
+
"""
|
|
2
|
+
blocks.py — Per-entity temporal block derivation and overlap set.
|
|
3
|
+
|
|
4
|
+
Purpose:
|
|
5
|
+
|
|
6
|
+
* Take group-level ``new_stepduration`` (v51 schema) and derive a
|
|
7
|
+
**block** for every node, unit and connection. A block is an
|
|
8
|
+
identifier that pairs the entity with a particular stepduration
|
|
9
|
+
schedule in the solve. When no group sets ``new_stepduration`` the
|
|
10
|
+
derivation collapses to a single block named ``"default"``, which is
|
|
11
|
+
bit-identical to pre-v51 behaviour.
|
|
12
|
+
* Precompute the cross-resolution **overlap set** — the mapping from
|
|
13
|
+
each (coarse block, coarse timestep) to the (fine block, fine
|
|
14
|
+
timestep) tuples it contains, with the area-fraction the fine step
|
|
15
|
+
contributes to the coarse row. The generalised node balance
|
|
16
|
+
consumes this set.
|
|
17
|
+
* Emit CSVs in ``solve_data/`` so downstream consumers can
|
|
18
|
+
index by block without re-running the derivation:
|
|
19
|
+
|
|
20
|
+
* ``entity_block.csv`` — one row per node.
|
|
21
|
+
* ``process_side_block.csv`` — two rows per process (source + sink).
|
|
22
|
+
* ``block_step_duration.csv`` — per (block, period, step) step
|
|
23
|
+
duration, analogous to
|
|
24
|
+
``steps_in_use.csv``.
|
|
25
|
+
* ``overlap_set.csv`` — (period, block_coarse, step_coarse,
|
|
26
|
+
block_fine, step_fine, fraction).
|
|
27
|
+
* ``block_step_previous.csv`` — per-block predecessor relations
|
|
28
|
+
analogous to ``step_previous.csv``
|
|
29
|
+
(Agent 1.4): for each
|
|
30
|
+
(block, period, step) row the
|
|
31
|
+
corresponding
|
|
32
|
+
``step_previous``,
|
|
33
|
+
``step_previous_within_timeset``,
|
|
34
|
+
``period_previous`` and
|
|
35
|
+
``step_previous_within_solve``.
|
|
36
|
+
* ``block_period_time_first.csv`` — per-block first step of each
|
|
37
|
+
period (Agent 1.4). Analogous
|
|
38
|
+
to ``first_timesteps.csv``.
|
|
39
|
+
* ``block_period_time_last.csv`` — per-block last step of each
|
|
40
|
+
period (Agent 1.4). Analogous
|
|
41
|
+
to ``last_timesteps.csv``.
|
|
42
|
+
|
|
43
|
+
These CSVs are consumed by the polars LP build.
|
|
44
|
+
|
|
45
|
+
Design notes
|
|
46
|
+
------------
|
|
47
|
+
* Block naming: an entity in a resolution-group takes that group's
|
|
48
|
+
name; everything else is in ``"default"``.
|
|
49
|
+
* Step duration of a block: the resolution-group's
|
|
50
|
+
``new_stepduration`` if set; else the solve's new_stepduration
|
|
51
|
+
(v50); else the finest step from the solve's timeline. The
|
|
52
|
+
``"default"`` block always takes the solve-level value (or the
|
|
53
|
+
raw timeline when unset).
|
|
54
|
+
* Per-process block: an explicit ``group__unit`` /
|
|
55
|
+
``group__connection`` membership in a resolution-group overrides
|
|
56
|
+
both sides to that block. Otherwise the source-side block follows
|
|
57
|
+
the source node and the sink-side block follows the sink node.
|
|
58
|
+
For direct (1var) methods the two sides agree in practice — when
|
|
59
|
+
they don't the finer side's resolution dominates (Agent 1.2 will
|
|
60
|
+
express this in the model).
|
|
61
|
+
* Overlap set: for **aligned subsets** (all coarse stepdurations are
|
|
62
|
+
integer multiples of the finest in the solve, and the finer timeline
|
|
63
|
+
cleanly partitions each coarse row), the fraction is always 1.0 and
|
|
64
|
+
non-matching rows are omitted. A ``NotImplementedError`` is raised
|
|
65
|
+
for the non-aligned case; the three-region LH2 test case is aligned
|
|
66
|
+
by construction.
|
|
67
|
+
"""
|
|
68
|
+
from __future__ import annotations
|
|
69
|
+
|
|
70
|
+
import csv
|
|
71
|
+
from collections import defaultdict
|
|
72
|
+
from dataclasses import dataclass, field
|
|
73
|
+
from pathlib import Path
|
|
74
|
+
from typing import TYPE_CHECKING, Iterable
|
|
75
|
+
|
|
76
|
+
import polars as pl
|
|
77
|
+
|
|
78
|
+
from flextool.engine_polars._solve_state import FlexToolConfigError
|
|
79
|
+
from flextool.engine_polars._emit_provider_io import _emit
|
|
80
|
+
from flextool.engine_polars.input import _provider_open
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Single-frame writer (canonical _write helper)
|
|
85
|
+
# ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
if TYPE_CHECKING: # pragma: no cover — import cycle guard only
|
|
89
|
+
from flextool.engine_polars._solve_config import SolveConfig
|
|
90
|
+
from flextool.engine_polars._timeline import TimelineConfig
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
# Public types
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
DEFAULT_BLOCK = "default"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# Direct (1var) and indirect (nvar) ct_method classification — matches
|
|
102
|
+
# ``set method_direct`` / ``set method_indirect`` in flextool_base.dat.
|
|
103
|
+
# Users write ct_method values in their DB (constant_efficiency, regular,
|
|
104
|
+
# …); the per-(ct, startup, fork) → method resolution happens in
|
|
105
|
+
# ``_write_process_method``. For block derivation we only need the
|
|
106
|
+
# single-var vs two-var distinction. Classification is at the ct_method
|
|
107
|
+
# level; an ``exact`` unit with fork_yes becomes ``method_2way_nvar_off``
|
|
108
|
+
# — still indirect — so the ct_method → direct/indirect mapping below
|
|
109
|
+
# holds regardless of fork.
|
|
110
|
+
_CT_METHOD_DIRECT: frozenset[str] = frozenset({
|
|
111
|
+
"constant_efficiency",
|
|
112
|
+
"min_load_efficiency",
|
|
113
|
+
"no_losses_no_variable_cost",
|
|
114
|
+
"variable_cost_only",
|
|
115
|
+
"unidirectional",
|
|
116
|
+
})
|
|
117
|
+
_CT_METHOD_INDIRECT: frozenset[str] = frozenset({
|
|
118
|
+
"regular",
|
|
119
|
+
"exact",
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _is_direct(ct_method: str) -> bool:
|
|
124
|
+
"""Return True for 1var-per-process (direct) methods.
|
|
125
|
+
|
|
126
|
+
Unknown ct_method values default to ``True`` — i.e. treat them as
|
|
127
|
+
direct — because the block-assignment logic for direct methods is
|
|
128
|
+
strictly more conservative (it picks the finer adjacent block),
|
|
129
|
+
keeping the per-side blocks identical for mis-classified processes.
|
|
130
|
+
"""
|
|
131
|
+
return ct_method in _CT_METHOD_DIRECT or ct_method not in _CT_METHOD_INDIRECT
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass
|
|
135
|
+
class BlockAssignments:
|
|
136
|
+
"""Per-entity block mappings for a single solve.
|
|
137
|
+
|
|
138
|
+
Attributes
|
|
139
|
+
----------
|
|
140
|
+
node_block : dict[str, str]
|
|
141
|
+
node_name → block_name.
|
|
142
|
+
process_block_in : dict[str, str]
|
|
143
|
+
process_name → source-side block_name.
|
|
144
|
+
process_block_out : dict[str, str]
|
|
145
|
+
process_name → sink-side block_name.
|
|
146
|
+
process_block : dict[str, str]
|
|
147
|
+
process_name → process's unified block. Equals the process's
|
|
148
|
+
explicit resolution-group block if set, else the finer of
|
|
149
|
+
``process_block_in`` / ``process_block_out``. Used to index
|
|
150
|
+
UC / startup / shutdown variables and their constraints so
|
|
151
|
+
that unit commitment lives at the process's own temporal
|
|
152
|
+
resolution. In the degenerate case (all entities on
|
|
153
|
+
``DEFAULT_BLOCK``) this equals ``DEFAULT_BLOCK`` for every
|
|
154
|
+
process.
|
|
155
|
+
block_step_duration : dict[str, float]
|
|
156
|
+
block_name → hours per step. ``DEFAULT_BLOCK`` always present.
|
|
157
|
+
"""
|
|
158
|
+
node_block: dict[str, str] = field(default_factory=dict)
|
|
159
|
+
process_block_in: dict[str, str] = field(default_factory=dict)
|
|
160
|
+
process_block_out: dict[str, str] = field(default_factory=dict)
|
|
161
|
+
process_block: dict[str, str] = field(default_factory=dict)
|
|
162
|
+
block_step_duration: dict[str, float] = field(default_factory=dict)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@dataclass
|
|
166
|
+
class OverlapSet:
|
|
167
|
+
"""Cross-resolution overlap rows.
|
|
168
|
+
|
|
169
|
+
Each row is ``(period, block_coarse, step_coarse, block_fine,
|
|
170
|
+
step_fine, fraction)``. Fractions follow the aligned-subsets
|
|
171
|
+
convention: always 1.0 when present, non-matching tuples omitted.
|
|
172
|
+
"""
|
|
173
|
+
rows: list[tuple[str, str, str, str, str, float]] = field(default_factory=list)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@dataclass
|
|
177
|
+
class BlockTimelines:
|
|
178
|
+
"""Per-block (period → [(step, duration_hours)]) timelines.
|
|
179
|
+
|
|
180
|
+
Used internally to emit ``block_step_duration.csv`` and to build
|
|
181
|
+
the overlap set.
|
|
182
|
+
"""
|
|
183
|
+
per_block: dict[str, dict[str, list[tuple[str, float]]]] = field(default_factory=dict)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@dataclass
|
|
187
|
+
class BlockPredecessors:
|
|
188
|
+
"""Per-block predecessor relations (Agent 1.4).
|
|
189
|
+
|
|
190
|
+
One row per ``(block, period, step)`` in each block's timeline,
|
|
191
|
+
carrying the successor→predecessor mapping analogous to
|
|
192
|
+
``dtttdt`` / ``step_previous.csv``. In the degenerate case (only
|
|
193
|
+
the ``"default"`` block) the row set equals the solve's existing
|
|
194
|
+
``dtttdt`` with the ``"default"`` tag prepended.
|
|
195
|
+
|
|
196
|
+
Row schema: ``(block, period, step, step_previous,
|
|
197
|
+
step_previous_within_timeset, period_previous,
|
|
198
|
+
step_previous_within_solve)``.
|
|
199
|
+
"""
|
|
200
|
+
rows: list[tuple[str, str, str, str, str, str, str]] = field(default_factory=list)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@dataclass
|
|
204
|
+
class BlockBoundaries:
|
|
205
|
+
"""Per-block first / last step of each period (Agent 1.4).
|
|
206
|
+
|
|
207
|
+
``first`` and ``last`` are lists of ``(block, period, step)`` rows,
|
|
208
|
+
covering every block in ``BlockAssignments.block_step_duration``.
|
|
209
|
+
In the degenerate case ``first`` and ``last`` for the ``"default"``
|
|
210
|
+
block equal the solve's ``period__time_first`` / ``period__time_last``
|
|
211
|
+
with the ``"default"`` tag prepended.
|
|
212
|
+
"""
|
|
213
|
+
first: list[tuple[str, str, str]] = field(default_factory=list)
|
|
214
|
+
last: list[tuple[str, str, str]] = field(default_factory=list)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ---------------------------------------------------------------------------
|
|
218
|
+
# Validation
|
|
219
|
+
# ---------------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def validate_group_membership(
|
|
223
|
+
group_unit: Iterable[tuple[str, str]],
|
|
224
|
+
group_connection: Iterable[tuple[str, str]],
|
|
225
|
+
group_node: Iterable[tuple[str, str]],
|
|
226
|
+
resolution_groups: dict[str, float],
|
|
227
|
+
decomposition_groups: dict[str, str],
|
|
228
|
+
reserve_upDown_group: Iterable[tuple[str, str, str]] | None = None,
|
|
229
|
+
process_reserve_upDown_node: Iterable[tuple[str, str, str, str]] | None = None,
|
|
230
|
+
) -> None:
|
|
231
|
+
"""Reject configurations with ambiguous block / region membership.
|
|
232
|
+
|
|
233
|
+
Rules
|
|
234
|
+
-----
|
|
235
|
+
Each entity (node / unit / connection) must be in **at most one**
|
|
236
|
+
resolution-group (a group with ``new_stepduration`` set) and
|
|
237
|
+
**at most one** decomposition-group (a group with
|
|
238
|
+
``decomposition_method != 'none'``). Membership in any number of
|
|
239
|
+
regular groups (CO2 caps, reserves, inertia …) is unconstrained.
|
|
240
|
+
|
|
241
|
+
Agent 1.7 additional rule: V1 restricts reserves to the default
|
|
242
|
+
(finest) temporal block. Any entity that participates in a reserve
|
|
243
|
+
(reserve-group member nodes, or processes listed in
|
|
244
|
+
``process_reserve_upDown_node``) must therefore NOT sit in a
|
|
245
|
+
resolution-group. Mixing reserves with coarse-resolution
|
|
246
|
+
participants creates subtle semantic issues around energy vs. power
|
|
247
|
+
aggregation; V1 forbids the configuration outright so the reserve
|
|
248
|
+
constraints in ``flextool.mod`` can stay at the fine ``dt`` index.
|
|
249
|
+
|
|
250
|
+
Args:
|
|
251
|
+
group_unit: ``(group, unit)`` membership tuples.
|
|
252
|
+
group_connection: ``(group, connection)`` membership tuples.
|
|
253
|
+
group_node: ``(group, node)`` membership tuples.
|
|
254
|
+
resolution_groups: group_name → new_stepduration hours. Only
|
|
255
|
+
keys matter (the set of resolution-groups).
|
|
256
|
+
decomposition_groups: group_name → decomposition_method value.
|
|
257
|
+
Only groups with a value other than ``"none"`` count as
|
|
258
|
+
decomposition-groups.
|
|
259
|
+
reserve_upDown_group: ``(reserve, upDown, group)`` rows defining
|
|
260
|
+
active reserves. When the iterable is empty the reserve
|
|
261
|
+
rule is a no-op.
|
|
262
|
+
process_reserve_upDown_node: ``(process, reserve, upDown, node)``
|
|
263
|
+
rows of processes that participate in reserves.
|
|
264
|
+
|
|
265
|
+
Raises:
|
|
266
|
+
FlexToolConfigError: if any entity is in two or more
|
|
267
|
+
resolution-groups, two or more decomposition-groups, or
|
|
268
|
+
is a reserve participant while sitting in a resolution-
|
|
269
|
+
group (Agent 1.7 V1 rule). The message names the entity
|
|
270
|
+
and both conflicting groups.
|
|
271
|
+
"""
|
|
272
|
+
res_set = set(resolution_groups.keys())
|
|
273
|
+
decomp_set = {g for g, m in decomposition_groups.items() if m and m != "none"}
|
|
274
|
+
|
|
275
|
+
# Build entity → list[group_in_resolution / in_decomposition] maps.
|
|
276
|
+
entity_res: dict[str, list[str]] = defaultdict(list)
|
|
277
|
+
entity_decomp: dict[str, list[str]] = defaultdict(list)
|
|
278
|
+
|
|
279
|
+
for members in (group_unit, group_connection, group_node):
|
|
280
|
+
for g, e in members:
|
|
281
|
+
if g in res_set:
|
|
282
|
+
entity_res[e].append(g)
|
|
283
|
+
if g in decomp_set:
|
|
284
|
+
entity_decomp[e].append(g)
|
|
285
|
+
|
|
286
|
+
errors: list[str] = []
|
|
287
|
+
for e, gs in entity_res.items():
|
|
288
|
+
if len(gs) > 1:
|
|
289
|
+
errors.append(
|
|
290
|
+
f"entity '{e}' is a member of multiple resolution-groups "
|
|
291
|
+
f"({sorted(set(gs))}); each entity may belong to at most "
|
|
292
|
+
f"one group with new_stepduration set."
|
|
293
|
+
)
|
|
294
|
+
for e, gs in entity_decomp.items():
|
|
295
|
+
if len(gs) > 1:
|
|
296
|
+
errors.append(
|
|
297
|
+
f"entity '{e}' is a member of multiple decomposition-groups "
|
|
298
|
+
f"({sorted(set(gs))}); each entity may belong to at most "
|
|
299
|
+
f"one group with decomposition_method != 'none'."
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
# Agent 1.7: reserve-block compatibility. Collect the set of
|
|
303
|
+
# reserve-participating entities (nodes whose reserve-groups are
|
|
304
|
+
# active, plus the processes and their nodes listed in
|
|
305
|
+
# ``process_reserve_upDown_node``) and reject any entity whose
|
|
306
|
+
# resolution-group is set (entity_res[e] non-empty).
|
|
307
|
+
reserve_groups: set[str] = set()
|
|
308
|
+
if reserve_upDown_group is not None:
|
|
309
|
+
for row in reserve_upDown_group:
|
|
310
|
+
if len(row) >= 3:
|
|
311
|
+
reserve_groups.add(row[2])
|
|
312
|
+
reserve_entities: set[str] = set()
|
|
313
|
+
# Nodes carried into a reserve via group membership.
|
|
314
|
+
for g, n in group_node:
|
|
315
|
+
if g in reserve_groups:
|
|
316
|
+
reserve_entities.add(n)
|
|
317
|
+
# Processes (and their nodes) explicitly listed as reserve
|
|
318
|
+
# participants.
|
|
319
|
+
if process_reserve_upDown_node is not None:
|
|
320
|
+
for row in process_reserve_upDown_node:
|
|
321
|
+
if len(row) >= 4:
|
|
322
|
+
reserve_entities.add(row[0]) # process
|
|
323
|
+
reserve_entities.add(row[3]) # node
|
|
324
|
+
|
|
325
|
+
for e in sorted(reserve_entities):
|
|
326
|
+
gs = entity_res.get(e, [])
|
|
327
|
+
if gs:
|
|
328
|
+
errors.append(
|
|
329
|
+
f"entity '{e}' participates in reserves but is a member of "
|
|
330
|
+
f"resolution-group(s) {sorted(set(gs))}; V1 restricts "
|
|
331
|
+
f"reserves to the default (finest) block — move '{e}' out "
|
|
332
|
+
f"of the resolution-group or drop the reserve participation."
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
if errors:
|
|
336
|
+
raise FlexToolConfigError("; ".join(errors))
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
# ---------------------------------------------------------------------------
|
|
340
|
+
# Block derivation
|
|
341
|
+
# ---------------------------------------------------------------------------
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _solve_step_duration(
|
|
345
|
+
solve: str,
|
|
346
|
+
solve_config: "SolveConfig | None",
|
|
347
|
+
timeline_config: "TimelineConfig | None",
|
|
348
|
+
) -> float:
|
|
349
|
+
"""Return the default-block step duration for *solve* in hours.
|
|
350
|
+
|
|
351
|
+
Priority (matches existing pre-v51 behaviour):
|
|
352
|
+
|
|
353
|
+
1. ``solve.new_stepduration`` (v50 parameter).
|
|
354
|
+
2. The smallest step duration that actually appears in any of the
|
|
355
|
+
solve's timelines. This catches the pre-v50 case where the
|
|
356
|
+
timeline already encodes the desired resolution.
|
|
357
|
+
|
|
358
|
+
When neither is available (unit-test fixture with only the bare
|
|
359
|
+
block dicts) falls back to 1.0. The return value is only used for
|
|
360
|
+
the ``DEFAULT_BLOCK`` entry in ``block_step_duration.csv`` and the
|
|
361
|
+
identity overlap rows; it does not participate in LP coefficients
|
|
362
|
+
in Agent 1.1.
|
|
363
|
+
"""
|
|
364
|
+
if solve_config is not None:
|
|
365
|
+
pass # type: ignore
|
|
366
|
+
# Solve-level new_stepduration (v50)
|
|
367
|
+
if timeline_config is not None:
|
|
368
|
+
raw = timeline_config.new_step_durations.get(solve)
|
|
369
|
+
if raw is not None:
|
|
370
|
+
try:
|
|
371
|
+
return float(raw)
|
|
372
|
+
except (TypeError, ValueError):
|
|
373
|
+
pass
|
|
374
|
+
|
|
375
|
+
# Fall back to the finest step in the solve's timelines.
|
|
376
|
+
if timeline_config is not None and solve_config is not None:
|
|
377
|
+
timesets_used = solve_config.timesets_used_by_solves.get(solve, [])
|
|
378
|
+
durations: list[float] = []
|
|
379
|
+
for _period, ts in timesets_used:
|
|
380
|
+
tl = timeline_config.timesets__timeline.get(ts)
|
|
381
|
+
if not tl:
|
|
382
|
+
continue
|
|
383
|
+
for step, dur in timeline_config.timelines.get(tl, []):
|
|
384
|
+
try:
|
|
385
|
+
durations.append(float(dur))
|
|
386
|
+
except (TypeError, ValueError):
|
|
387
|
+
continue
|
|
388
|
+
if durations:
|
|
389
|
+
return min(durations)
|
|
390
|
+
return 1.0
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _assign_entity_to_group(
|
|
394
|
+
entity: str,
|
|
395
|
+
memberships: Iterable[tuple[str, str]],
|
|
396
|
+
resolution_groups: dict[str, float],
|
|
397
|
+
) -> str | None:
|
|
398
|
+
"""Return the resolution-group *entity* belongs to, or None.
|
|
399
|
+
|
|
400
|
+
Linear scan kept for backwards compatibility with callers that pass a
|
|
401
|
+
raw membership iterable. Hot per-entity-loop callers should build a
|
|
402
|
+
lookup once via :func:`_build_membership_index` instead — the linear
|
|
403
|
+
scan is O(M) per entity and turns the surrounding loop into O(N × M)
|
|
404
|
+
when called inside a `for entity in entities:` block.
|
|
405
|
+
"""
|
|
406
|
+
for g, e in memberships:
|
|
407
|
+
if e == entity and g in resolution_groups:
|
|
408
|
+
return g
|
|
409
|
+
return None
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _build_membership_index(
|
|
413
|
+
memberships: Iterable[tuple[str, str]],
|
|
414
|
+
resolution_groups: dict[str, float],
|
|
415
|
+
) -> dict[str, str]:
|
|
416
|
+
"""Pre-build {entity -> first resolution_group} so per-entity lookups
|
|
417
|
+
in `derive_blocks` go from O(M) linear scan to O(1) dict lookup.
|
|
418
|
+
|
|
419
|
+
Preserves :func:`_assign_entity_to_group`'s "first match wins"
|
|
420
|
+
semantics: when an entity appears in multiple resolution-groups, the
|
|
421
|
+
first occurrence in *memberships* iteration order wins, matching the
|
|
422
|
+
behaviour of the linear-scan helper.
|
|
423
|
+
"""
|
|
424
|
+
idx: dict[str, str] = {}
|
|
425
|
+
for g, e in memberships:
|
|
426
|
+
if e in idx:
|
|
427
|
+
continue
|
|
428
|
+
if g in resolution_groups:
|
|
429
|
+
idx[e] = g
|
|
430
|
+
return idx
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def derive_blocks(
|
|
434
|
+
solve: str,
|
|
435
|
+
solve_config: "SolveConfig | None",
|
|
436
|
+
timeline_config: "TimelineConfig | None",
|
|
437
|
+
nodes: Iterable[str],
|
|
438
|
+
units: Iterable[str],
|
|
439
|
+
connections: Iterable[str],
|
|
440
|
+
resolution_groups: dict[str, float],
|
|
441
|
+
group_unit: Iterable[tuple[str, str]],
|
|
442
|
+
group_connection: Iterable[tuple[str, str]],
|
|
443
|
+
group_node: Iterable[tuple[str, str]],
|
|
444
|
+
process_source_sink: Iterable[tuple[str, str, str]],
|
|
445
|
+
process_ct_method: dict[str, str],
|
|
446
|
+
) -> BlockAssignments:
|
|
447
|
+
"""Compute per-entity block assignments for *solve*.
|
|
448
|
+
|
|
449
|
+
Args:
|
|
450
|
+
solve: Solve name (for ``new_stepduration`` lookup).
|
|
451
|
+
solve_config: Live SolveConfig (optional — None in unit tests).
|
|
452
|
+
timeline_config: Live TimelineConfig (optional — None in unit
|
|
453
|
+
tests).
|
|
454
|
+
nodes: Iterable of node names in the solve.
|
|
455
|
+
units: Iterable of unit names in the solve.
|
|
456
|
+
connections: Iterable of connection names in the solve.
|
|
457
|
+
resolution_groups: group_name → hours (``new_stepduration``).
|
|
458
|
+
Used as the set of resolution-groups and for
|
|
459
|
+
``block_step_duration`` lookup.
|
|
460
|
+
group_unit: ``(group, unit)`` membership rows.
|
|
461
|
+
group_connection: ``(group, connection)`` membership rows.
|
|
462
|
+
group_node: ``(group, node)`` membership rows.
|
|
463
|
+
process_source_sink: ``(process, source_node, sink_node)`` rows.
|
|
464
|
+
A process can appear multiple times for a multi-source or
|
|
465
|
+
multi-sink setup (fork_yes). The first source/sink seen is
|
|
466
|
+
canonical for block assignment — per-flow blocks are
|
|
467
|
+
derived from node_block in Agent 1.3.
|
|
468
|
+
process_ct_method: process_name → ct_method string (raw
|
|
469
|
+
user-facing values: constant_efficiency, regular, exact,
|
|
470
|
+
…). Drives direct vs indirect classification.
|
|
471
|
+
|
|
472
|
+
Returns:
|
|
473
|
+
BlockAssignments with node_block, process_block_in,
|
|
474
|
+
process_block_out and block_step_duration populated.
|
|
475
|
+
"""
|
|
476
|
+
gu = list(group_unit)
|
|
477
|
+
gc = list(group_connection)
|
|
478
|
+
gn = list(group_node)
|
|
479
|
+
|
|
480
|
+
# Step-duration table for all blocks seen so far. Always includes
|
|
481
|
+
# the default block.
|
|
482
|
+
default_step_duration = _solve_step_duration(
|
|
483
|
+
solve, solve_config, timeline_config
|
|
484
|
+
)
|
|
485
|
+
block_step_duration: dict[str, float] = {DEFAULT_BLOCK: default_step_duration}
|
|
486
|
+
for g, dur in resolution_groups.items():
|
|
487
|
+
try:
|
|
488
|
+
block_step_duration[g] = float(dur)
|
|
489
|
+
except (TypeError, ValueError):
|
|
490
|
+
continue
|
|
491
|
+
|
|
492
|
+
# --- Node block ----------------------------------------------------
|
|
493
|
+
# Pre-build node→group lookup once: was O(|nodes| × |gn|) linear scans
|
|
494
|
+
# per node (quadratic in entity count for models where ~every node is
|
|
495
|
+
# tagged). Now O(|gn|) once + O(|nodes|) lookups.
|
|
496
|
+
node_membership_index = _build_membership_index(gn, resolution_groups)
|
|
497
|
+
node_block: dict[str, str] = {}
|
|
498
|
+
for n in nodes:
|
|
499
|
+
g = node_membership_index.get(n)
|
|
500
|
+
node_block[n] = g if g is not None else DEFAULT_BLOCK
|
|
501
|
+
|
|
502
|
+
# --- Process block(s) ----------------------------------------------
|
|
503
|
+
# First source and first sink per process (preserve iteration order).
|
|
504
|
+
first_source: dict[str, str] = {}
|
|
505
|
+
first_sink: dict[str, str] = {}
|
|
506
|
+
for p, s, k in process_source_sink:
|
|
507
|
+
first_source.setdefault(p, s)
|
|
508
|
+
first_sink.setdefault(p, k)
|
|
509
|
+
|
|
510
|
+
process_block_in: dict[str, str] = {}
|
|
511
|
+
process_block_out: dict[str, str] = {}
|
|
512
|
+
process_block: dict[str, str] = {}
|
|
513
|
+
|
|
514
|
+
def _finer(a: str, b: str) -> str:
|
|
515
|
+
"""Pick the block with the shorter step duration.
|
|
516
|
+
|
|
517
|
+
Ties broken in favour of *a* to keep the iteration-order
|
|
518
|
+
determinism callers rely on.
|
|
519
|
+
"""
|
|
520
|
+
da = block_step_duration.get(a, default_step_duration)
|
|
521
|
+
db = block_step_duration.get(b, default_step_duration)
|
|
522
|
+
return a if da <= db else b
|
|
523
|
+
|
|
524
|
+
# Pre-build process→group lookup once across both unit and connection
|
|
525
|
+
# membership tables. Was an O((|units|+|connections|) × (|gu|+|gc|))
|
|
526
|
+
# nested scan with `list(gu) + list(gc)` rebuilt every iteration.
|
|
527
|
+
process_memberships: list[tuple[str, str]] = list(gu) + list(gc)
|
|
528
|
+
process_membership_index = _build_membership_index(
|
|
529
|
+
process_memberships, resolution_groups
|
|
530
|
+
)
|
|
531
|
+
|
|
532
|
+
for p in list(units) + list(connections):
|
|
533
|
+
# Check explicit unit / connection membership first (same code path —
|
|
534
|
+
# both lists get scanned).
|
|
535
|
+
explicit_block = process_membership_index.get(p)
|
|
536
|
+
if explicit_block is not None:
|
|
537
|
+
process_block_in[p] = explicit_block
|
|
538
|
+
process_block_out[p] = explicit_block
|
|
539
|
+
process_block[p] = explicit_block
|
|
540
|
+
continue
|
|
541
|
+
|
|
542
|
+
src = first_source.get(p)
|
|
543
|
+
snk = first_sink.get(p)
|
|
544
|
+
src_block = node_block.get(src, DEFAULT_BLOCK) if src else DEFAULT_BLOCK
|
|
545
|
+
snk_block = node_block.get(snk, DEFAULT_BLOCK) if snk else DEFAULT_BLOCK
|
|
546
|
+
|
|
547
|
+
ct = process_ct_method.get(p, "")
|
|
548
|
+
if _is_direct(ct):
|
|
549
|
+
# One flow variable per process; put it on the finer side.
|
|
550
|
+
finer = _finer(src_block, snk_block)
|
|
551
|
+
process_block_in[p] = finer
|
|
552
|
+
process_block_out[p] = finer
|
|
553
|
+
# Agent 1.6: process's unified block for UC / ramp / profile.
|
|
554
|
+
# Direct methods already collapsed both sides to the finer
|
|
555
|
+
# one, so the process block matches.
|
|
556
|
+
process_block[p] = finer
|
|
557
|
+
else:
|
|
558
|
+
# Indirect (nvar) — flows already split by side.
|
|
559
|
+
process_block_in[p] = src_block
|
|
560
|
+
process_block_out[p] = snk_block
|
|
561
|
+
# Agent 1.6: process's unified block for UC / ramp / profile.
|
|
562
|
+
# No explicit process-level resolution-group override, so
|
|
563
|
+
# fall back to the finer of the two sides.
|
|
564
|
+
process_block[p] = _finer(src_block, snk_block)
|
|
565
|
+
|
|
566
|
+
return BlockAssignments(
|
|
567
|
+
node_block=node_block,
|
|
568
|
+
process_block_in=process_block_in,
|
|
569
|
+
process_block_out=process_block_out,
|
|
570
|
+
process_block=process_block,
|
|
571
|
+
block_step_duration=block_step_duration,
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
# ---------------------------------------------------------------------------
|
|
576
|
+
# Per-block timelines
|
|
577
|
+
# ---------------------------------------------------------------------------
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _build_block_timelines(
|
|
581
|
+
solve: str,
|
|
582
|
+
solve_config: "SolveConfig | None",
|
|
583
|
+
timeline_config: "TimelineConfig | None",
|
|
584
|
+
block_assignments: BlockAssignments,
|
|
585
|
+
active_time_list: dict[str, list] | None = None,
|
|
586
|
+
) -> BlockTimelines:
|
|
587
|
+
"""Build per-block timelines for *solve*.
|
|
588
|
+
|
|
589
|
+
Each block's timeline is a dict of ``period → [(step, duration)]``
|
|
590
|
+
at that block's stepduration. For the default block this is the
|
|
591
|
+
same list of (timestep, duration) rows the rest of the solve data
|
|
592
|
+
already uses (``steps_in_use.csv``). For a resolution-group block,
|
|
593
|
+
the timeline is aggregated to that block's step count.
|
|
594
|
+
|
|
595
|
+
Aggregation rule
|
|
596
|
+
----------------
|
|
597
|
+
Every timestep in the default timeline is assigned to the coarse
|
|
598
|
+
block row whose step duration it contributes to. Currently this
|
|
599
|
+
requires the coarse step duration to be an integer multiple of the
|
|
600
|
+
default step duration (aligned subsets). The non-aligned case (e.g.
|
|
601
|
+
default=2h, coarse=3h) is surfaced as ``NotImplementedError`` —
|
|
602
|
+
Agent 1.3 will revisit it when the model actually consumes these
|
|
603
|
+
fractions.
|
|
604
|
+
"""
|
|
605
|
+
per_block: dict[str, dict[str, list[tuple[str, float]]]] = {}
|
|
606
|
+
|
|
607
|
+
# --- Source timeline (default block) ---------------------------------
|
|
608
|
+
# Priority: active_time_list from the orchestration loop (has the
|
|
609
|
+
# trimmed per-solve windowing); otherwise reconstruct from the
|
|
610
|
+
# timeline_config.
|
|
611
|
+
block_assignments.block_step_duration[DEFAULT_BLOCK]
|
|
612
|
+
default_periods: dict[str, list[tuple[str, float]]] = {}
|
|
613
|
+
|
|
614
|
+
if active_time_list is not None:
|
|
615
|
+
for period, rows in active_time_list.items():
|
|
616
|
+
default_periods[period] = [
|
|
617
|
+
(entry.timestep, float(entry.duration)) for entry in rows
|
|
618
|
+
]
|
|
619
|
+
elif solve_config is not None and timeline_config is not None:
|
|
620
|
+
for period, ts in solve_config.timesets_used_by_solves.get(solve, []):
|
|
621
|
+
tl = timeline_config.timesets__timeline.get(ts)
|
|
622
|
+
if not tl:
|
|
623
|
+
continue
|
|
624
|
+
tl_rows = timeline_config.timelines.get(tl, [])
|
|
625
|
+
default_periods.setdefault(period, [])
|
|
626
|
+
for step, dur in tl_rows:
|
|
627
|
+
default_periods[period].append((step, float(dur)))
|
|
628
|
+
else:
|
|
629
|
+
# Unit-test mode: no timelines known. Caller must set them.
|
|
630
|
+
pass
|
|
631
|
+
|
|
632
|
+
per_block[DEFAULT_BLOCK] = default_periods
|
|
633
|
+
|
|
634
|
+
# --- Aggregate per-resolution-group block ----------------------------
|
|
635
|
+
for block, dur in block_assignments.block_step_duration.items():
|
|
636
|
+
if block == DEFAULT_BLOCK:
|
|
637
|
+
continue
|
|
638
|
+
# Skip blocks that no entity is assigned to (would produce
|
|
639
|
+
# empty rows); consumers still get the duration via
|
|
640
|
+
# block_step_duration.
|
|
641
|
+
if (
|
|
642
|
+
block not in block_assignments.node_block.values()
|
|
643
|
+
and block not in block_assignments.process_block_in.values()
|
|
644
|
+
and block not in block_assignments.process_block_out.values()
|
|
645
|
+
and block not in block_assignments.process_block.values()
|
|
646
|
+
):
|
|
647
|
+
continue
|
|
648
|
+
block_rows: dict[str, list[tuple[str, float]]] = {}
|
|
649
|
+
for period, rows in default_periods.items():
|
|
650
|
+
aggregated = _aggregate_timeline(rows, coarse_duration=float(dur))
|
|
651
|
+
block_rows[period] = aggregated
|
|
652
|
+
per_block[block] = block_rows
|
|
653
|
+
|
|
654
|
+
return BlockTimelines(per_block=per_block)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _aggregate_timeline(
|
|
658
|
+
rows: list[tuple[str, float]], coarse_duration: float,
|
|
659
|
+
) -> list[tuple[str, float]]:
|
|
660
|
+
"""Aggregate a fine-resolution timeline to *coarse_duration* hours.
|
|
661
|
+
|
|
662
|
+
Emits one coarse row per accumulated block of fine rows whose
|
|
663
|
+
durations sum to exactly ``coarse_duration``. The coarse row's
|
|
664
|
+
label is the label of the first fine step it covers (matches the
|
|
665
|
+
pattern used by ``create_timeline_from_timestep_duration``).
|
|
666
|
+
|
|
667
|
+
Raises ``NotImplementedError`` when a block of fine rows cannot be
|
|
668
|
+
cleanly packed — i.e. the coarse duration is not an integer multiple
|
|
669
|
+
of the fine row it straddles. The three-region LH2 example is
|
|
670
|
+
aligned-subsets by construction; non-aligned support is deferred.
|
|
671
|
+
"""
|
|
672
|
+
out: list[tuple[str, float]] = []
|
|
673
|
+
i = 0
|
|
674
|
+
n = len(rows)
|
|
675
|
+
eps = 1e-9
|
|
676
|
+
while i < n:
|
|
677
|
+
acc = 0.0
|
|
678
|
+
first_label = rows[i][0]
|
|
679
|
+
j = i
|
|
680
|
+
while j < n and acc + float(rows[j][1]) <= coarse_duration + eps:
|
|
681
|
+
acc += float(rows[j][1])
|
|
682
|
+
j += 1
|
|
683
|
+
if abs(acc - coarse_duration) <= eps:
|
|
684
|
+
break
|
|
685
|
+
if abs(acc - coarse_duration) > eps:
|
|
686
|
+
raise NotImplementedError(
|
|
687
|
+
f"Non-aligned subset: fine rows starting at "
|
|
688
|
+
f"'{first_label}' accumulate to {acc}h but coarse "
|
|
689
|
+
f"block stepduration is {coarse_duration}h. "
|
|
690
|
+
f"Non-integer multiples of the default step are not "
|
|
691
|
+
f"supported yet (aligned-subsets only)."
|
|
692
|
+
)
|
|
693
|
+
out.append((first_label, coarse_duration))
|
|
694
|
+
i = j
|
|
695
|
+
return out
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
# ---------------------------------------------------------------------------
|
|
699
|
+
# Overlap set
|
|
700
|
+
# ---------------------------------------------------------------------------
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def derive_overlap_set(
|
|
704
|
+
solve: str,
|
|
705
|
+
block_assignments: BlockAssignments,
|
|
706
|
+
block_timelines: BlockTimelines | None = None,
|
|
707
|
+
solve_config: "SolveConfig | None" = None,
|
|
708
|
+
timeline_config: "TimelineConfig | None" = None,
|
|
709
|
+
active_time_list: dict[str, list] | None = None,
|
|
710
|
+
) -> OverlapSet:
|
|
711
|
+
"""Compute the overlap rows between every pair of blocks in the solve.
|
|
712
|
+
|
|
713
|
+
Row schema: ``(period, block_coarse, step_coarse, block_fine,
|
|
714
|
+
step_fine, fraction)``.
|
|
715
|
+
|
|
716
|
+
The aligned-subsets case emits fraction = 1.0 for every fine step
|
|
717
|
+
that falls within a coarse step (same calendar moment), and omits
|
|
718
|
+
non-matching rows. The degenerate case — only the default block
|
|
719
|
+
exists — emits identity rows ``(d, "default", t, "default", t,
|
|
720
|
+
1.0)``.
|
|
721
|
+
|
|
722
|
+
Args:
|
|
723
|
+
solve: Solve name.
|
|
724
|
+
block_assignments: From :func:`derive_blocks`.
|
|
725
|
+
block_timelines: Optional precomputed timelines; when ``None``
|
|
726
|
+
this function calls :func:`_build_block_timelines`.
|
|
727
|
+
solve_config / timeline_config: Passed through to the timeline
|
|
728
|
+
builder when *block_timelines* is ``None``.
|
|
729
|
+
active_time_list: Passed through to the timeline builder;
|
|
730
|
+
typically the ``active_time_lists[solve]`` value from the
|
|
731
|
+
orchestration loop.
|
|
732
|
+
"""
|
|
733
|
+
if block_timelines is None:
|
|
734
|
+
block_timelines = _build_block_timelines(
|
|
735
|
+
solve=solve,
|
|
736
|
+
solve_config=solve_config,
|
|
737
|
+
timeline_config=timeline_config,
|
|
738
|
+
block_assignments=block_assignments,
|
|
739
|
+
active_time_list=active_time_list,
|
|
740
|
+
)
|
|
741
|
+
per_block = block_timelines.per_block
|
|
742
|
+
default_rows = per_block.get(DEFAULT_BLOCK, {})
|
|
743
|
+
|
|
744
|
+
rows: list[tuple[str, str, str, str, str, float]] = []
|
|
745
|
+
|
|
746
|
+
# Degenerate case: only the default block exists → identity rows.
|
|
747
|
+
non_default = [b for b in per_block if b != DEFAULT_BLOCK]
|
|
748
|
+
if not non_default:
|
|
749
|
+
for period, pr in default_rows.items():
|
|
750
|
+
for step, _dur in pr:
|
|
751
|
+
rows.append((period, DEFAULT_BLOCK, step, DEFAULT_BLOCK, step, 1.0))
|
|
752
|
+
return OverlapSet(rows=rows)
|
|
753
|
+
|
|
754
|
+
# Default-against-default identity rows are always emitted too so
|
|
755
|
+
# that constraints indexed on ``(b_coarse, b_fine) = (default,
|
|
756
|
+
# default)`` have the right coverage.
|
|
757
|
+
for period, pr in default_rows.items():
|
|
758
|
+
for step, _dur in pr:
|
|
759
|
+
rows.append((period, DEFAULT_BLOCK, step, DEFAULT_BLOCK, step, 1.0))
|
|
760
|
+
|
|
761
|
+
# For every resolution-group block, emit coarse↔fine rows. We
|
|
762
|
+
# consider "fine" = default (and any block strictly finer than
|
|
763
|
+
# coarse); "coarse" = the resolution-group block itself. Every
|
|
764
|
+
# fine step that falls inside a coarse step becomes an overlap row.
|
|
765
|
+
#
|
|
766
|
+
# Agent 1.9 fix: also emit *self-identity* rows for every
|
|
767
|
+
# non-default block (e.g. ``(d, daily, t, daily, t, 1.0)``). The
|
|
768
|
+
# generalised node-balance constraints in ``flextool.mod`` look up
|
|
769
|
+
# ``(d, b_c, tc, b_f, t_f) in overlap`` with ``b_c = b_n`` (node
|
|
770
|
+
# block) and ``b_f`` = the process-side block. When both the node
|
|
771
|
+
# AND the process side live on the same coarse block (e.g. an H2
|
|
772
|
+
# node and a liquefier flow both at ``daily``) the self-identity
|
|
773
|
+
# row is what carries the contribution. Agent 1.1 only emitted
|
|
774
|
+
# ``(default, default)`` self-identity which is fine for the
|
|
775
|
+
# degenerate case but silently zeroes out coarse-coarse couplings.
|
|
776
|
+
for coarse in non_default:
|
|
777
|
+
coarse_rows = per_block.get(coarse, {})
|
|
778
|
+
# Self-identity: every (period, step) on the coarse block maps
|
|
779
|
+
# onto itself with fraction 1.0. Without this row, a
|
|
780
|
+
# daily→daily flow would produce no contribution to the daily
|
|
781
|
+
# node balance.
|
|
782
|
+
for period, c_rows in coarse_rows.items():
|
|
783
|
+
for coarse_step, _coarse_dur in c_rows:
|
|
784
|
+
rows.append(
|
|
785
|
+
(period, coarse, coarse_step, coarse, coarse_step, 1.0)
|
|
786
|
+
)
|
|
787
|
+
fine_rows_per_period = default_rows
|
|
788
|
+
for period, c_rows in coarse_rows.items():
|
|
789
|
+
# Walk the fine timeline in parallel, packing fine rows
|
|
790
|
+
# into the coarse row until its duration is met.
|
|
791
|
+
fine = fine_rows_per_period.get(period, [])
|
|
792
|
+
f_idx = 0
|
|
793
|
+
for coarse_step, coarse_dur in c_rows:
|
|
794
|
+
remaining = float(coarse_dur)
|
|
795
|
+
eps = 1e-9
|
|
796
|
+
while remaining > eps and f_idx < len(fine):
|
|
797
|
+
fine_step, fine_dur = fine[f_idx]
|
|
798
|
+
# Aligned-subsets assumption: every fine row is fully
|
|
799
|
+
# contained in the current coarse row, so we emit the
|
|
800
|
+
# 1.0 overlap and advance.
|
|
801
|
+
rows.append(
|
|
802
|
+
(period, coarse, coarse_step, DEFAULT_BLOCK, fine_step, 1.0)
|
|
803
|
+
)
|
|
804
|
+
# Symmetric row: default↔coarse mapping for callers
|
|
805
|
+
# that query by fine block.
|
|
806
|
+
rows.append(
|
|
807
|
+
(period, DEFAULT_BLOCK, fine_step, coarse, coarse_step, 1.0)
|
|
808
|
+
)
|
|
809
|
+
remaining -= float(fine_dur)
|
|
810
|
+
f_idx += 1
|
|
811
|
+
if remaining > eps:
|
|
812
|
+
raise NotImplementedError(
|
|
813
|
+
f"overlap_set: coarse step '{coarse_step}' on "
|
|
814
|
+
f"block '{coarse}' left {remaining}h unmatched "
|
|
815
|
+
f"at end of fine timeline — non-aligned subsets "
|
|
816
|
+
f"are not supported yet."
|
|
817
|
+
)
|
|
818
|
+
|
|
819
|
+
# Pairs of resolution-group blocks (coarse vs finer-coarse): emit
|
|
820
|
+
# an overlap row for every shared fine step. Handles e.g. a 24h
|
|
821
|
+
# block coexisting with a 6h block. Implemented by intersecting
|
|
822
|
+
# each block's coverage against the default-block steps.
|
|
823
|
+
if len(non_default) >= 2:
|
|
824
|
+
# Build (period, step) → block coverage for each non-default
|
|
825
|
+
# block so we can cross-reference.
|
|
826
|
+
coverage: dict[str, dict[str, dict[str, str]]] = {}
|
|
827
|
+
# coverage[block][period][fine_step] = coarse_step_that_covers_it
|
|
828
|
+
for block in non_default:
|
|
829
|
+
coverage[block] = {}
|
|
830
|
+
b_rows = per_block.get(block, {})
|
|
831
|
+
for period, c_rows in b_rows.items():
|
|
832
|
+
coverage[block][period] = {}
|
|
833
|
+
fine = default_rows.get(period, [])
|
|
834
|
+
f_idx = 0
|
|
835
|
+
for coarse_step, coarse_dur in c_rows:
|
|
836
|
+
remaining = float(coarse_dur)
|
|
837
|
+
eps = 1e-9
|
|
838
|
+
while remaining > eps and f_idx < len(fine):
|
|
839
|
+
fine_step, fine_dur = fine[f_idx]
|
|
840
|
+
coverage[block][period][fine_step] = coarse_step
|
|
841
|
+
remaining -= float(fine_dur)
|
|
842
|
+
f_idx += 1
|
|
843
|
+
for a in non_default:
|
|
844
|
+
for b in non_default:
|
|
845
|
+
if a == b:
|
|
846
|
+
continue
|
|
847
|
+
for period, a_cov in coverage[a].items():
|
|
848
|
+
b_cov = coverage[b].get(period, {})
|
|
849
|
+
for fine_step, a_step in a_cov.items():
|
|
850
|
+
b_step = b_cov.get(fine_step)
|
|
851
|
+
if b_step is None:
|
|
852
|
+
continue
|
|
853
|
+
# Identify which is coarser by step-duration.
|
|
854
|
+
a_dur = block_assignments.block_step_duration.get(a, 0.0)
|
|
855
|
+
b_dur = block_assignments.block_step_duration.get(b, 0.0)
|
|
856
|
+
# Only emit coarse→fine pairs to avoid duplication.
|
|
857
|
+
if a_dur > b_dur:
|
|
858
|
+
rows.append((period, a, a_step, b, b_step, 1.0))
|
|
859
|
+
elif a_dur == b_dur and a < b:
|
|
860
|
+
# Same duration, both cover the same fine
|
|
861
|
+
# steps — emit the canonical direction.
|
|
862
|
+
rows.append((period, a, a_step, b, b_step, 1.0))
|
|
863
|
+
# Deduplicate while preserving order: two different fine steps
|
|
864
|
+
# contained in the same coarse pair yields the same (a_step,
|
|
865
|
+
# b_step) pair every iteration.
|
|
866
|
+
seen: set[tuple] = set()
|
|
867
|
+
deduped: list[tuple[str, str, str, str, str, float]] = []
|
|
868
|
+
for row in rows:
|
|
869
|
+
if row in seen:
|
|
870
|
+
continue
|
|
871
|
+
seen.add(row)
|
|
872
|
+
deduped.append(row)
|
|
873
|
+
return OverlapSet(rows=deduped)
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
# ---------------------------------------------------------------------------
|
|
877
|
+
# Per-block predecessors + boundaries (Agent 1.4)
|
|
878
|
+
# ---------------------------------------------------------------------------
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def derive_block_predecessors(
|
|
882
|
+
solve: str,
|
|
883
|
+
block_assignments: BlockAssignments,
|
|
884
|
+
block_timelines: BlockTimelines,
|
|
885
|
+
default_jump_list: Iterable[tuple] | None = None,
|
|
886
|
+
) -> BlockPredecessors:
|
|
887
|
+
"""Compute per-block predecessor relations analogous to ``dtttdt``.
|
|
888
|
+
|
|
889
|
+
For the ``"default"`` block the predecessor rows are taken straight
|
|
890
|
+
from *default_jump_list* — the orchestration-level
|
|
891
|
+
``jump_list[solve]`` produced by :func:`make_step_jump` — and
|
|
892
|
+
tagged with the ``"default"`` block label. This guarantees the
|
|
893
|
+
degenerate case is bit-identical to pre-v51 behaviour: every row
|
|
894
|
+
carries exactly the same ``(step_previous,
|
|
895
|
+
step_previous_within_timeset, period_previous,
|
|
896
|
+
step_previous_within_solve)`` values consumed from
|
|
897
|
+
``step_previous.csv``.
|
|
898
|
+
|
|
899
|
+
For every non-default block we derive the predecessor relations
|
|
900
|
+
from that block's own aggregated timeline (``block_timelines``).
|
|
901
|
+
The block's timeline is a per-period ordered list of
|
|
902
|
+
``(step, duration)`` rows; the predecessor of step ``i`` in
|
|
903
|
+
period ``p`` is step ``i - 1`` in the same period (``jump = 1``),
|
|
904
|
+
except the first step of the first period which wraps cyclically
|
|
905
|
+
to the last step of the last period (``jump = -N + 1``), and the
|
|
906
|
+
first step of any subsequent period which points at the last step
|
|
907
|
+
of the previous period. This mirrors the ``make_step_jump``
|
|
908
|
+
behaviour — block aggregation preserves the simple cyclic pattern
|
|
909
|
+
because the aligned-subsets assumption collapses every block
|
|
910
|
+
internally to one monotone timestep sequence per period.
|
|
911
|
+
|
|
912
|
+
Args:
|
|
913
|
+
solve: Solve name (unused for now — kept for symmetry with
|
|
914
|
+
sibling functions).
|
|
915
|
+
block_assignments: From :func:`derive_blocks`.
|
|
916
|
+
block_timelines: From :func:`_build_block_timelines`.
|
|
917
|
+
default_jump_list: Iterable of 7-tuples as produced by
|
|
918
|
+
:func:`flextool.engine_polars._timeline.make_step_jump`.
|
|
919
|
+
When ``None`` the default-block rows fall back to the
|
|
920
|
+
simple cyclic pattern used for non-default blocks — this
|
|
921
|
+
only matters for unit tests; the orchestration loop always
|
|
922
|
+
passes the real list.
|
|
923
|
+
|
|
924
|
+
Returns:
|
|
925
|
+
BlockPredecessors. ``rows`` preserves iteration order:
|
|
926
|
+
default block first, then resolution-group blocks in insertion
|
|
927
|
+
order.
|
|
928
|
+
"""
|
|
929
|
+
rows: list[tuple[str, str, str, str, str, str, str]] = []
|
|
930
|
+
|
|
931
|
+
# --- Default block ------------------------------------------------------
|
|
932
|
+
if default_jump_list is not None:
|
|
933
|
+
for entry in default_jump_list:
|
|
934
|
+
# jump_list entries are 7-tuples (period, step, previous,
|
|
935
|
+
# previous_within_timeset, previous_period,
|
|
936
|
+
# previous_within_solve, jump). Re-project onto the block-
|
|
937
|
+
# tagged 7-tuple (block, period, step, previous,
|
|
938
|
+
# previous_within_timeset, previous_period,
|
|
939
|
+
# previous_within_solve).
|
|
940
|
+
period = entry[0]
|
|
941
|
+
step = entry[1]
|
|
942
|
+
previous = entry[2]
|
|
943
|
+
previous_within_timeset = entry[3]
|
|
944
|
+
previous_period = entry[4]
|
|
945
|
+
previous_within_solve = entry[5]
|
|
946
|
+
rows.append(
|
|
947
|
+
(
|
|
948
|
+
DEFAULT_BLOCK,
|
|
949
|
+
period,
|
|
950
|
+
step,
|
|
951
|
+
previous,
|
|
952
|
+
previous_within_timeset,
|
|
953
|
+
previous_period,
|
|
954
|
+
previous_within_solve,
|
|
955
|
+
)
|
|
956
|
+
)
|
|
957
|
+
else:
|
|
958
|
+
default_rows = block_timelines.per_block.get(DEFAULT_BLOCK, {})
|
|
959
|
+
rows.extend(_cyclic_block_predecessors(DEFAULT_BLOCK, default_rows))
|
|
960
|
+
|
|
961
|
+
# --- Non-default blocks -------------------------------------------------
|
|
962
|
+
for block in block_assignments.block_step_duration:
|
|
963
|
+
if block == DEFAULT_BLOCK:
|
|
964
|
+
continue
|
|
965
|
+
block_rows_per_period = block_timelines.per_block.get(block, {})
|
|
966
|
+
if not block_rows_per_period:
|
|
967
|
+
# Block declared in block_step_duration but no entity is on
|
|
968
|
+
# it — skip to avoid empty period rows.
|
|
969
|
+
continue
|
|
970
|
+
rows.extend(_cyclic_block_predecessors(block, block_rows_per_period))
|
|
971
|
+
|
|
972
|
+
return BlockPredecessors(rows=rows)
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _cyclic_block_predecessors(
|
|
976
|
+
block: str,
|
|
977
|
+
per_period_rows: dict[str, list[tuple[str, float]]],
|
|
978
|
+
) -> list[tuple[str, str, str, str, str, str, str]]:
|
|
979
|
+
"""Cyclic predecessor pattern over a block's aggregated timeline.
|
|
980
|
+
|
|
981
|
+
Cycles within the whole block's timeline: the first step of the
|
|
982
|
+
first period wraps to the last step of the last period. Within a
|
|
983
|
+
period interior steps point at the preceding step (jump=1), and
|
|
984
|
+
the first step of non-first periods points at the last step of
|
|
985
|
+
the previous period (jump across periods).
|
|
986
|
+
|
|
987
|
+
The ``step_previous_within_timeset`` column follows the pre-v51
|
|
988
|
+
convention: it equals ``step_previous`` for interior steps
|
|
989
|
+
(rows where the previous step is the immediately adjacent row in
|
|
990
|
+
the same period). For first-of-period rows — where the jump
|
|
991
|
+
crosses a period boundary — it's pinned at the current period's
|
|
992
|
+
block-last step (matching the ``block_last.timestep`` projection
|
|
993
|
+
used by :func:`make_step_jump`). In the degenerate case this
|
|
994
|
+
column is populated from the real ``jump_list`` instead of this
|
|
995
|
+
fallback, so the bit-identical promise still holds.
|
|
996
|
+
"""
|
|
997
|
+
periods = list(per_period_rows.keys())
|
|
998
|
+
if not periods:
|
|
999
|
+
return []
|
|
1000
|
+
first_period = periods[0]
|
|
1001
|
+
last_period = periods[-1]
|
|
1002
|
+
|
|
1003
|
+
out: list[tuple[str, str, str, str, str, str, str]] = []
|
|
1004
|
+
for pi, period in enumerate(periods):
|
|
1005
|
+
rows = per_period_rows[period]
|
|
1006
|
+
if not rows:
|
|
1007
|
+
continue
|
|
1008
|
+
block_last_step = rows[-1][0]
|
|
1009
|
+
for si, (step, _dur) in enumerate(rows):
|
|
1010
|
+
if si > 0:
|
|
1011
|
+
prev_step = rows[si - 1][0]
|
|
1012
|
+
prev_within_ts = prev_step
|
|
1013
|
+
prev_period = period
|
|
1014
|
+
prev_within_solve = prev_step
|
|
1015
|
+
else:
|
|
1016
|
+
# First step of the period — cross-period predecessor.
|
|
1017
|
+
if period == first_period:
|
|
1018
|
+
# Wrap to last period's last step (cyclic).
|
|
1019
|
+
prev_period_rows = per_period_rows[last_period]
|
|
1020
|
+
prev_step = prev_period_rows[-1][0] if prev_period_rows else step
|
|
1021
|
+
prev_period = last_period
|
|
1022
|
+
prev_within_solve = prev_step
|
|
1023
|
+
prev_within_ts = block_last_step
|
|
1024
|
+
else:
|
|
1025
|
+
prev_period = periods[pi - 1]
|
|
1026
|
+
prev_rows = per_period_rows[prev_period]
|
|
1027
|
+
prev_step = prev_rows[-1][0] if prev_rows else step
|
|
1028
|
+
prev_within_solve = prev_step
|
|
1029
|
+
prev_within_ts = block_last_step
|
|
1030
|
+
out.append(
|
|
1031
|
+
(
|
|
1032
|
+
block,
|
|
1033
|
+
period,
|
|
1034
|
+
step,
|
|
1035
|
+
prev_step,
|
|
1036
|
+
prev_within_ts,
|
|
1037
|
+
prev_period,
|
|
1038
|
+
prev_within_solve,
|
|
1039
|
+
)
|
|
1040
|
+
)
|
|
1041
|
+
return out
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def derive_block_boundaries(
|
|
1045
|
+
block_assignments: BlockAssignments,
|
|
1046
|
+
block_timelines: BlockTimelines,
|
|
1047
|
+
) -> BlockBoundaries:
|
|
1048
|
+
"""Compute per-block first and last step of each period.
|
|
1049
|
+
|
|
1050
|
+
Mirrors the solve-level ``period__time_first`` /
|
|
1051
|
+
``period__time_last`` sets, but keyed at the block level. In the
|
|
1052
|
+
degenerate (default-block-only) case ``first`` / ``last`` for
|
|
1053
|
+
``"default"`` equal the pre-v51 sets with the block tag prepended.
|
|
1054
|
+
"""
|
|
1055
|
+
first: list[tuple[str, str, str]] = []
|
|
1056
|
+
last: list[tuple[str, str, str]] = []
|
|
1057
|
+
for block in block_assignments.block_step_duration:
|
|
1058
|
+
rows_per_period = block_timelines.per_block.get(block, {})
|
|
1059
|
+
for period, rows in rows_per_period.items():
|
|
1060
|
+
if not rows:
|
|
1061
|
+
continue
|
|
1062
|
+
first.append((block, period, rows[0][0]))
|
|
1063
|
+
last.append((block, period, rows[-1][0]))
|
|
1064
|
+
return BlockBoundaries(first=first, last=last)
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
# ---------------------------------------------------------------------------
|
|
1068
|
+
# CSV emission
|
|
1069
|
+
# ---------------------------------------------------------------------------
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _entity_block_frame(block_assignments: BlockAssignments) -> pl.DataFrame:
|
|
1073
|
+
rows = list(block_assignments.node_block.items())
|
|
1074
|
+
return pl.DataFrame(
|
|
1075
|
+
{
|
|
1076
|
+
"entity": [r[0] for r in rows],
|
|
1077
|
+
"block": [r[1] for r in rows],
|
|
1078
|
+
},
|
|
1079
|
+
schema={"entity": pl.Utf8, "block": pl.Utf8},
|
|
1080
|
+
)
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _process_side_block_frame(
|
|
1084
|
+
block_assignments: BlockAssignments,
|
|
1085
|
+
) -> pl.DataFrame:
|
|
1086
|
+
procs: list[str] = []
|
|
1087
|
+
sides: list[str] = []
|
|
1088
|
+
blocks: list[str] = []
|
|
1089
|
+
for process, block in block_assignments.process_block_in.items():
|
|
1090
|
+
procs.append(process)
|
|
1091
|
+
sides.append("source")
|
|
1092
|
+
blocks.append(block)
|
|
1093
|
+
for process, block in block_assignments.process_block_out.items():
|
|
1094
|
+
procs.append(process)
|
|
1095
|
+
sides.append("sink")
|
|
1096
|
+
blocks.append(block)
|
|
1097
|
+
return pl.DataFrame(
|
|
1098
|
+
{"process": procs, "side": sides, "block": blocks},
|
|
1099
|
+
schema={"process": pl.Utf8, "side": pl.Utf8, "block": pl.Utf8},
|
|
1100
|
+
)
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
def _process_block_frame(
|
|
1104
|
+
block_assignments: BlockAssignments,
|
|
1105
|
+
) -> pl.DataFrame:
|
|
1106
|
+
rows = list(block_assignments.process_block.items())
|
|
1107
|
+
return pl.DataFrame(
|
|
1108
|
+
{
|
|
1109
|
+
"process": [r[0] for r in rows],
|
|
1110
|
+
"block": [r[1] for r in rows],
|
|
1111
|
+
},
|
|
1112
|
+
schema={"process": pl.Utf8, "block": pl.Utf8},
|
|
1113
|
+
)
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def _block_step_duration_frame(
|
|
1117
|
+
block_timelines: BlockTimelines | None,
|
|
1118
|
+
) -> pl.DataFrame:
|
|
1119
|
+
blocks: list[str] = []
|
|
1120
|
+
periods: list[str] = []
|
|
1121
|
+
steps: list[str] = []
|
|
1122
|
+
durs: list[float] = []
|
|
1123
|
+
if block_timelines is not None:
|
|
1124
|
+
for block, period_rows in block_timelines.per_block.items():
|
|
1125
|
+
for period, rows in period_rows.items():
|
|
1126
|
+
for step, dur in rows:
|
|
1127
|
+
blocks.append(block)
|
|
1128
|
+
periods.append(period)
|
|
1129
|
+
steps.append(step)
|
|
1130
|
+
durs.append(float(dur))
|
|
1131
|
+
return pl.DataFrame(
|
|
1132
|
+
{
|
|
1133
|
+
"block": blocks, "period": periods,
|
|
1134
|
+
"step": steps, "step_duration": durs,
|
|
1135
|
+
},
|
|
1136
|
+
schema={
|
|
1137
|
+
"block": pl.Utf8, "period": pl.Utf8,
|
|
1138
|
+
"step": pl.Utf8, "step_duration": pl.Float64,
|
|
1139
|
+
},
|
|
1140
|
+
)
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def _overlap_set_frame(overlap_set: OverlapSet) -> pl.DataFrame:
|
|
1144
|
+
rows = overlap_set.rows
|
|
1145
|
+
return pl.DataFrame(
|
|
1146
|
+
{
|
|
1147
|
+
"period": [r[0] for r in rows],
|
|
1148
|
+
"block_coarse": [r[1] for r in rows],
|
|
1149
|
+
"step_coarse": [r[2] for r in rows],
|
|
1150
|
+
"block_fine": [r[3] for r in rows],
|
|
1151
|
+
"step_fine": [r[4] for r in rows],
|
|
1152
|
+
"fraction": [r[5] for r in rows],
|
|
1153
|
+
},
|
|
1154
|
+
schema={
|
|
1155
|
+
"period": pl.Utf8, "block_coarse": pl.Utf8,
|
|
1156
|
+
"step_coarse": pl.Utf8, "block_fine": pl.Utf8,
|
|
1157
|
+
"step_fine": pl.Utf8, "fraction": pl.Float64,
|
|
1158
|
+
},
|
|
1159
|
+
)
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def _block_step_previous_frame(
|
|
1163
|
+
block_predecessors: BlockPredecessors | None,
|
|
1164
|
+
) -> pl.DataFrame:
|
|
1165
|
+
rows = block_predecessors.rows if block_predecessors is not None else []
|
|
1166
|
+
return pl.DataFrame(
|
|
1167
|
+
{
|
|
1168
|
+
"block": [r[0] for r in rows],
|
|
1169
|
+
"period": [r[1] for r in rows],
|
|
1170
|
+
"step": [r[2] for r in rows],
|
|
1171
|
+
"step_previous": [r[3] for r in rows],
|
|
1172
|
+
"step_previous_within_timeset": [r[4] for r in rows],
|
|
1173
|
+
"period_previous": [r[5] for r in rows],
|
|
1174
|
+
"step_previous_within_solve": [r[6] for r in rows],
|
|
1175
|
+
},
|
|
1176
|
+
schema={
|
|
1177
|
+
"block": pl.Utf8, "period": pl.Utf8, "step": pl.Utf8,
|
|
1178
|
+
"step_previous": pl.Utf8,
|
|
1179
|
+
"step_previous_within_timeset": pl.Utf8,
|
|
1180
|
+
"period_previous": pl.Utf8,
|
|
1181
|
+
"step_previous_within_solve": pl.Utf8,
|
|
1182
|
+
},
|
|
1183
|
+
)
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _block_period_time_frame(
|
|
1187
|
+
rows: list[tuple[str, str, str]],
|
|
1188
|
+
) -> pl.DataFrame:
|
|
1189
|
+
return pl.DataFrame(
|
|
1190
|
+
{
|
|
1191
|
+
"block": [r[0] for r in rows],
|
|
1192
|
+
"period": [r[1] for r in rows],
|
|
1193
|
+
"step": [r[2] for r in rows],
|
|
1194
|
+
},
|
|
1195
|
+
schema={"block": pl.Utf8, "period": pl.Utf8, "step": pl.Utf8},
|
|
1196
|
+
)
|
|
1197
|
+
|
|
1198
|
+
|
|
1199
|
+
def emit_block_data(
|
|
1200
|
+
block_assignments: BlockAssignments,
|
|
1201
|
+
overlap_set: OverlapSet,
|
|
1202
|
+
block_timelines: BlockTimelines | None,
|
|
1203
|
+
solve_data_dir: Path,
|
|
1204
|
+
block_predecessors: BlockPredecessors | None = None,
|
|
1205
|
+
block_boundaries: BlockBoundaries | None = None,
|
|
1206
|
+
*, provider,
|
|
1207
|
+
) -> None:
|
|
1208
|
+
"""Provider-emitting twin of :func:`write_block_data`.
|
|
1209
|
+
|
|
1210
|
+
Emits the eight block frames under ``solve_data/<basename>`` keys
|
|
1211
|
+
via :func:`_emit` (dual-key registration: basename and
|
|
1212
|
+
parent/basename). *solve_data_dir* is retained for signature parity
|
|
1213
|
+
with :func:`write_block_data`; the Provider is the only sink.
|
|
1214
|
+
"""
|
|
1215
|
+
del solve_data_dir
|
|
1216
|
+
_emit(provider, "solve_data/entity_block.csv",
|
|
1217
|
+
_entity_block_frame(block_assignments))
|
|
1218
|
+
_emit(provider, "solve_data/process_side_block.csv",
|
|
1219
|
+
_process_side_block_frame(block_assignments))
|
|
1220
|
+
_emit(provider, "solve_data/process_block.csv",
|
|
1221
|
+
_process_block_frame(block_assignments))
|
|
1222
|
+
_emit(provider, "solve_data/block_step_duration.csv",
|
|
1223
|
+
_block_step_duration_frame(block_timelines))
|
|
1224
|
+
_emit(provider, "solve_data/overlap_set.csv",
|
|
1225
|
+
_overlap_set_frame(overlap_set))
|
|
1226
|
+
_emit(provider, "solve_data/block_step_previous.csv",
|
|
1227
|
+
_block_step_previous_frame(block_predecessors))
|
|
1228
|
+
first_rows = (
|
|
1229
|
+
block_boundaries.first if block_boundaries is not None else []
|
|
1230
|
+
)
|
|
1231
|
+
last_rows = (
|
|
1232
|
+
block_boundaries.last if block_boundaries is not None else []
|
|
1233
|
+
)
|
|
1234
|
+
_emit(provider, "solve_data/block_period_time_first.csv",
|
|
1235
|
+
_block_period_time_frame(first_rows))
|
|
1236
|
+
_emit(provider, "solve_data/block_period_time_last.csv",
|
|
1237
|
+
_block_period_time_frame(last_rows))
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
# ---------------------------------------------------------------------------
|
|
1241
|
+
# Orchestration hook
|
|
1242
|
+
# ---------------------------------------------------------------------------
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
def _cell_str(value: "object | None") -> str:
|
|
1246
|
+
"""Reproduce a ``csv.reader`` cell string for a native frame value.
|
|
1247
|
+
|
|
1248
|
+
``DataFrame.write_csv`` renders ``null`` as the empty string and every
|
|
1249
|
+
other scalar as its textual form; ``csv.reader`` then reads those
|
|
1250
|
+
strings back. Mirror that here so the row-of-strings shape stays
|
|
1251
|
+
byte-identical to the legacy CSV round-trip the block-assembly code
|
|
1252
|
+
indexes positionally and compares against.
|
|
1253
|
+
"""
|
|
1254
|
+
return "" if value is None else str(value)
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
def _read_input_rows(
|
|
1258
|
+
inp: Path,
|
|
1259
|
+
basename: str,
|
|
1260
|
+
provider: "object | None" = None,
|
|
1261
|
+
) -> list[list[str]]:
|
|
1262
|
+
"""Read rows from ``inp/<basename>.csv``, falling back to *provider*.
|
|
1263
|
+
|
|
1264
|
+
Returns a list of row lists with the header stripped. When the
|
|
1265
|
+
Provider is supplied and the disk file is absent / empty, looks up
|
|
1266
|
+
the frame under the canonical ``input/<stem>`` key the cascade uses.
|
|
1267
|
+
|
|
1268
|
+
Returning the same row-of-strings shape as the legacy ``csv.reader``
|
|
1269
|
+
loops keeps the existing assembly code byte-identical.
|
|
1270
|
+
"""
|
|
1271
|
+
path = inp / basename
|
|
1272
|
+
stem = basename[:-4] if basename.endswith(".csv") else basename
|
|
1273
|
+
# Provider arm: read the in-memory frame natively instead of
|
|
1274
|
+
# round-tripping through ``DataFrame.write_csv`` + ``csv.reader``.
|
|
1275
|
+
# Try the canonical ``input/<stem>`` first (post-Δ.20 input writer),
|
|
1276
|
+
# then the bare ``<stem>`` for the legacy alias; first non-None
|
|
1277
|
+
# frame wins. ``provider.get`` returns DATA rows only (no header),
|
|
1278
|
+
# so no header is skipped. Frames never carry empty rows, so the
|
|
1279
|
+
# legacy ``if row`` blank-row filter is a no-op here. When neither
|
|
1280
|
+
# key resolves, fall through to the unchanged disk-fallback arm.
|
|
1281
|
+
if provider is not None:
|
|
1282
|
+
for key in (f"input/{stem}", stem):
|
|
1283
|
+
try:
|
|
1284
|
+
df = provider.get(key)
|
|
1285
|
+
except Exception: # pragma: no cover — defensive only
|
|
1286
|
+
df = None
|
|
1287
|
+
if df is None:
|
|
1288
|
+
continue
|
|
1289
|
+
return [[_cell_str(c) for c in row] for row in df.iter_rows()]
|
|
1290
|
+
fh = _provider_open(None, basename, path)
|
|
1291
|
+
if fh is None:
|
|
1292
|
+
return []
|
|
1293
|
+
with fh as f:
|
|
1294
|
+
reader = csv.reader(f)
|
|
1295
|
+
next(reader, None)
|
|
1296
|
+
return [row for row in reader if row]
|
|
1297
|
+
|
|
1298
|
+
|
|
1299
|
+
def emit_block_data_for_solve(
|
|
1300
|
+
solve: str,
|
|
1301
|
+
solve_config: "SolveConfig",
|
|
1302
|
+
timeline_config: "TimelineConfig",
|
|
1303
|
+
work_folder: Path,
|
|
1304
|
+
active_time_list: dict[str, list] | None = None,
|
|
1305
|
+
default_jump_list: Iterable[tuple] | None = None,
|
|
1306
|
+
provider: "object | None" = None,
|
|
1307
|
+
*, emit_provider,
|
|
1308
|
+
) -> BlockAssignments:
|
|
1309
|
+
"""Provider-emitting twin of :func:`write_block_data_for_solve`.
|
|
1310
|
+
|
|
1311
|
+
Mirrors the orchestration hook end-to-end but routes the final
|
|
1312
|
+
eight CSV emissions through :func:`emit_block_data` (Provider) rather
|
|
1313
|
+
than :func:`write_block_data` (disk).
|
|
1314
|
+
|
|
1315
|
+
The ``provider`` keyword is retained as the *input* Provider —
|
|
1316
|
+
:func:`_read_input_rows` consults it for ``input/*.csv`` lookups.
|
|
1317
|
+
The new keyword-only ``emit_provider`` is the *output* Provider into
|
|
1318
|
+
which the eight block frames are emitted.
|
|
1319
|
+
"""
|
|
1320
|
+
wf = Path(work_folder)
|
|
1321
|
+
inp = wf / "input"
|
|
1322
|
+
|
|
1323
|
+
nodes: list[str] = []
|
|
1324
|
+
units: list[str] = []
|
|
1325
|
+
connections: list[str] = []
|
|
1326
|
+
for row in _read_input_rows(inp, "node.csv", provider):
|
|
1327
|
+
if row:
|
|
1328
|
+
nodes.append(row[0])
|
|
1329
|
+
for row in _read_input_rows(inp, "process_unit.csv", provider):
|
|
1330
|
+
if row:
|
|
1331
|
+
units.append(row[0])
|
|
1332
|
+
for row in _read_input_rows(inp, "process_connection.csv", provider):
|
|
1333
|
+
if row:
|
|
1334
|
+
connections.append(row[0])
|
|
1335
|
+
if not nodes and not units and not connections:
|
|
1336
|
+
for row in _read_input_rows(inp, "entity.csv", provider):
|
|
1337
|
+
if row:
|
|
1338
|
+
nodes.append(row[0])
|
|
1339
|
+
|
|
1340
|
+
resolution_groups: dict[str, float] = {}
|
|
1341
|
+
p_group_rows = _read_input_rows(inp, "p_group.csv", provider)
|
|
1342
|
+
p_group_csv = inp / "p_group.csv"
|
|
1343
|
+
p_group_header: list[str] | None = None
|
|
1344
|
+
# Provider arm: the frame carries the column NAMES, so the "header"
|
|
1345
|
+
# is ``list(df.columns)`` — exactly what ``next(csv.reader(f), None)``
|
|
1346
|
+
# returned for the header row. Same key-candidate order
|
|
1347
|
+
# (``input/p_group`` before bare ``p_group``); first non-None frame
|
|
1348
|
+
# wins. When no frame resolves, fall to the unchanged disk arm.
|
|
1349
|
+
if provider is not None:
|
|
1350
|
+
for key in ("input/p_group", "p_group"):
|
|
1351
|
+
try:
|
|
1352
|
+
df = provider.get(key)
|
|
1353
|
+
except Exception: # pragma: no cover — defensive only
|
|
1354
|
+
df = None
|
|
1355
|
+
if df is not None:
|
|
1356
|
+
p_group_header = list(df.columns)
|
|
1357
|
+
break
|
|
1358
|
+
if p_group_header is None:
|
|
1359
|
+
p_group_fh = _provider_open(None, "p_group", p_group_csv)
|
|
1360
|
+
if p_group_fh is not None:
|
|
1361
|
+
with p_group_fh as f:
|
|
1362
|
+
p_group_header = next(csv.reader(f), None)
|
|
1363
|
+
if p_group_header is None:
|
|
1364
|
+
p_group_header = ["group", "groupParam", "p_group"]
|
|
1365
|
+
|
|
1366
|
+
def _row_dict(row: list[str], header: list[str]) -> dict[str, str]:
|
|
1367
|
+
return {h: (row[i] if i < len(row) else "")
|
|
1368
|
+
for i, h in enumerate(header)}
|
|
1369
|
+
|
|
1370
|
+
for row in p_group_rows:
|
|
1371
|
+
r = _row_dict(row, p_group_header)
|
|
1372
|
+
if r.get("groupParam") == "new_stepduration":
|
|
1373
|
+
try:
|
|
1374
|
+
resolution_groups[r["group"]] = float(r["p_group"])
|
|
1375
|
+
except (TypeError, ValueError):
|
|
1376
|
+
continue
|
|
1377
|
+
decomposition_groups: dict[str, str] = {}
|
|
1378
|
+
p_group_decomp_csv = inp / "p_group_decomposition.csv"
|
|
1379
|
+
p_group_decomp_header: list[str] | None = None
|
|
1380
|
+
# Same Provider-first / disk-fallback shape as the ``p_group`` header
|
|
1381
|
+
# probe above: the frame's column NAMES are the header. Same
|
|
1382
|
+
# key-candidate order; first non-None frame wins; otherwise fall to
|
|
1383
|
+
# the unchanged disk arm.
|
|
1384
|
+
if provider is not None:
|
|
1385
|
+
for key in ("input/p_group_decomposition", "p_group_decomposition"):
|
|
1386
|
+
try:
|
|
1387
|
+
df = provider.get(key)
|
|
1388
|
+
except Exception: # pragma: no cover — defensive only
|
|
1389
|
+
df = None
|
|
1390
|
+
if df is not None:
|
|
1391
|
+
p_group_decomp_header = list(df.columns)
|
|
1392
|
+
break
|
|
1393
|
+
if p_group_decomp_header is None:
|
|
1394
|
+
p_group_decomp_fh = _provider_open(
|
|
1395
|
+
None, "p_group_decomposition", p_group_decomp_csv,
|
|
1396
|
+
)
|
|
1397
|
+
if p_group_decomp_fh is not None:
|
|
1398
|
+
with p_group_decomp_fh as f:
|
|
1399
|
+
p_group_decomp_header = next(csv.reader(f), None)
|
|
1400
|
+
if p_group_decomp_header is None:
|
|
1401
|
+
p_group_decomp_header = ["group", "groupParam", "p_group"]
|
|
1402
|
+
for row in _read_input_rows(inp, "p_group_decomposition.csv", provider):
|
|
1403
|
+
r = _row_dict(row, p_group_decomp_header)
|
|
1404
|
+
if r.get("groupParam") == "decomposition_method":
|
|
1405
|
+
decomposition_groups[r["group"]] = str(r["p_group"])
|
|
1406
|
+
if not decomposition_groups:
|
|
1407
|
+
for row in p_group_rows:
|
|
1408
|
+
r = _row_dict(row, p_group_header)
|
|
1409
|
+
if r.get("groupParam") == "decomposition_method":
|
|
1410
|
+
decomposition_groups[r["group"]] = str(r["p_group"])
|
|
1411
|
+
|
|
1412
|
+
group_node: list[tuple[str, str]] = []
|
|
1413
|
+
group_process: list[tuple[str, str]] = []
|
|
1414
|
+
for row in _read_input_rows(inp, "group__node.csv", provider):
|
|
1415
|
+
if len(row) >= 2:
|
|
1416
|
+
group_node.append((row[0], row[1]))
|
|
1417
|
+
for row in _read_input_rows(inp, "group__process.csv", provider):
|
|
1418
|
+
if len(row) >= 2:
|
|
1419
|
+
group_process.append((row[0], row[1]))
|
|
1420
|
+
|
|
1421
|
+
unit_set = set(units)
|
|
1422
|
+
conn_set = set(connections)
|
|
1423
|
+
group_unit = [(g, p) for g, p in group_process if p in unit_set]
|
|
1424
|
+
group_connection = [(g, p) for g, p in group_process if p in conn_set]
|
|
1425
|
+
|
|
1426
|
+
reserve_upDown_group: list[tuple[str, str, str]] = []
|
|
1427
|
+
for row in _read_input_rows(inp, "reserve__upDown__group__method.csv",
|
|
1428
|
+
provider):
|
|
1429
|
+
if len(row) >= 4 and row[3] != "no_reserve":
|
|
1430
|
+
reserve_upDown_group.append((row[0], row[1], row[2]))
|
|
1431
|
+
process_reserve_upDown_node: list[tuple[str, str, str, str]] = []
|
|
1432
|
+
for row in _read_input_rows(inp, "process__reserve__upDown__node.csv",
|
|
1433
|
+
provider):
|
|
1434
|
+
if len(row) >= 4:
|
|
1435
|
+
process_reserve_upDown_node.append(
|
|
1436
|
+
(row[0], row[1], row[2], row[3])
|
|
1437
|
+
)
|
|
1438
|
+
|
|
1439
|
+
validate_group_membership(
|
|
1440
|
+
group_unit, group_connection, group_node,
|
|
1441
|
+
resolution_groups, decomposition_groups,
|
|
1442
|
+
reserve_upDown_group=reserve_upDown_group,
|
|
1443
|
+
process_reserve_upDown_node=process_reserve_upDown_node,
|
|
1444
|
+
)
|
|
1445
|
+
|
|
1446
|
+
process_source_sink: list[tuple[str, str, str]] = []
|
|
1447
|
+
sources: dict[str, list[str]] = defaultdict(list)
|
|
1448
|
+
sinks: dict[str, list[str]] = defaultdict(list)
|
|
1449
|
+
for row in _read_input_rows(inp, "process__source.csv", provider):
|
|
1450
|
+
if len(row) >= 2:
|
|
1451
|
+
sources[row[0]].append(row[1])
|
|
1452
|
+
for row in _read_input_rows(inp, "process__sink.csv", provider):
|
|
1453
|
+
if len(row) >= 2:
|
|
1454
|
+
sinks[row[0]].append(row[1])
|
|
1455
|
+
for p in set(list(sources.keys()) + list(sinks.keys())):
|
|
1456
|
+
src_list = sources.get(p, [""])
|
|
1457
|
+
snk_list = sinks.get(p, [""])
|
|
1458
|
+
process_source_sink.append((p, src_list[0], snk_list[0]))
|
|
1459
|
+
|
|
1460
|
+
process_ct_method: dict[str, str] = {}
|
|
1461
|
+
for row in _read_input_rows(inp, "process__ct_method.csv", provider):
|
|
1462
|
+
if len(row) >= 2:
|
|
1463
|
+
process_ct_method[row[0]] = row[1]
|
|
1464
|
+
|
|
1465
|
+
block_assignments = derive_blocks(
|
|
1466
|
+
solve=solve,
|
|
1467
|
+
solve_config=solve_config,
|
|
1468
|
+
timeline_config=timeline_config,
|
|
1469
|
+
nodes=nodes,
|
|
1470
|
+
units=units,
|
|
1471
|
+
connections=connections,
|
|
1472
|
+
resolution_groups=resolution_groups,
|
|
1473
|
+
group_unit=group_unit,
|
|
1474
|
+
group_connection=group_connection,
|
|
1475
|
+
group_node=group_node,
|
|
1476
|
+
process_source_sink=process_source_sink,
|
|
1477
|
+
process_ct_method=process_ct_method,
|
|
1478
|
+
)
|
|
1479
|
+
|
|
1480
|
+
block_timelines = _build_block_timelines(
|
|
1481
|
+
solve=solve,
|
|
1482
|
+
solve_config=solve_config,
|
|
1483
|
+
timeline_config=timeline_config,
|
|
1484
|
+
block_assignments=block_assignments,
|
|
1485
|
+
active_time_list=active_time_list,
|
|
1486
|
+
)
|
|
1487
|
+
|
|
1488
|
+
overlap = derive_overlap_set(
|
|
1489
|
+
solve=solve,
|
|
1490
|
+
block_assignments=block_assignments,
|
|
1491
|
+
block_timelines=block_timelines,
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
block_predecessors = derive_block_predecessors(
|
|
1495
|
+
solve=solve,
|
|
1496
|
+
block_assignments=block_assignments,
|
|
1497
|
+
block_timelines=block_timelines,
|
|
1498
|
+
default_jump_list=default_jump_list,
|
|
1499
|
+
)
|
|
1500
|
+
block_boundaries = derive_block_boundaries(
|
|
1501
|
+
block_assignments=block_assignments,
|
|
1502
|
+
block_timelines=block_timelines,
|
|
1503
|
+
)
|
|
1504
|
+
|
|
1505
|
+
emit_block_data(
|
|
1506
|
+
block_assignments=block_assignments,
|
|
1507
|
+
overlap_set=overlap,
|
|
1508
|
+
block_timelines=block_timelines,
|
|
1509
|
+
solve_data_dir=wf / "solve_data",
|
|
1510
|
+
block_predecessors=block_predecessors,
|
|
1511
|
+
block_boundaries=block_boundaries,
|
|
1512
|
+
provider=emit_provider,
|
|
1513
|
+
)
|
|
1514
|
+
|
|
1515
|
+
return block_assignments
|