flextool 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flextool/__init__.py +41 -0
- flextool/_mem_sampler.py +193 -0
- flextool/_resources.py +43 -0
- flextool/calibrate/__init__.py +51 -0
- flextool/calibrate/__main__.py +11 -0
- flextool/calibrate/_cli.py +316 -0
- flextool/calibrate/_db_alt.py +166 -0
- flextool/calibrate/_final_outputs.py +110 -0
- flextool/calibrate/_guard.py +151 -0
- flextool/calibrate/_loop.py +558 -0
- flextool/calibrate/_readers.py +223 -0
- flextool/calibrate/_report.py +263 -0
- flextool/calibrate/_sizing.py +699 -0
- flextool/calibrate/_solve.py +134 -0
- flextool/calibrate/_solve_status.py +495 -0
- flextool/cli/__init__.py +9 -0
- flextool/cli/_console.py +51 -0
- flextool/cli/_timing.py +147 -0
- flextool/cli/cmd_execute_flextool_workflow.py +187 -0
- flextool/cli/cmd_export_to_tabular.py +56 -0
- flextool/cli/cmd_import_sensitivities.py +75 -0
- flextool/cli/cmd_migrate_database.py +13 -0
- flextool/cli/cmd_open_results_db.py +269 -0
- flextool/cli/cmd_read_matpower.py +66 -0
- flextool/cli/cmd_read_old_flextool.py +63 -0
- flextool/cli/cmd_read_self_describing_tabular_input.py +50 -0
- flextool/cli/cmd_read_tabular_input.py +81 -0
- flextool/cli/cmd_run_flextool.py +1095 -0
- flextool/cli/cmd_scenario_results.py +284 -0
- flextool/cli/cmd_solve_mps.py +169 -0
- flextool/cli/cmd_update_flextool.py +17 -0
- flextool/cli/cmd_write_outputs.py +125 -0
- flextool/common_utils/__init__.py +1 -0
- flextool/common_utils/plot_mem_shape.py +77 -0
- flextool/common_utils/precision.py +451 -0
- flextool/decomposition/__init__.py +0 -0
- flextool/decomposition/region_decomposition.py +128 -0
- flextool/decomposition/region_filter.py +1261 -0
- flextool/engine_polars/__init__.py +110 -0
- flextool/engine_polars/_axis_enums.py +742 -0
- flextool/engine_polars/_benders.py +3462 -0
- flextool/engine_polars/_block_layout.py +1479 -0
- flextool/engine_polars/_blocks.py +1515 -0
- flextool/engine_polars/_commodity_ladder.py +660 -0
- flextool/engine_polars/_cumulative_invest.py +1165 -0
- flextool/engine_polars/_db_loader.py +153 -0
- flextool/engine_polars/_db_reader.py +127 -0
- flextool/engine_polars/_dc_power_flow.py +445 -0
- flextool/engine_polars/_delay.py +442 -0
- flextool/engine_polars/_derived_arithmetic.py +432 -0
- flextool/engine_polars/_derived_block.py +990 -0
- flextool/engine_polars/_derived_branch.py +769 -0
- flextool/engine_polars/_derived_existing.py +1353 -0
- flextool/engine_polars/_derived_npv.py +1297 -0
- flextool/engine_polars/_derived_params.py +9850 -0
- flextool/engine_polars/_derived_profile.py +881 -0
- flextool/engine_polars/_derived_walks.py +276 -0
- flextool/engine_polars/_determinism.py +70 -0
- flextool/engine_polars/_direct_params.py +2186 -0
- flextool/engine_polars/_dump_csvs.py +1009 -0
- flextool/engine_polars/_emit_arc_unions.py +1631 -0
- flextool/engine_polars/_emit_calc_params.py +729 -0
- flextool/engine_polars/_emit_chain_params.py +709 -0
- flextool/engine_polars/_emit_co2_accumulators.py +400 -0
- flextool/engine_polars/_emit_dispatchers.py +690 -0
- flextool/engine_polars/_emit_energy_margin.py +125 -0
- flextool/engine_polars/_emit_energy_margin_adder.py +290 -0
- flextool/engine_polars/_emit_entity_annual.py +428 -0
- flextool/engine_polars/_emit_inflow_scaling.py +1420 -0
- flextool/engine_polars/_emit_leaf_sets.py +550 -0
- flextool/engine_polars/_emit_lp_scaling.py +665 -0
- flextool/engine_polars/_emit_mid_sets.py +859 -0
- flextool/engine_polars/_emit_pdt_params.py +759 -0
- flextool/engine_polars/_emit_per_solve.py +774 -0
- flextool/engine_polars/_emit_period_calc.py +504 -0
- flextool/engine_polars/_emit_period_params.py +2398 -0
- flextool/engine_polars/_emit_provider_io.py +141 -0
- flextool/engine_polars/_emit_reserve.py +574 -0
- flextool/engine_polars/_emit_solve_time.py +311 -0
- flextool/engine_polars/_emit_solve_writers.py +1249 -0
- flextool/engine_polars/_flex_data_accumulator.py +388 -0
- flextool/engine_polars/_flex_data_provider.py +478 -0
- flextool/engine_polars/_group_slack.py +1253 -0
- flextool/engine_polars/_inmemory_reader.py +140 -0
- flextool/engine_polars/_input_source.py +336 -0
- flextool/engine_polars/_invest_seeds.py +191 -0
- flextool/engine_polars/_native_input_writer.py +100 -0
- flextool/engine_polars/_native_run_model.py +1348 -0
- flextool/engine_polars/_orchestration.py +4314 -0
- flextool/engine_polars/_output_writer.py +439 -0
- flextool/engine_polars/_param_shapes.py +1595 -0
- flextool/engine_polars/_parquet_bundle.py +723 -0
- flextool/engine_polars/_pdt_join.py +167 -0
- flextool/engine_polars/_pdt_lookup.py +547 -0
- flextool/engine_polars/_per_solve_sets.py +335 -0
- flextool/engine_polars/_projection_params.py +2056 -0
- flextool/engine_polars/_provider_keys.py +173 -0
- flextool/engine_polars/_provider_translators.py +225 -0
- flextool/engine_polars/_recursive_solve.py +703 -0
- flextool/engine_polars/_region_filter.py +2508 -0
- flextool/engine_polars/_reserve.py +649 -0
- flextool/engine_polars/_solve_acceptance.py +331 -0
- flextool/engine_polars/_solve_config.py +1001 -0
- flextool/engine_polars/_solve_context.py +885 -0
- flextool/engine_polars/_solve_handoff.py +164 -0
- flextool/engine_polars/_solve_state.py +232 -0
- flextool/engine_polars/_solver_base.py +36 -0
- flextool/engine_polars/_solver_dispatch.py +511 -0
- flextool/engine_polars/_spinedb_reader.py +1165 -0
- flextool/engine_polars/_stochastic.py +593 -0
- flextool/engine_polars/_subprocess_solve.py +1838 -0
- flextool/engine_polars/_timeline.py +1416 -0
- flextool/engine_polars/_vectorize.py +438 -0
- flextool/engine_polars/_warm.py +858 -0
- flextool/engine_polars/autoscale/__init__.py +107 -0
- flextool/engine_polars/autoscale/_config.py +218 -0
- flextool/engine_polars/autoscale/_layer2.py +1253 -0
- flextool/engine_polars/autoscale/_layer2_types.py +584 -0
- flextool/engine_polars/autoscale/_quantity_types.py +621 -0
- flextool/engine_polars/autoscale/_report.py +336 -0
- flextool/engine_polars/chain.py +259 -0
- flextool/engine_polars/input.py +6638 -0
- flextool/engine_polars/model.py +4754 -0
- flextool/env_check.py +388 -0
- flextool/export_to_tabular/__init__.py +5 -0
- flextool/export_to_tabular/db_reader.py +224 -0
- flextool/export_to_tabular/excel_writer.py +3559 -0
- flextool/export_to_tabular/export_settings.yaml +377 -0
- flextool/export_to_tabular/export_to_excel.py +227 -0
- flextool/export_to_tabular/formatting.py +543 -0
- flextool/export_to_tabular/sheet_config.py +876 -0
- flextool/gui/__init__.py +0 -0
- flextool/gui/__main__.py +118 -0
- flextool/gui/calibrate_commands.py +184 -0
- flextool/gui/calibrate_jobs.py +424 -0
- flextool/gui/check_tree.py +142 -0
- flextool/gui/cli_format.py +83 -0
- flextool/gui/config_parser.py +68 -0
- flextool/gui/data_models.py +362 -0
- flextool/gui/db_editor_integration.py +202 -0
- flextool/gui/db_version_check.py +269 -0
- flextool/gui/dialogs/__init__.py +0 -0
- flextool/gui/dialogs/add_dialog.py +1098 -0
- flextool/gui/dialogs/calibrate_dialog.py +1259 -0
- flextool/gui/dialogs/file_picker.py +473 -0
- flextool/gui/dialogs/group_picker.py +299 -0
- flextool/gui/dialogs/migration_consent_dialog.py +106 -0
- flextool/gui/dialogs/migration_progress_dialog.py +237 -0
- flextool/gui/dialogs/plot_dialog.py +459 -0
- flextool/gui/dialogs/plot_settings_picker.py +2184 -0
- flextool/gui/dialogs/project_dialog.py +426 -0
- flextool/gui/dialogs/update_dialog.py +212 -0
- flextool/gui/downsampling.py +88 -0
- flextool/gui/error_handling.py +50 -0
- flextool/gui/execution_manager.py +1715 -0
- flextool/gui/execution_window.py +1377 -0
- flextool/gui/hover_tooltip.py +111 -0
- flextool/gui/input_sources.py +730 -0
- flextool/gui/main_window.py +6181 -0
- flextool/gui/network_graph.py +215 -0
- flextool/gui/output_actions.py +393 -0
- flextool/gui/output_log_window.py +159 -0
- flextool/gui/platform_utils.py +421 -0
- flextool/gui/plot_cache.py +88 -0
- flextool/gui/plot_canvas.py +543 -0
- flextool/gui/plot_config_reader.py +272 -0
- flextool/gui/project_utils.py +100 -0
- flextool/gui/result_viewer.py +4394 -0
- flextool/gui/scenario_key.py +162 -0
- flextool/gui/scenario_lists.py +516 -0
- flextool/gui/settings_io.py +360 -0
- flextool/gui/solve_reader.py +103 -0
- flextool/gui/tree_reorder.py +88 -0
- flextool/gui/ui_metrics.py +420 -0
- flextool/input_derivation/__init__.py +281 -0
- flextool/input_derivation/_commodity_ladder.py +375 -0
- flextool/input_derivation/_commodity_ladder_sets.py +70 -0
- flextool/input_derivation/_dc_power_flow.py +377 -0
- flextool/input_derivation/_method_constants.py +77 -0
- flextool/input_derivation/_process_method.py +258 -0
- flextool/input_derivation/_specs.py +1026 -0
- flextool/input_derivation/_validators.py +321 -0
- flextool/lean_parquet.py +159 -0
- flextool/model_builder/__init__.py +5 -0
- flextool/model_builder/build_model.py +589 -0
- flextool/model_builder/encoding.py +67 -0
- flextool/model_builder/names.py +34 -0
- flextool/model_builder/profiles.py +129 -0
- flextool/plot_outputs/__init__.py +14 -0
- flextool/plot_outputs/axis_helpers.py +355 -0
- flextool/plot_outputs/color_template.py +888 -0
- flextool/plot_outputs/config.py +171 -0
- flextool/plot_outputs/format_helpers.py +345 -0
- flextool/plot_outputs/legend_helpers.py +143 -0
- flextool/plot_outputs/orchestrator.py +1141 -0
- flextool/plot_outputs/perf.py +37 -0
- flextool/plot_outputs/plan.py +1787 -0
- flextool/plot_outputs/plot_bars.py +1510 -0
- flextool/plot_outputs/plot_bars_detail.py +753 -0
- flextool/plot_outputs/plot_lines.py +951 -0
- flextool/plot_outputs/shared_manifest.py +564 -0
- flextool/plot_outputs/subplot_helpers.py +137 -0
- flextool/process_inputs/__init__.py +188 -0
- flextool/process_inputs/import_old_excel_input.json +4159 -0
- flextool/process_inputs/read_matpower.py +451 -0
- flextool/process_inputs/read_old_flextool.py +1288 -0
- flextool/process_inputs/read_self_describing_excel.py +1423 -0
- flextool/process_inputs/read_tabular_with_specification.py +1114 -0
- flextool/process_inputs/write_old_flextool_to_db.py +3077 -0
- flextool/process_inputs/write_self_describing_to_db.py +977 -0
- flextool/process_inputs/write_to_input_db.py +269 -0
- flextool/process_outputs/__init__.py +7 -0
- flextool/process_outputs/_annualize.py +55 -0
- flextool/process_outputs/_inmemory_helpers.py +292 -0
- flextool/process_outputs/_output_meta.py +672 -0
- flextool/process_outputs/calc_capacity_flows.py +107 -0
- flextool/process_outputs/calc_connections.py +136 -0
- flextool/process_outputs/calc_costs.py +260 -0
- flextool/process_outputs/calc_group_flows.py +192 -0
- flextool/process_outputs/calc_slacks.py +103 -0
- flextool/process_outputs/calc_storage_vre.py +160 -0
- flextool/process_outputs/drop_levels.py +208 -0
- flextool/process_outputs/handoff_writers.py +1315 -0
- flextool/process_outputs/out_ancillary.py +544 -0
- flextool/process_outputs/out_capacity.py +179 -0
- flextool/process_outputs/out_costs.py +334 -0
- flextool/process_outputs/out_flowgroup.py +189 -0
- flextool/process_outputs/out_flows.py +301 -0
- flextool/process_outputs/out_group.py +475 -0
- flextool/process_outputs/out_node.py +190 -0
- flextool/process_outputs/persist_realized_slice.py +601 -0
- flextool/process_outputs/process_results.py +24 -0
- flextool/process_outputs/read_highs_solution.py +2256 -0
- flextool/process_outputs/read_parameters.py +1799 -0
- flextool/process_outputs/read_sets.py +1095 -0
- flextool/process_outputs/read_variables.py +553 -0
- flextool/process_outputs/solve_order.py +81 -0
- flextool/process_outputs/spinedb_replay.py +412 -0
- flextool/process_outputs/union_realized_slice.py +224 -0
- flextool/process_outputs/write_outputs.py +1286 -0
- flextool/process_outputs/write_spinedb.py +1267 -0
- flextool/representative_periods/__init__.py +5 -0
- flextool/representative_periods/clustering.py +165 -0
- flextool/representative_periods/force_include.py +563 -0
- flextool/representative_periods/netload.py +365 -0
- flextool/representative_periods/netload_inputs.py +345 -0
- flextool/representative_periods/netload_iterate.py +722 -0
- flextool/representative_periods/preprocess.py +948 -0
- flextool/representative_periods/scenario_stack.py +195 -0
- flextool/representative_periods/weights.py +124 -0
- flextool/scenario_comparison/__init__.py +13 -0
- flextool/scenario_comparison/config_builder.py +158 -0
- flextool/scenario_comparison/constants.py +20 -0
- flextool/scenario_comparison/data_models.py +222 -0
- flextool/scenario_comparison/db_reader.py +399 -0
- flextool/scenario_comparison/dispatch_data.py +1002 -0
- flextool/scenario_comparison/dispatch_mappings.py +205 -0
- flextool/scenario_comparison/dispatch_plots.py +691 -0
- flextool/scenario_comparison/input_entity_colors.py +319 -0
- flextool/scenario_comparison/orchestrator.py +453 -0
- flextool/scenario_comparison/plan_union.py +244 -0
- flextool/scenario_comparison/plot_settings_seed.py +205 -0
- flextool/schemas/AXIS_CONTRACT.md +71 -0
- flextool/schemas/canonical_databases/howto_aggregate_output.json +6225 -0
- flextool/schemas/canonical_databases/howto_connections.json +5606 -0
- flextool/schemas/canonical_databases/howto_demand.json +5518 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir.json +6239 -0
- flextool/schemas/canonical_databases/howto_hydro_reservoir_with_pump.json +5933 -0
- flextool/schemas/canonical_databases/howto_non_sync_and_curtailment.json +5794 -0
- flextool/schemas/canonical_databases/howto_ramp_and_start_up.json +5707 -0
- flextool/schemas/canonical_databases/howto_stochastics.json +6032 -0
- flextool/schemas/canonical_databases/templates_examples.json +13532 -0
- flextool/schemas/canonical_databases/templates_time_settings_only.json +5340 -0
- flextool/schemas/comparison_settings_template.json +197 -0
- flextool/schemas/default_plot_settings.yaml +260 -0
- flextool/schemas/default_plots.yaml +2293 -0
- flextool/schemas/flextool_axis_contract.json +303 -0
- flextool/schemas/flextool_axis_contract.schema.json +247 -0
- flextool/schemas/old_flextool_import_template.json +4443 -0
- flextool/schemas/output_info_template.json +48 -0
- flextool/schemas/output_settings_template.json +256 -0
- flextool/schemas/pre_v26/flextool_template_constant_default.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_default_optional_output.json +2152 -0
- flextool/schemas/pre_v26/flextool_template_default_value.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_drop_down.json +2080 -0
- flextool/schemas/pre_v26/flextool_template_lifetime_method.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_optional_outputs.json +2094 -0
- flextool/schemas/pre_v26/flextool_template_output_node_flows.json +2105 -0
- flextool/schemas/pre_v26/flextool_template_results_master.json +493 -0
- flextool/schemas/pre_v26/flextool_template_rolling_start_remove.json +2087 -0
- flextool/schemas/pre_v26/flextool_template_rolling_window.json +2059 -0
- flextool/schemas/pre_v26/flextool_template_storage_binding_defaults.json +46 -0
- flextool/schemas/pre_v26/flextool_template_v2.json +1990 -0
- flextool/schemas/pre_v26/flextool_template_v25.json +3864 -0
- flextool/schemas/spinedb_results_schema.json +581 -0
- flextool/schemas/spinedb_schema.json +4636 -0
- flextool/solver_config/copt.opt.template +18 -0
- flextool/solver_config/cplex.opt.template +25 -0
- flextool/solver_config/gurobi.opt.template +18 -0
- flextool/solver_config/highs.opt.template +18 -0
- flextool/solver_config/xpress.opt.template +26 -0
- flextool/spinedb_backend/__init__.py +26 -0
- flextool/spinedb_backend/_axis_enums.py +1119 -0
- flextool/spinedb_backend/_backend.py +1139 -0
- flextool/update_flextool/__init__.py +12 -0
- flextool/update_flextool/canonical_databases.py +251 -0
- flextool/update_flextool/db_migration.py +7108 -0
- flextool/update_flextool/ensure_settings_db.py +138 -0
- flextool/update_flextool/export_database.py +103 -0
- flextool/update_flextool/extend_tests_fixture.py +772 -0
- flextool/update_flextool/generate_canonical.py +274 -0
- flextool/update_flextool/initialize_database.py +42 -0
- flextool/update_flextool/install_info.py +225 -0
- flextool/update_flextool/self_update.py +464 -0
- flextool/update_flextool/sync_master_json_template.py +125 -0
- flextool/update_flextool/test_fixtures.py +187 -0
- flextool-4.0.0.dist-info/METADATA +217 -0
- flextool-4.0.0.dist-info/RECORD +322 -0
- flextool-4.0.0.dist-info/WHEEL +5 -0
- flextool-4.0.0.dist-info/entry_points.txt +17 -0
- flextool-4.0.0.dist-info/licenses/LICENSE.txt +19 -0
- flextool-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1787 @@
|
|
|
1
|
+
"""PlotPlan -- pre-computed plot plans for instant rendering.
|
|
2
|
+
|
|
3
|
+
A PlotPlan captures everything needed to render any file page of a plot
|
|
4
|
+
without re-running dimension rules, layout computation, or color mapping.
|
|
5
|
+
Plans are saved alongside parquet files and loaded by the viewer.
|
|
6
|
+
|
|
7
|
+
File format:
|
|
8
|
+
{result_key}__{sub_config}_plan.json -- metadata, layout, colors, batch structure
|
|
9
|
+
{result_key}__{sub_config}_plan.parquet -- processed DataFrame (post dimension rules)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import TYPE_CHECKING, Any
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
|
|
22
|
+
from flextool.lean_parquet import read_lean_parquet, write_lean_parquet
|
|
23
|
+
import pandas as pd
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from matplotlib.figure import Figure
|
|
27
|
+
|
|
28
|
+
from flextool.plot_outputs.config import PlotConfig
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# Bump when the on-disk PlotPlan schema or pagination semantics change so
|
|
34
|
+
# stale plans get recomputed instead of silently mis-applied. Old JSONs
|
|
35
|
+
# without a schema_version (or with a lower one) are ignored by
|
|
36
|
+
# load_plot_plan and the GUI's compute_live_plan path takes over.
|
|
37
|
+
PLAN_SCHEMA_VERSION = 3
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
# JSON helper: make numpy / tuple types serializable
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
def _json_safe(obj: Any) -> Any:
|
|
45
|
+
"""Recursively convert numpy scalars, tuples, etc. into JSON-safe types."""
|
|
46
|
+
if obj is None:
|
|
47
|
+
return None
|
|
48
|
+
if isinstance(obj, (np.integer,)):
|
|
49
|
+
return int(obj)
|
|
50
|
+
if isinstance(obj, (np.floating,)):
|
|
51
|
+
return float(obj)
|
|
52
|
+
if isinstance(obj, np.ndarray):
|
|
53
|
+
return obj.tolist()
|
|
54
|
+
if isinstance(obj, tuple):
|
|
55
|
+
return list(obj)
|
|
56
|
+
if isinstance(obj, dict):
|
|
57
|
+
return {str(k): _json_safe(v) for k, v in obj.items()}
|
|
58
|
+
if isinstance(obj, list):
|
|
59
|
+
return [_json_safe(v) for v in obj]
|
|
60
|
+
return obj
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
# Time-axis label extraction (shared by planner + loader)
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
def _extract_time_labels(
|
|
68
|
+
df: pd.DataFrame,
|
|
69
|
+
) -> tuple[list[str], list[str] | None, int]:
|
|
70
|
+
"""Extract x-axis time values and optional period labels from a DataFrame.
|
|
71
|
+
|
|
72
|
+
- ``time_values``: last MultiIndex level as strings, or the whole index
|
|
73
|
+
when the DataFrame has a plain Index.
|
|
74
|
+
- ``period_labels``: values of a level named (case-insensitive) ``period``
|
|
75
|
+
or ``solve_period`` when the index is a MultiIndex; otherwise ``None``.
|
|
76
|
+
- ``max_period_label_len``: longest period label length, or 0 when there
|
|
77
|
+
are no period labels.
|
|
78
|
+
|
|
79
|
+
Mirrors the extraction pattern used in ``_compute_time_plan`` so both the
|
|
80
|
+
planner and the plan loader agree on how these fields are derived.
|
|
81
|
+
"""
|
|
82
|
+
period_labels: list[str] | None = None
|
|
83
|
+
if isinstance(df.index, pd.MultiIndex):
|
|
84
|
+
time_values = df.index.get_level_values(-1).astype(str).tolist()
|
|
85
|
+
for lvl_i, name in enumerate(df.index.names[:-1]):
|
|
86
|
+
if name and str(name).lower() in ('period', 'solve_period'):
|
|
87
|
+
period_labels = (
|
|
88
|
+
df.index.get_level_values(lvl_i).astype(str).tolist()
|
|
89
|
+
)
|
|
90
|
+
break
|
|
91
|
+
else:
|
|
92
|
+
time_values = df.index.astype(str).tolist()
|
|
93
|
+
max_len = (
|
|
94
|
+
max((len(lbl) for lbl in period_labels), default=0)
|
|
95
|
+
if period_labels else 0
|
|
96
|
+
)
|
|
97
|
+
return time_values, period_labels, max_len
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
# PlotPlan dataclass
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
@dataclass
|
|
105
|
+
class PlotPlan:
|
|
106
|
+
"""Pre-computed plan for rendering a set of figures."""
|
|
107
|
+
|
|
108
|
+
chart_type: str # 'lines', 'stack', 'bar'
|
|
109
|
+
plot_name: str
|
|
110
|
+
total_file_count: int
|
|
111
|
+
|
|
112
|
+
# The processed DataFrame -- post dimension rules, time-sliced,
|
|
113
|
+
# file-member filtered, zero-filtered, multiplied.
|
|
114
|
+
# For bar charts: index = bar labels, columns = data.
|
|
115
|
+
# For time charts: index = time, columns = data.
|
|
116
|
+
processed_df: pd.DataFrame
|
|
117
|
+
|
|
118
|
+
# Effective plots structure: list of (title, column_labels_list)
|
|
119
|
+
# column_labels_list identifies which columns of processed_df belong
|
|
120
|
+
# to this subplot.
|
|
121
|
+
# For bars: (title, {rows: [...], cols: [...]}) with row and column selectors.
|
|
122
|
+
# For time-series: (title, column_labels_list).
|
|
123
|
+
effective_plot_specs: list[tuple[str | None, list | dict]]
|
|
124
|
+
|
|
125
|
+
# Batch structure: file_batches[i] = list of indices into effective_plot_specs
|
|
126
|
+
file_batches: list[list[int]]
|
|
127
|
+
|
|
128
|
+
# Visual config
|
|
129
|
+
shared_color_map: dict[str, tuple] | None = None
|
|
130
|
+
axis_bounds: list | None = None
|
|
131
|
+
|
|
132
|
+
# Color-template hints carried so the viewer can rebuild ONLY the
|
|
133
|
+
# shared_color_map (colors + file order) in place when the user edits
|
|
134
|
+
# the project's plot_settings.yaml — without recomputing dimension
|
|
135
|
+
# rules, layout, or the processed DataFrame. Mirror cfg.color_category /
|
|
136
|
+
# cfg.color_entity_class. Both None for plots that don't color by
|
|
137
|
+
# template (or non-shared legends).
|
|
138
|
+
color_category: str | None = None
|
|
139
|
+
color_entity_class: str | None = None
|
|
140
|
+
|
|
141
|
+
# For simple (non-stacked, non-grouped) bar charts only: the column-index
|
|
142
|
+
# level whose value supplies each bar's color, resolved from
|
|
143
|
+
# ``color_entity_class`` via ``resolve_color_bar_level``. Lines / stacked /
|
|
144
|
+
# grouped charts don't need this — their colored level is fixed by a role
|
|
145
|
+
# char. ``None`` => bars fall back to the default single color.
|
|
146
|
+
color_bar_level: str | None = None
|
|
147
|
+
|
|
148
|
+
# Layout (stored as plain dict for JSON serialization)
|
|
149
|
+
layout_type: str = '' # 'line' or 'bar'
|
|
150
|
+
layout_params: dict = field(default_factory=dict)
|
|
151
|
+
|
|
152
|
+
# Figure builder parameters
|
|
153
|
+
sub_levels: list[int] = field(default_factory=list)
|
|
154
|
+
item_level_names: list[str] = field(default_factory=list)
|
|
155
|
+
time_index_values: list[str] | None = None # serialized time index
|
|
156
|
+
period_labels: list[str] | None = None # period name per x-position
|
|
157
|
+
max_period_label_len: int = 0 # longest period label (for overlap check)
|
|
158
|
+
|
|
159
|
+
# Config params for figure builder
|
|
160
|
+
subplots_per_row: int = 2
|
|
161
|
+
legend_position: str = 'right'
|
|
162
|
+
xlabel: str | None = None
|
|
163
|
+
ylabel: str | None = None
|
|
164
|
+
bar_orientation: str = 'horizontal'
|
|
165
|
+
axis_tick_format: str = 'dynamic'
|
|
166
|
+
always_include_zero_in_axis: bool = True
|
|
167
|
+
value_label: str | None = None
|
|
168
|
+
base_bar_length: float = 4.0
|
|
169
|
+
skip_data_with_only_zeroes: bool = False
|
|
170
|
+
|
|
171
|
+
# Per-subplot global y-axis range across full time series.
|
|
172
|
+
# List of (min, max) tuples, one per effective_plot_spec entry.
|
|
173
|
+
# Used by the result viewer to keep y-axis stable when scrolling.
|
|
174
|
+
subplot_y_ranges: list[tuple[float, float]] = field(default_factory=list)
|
|
175
|
+
|
|
176
|
+
# Bar-specific level metadata
|
|
177
|
+
stack_levels: list[int] = field(default_factory=list)
|
|
178
|
+
stack_level_names: list[str] = field(default_factory=list)
|
|
179
|
+
expand_axis_levels: list[int] = field(default_factory=list)
|
|
180
|
+
expand_axis_level_names: list[str] = field(default_factory=list)
|
|
181
|
+
grouped_bar_levels: list[int] = field(default_factory=list)
|
|
182
|
+
grouped_bar_level_names: list[str] = field(default_factory=list)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ---------------------------------------------------------------------------
|
|
186
|
+
# save / load
|
|
187
|
+
# ---------------------------------------------------------------------------
|
|
188
|
+
|
|
189
|
+
def save_plot_plan(
|
|
190
|
+
plan: PlotPlan, output_dir: Path, result_key: str, sub_config: str,
|
|
191
|
+
) -> None:
|
|
192
|
+
"""Save a PlotPlan to disk as JSON metadata + parquet data."""
|
|
193
|
+
output_dir = Path(output_dir)
|
|
194
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
195
|
+
|
|
196
|
+
prefix = f"{result_key}__{sub_config}"
|
|
197
|
+
|
|
198
|
+
# write_lean_parquet handles single-level MultiIndex correctly, so no
|
|
199
|
+
# flattening is needed (the old pandas-metadata approach required it).
|
|
200
|
+
write_lean_parquet(plan.processed_df, output_dir / f"{prefix}_plan.parquet")
|
|
201
|
+
|
|
202
|
+
# Serialize effective_plot_specs: titles may be None, selectors are lists
|
|
203
|
+
# of scalars or tuples. JSON-encode tuples as lists.
|
|
204
|
+
specs_json: list[list] = []
|
|
205
|
+
for title, selector in plan.effective_plot_specs:
|
|
206
|
+
specs_json.append([title, _json_safe(selector)])
|
|
207
|
+
|
|
208
|
+
# Build JSON-serializable metadata
|
|
209
|
+
meta: dict[str, Any] = {
|
|
210
|
+
'schema_version': PLAN_SCHEMA_VERSION,
|
|
211
|
+
'chart_type': plan.chart_type,
|
|
212
|
+
'plot_name': plan.plot_name,
|
|
213
|
+
'total_file_count': plan.total_file_count,
|
|
214
|
+
'effective_plot_specs': specs_json,
|
|
215
|
+
'file_batches': plan.file_batches,
|
|
216
|
+
'shared_color_map': (
|
|
217
|
+
{k: list(v) for k, v in plan.shared_color_map.items()}
|
|
218
|
+
if plan.shared_color_map else None
|
|
219
|
+
),
|
|
220
|
+
'color_category': plan.color_category,
|
|
221
|
+
'color_entity_class': plan.color_entity_class,
|
|
222
|
+
'color_bar_level': plan.color_bar_level,
|
|
223
|
+
'axis_bounds': _json_safe(plan.axis_bounds),
|
|
224
|
+
'layout_type': plan.layout_type,
|
|
225
|
+
'layout_params': _json_safe(plan.layout_params),
|
|
226
|
+
'sub_levels': plan.sub_levels,
|
|
227
|
+
'item_level_names': plan.item_level_names,
|
|
228
|
+
# time_index_values / period_labels / max_period_label_len are
|
|
229
|
+
# intentionally NOT serialized — ``load_plot_plan`` reconstructs
|
|
230
|
+
# them from ``processed_df.index`` via ``_extract_time_labels``.
|
|
231
|
+
# Old JSON files that still carry these keys are accepted on load
|
|
232
|
+
# so rolling out this change doesn't force a plan regen.
|
|
233
|
+
'subplots_per_row': plan.subplots_per_row,
|
|
234
|
+
'legend_position': plan.legend_position,
|
|
235
|
+
'xlabel': plan.xlabel,
|
|
236
|
+
'ylabel': plan.ylabel,
|
|
237
|
+
'bar_orientation': plan.bar_orientation,
|
|
238
|
+
'axis_tick_format': plan.axis_tick_format,
|
|
239
|
+
'always_include_zero_in_axis': plan.always_include_zero_in_axis,
|
|
240
|
+
'value_label': plan.value_label,
|
|
241
|
+
'base_bar_length': plan.base_bar_length,
|
|
242
|
+
'skip_data_with_only_zeroes': plan.skip_data_with_only_zeroes,
|
|
243
|
+
'stack_levels': plan.stack_levels,
|
|
244
|
+
'stack_level_names': plan.stack_level_names,
|
|
245
|
+
'expand_axis_levels': plan.expand_axis_levels,
|
|
246
|
+
'expand_axis_level_names': plan.expand_axis_level_names,
|
|
247
|
+
'grouped_bar_levels': plan.grouped_bar_levels,
|
|
248
|
+
'grouped_bar_level_names': plan.grouped_bar_level_names,
|
|
249
|
+
'subplot_y_ranges': [list(r) for r in plan.subplot_y_ranges],
|
|
250
|
+
# Legacy: lean_parquet handles single-level MultiIndex now; kept
|
|
251
|
+
# so old plan JSON files still load via the reconstruction below.
|
|
252
|
+
'col_was_single_multi': False,
|
|
253
|
+
'idx_was_single_multi': False,
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
with open(output_dir / f"{prefix}_plan.json", 'w') as f:
|
|
257
|
+
json.dump(meta, f, indent=2, default=_json_safe)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def load_plot_plan(
|
|
261
|
+
plan_dir: Path, result_key: str, sub_config: str,
|
|
262
|
+
) -> PlotPlan | None:
|
|
263
|
+
"""Load a PlotPlan from disk. Returns None if files don't exist."""
|
|
264
|
+
plan_dir = Path(plan_dir)
|
|
265
|
+
prefix = f"{result_key}__{sub_config}"
|
|
266
|
+
|
|
267
|
+
json_path = plan_dir / f"{prefix}_plan.json"
|
|
268
|
+
parquet_path = plan_dir / f"{prefix}_plan.parquet"
|
|
269
|
+
|
|
270
|
+
if not json_path.exists() or not parquet_path.exists():
|
|
271
|
+
return None
|
|
272
|
+
|
|
273
|
+
try:
|
|
274
|
+
with open(json_path, 'r', encoding='utf-8') as f:
|
|
275
|
+
meta = json.load(f)
|
|
276
|
+
|
|
277
|
+
# Reject stale schemas so a code change that affects pagination
|
|
278
|
+
# or plan content does not silently re-use plans saved before
|
|
279
|
+
# the change. compute_live_plan will rebuild from current code.
|
|
280
|
+
if meta.get('schema_version', 0) != PLAN_SCHEMA_VERSION:
|
|
281
|
+
return None
|
|
282
|
+
|
|
283
|
+
df = read_lean_parquet(parquet_path)
|
|
284
|
+
|
|
285
|
+
# Reconstruct single-level MultiIndex if it was flattened on save
|
|
286
|
+
if meta.get('col_was_single_multi', False):
|
|
287
|
+
df.columns = pd.MultiIndex.from_arrays(
|
|
288
|
+
[df.columns], names=[df.columns.name]
|
|
289
|
+
)
|
|
290
|
+
if meta.get('idx_was_single_multi', False):
|
|
291
|
+
df.index = pd.MultiIndex.from_arrays(
|
|
292
|
+
[df.index], names=[df.index.name]
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
# Reconstruct color map tuples
|
|
296
|
+
color_map = None
|
|
297
|
+
if meta.get('shared_color_map'):
|
|
298
|
+
color_map = {k: tuple(v) for k, v in meta['shared_color_map'].items()}
|
|
299
|
+
|
|
300
|
+
# Reconstruct effective_plot_specs as list of (title, selector)
|
|
301
|
+
raw_specs = meta.get('effective_plot_specs', [])
|
|
302
|
+
effective_plot_specs: list[tuple[str | None, list]] = [
|
|
303
|
+
(s[0], s[1]) for s in raw_specs
|
|
304
|
+
]
|
|
305
|
+
|
|
306
|
+
chart_type = meta['chart_type']
|
|
307
|
+
# Time-axis fields: prefer values carried in old-format JSON (so
|
|
308
|
+
# rolling out the "don't serialize them" change doesn't force a
|
|
309
|
+
# plan regen). For new-format JSON, reconstruct from
|
|
310
|
+
# ``processed_df.index`` — but only for time-based charts. Bar
|
|
311
|
+
# charts never populate these (matching ``_compute_bar_plan``
|
|
312
|
+
# semantics), so we leave them at their None/0 defaults.
|
|
313
|
+
has_legacy_time_fields = (
|
|
314
|
+
'time_index_values' in meta
|
|
315
|
+
or 'period_labels' in meta
|
|
316
|
+
or 'max_period_label_len' in meta
|
|
317
|
+
)
|
|
318
|
+
if has_legacy_time_fields or chart_type == 'bar':
|
|
319
|
+
time_index_values = meta.get('time_index_values')
|
|
320
|
+
period_labels = meta.get('period_labels')
|
|
321
|
+
max_period_label_len = meta.get('max_period_label_len', 0)
|
|
322
|
+
else:
|
|
323
|
+
time_index_values, period_labels, max_period_label_len = (
|
|
324
|
+
_extract_time_labels(df)
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
plan = PlotPlan(
|
|
328
|
+
chart_type=chart_type,
|
|
329
|
+
plot_name=meta['plot_name'],
|
|
330
|
+
total_file_count=meta['total_file_count'],
|
|
331
|
+
processed_df=df,
|
|
332
|
+
effective_plot_specs=effective_plot_specs,
|
|
333
|
+
file_batches=meta['file_batches'],
|
|
334
|
+
shared_color_map=color_map,
|
|
335
|
+
color_category=meta.get('color_category'),
|
|
336
|
+
color_entity_class=meta.get('color_entity_class'),
|
|
337
|
+
color_bar_level=meta.get('color_bar_level'),
|
|
338
|
+
axis_bounds=meta.get('axis_bounds'),
|
|
339
|
+
layout_type=meta.get('layout_type', ''),
|
|
340
|
+
layout_params=meta.get('layout_params', {}),
|
|
341
|
+
sub_levels=meta.get('sub_levels', []),
|
|
342
|
+
item_level_names=meta.get('item_level_names', []),
|
|
343
|
+
time_index_values=time_index_values,
|
|
344
|
+
period_labels=period_labels,
|
|
345
|
+
max_period_label_len=max_period_label_len,
|
|
346
|
+
subplot_y_ranges=[tuple(r) for r in meta.get('subplot_y_ranges', [])],
|
|
347
|
+
subplots_per_row=meta.get('subplots_per_row', 2),
|
|
348
|
+
legend_position=meta.get('legend_position', 'right'),
|
|
349
|
+
xlabel=meta.get('xlabel'),
|
|
350
|
+
ylabel=meta.get('ylabel'),
|
|
351
|
+
bar_orientation=meta.get('bar_orientation', 'horizontal'),
|
|
352
|
+
axis_tick_format=meta.get('axis_tick_format', 'dynamic'),
|
|
353
|
+
always_include_zero_in_axis=meta.get('always_include_zero_in_axis', True),
|
|
354
|
+
value_label=meta.get('value_label'),
|
|
355
|
+
base_bar_length=meta.get('base_bar_length', 4.0),
|
|
356
|
+
skip_data_with_only_zeroes=meta.get('skip_data_with_only_zeroes', False),
|
|
357
|
+
stack_levels=meta.get('stack_levels', []),
|
|
358
|
+
stack_level_names=meta.get('stack_level_names', []),
|
|
359
|
+
expand_axis_levels=meta.get('expand_axis_levels', []),
|
|
360
|
+
expand_axis_level_names=meta.get('expand_axis_level_names', []),
|
|
361
|
+
grouped_bar_levels=meta.get('grouped_bar_levels', []),
|
|
362
|
+
grouped_bar_level_names=meta.get('grouped_bar_level_names', []),
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
# Compute subplot_y_ranges from data if not stored in the plan file
|
|
366
|
+
if not plan.subplot_y_ranges and plan.chart_type in ('lines', 'stack'):
|
|
367
|
+
_backfill_subplot_y_ranges(plan)
|
|
368
|
+
|
|
369
|
+
return plan
|
|
370
|
+
except Exception as exc:
|
|
371
|
+
logger.warning("Failed to load plot plan %s/%s: %s", result_key, sub_config, exc)
|
|
372
|
+
return None
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _add_y_buffer(lo: float, hi: float, fraction: float = 0.05) -> tuple[float, float]:
|
|
376
|
+
"""Add a small buffer to y-axis range so extremes are fully visible."""
|
|
377
|
+
span = hi - lo
|
|
378
|
+
if span == 0:
|
|
379
|
+
return (lo - 0.5, hi + 0.5) if lo == 0 else (lo * 0.95, hi * 1.05)
|
|
380
|
+
return (lo - span * fraction, hi + span * fraction)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _backfill_subplot_y_ranges(plan: PlotPlan) -> None:
|
|
384
|
+
"""Compute subplot_y_ranges from the plan's processed_df.
|
|
385
|
+
|
|
386
|
+
Called for plans loaded from disk that predate the subplot_y_ranges field.
|
|
387
|
+
"""
|
|
388
|
+
include_zero = plan.always_include_zero_in_axis
|
|
389
|
+
ranges: list[tuple[float, float]] = []
|
|
390
|
+
for _title, selector in plan.effective_plot_specs:
|
|
391
|
+
df_sub = _select_time_columns(plan.processed_df, selector)
|
|
392
|
+
lo, hi = 0.0, 0.0
|
|
393
|
+
if isinstance(df_sub, pd.Series):
|
|
394
|
+
vals = df_sub.dropna()
|
|
395
|
+
if len(vals):
|
|
396
|
+
lo, hi = float(vals.min()), float(vals.max())
|
|
397
|
+
elif isinstance(df_sub, pd.DataFrame):
|
|
398
|
+
num = df_sub.select_dtypes(include='number')
|
|
399
|
+
if not num.empty:
|
|
400
|
+
lo, hi = float(num.min().min()), float(num.max().max())
|
|
401
|
+
if include_zero:
|
|
402
|
+
lo = min(lo, 0.0)
|
|
403
|
+
hi = max(hi, 0.0)
|
|
404
|
+
ranges.append(_add_y_buffer(lo, hi))
|
|
405
|
+
plan.subplot_y_ranges = ranges
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# ---------------------------------------------------------------------------
|
|
409
|
+
# build_figure_from_plan (the fast path)
|
|
410
|
+
# ---------------------------------------------------------------------------
|
|
411
|
+
|
|
412
|
+
def _select_bar_rows(df: pd.DataFrame, selector: list) -> pd.DataFrame:
|
|
413
|
+
"""Select rows from a bar DataFrame using the stored index labels."""
|
|
414
|
+
if not selector:
|
|
415
|
+
return df
|
|
416
|
+
# Reconstruct tuples for MultiIndex row selection
|
|
417
|
+
if isinstance(df.index, pd.MultiIndex):
|
|
418
|
+
idx_labels = [tuple(s) if isinstance(s, list) else s for s in selector]
|
|
419
|
+
mask = df.index.isin(idx_labels)
|
|
420
|
+
return df.loc[mask]
|
|
421
|
+
# Unwrap single-element lists that originate from a single-level MultiIndex
|
|
422
|
+
# encoded via _encode_row_selector then JSON-roundtripped.
|
|
423
|
+
flat_sel = [s[0] if isinstance(s, list) and len(s) == 1 else s for s in selector]
|
|
424
|
+
return df.loc[df.index.isin(flat_sel)]
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _select_time_columns(df: pd.DataFrame, selector: list) -> pd.DataFrame:
|
|
428
|
+
"""Select columns from a time-series DataFrame using stored column labels."""
|
|
429
|
+
if selector is None or len(selector) == 0:
|
|
430
|
+
return df
|
|
431
|
+
if isinstance(df.columns, pd.MultiIndex):
|
|
432
|
+
col_tuples = [tuple(c) if isinstance(c, list) else c for c in selector]
|
|
433
|
+
try:
|
|
434
|
+
mask = df.columns.isin(col_tuples)
|
|
435
|
+
except (AssertionError, ValueError):
|
|
436
|
+
# MultiIndex nlevels mismatch — fall back to string matching
|
|
437
|
+
col_strs = {str(c) for c in col_tuples}
|
|
438
|
+
mask = pd.array([str(c) in col_strs for c in df.columns], dtype=bool)
|
|
439
|
+
if mask.any():
|
|
440
|
+
return df.loc[:, mask]
|
|
441
|
+
# Fallback: try matching as strings
|
|
442
|
+
col_strs = [str(c) for c in col_tuples]
|
|
443
|
+
str_cols = [str(c) for c in df.columns]
|
|
444
|
+
mask2 = pd.Index(str_cols).isin(col_strs)
|
|
445
|
+
if mask2.any():
|
|
446
|
+
return df.loc[:, mask2]
|
|
447
|
+
return df # no match at all — return full DataFrame rather than empty
|
|
448
|
+
# Non-MultiIndex columns: unwrap single-element lists from JSON round-trip
|
|
449
|
+
flat_sel = []
|
|
450
|
+
for s in selector:
|
|
451
|
+
if isinstance(s, list):
|
|
452
|
+
flat_sel.append(s[0] if len(s) == 1 else tuple(s))
|
|
453
|
+
else:
|
|
454
|
+
flat_sel.append(s)
|
|
455
|
+
valid = [s for s in flat_sel if s in df.columns]
|
|
456
|
+
return df[valid] if valid else df
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def build_figure_from_plan(
|
|
460
|
+
plan: PlotPlan,
|
|
461
|
+
file_index: int = 0,
|
|
462
|
+
plot_rows: tuple[int, int] | None = None,
|
|
463
|
+
) -> 'Figure | None':
|
|
464
|
+
"""Build a single Figure from a pre-computed PlotPlan.
|
|
465
|
+
|
|
466
|
+
This is the fast path -- no dimension rules, no layout computation,
|
|
467
|
+
just direct figure building from pre-computed data.
|
|
468
|
+
|
|
469
|
+
For time-series plans whose *processed_df* covers the full timeline,
|
|
470
|
+
pass *plot_rows* ``(start, start + duration)`` to display a sub-range.
|
|
471
|
+
"""
|
|
472
|
+
from flextool.plot_outputs.subplot_helpers import LineLayoutParams, BarLayoutParams
|
|
473
|
+
|
|
474
|
+
if file_index >= plan.total_file_count or file_index < 0:
|
|
475
|
+
return None
|
|
476
|
+
|
|
477
|
+
# For time-series plans, slice to the requested window.
|
|
478
|
+
# When slicing, use pre-computed global y-ranges so the axis stays stable.
|
|
479
|
+
# Skip slicing when the range covers the full data (nothing to slice).
|
|
480
|
+
processed_df = plan.processed_df
|
|
481
|
+
time_vals = plan.time_index_values
|
|
482
|
+
period_labels = plan.period_labels
|
|
483
|
+
use_global_y_ranges = False
|
|
484
|
+
# *expected_x_length* pins the time-axis xlim to (start, end) regardless
|
|
485
|
+
# of how short the underlying data turns out to be — keeps the time
|
|
486
|
+
# axis stable when scenarios with different model horizons are compared
|
|
487
|
+
# side by side. Only meaningful for time-series charts.
|
|
488
|
+
expected_x_length: int | None = None
|
|
489
|
+
if plot_rows is not None and plan.chart_type != 'bar':
|
|
490
|
+
start, end = plot_rows
|
|
491
|
+
expected_x_length = max(0, end - start)
|
|
492
|
+
actually_slicing = start > 0 or end < len(processed_df)
|
|
493
|
+
if actually_slicing:
|
|
494
|
+
processed_df = processed_df.iloc[start:end]
|
|
495
|
+
if time_vals is not None:
|
|
496
|
+
time_vals = time_vals[start:end]
|
|
497
|
+
if period_labels is not None:
|
|
498
|
+
period_labels = period_labels[start:end]
|
|
499
|
+
if plan.subplot_y_ranges:
|
|
500
|
+
use_global_y_ranges = True
|
|
501
|
+
|
|
502
|
+
# Reconstruct effective_plots for the requested batch
|
|
503
|
+
batch_indices = plan.file_batches[file_index]
|
|
504
|
+
effective_plots: list[tuple[str | None, pd.DataFrame]] = []
|
|
505
|
+
for idx in batch_indices:
|
|
506
|
+
title, selector = plan.effective_plot_specs[idx]
|
|
507
|
+
if plan.chart_type == 'bar':
|
|
508
|
+
# selector may be a dict with 'rows'/'cols' keys (new format)
|
|
509
|
+
# or a plain list of row labels (old format, backward compat)
|
|
510
|
+
if isinstance(selector, dict):
|
|
511
|
+
df_sub = _select_bar_rows(processed_df, selector.get('rows', []))
|
|
512
|
+
col_sel = selector.get('cols')
|
|
513
|
+
if col_sel:
|
|
514
|
+
df_sub = _select_time_columns(df_sub, col_sel)
|
|
515
|
+
else:
|
|
516
|
+
df_sub = _select_bar_rows(processed_df, selector)
|
|
517
|
+
else:
|
|
518
|
+
df_sub = _select_time_columns(processed_df, selector)
|
|
519
|
+
effective_plots.append((title, df_sub))
|
|
520
|
+
|
|
521
|
+
if not effective_plots:
|
|
522
|
+
return None
|
|
523
|
+
|
|
524
|
+
# Reconstruct time_index
|
|
525
|
+
time_index = None
|
|
526
|
+
if time_vals is not None:
|
|
527
|
+
time_index = pd.Index(time_vals)
|
|
528
|
+
|
|
529
|
+
# Reconstruct layout
|
|
530
|
+
if plan.layout_type == 'line':
|
|
531
|
+
layout = LineLayoutParams(**plan.layout_params)
|
|
532
|
+
elif plan.layout_type == 'bar':
|
|
533
|
+
# The category-label margin is computed once at plan-build time
|
|
534
|
+
# (_compute_bar_layout → _bar_label_width_inches: the longest label
|
|
535
|
+
# ACTUALLY drawn across all pages, not a longest-group + longest-bar
|
|
536
|
+
# sum) and stored, so every page reserves the same margin — the plot
|
|
537
|
+
# area stays at the same horizontal position when paging, and render
|
|
538
|
+
# stays fast (no recompute).
|
|
539
|
+
layout = BarLayoutParams(**plan.layout_params)
|
|
540
|
+
else:
|
|
541
|
+
return None
|
|
542
|
+
|
|
543
|
+
# Determine axis bounds — use global y-ranges when time-slicing
|
|
544
|
+
axis_bounds = plan.axis_bounds
|
|
545
|
+
if use_global_y_ranges and plan.subplot_y_ranges:
|
|
546
|
+
# Slice to match the current batch (batch_indices are global positions)
|
|
547
|
+
axis_bounds = [
|
|
548
|
+
plan.subplot_y_ranges[i]
|
|
549
|
+
for i in batch_indices
|
|
550
|
+
if i < len(plan.subplot_y_ranges)
|
|
551
|
+
]
|
|
552
|
+
|
|
553
|
+
# Build the figure
|
|
554
|
+
if plan.chart_type == 'lines':
|
|
555
|
+
from flextool.plot_outputs.plot_lines import _build_lines_figure
|
|
556
|
+
return _build_lines_figure(
|
|
557
|
+
effective_plots, plan.plot_name, plan.sub_levels,
|
|
558
|
+
plan.item_level_names, time_index,
|
|
559
|
+
plan.subplots_per_row, plan.legend_position,
|
|
560
|
+
plan.xlabel, plan.ylabel,
|
|
561
|
+
axis_bounds, plan.axis_tick_format,
|
|
562
|
+
plan.always_include_zero_in_axis,
|
|
563
|
+
layout, plan.shared_color_map,
|
|
564
|
+
period_labels=period_labels,
|
|
565
|
+
expected_x_length=expected_x_length,
|
|
566
|
+
)
|
|
567
|
+
elif plan.chart_type == 'stack':
|
|
568
|
+
from flextool.plot_outputs.plot_lines import _build_stack_figure
|
|
569
|
+
return _build_stack_figure(
|
|
570
|
+
effective_plots, plan.plot_name, plan.sub_levels,
|
|
571
|
+
plan.item_level_names, time_index,
|
|
572
|
+
plan.subplots_per_row, plan.legend_position,
|
|
573
|
+
plan.xlabel, plan.ylabel,
|
|
574
|
+
axis_bounds, plan.axis_tick_format,
|
|
575
|
+
plan.always_include_zero_in_axis,
|
|
576
|
+
layout, plan.shared_color_map,
|
|
577
|
+
period_labels=period_labels,
|
|
578
|
+
expected_x_length=expected_x_length,
|
|
579
|
+
)
|
|
580
|
+
elif plan.chart_type == 'bar':
|
|
581
|
+
from flextool.plot_outputs.plot_bars import _build_bar_figure
|
|
582
|
+
|
|
583
|
+
# Resolve value_label
|
|
584
|
+
value_fmt = None
|
|
585
|
+
if plan.value_label is True or plan.value_label == 'true':
|
|
586
|
+
value_fmt = 'dynamic'
|
|
587
|
+
elif plan.value_label:
|
|
588
|
+
value_fmt = str(plan.value_label)
|
|
589
|
+
|
|
590
|
+
batch_title = (
|
|
591
|
+
f"{plan.plot_name} ({file_index + 1}/{plan.total_file_count})"
|
|
592
|
+
if plan.total_file_count > 1 else plan.plot_name
|
|
593
|
+
)
|
|
594
|
+
return _build_bar_figure(
|
|
595
|
+
effective_plots, plan.processed_df, batch_title, '',
|
|
596
|
+
plan.stack_levels, plan.stack_level_names,
|
|
597
|
+
plan.expand_axis_levels, plan.expand_axis_level_names,
|
|
598
|
+
plan.sub_levels,
|
|
599
|
+
plan.grouped_bar_levels, plan.grouped_bar_level_names,
|
|
600
|
+
plan.legend_position, plan.subplots_per_row,
|
|
601
|
+
plan.xlabel, plan.ylabel,
|
|
602
|
+
plan.bar_orientation, plan.base_bar_length,
|
|
603
|
+
value_fmt,
|
|
604
|
+
plan.axis_bounds, plan.axis_tick_format,
|
|
605
|
+
plan.always_include_zero_in_axis,
|
|
606
|
+
layout, plan.shared_color_map,
|
|
607
|
+
plan.skip_data_with_only_zeroes,
|
|
608
|
+
color_bar_level=plan.color_bar_level,
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
return None
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
# ---------------------------------------------------------------------------
|
|
615
|
+
# _encode_column_selector / _encode_row_selector
|
|
616
|
+
# ---------------------------------------------------------------------------
|
|
617
|
+
|
|
618
|
+
def _encode_column_selector(
|
|
619
|
+
df_sub: pd.DataFrame,
|
|
620
|
+
df_full: pd.DataFrame,
|
|
621
|
+
sub_levels: list[int] | None = None,
|
|
622
|
+
sub_value=None,
|
|
623
|
+
) -> list:
|
|
624
|
+
"""Encode columns of df_sub as a JSON-serializable selector list.
|
|
625
|
+
|
|
626
|
+
The selector must be usable with ``df_full.columns.isin(...)`` at
|
|
627
|
+
reconstruction time. ``_extract_subplot_data`` applies ``xs`` to
|
|
628
|
+
df_full, which drops the sub-levels, so df_sub's column tuples are
|
|
629
|
+
NARROWER than df_full's. When *sub_levels* and *sub_value* are
|
|
630
|
+
provided, we reinsert the sub values at the correct level positions
|
|
631
|
+
so the stored selector matches df_full's full-width tuples.
|
|
632
|
+
"""
|
|
633
|
+
if not isinstance(df_full.columns, pd.MultiIndex):
|
|
634
|
+
return df_sub.columns.tolist()
|
|
635
|
+
if not sub_levels or sub_value is None:
|
|
636
|
+
# Nothing was dropped (or caller didn't ask) — df_sub already
|
|
637
|
+
# matches df_full's level count.
|
|
638
|
+
return [list(c) if isinstance(c, tuple) else [c] for c in df_sub.columns]
|
|
639
|
+
sub_vals = sub_value if isinstance(sub_value, tuple) else (sub_value,)
|
|
640
|
+
if len(sub_vals) != len(sub_levels):
|
|
641
|
+
# Mismatch — best effort, fall back to reduced form.
|
|
642
|
+
return [list(c) if isinstance(c, tuple) else [c] for c in df_sub.columns]
|
|
643
|
+
sub_positions = dict(zip(sub_levels, sub_vals))
|
|
644
|
+
full_width = df_full.columns.nlevels
|
|
645
|
+
result: list[list] = []
|
|
646
|
+
for c in df_sub.columns:
|
|
647
|
+
reduced = list(c) if isinstance(c, tuple) else [c]
|
|
648
|
+
full: list = [None] * full_width
|
|
649
|
+
for lvl, val in sub_positions.items():
|
|
650
|
+
if 0 <= lvl < full_width:
|
|
651
|
+
full[lvl] = val
|
|
652
|
+
r_iter = iter(reduced)
|
|
653
|
+
for i in range(full_width):
|
|
654
|
+
if full[i] is None:
|
|
655
|
+
try:
|
|
656
|
+
full[i] = next(r_iter)
|
|
657
|
+
except StopIteration:
|
|
658
|
+
break
|
|
659
|
+
result.append(full)
|
|
660
|
+
return result
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def _encode_row_selector(df_sub: pd.DataFrame) -> list:
|
|
664
|
+
"""Encode the row index labels of df_sub for bar charts."""
|
|
665
|
+
if isinstance(df_sub.index, pd.MultiIndex):
|
|
666
|
+
return [list(idx) for idx in df_sub.index]
|
|
667
|
+
return df_sub.index.tolist()
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
# ---------------------------------------------------------------------------
|
|
671
|
+
# _compute_time_plan / _compute_bar_plan
|
|
672
|
+
# ---------------------------------------------------------------------------
|
|
673
|
+
|
|
674
|
+
def _compute_time_plan(
|
|
675
|
+
df_fm: pd.DataFrame,
|
|
676
|
+
effective_plot_name: str,
|
|
677
|
+
cfg: 'PlotConfig',
|
|
678
|
+
fm_stack_levels: list[int],
|
|
679
|
+
fm_subplot_levels: list[int],
|
|
680
|
+
fm_line_levels: list[int],
|
|
681
|
+
axis_bounds,
|
|
682
|
+
plot_rows: tuple[int, int],
|
|
683
|
+
color_path: Path | None = None,
|
|
684
|
+
) -> PlotPlan | None:
|
|
685
|
+
"""Compute a PlotPlan for a time-series (lines or stack) chart.
|
|
686
|
+
|
|
687
|
+
Replicates the planning steps of build_line_figures() and
|
|
688
|
+
build_stack_figures() without actually creating matplotlib Figures.
|
|
689
|
+
"""
|
|
690
|
+
from flextool.plot_outputs.plot_lines import (
|
|
691
|
+
_build_effective_plots, _compute_line_layout, _make_file_batches,
|
|
692
|
+
_get_column_items,
|
|
693
|
+
)
|
|
694
|
+
from flextool.plot_outputs.legend_helpers import build_shared_color_map
|
|
695
|
+
from flextool.plot_outputs.color_template import (
|
|
696
|
+
load_color_template,
|
|
697
|
+
order_labels_by_template,
|
|
698
|
+
)
|
|
699
|
+
from flextool.plot_outputs.subplot_helpers import _extract_subplot_data
|
|
700
|
+
|
|
701
|
+
# Determine chart sub-type
|
|
702
|
+
is_stack = bool(fm_stack_levels)
|
|
703
|
+
chart_type = 'stack' if is_stack else 'lines'
|
|
704
|
+
item_levels = fm_stack_levels if is_stack else fm_line_levels
|
|
705
|
+
|
|
706
|
+
# Convert level indices to level names
|
|
707
|
+
if isinstance(df_fm.columns, pd.MultiIndex):
|
|
708
|
+
item_level_names = [df_fm.columns.names[i] for i in item_levels]
|
|
709
|
+
else:
|
|
710
|
+
item_level_names = item_levels
|
|
711
|
+
|
|
712
|
+
# Get x-axis index, and the period label per x-position when the
|
|
713
|
+
# DataFrame has a ``period`` / ``solve_period`` MultiIndex level.
|
|
714
|
+
# The secondary period tick row on the x-axis depends on this being
|
|
715
|
+
# populated here and carried through the plan. Shared helper keeps
|
|
716
|
+
# this in sync with the loader's reconstruction path.
|
|
717
|
+
time_values, period_labels, max_period_label_len = _extract_time_labels(df_fm)
|
|
718
|
+
|
|
719
|
+
# Determine max items
|
|
720
|
+
default_max_items = 10
|
|
721
|
+
max_items = (
|
|
722
|
+
cfg.max_items_per_plot
|
|
723
|
+
if cfg.max_items_per_plot is not None
|
|
724
|
+
else default_max_items
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
# Build effective_plots with item splitting, tracking which sub-value
|
|
728
|
+
# each chunk came from so we can reconstruct full df_fm column tuples
|
|
729
|
+
# later (see _encode_column_selector).
|
|
730
|
+
from flextool.plot_outputs.subplot_helpers import _get_unique_levels, _sort_subs
|
|
731
|
+
if fm_subplot_levels:
|
|
732
|
+
subs_for_iter = _sort_subs(_get_unique_levels(df_fm.columns, fm_subplot_levels))
|
|
733
|
+
else:
|
|
734
|
+
subs_for_iter = [None]
|
|
735
|
+
effective_plots = []
|
|
736
|
+
chunk_subs: list = []
|
|
737
|
+
for sub in subs_for_iter:
|
|
738
|
+
if sub is None:
|
|
739
|
+
df_for_sub = df_fm
|
|
740
|
+
per_sub_levels: list[int] = []
|
|
741
|
+
else:
|
|
742
|
+
df_for_sub = _extract_subplot_data(df_fm, sub, fm_subplot_levels)
|
|
743
|
+
per_sub_levels = [] # already extracted; no further sub_levels inside
|
|
744
|
+
per_sub_plots = _build_effective_plots(
|
|
745
|
+
df_for_sub, per_sub_levels, item_level_names, max_items,
|
|
746
|
+
subplots_by_magnitudes=cfg.subplots_by_magnitudes if not is_stack else False,
|
|
747
|
+
)
|
|
748
|
+
if sub is not None:
|
|
749
|
+
base_title = (
|
|
750
|
+
' | '.join(str(v) for v in sub) if isinstance(sub, tuple)
|
|
751
|
+
else str(sub)
|
|
752
|
+
)
|
|
753
|
+
retitled = []
|
|
754
|
+
for t, d in per_sub_plots:
|
|
755
|
+
if t is None:
|
|
756
|
+
retitled.append((base_title, d))
|
|
757
|
+
elif str(t).startswith('None'):
|
|
758
|
+
# chunk indicator appended after a base title of None
|
|
759
|
+
retitled.append((base_title + str(t)[len('None'):], d))
|
|
760
|
+
else:
|
|
761
|
+
retitled.append((t, d))
|
|
762
|
+
per_sub_plots = retitled
|
|
763
|
+
effective_plots.extend(per_sub_plots)
|
|
764
|
+
chunk_subs.extend([sub] * len(per_sub_plots))
|
|
765
|
+
if not effective_plots:
|
|
766
|
+
return None
|
|
767
|
+
|
|
768
|
+
# Build shared color map. When the colored (legend) series IS the
|
|
769
|
+
# scenario dim — comparison plots with scenario_rule 'l'/'s' — color it
|
|
770
|
+
# from the project's scenarios section instead of the entity/category
|
|
771
|
+
# hint (which describes a different, non-legend dim here).
|
|
772
|
+
shared_color_map = None
|
|
773
|
+
use_scenario = (
|
|
774
|
+
len(item_level_names) == 1 and str(item_level_names[0]) == 'scenario'
|
|
775
|
+
)
|
|
776
|
+
if (cfg.legend == 'shared' or cfg.color_category or cfg.color_entity_class
|
|
777
|
+
or use_scenario) and item_level_names:
|
|
778
|
+
cat = None if use_scenario else cfg.color_category
|
|
779
|
+
ent = None if use_scenario else cfg.color_entity_class
|
|
780
|
+
all_labels: list[str] = []
|
|
781
|
+
for _, df_sub in effective_plots:
|
|
782
|
+
for item in _get_column_items(df_sub, item_level_names):
|
|
783
|
+
label = str(item)
|
|
784
|
+
if label not in all_labels:
|
|
785
|
+
all_labels.append(label)
|
|
786
|
+
template = load_color_template(color_path)
|
|
787
|
+
# Listed labels first in file order; unlisted appended alphabetically.
|
|
788
|
+
all_labels = order_labels_by_template(
|
|
789
|
+
all_labels, template,
|
|
790
|
+
category=cat, entity_class=ent, scenario=use_scenario,
|
|
791
|
+
)
|
|
792
|
+
shared_color_map = build_shared_color_map(
|
|
793
|
+
all_labels, color_template=template,
|
|
794
|
+
category=cat, entity_class=ent, scenario=use_scenario,
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
# Compute layout
|
|
798
|
+
layout = _compute_line_layout(
|
|
799
|
+
effective_plots, item_level_names,
|
|
800
|
+
cfg.legend, cfg.subplots_per_row,
|
|
801
|
+
6, # base_width_per_col default
|
|
802
|
+
cfg.base_length,
|
|
803
|
+
cfg.axis_tick_format,
|
|
804
|
+
)
|
|
805
|
+
|
|
806
|
+
# Split into file batches
|
|
807
|
+
file_batches_raw = _make_file_batches(
|
|
808
|
+
effective_plots, cfg.max_subplots_per_file, None, '', effective_plot_name,
|
|
809
|
+
)
|
|
810
|
+
total_file_count = len(file_batches_raw)
|
|
811
|
+
|
|
812
|
+
# Encode effective_plot_specs and build batch index lists. Pass the
|
|
813
|
+
# per-chunk sub-value so the encoder can reinsert dropped sub-levels
|
|
814
|
+
# and produce full df_fm column tuples.
|
|
815
|
+
effective_plot_specs: list[tuple[str | None, list]] = []
|
|
816
|
+
for (title, df_sub), chunk_sub in zip(effective_plots, chunk_subs):
|
|
817
|
+
selector = _encode_column_selector(
|
|
818
|
+
df_sub, df_fm, fm_subplot_levels, chunk_sub,
|
|
819
|
+
)
|
|
820
|
+
effective_plot_specs.append((title, selector))
|
|
821
|
+
|
|
822
|
+
file_batches: list[list[int]] = []
|
|
823
|
+
# _make_file_batches returns [(batch_list, filepath), ...]
|
|
824
|
+
# We need to map each batch's items back to effective_plot indices.
|
|
825
|
+
offset = 0
|
|
826
|
+
for batch, _filepath in file_batches_raw:
|
|
827
|
+
batch_size = len(batch)
|
|
828
|
+
file_batches.append(list(range(offset, offset + batch_size)))
|
|
829
|
+
offset += batch_size
|
|
830
|
+
|
|
831
|
+
# Compute per-subplot global y-axis ranges from full time series.
|
|
832
|
+
# For stacked charts the visual ceiling is the column-wise stacked
|
|
833
|
+
# sum (positives stack up, negatives stack down), not the per-series
|
|
834
|
+
# max — using the per-series max here clips peaks when multiple
|
|
835
|
+
# series are non-zero at the same timestep.
|
|
836
|
+
include_zero = cfg.always_include_zero_in_axis
|
|
837
|
+
subplot_y_ranges: list[tuple[float, float]] = []
|
|
838
|
+
for _title, df_sub in effective_plots:
|
|
839
|
+
lo, hi = 0.0, 0.0
|
|
840
|
+
if isinstance(df_sub, pd.Series):
|
|
841
|
+
vals = df_sub.dropna()
|
|
842
|
+
if len(vals):
|
|
843
|
+
lo, hi = float(vals.min()), float(vals.max())
|
|
844
|
+
elif isinstance(df_sub, pd.DataFrame):
|
|
845
|
+
num = df_sub.select_dtypes(include='number')
|
|
846
|
+
if not num.empty:
|
|
847
|
+
if is_stack:
|
|
848
|
+
pos_stack = num.clip(lower=0).sum(axis=1)
|
|
849
|
+
neg_stack = num.clip(upper=0).sum(axis=1)
|
|
850
|
+
lo, hi = float(neg_stack.min()), float(pos_stack.max())
|
|
851
|
+
else:
|
|
852
|
+
lo, hi = float(num.min().min()), float(num.max().max())
|
|
853
|
+
if include_zero:
|
|
854
|
+
lo = min(lo, 0.0)
|
|
855
|
+
hi = max(hi, 0.0)
|
|
856
|
+
subplot_y_ranges.append(_add_y_buffer(lo, hi))
|
|
857
|
+
|
|
858
|
+
# Serialize layout
|
|
859
|
+
layout_params = {
|
|
860
|
+
'value_label_width': layout.value_label_width,
|
|
861
|
+
'legend_width': layout.legend_width,
|
|
862
|
+
'base_width': layout.base_width,
|
|
863
|
+
'subplot_height': layout.subplot_height,
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
return PlotPlan(
|
|
867
|
+
chart_type=chart_type,
|
|
868
|
+
plot_name=effective_plot_name,
|
|
869
|
+
total_file_count=total_file_count,
|
|
870
|
+
processed_df=df_fm,
|
|
871
|
+
effective_plot_specs=effective_plot_specs,
|
|
872
|
+
file_batches=file_batches,
|
|
873
|
+
shared_color_map=shared_color_map,
|
|
874
|
+
color_category=cfg.color_category,
|
|
875
|
+
color_entity_class=cfg.color_entity_class,
|
|
876
|
+
axis_bounds=axis_bounds,
|
|
877
|
+
layout_type='line',
|
|
878
|
+
layout_params=layout_params,
|
|
879
|
+
sub_levels=fm_subplot_levels,
|
|
880
|
+
item_level_names=item_level_names,
|
|
881
|
+
time_index_values=time_values,
|
|
882
|
+
period_labels=period_labels,
|
|
883
|
+
max_period_label_len=max_period_label_len,
|
|
884
|
+
subplots_per_row=cfg.subplots_per_row,
|
|
885
|
+
legend_position=cfg.legend,
|
|
886
|
+
xlabel=cfg.xlabel,
|
|
887
|
+
ylabel=cfg.ylabel,
|
|
888
|
+
axis_tick_format=cfg.axis_tick_format,
|
|
889
|
+
always_include_zero_in_axis=cfg.always_include_zero_in_axis,
|
|
890
|
+
subplot_y_ranges=subplot_y_ranges,
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
# Engine-internal map from a column/index level NAME (set by the output
|
|
895
|
+
# processing, see ``flextool/process_outputs/*.py`` ``.columns.names = [...]``)
|
|
896
|
+
# to the entity class(es) that level can carry. Some levels are *mixed*: a
|
|
897
|
+
# ``process`` is a unit or a connection, and a ``group`` is a nodeGroup or a
|
|
898
|
+
# flowGroup (the entity-class split keyed them apart). This is the inverse of
|
|
899
|
+
# the masking the engine already owns, so a plot author writes the class hint
|
|
900
|
+
# (``color_entity_class: unit`` / ``group``) and never an internal level name.
|
|
901
|
+
# A hint that is itself a mixed key (``process`` / ``group``) resolves each
|
|
902
|
+
# value against both constituent classes.
|
|
903
|
+
_LEVEL_NAME_TO_CLASSES: dict[str, tuple[str, ...]] = {
|
|
904
|
+
'process': ('unit', 'connection'),
|
|
905
|
+
'unit': ('unit',),
|
|
906
|
+
'connection': ('connection',),
|
|
907
|
+
'node': ('node',),
|
|
908
|
+
'source': ('node',),
|
|
909
|
+
'sink': ('node',),
|
|
910
|
+
'group': ('nodeGroup', 'flowGroup'),
|
|
911
|
+
'node_group': ('nodeGroup',),
|
|
912
|
+
'flow_group': ('flowGroup',),
|
|
913
|
+
'nodeGroup': ('nodeGroup',),
|
|
914
|
+
'flowGroup': ('flowGroup',),
|
|
915
|
+
'commodity': ('commodity',),
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def resolve_color_bar_level(
|
|
920
|
+
level_names: list, entity_class: str | None
|
|
921
|
+
) -> str | None:
|
|
922
|
+
"""Pick the column/index level that carries ``entity_class`` for a bar.
|
|
923
|
+
|
|
924
|
+
Lines / stacked / grouped charts designate their colored level via a
|
|
925
|
+
role char (``l`` / ``s`` / ``g``); a simple bar has none, so given
|
|
926
|
+
``color_entity_class: unit`` the engine must *find* the level holding
|
|
927
|
+
units. A level matches when its class-set and the hint's class-set
|
|
928
|
+
overlap — so ``unit`` matches the mixed ``process`` level, ``group``
|
|
929
|
+
(= nodeGroup+flowGroup) matches a ``group`` or ``node_group`` level, and
|
|
930
|
+
a literal name match always wins. Returns the first match
|
|
931
|
+
(``process__source__sink`` never reaches plotting, so first-match is a
|
|
932
|
+
safe tiebreak), or ``None`` when nothing matches / no class is set.
|
|
933
|
+
"""
|
|
934
|
+
if not entity_class:
|
|
935
|
+
return None
|
|
936
|
+
want = set(_LEVEL_NAME_TO_CLASSES.get(entity_class, (entity_class,)))
|
|
937
|
+
for name in level_names:
|
|
938
|
+
have = set(_LEVEL_NAME_TO_CLASSES.get(name, (name,)))
|
|
939
|
+
if name == entity_class or (want & have):
|
|
940
|
+
return name
|
|
941
|
+
return None
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def _color_level_values(df_sub: pd.DataFrame, level: str) -> list:
|
|
945
|
+
"""Distinct values of ``level`` in ``df_sub``, from columns or index.
|
|
946
|
+
|
|
947
|
+
The color-bar level may sit on the columns (expand / subplot dim) or on
|
|
948
|
+
the row index (a ``b``-role dim moved there by the fm-transform). Returns
|
|
949
|
+
``[]`` when the level is on neither (e.g. a level dropped from this slice).
|
|
950
|
+
"""
|
|
951
|
+
col_multi = isinstance(df_sub.columns, pd.MultiIndex)
|
|
952
|
+
col_names = list(df_sub.columns.names) if col_multi else [df_sub.columns.name]
|
|
953
|
+
if level in col_names:
|
|
954
|
+
if col_multi:
|
|
955
|
+
return df_sub.columns.get_level_values(level).unique().tolist()
|
|
956
|
+
return df_sub.columns.unique().tolist()
|
|
957
|
+
idx_multi = isinstance(df_sub.index, pd.MultiIndex)
|
|
958
|
+
idx_names = list(df_sub.index.names) if idx_multi else [df_sub.index.name]
|
|
959
|
+
if level in idx_names:
|
|
960
|
+
if idx_multi:
|
|
961
|
+
return df_sub.index.get_level_values(level).unique().tolist()
|
|
962
|
+
return df_sub.index.unique().tolist()
|
|
963
|
+
return []
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _multiclass_color_map(
|
|
967
|
+
labels: list, color_template: dict | None, classes: tuple,
|
|
968
|
+
) -> dict:
|
|
969
|
+
"""Color map for labels that resolve against more than one entity class.
|
|
970
|
+
|
|
971
|
+
For a mixed level like ``process`` (each value is *either* a unit or a
|
|
972
|
+
connection), resolve every label against ``classes`` in order — first
|
|
973
|
+
section that lists it wins — and give the rest tab10/tab20 palette
|
|
974
|
+
colors in input order (mirroring ``build_shared_color_map``'s fallback).
|
|
975
|
+
"""
|
|
976
|
+
import matplotlib.pyplot as plt
|
|
977
|
+
from flextool.plot_outputs.color_template import resolve_label_color
|
|
978
|
+
|
|
979
|
+
template = color_template or {}
|
|
980
|
+
resolved: dict = {}
|
|
981
|
+
palette_needed = 0
|
|
982
|
+
for lbl in labels:
|
|
983
|
+
col = None
|
|
984
|
+
for cls in classes:
|
|
985
|
+
col = resolve_label_color(lbl, template, category=None, entity_class=cls)
|
|
986
|
+
if col is not None:
|
|
987
|
+
break
|
|
988
|
+
resolved[lbl] = col
|
|
989
|
+
if col is None:
|
|
990
|
+
palette_needed += 1
|
|
991
|
+
cmap_colors = (
|
|
992
|
+
plt.colormaps['tab10'].colors if palette_needed <= 10
|
|
993
|
+
else plt.colormaps['tab20'].colors
|
|
994
|
+
)
|
|
995
|
+
out: dict = {}
|
|
996
|
+
pi = 0
|
|
997
|
+
for lbl in labels:
|
|
998
|
+
if resolved[lbl] is not None:
|
|
999
|
+
out[lbl] = resolved[lbl]
|
|
1000
|
+
else:
|
|
1001
|
+
out[lbl] = cmap_colors[pi % len(cmap_colors)]
|
|
1002
|
+
pi += 1
|
|
1003
|
+
return out
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def build_simple_bar_color_map(
|
|
1007
|
+
df_fm: pd.DataFrame,
|
|
1008
|
+
entity_class: str | None,
|
|
1009
|
+
color_template: dict | None,
|
|
1010
|
+
) -> tuple[dict | None, str | None]:
|
|
1011
|
+
"""Color map + color level for a simple (non-stacked, non-grouped) bar.
|
|
1012
|
+
|
|
1013
|
+
Resolves the column/index level carrying ``entity_class`` and builds a
|
|
1014
|
+
``{value: rgb}`` map over that level's distinct values (read from the
|
|
1015
|
+
full ``df_fm`` so subplot levels — which subplot extraction would drop —
|
|
1016
|
+
are still captured), ordered by the picker's file order. Shared by the
|
|
1017
|
+
plan path (``_compute_bar_plan``) and the live render path
|
|
1018
|
+
(``build_bar_figures``) so both color identically.
|
|
1019
|
+
|
|
1020
|
+
``entity_class`` may name a mixed level: ``process`` carries both units
|
|
1021
|
+
and connections, so each value is resolved against ``unit`` then
|
|
1022
|
+
``connection`` — the one place a plot author writes ``process``. Returns
|
|
1023
|
+
``(None, None)`` when no class is set or no level carries it.
|
|
1024
|
+
"""
|
|
1025
|
+
from flextool.plot_outputs.legend_helpers import build_shared_color_map
|
|
1026
|
+
from flextool.plot_outputs.color_template import order_labels_by_template
|
|
1027
|
+
|
|
1028
|
+
col_names = list(df_fm.columns.names) if isinstance(
|
|
1029
|
+
df_fm.columns, pd.MultiIndex) else [df_fm.columns.name]
|
|
1030
|
+
idx_names = list(df_fm.index.names) if isinstance(
|
|
1031
|
+
df_fm.index, pd.MultiIndex) else [df_fm.index.name]
|
|
1032
|
+
# Index first: a 'b'-role dim (sum/total variant) lives on the index;
|
|
1033
|
+
# an expand/subplot dim (period variant) lives on a column.
|
|
1034
|
+
color_bar_level = resolve_color_bar_level(idx_names + col_names, entity_class)
|
|
1035
|
+
if not color_bar_level:
|
|
1036
|
+
return None, None
|
|
1037
|
+
all_labels: list[str] = []
|
|
1038
|
+
for v in _color_level_values(df_fm, color_bar_level):
|
|
1039
|
+
label = str(v)
|
|
1040
|
+
if label not in all_labels:
|
|
1041
|
+
all_labels.append(label)
|
|
1042
|
+
|
|
1043
|
+
# A mixed level (process) resolves against several classes; a plain one
|
|
1044
|
+
# against itself.
|
|
1045
|
+
classes = _LEVEL_NAME_TO_CLASSES.get(entity_class, (entity_class,))
|
|
1046
|
+
if len(classes) > 1:
|
|
1047
|
+
shared_color_map = _multiclass_color_map(all_labels, color_template, classes)
|
|
1048
|
+
else:
|
|
1049
|
+
cls = classes[0]
|
|
1050
|
+
all_labels = order_labels_by_template(
|
|
1051
|
+
all_labels, color_template or {}, category=None, entity_class=cls,
|
|
1052
|
+
)
|
|
1053
|
+
shared_color_map = build_shared_color_map(
|
|
1054
|
+
all_labels, color_template=color_template, category=None, entity_class=cls,
|
|
1055
|
+
)
|
|
1056
|
+
return shared_color_map, color_bar_level
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def _compute_bar_plan(
|
|
1060
|
+
df_fm: pd.DataFrame,
|
|
1061
|
+
effective_plot_name: str,
|
|
1062
|
+
cfg: 'PlotConfig',
|
|
1063
|
+
fm_stack_levels: list[int],
|
|
1064
|
+
fm_expand_axis_levels: list[int],
|
|
1065
|
+
fm_subplot_levels: list[int],
|
|
1066
|
+
fm_grouped_bar_levels: list[int],
|
|
1067
|
+
axis_bounds,
|
|
1068
|
+
color_path: Path | None = None,
|
|
1069
|
+
) -> PlotPlan | None:
|
|
1070
|
+
"""Compute a PlotPlan for a bar chart.
|
|
1071
|
+
|
|
1072
|
+
Replicates the planning steps of build_bar_figures() without actually
|
|
1073
|
+
creating matplotlib Figures.
|
|
1074
|
+
"""
|
|
1075
|
+
from flextool.plot_outputs.plot_bars import _compute_bar_layout
|
|
1076
|
+
from flextool.plot_outputs.subplot_helpers import (
|
|
1077
|
+
_get_unique_levels, _extract_subplot_data, _sort_subs,
|
|
1078
|
+
)
|
|
1079
|
+
from flextool.plot_outputs.legend_helpers import build_shared_color_map, _format_legend_labels
|
|
1080
|
+
from flextool.plot_outputs.color_template import (
|
|
1081
|
+
load_color_template,
|
|
1082
|
+
order_labels_by_template,
|
|
1083
|
+
)
|
|
1084
|
+
|
|
1085
|
+
sub_levels = fm_subplot_levels or []
|
|
1086
|
+
stack_levels = fm_stack_levels or []
|
|
1087
|
+
grouped_bar_levels = fm_grouped_bar_levels or []
|
|
1088
|
+
expand_axis_levels = fm_expand_axis_levels or []
|
|
1089
|
+
|
|
1090
|
+
# Convert level indices to names
|
|
1091
|
+
if isinstance(df_fm.columns, pd.MultiIndex):
|
|
1092
|
+
stack_level_names = [df_fm.columns.names[i] for i in stack_levels] if stack_levels else []
|
|
1093
|
+
expand_axis_level_names = [df_fm.columns.names[i] for i in expand_axis_levels] if expand_axis_levels else []
|
|
1094
|
+
grouped_bar_level_names = [df_fm.columns.names[i] for i in grouped_bar_levels] if grouped_bar_levels else []
|
|
1095
|
+
else:
|
|
1096
|
+
stack_level_names = stack_levels
|
|
1097
|
+
expand_axis_level_names = [df_fm.columns.name] if expand_axis_levels else []
|
|
1098
|
+
grouped_bar_level_names = [df_fm.columns.name] if grouped_bar_levels else []
|
|
1099
|
+
|
|
1100
|
+
subs = _sort_subs(_get_unique_levels(df_fm.columns, sub_levels))
|
|
1101
|
+
|
|
1102
|
+
# Compute expand-group count. When columns is a plain (single-level)
|
|
1103
|
+
# Index, each unique column value still acts as one expand group at
|
|
1104
|
+
# render time (see ``_build_bar_figure`` group-iteration loop), so the
|
|
1105
|
+
# visual bar count is ``n_rows × n_unique_cols`` — not ``n_rows`` as
|
|
1106
|
+
# the old code assumed. Without this, pagination undercounts and a
|
|
1107
|
+
# 175×245 frame collapses to "175 items" and renders 42 875 bars in
|
|
1108
|
+
# one figure (~2 350-inch height, ~290 MP bitmap → GUI hard hang).
|
|
1109
|
+
if expand_axis_levels:
|
|
1110
|
+
if isinstance(df_fm.columns, pd.MultiIndex):
|
|
1111
|
+
if len(expand_axis_level_names) == 1:
|
|
1112
|
+
expand_level_name = expand_axis_level_names[0]
|
|
1113
|
+
n_expand_groups = len(df_fm.columns.get_level_values(expand_level_name).unique())
|
|
1114
|
+
else:
|
|
1115
|
+
expand_frame = df_fm.columns.to_frame()[expand_axis_level_names].drop_duplicates()
|
|
1116
|
+
n_expand_groups = len(expand_frame)
|
|
1117
|
+
else:
|
|
1118
|
+
# Single-level columns: each unique value is its own expand group.
|
|
1119
|
+
n_expand_groups = len(df_fm.columns.unique())
|
|
1120
|
+
else:
|
|
1121
|
+
n_expand_groups = 1
|
|
1122
|
+
|
|
1123
|
+
# Determine max items
|
|
1124
|
+
default_max_items = 10
|
|
1125
|
+
max_items = (
|
|
1126
|
+
cfg.max_items_per_plot
|
|
1127
|
+
if cfg.max_items_per_plot is not None
|
|
1128
|
+
else default_max_items
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
# Build effective_plots (mirrors build_bar_figures logic).
|
|
1132
|
+
# Split subplots that exceed max_items_per_plot visual bar labels
|
|
1133
|
+
# (n_rows * n_expand_groups).
|
|
1134
|
+
effective_plots: list[tuple[str | None, pd.DataFrame]] = []
|
|
1135
|
+
# Parallel list: which sub-value each chunk was extracted from, so
|
|
1136
|
+
# the column selector encoder can reinsert it and produce tuples
|
|
1137
|
+
# that match df_fm.columns at reconstruction time.
|
|
1138
|
+
chunk_subs: list = []
|
|
1139
|
+
for sub in subs:
|
|
1140
|
+
df_sub = _extract_subplot_data(df_fm, sub, sub_levels)
|
|
1141
|
+
df_sub = df_sub.dropna(how='all')
|
|
1142
|
+
if df_sub.empty:
|
|
1143
|
+
continue
|
|
1144
|
+
df_sub = df_sub.fillna(0)
|
|
1145
|
+
title = (
|
|
1146
|
+
' | '.join(str(v) for v in sub) if isinstance(sub, tuple)
|
|
1147
|
+
else str(sub) if sub is not None else None
|
|
1148
|
+
)
|
|
1149
|
+
n_rows = len(df_sub)
|
|
1150
|
+
|
|
1151
|
+
def _add(label: str | None, df_chunk: pd.DataFrame) -> None:
|
|
1152
|
+
effective_plots.append((label, df_chunk))
|
|
1153
|
+
chunk_subs.append(sub)
|
|
1154
|
+
|
|
1155
|
+
if not max_items:
|
|
1156
|
+
_add(title, df_sub)
|
|
1157
|
+
continue
|
|
1158
|
+
|
|
1159
|
+
total_items = n_rows * max(n_expand_groups, 1)
|
|
1160
|
+
if total_items <= max_items:
|
|
1161
|
+
_add(title, df_sub)
|
|
1162
|
+
continue
|
|
1163
|
+
|
|
1164
|
+
if expand_axis_levels and expand_axis_level_names and n_expand_groups > 1:
|
|
1165
|
+
expand_level_name_local = expand_axis_level_names[0]
|
|
1166
|
+
max_groups = max(1, max_items // max(n_rows, 1))
|
|
1167
|
+
if isinstance(df_sub.columns, pd.MultiIndex):
|
|
1168
|
+
all_groups = df_sub.columns.get_level_values(expand_level_name_local).unique().tolist()
|
|
1169
|
+
else:
|
|
1170
|
+
all_groups = df_sub.columns.unique().tolist()
|
|
1171
|
+
for gi, grp_start in enumerate(range(0, len(all_groups), max_groups)):
|
|
1172
|
+
grp_chunk = all_groups[grp_start:grp_start + max_groups]
|
|
1173
|
+
if isinstance(df_sub.columns, pd.MultiIndex):
|
|
1174
|
+
mask = df_sub.columns.get_level_values(expand_level_name_local).isin(grp_chunk)
|
|
1175
|
+
else:
|
|
1176
|
+
mask = df_sub.columns.isin(grp_chunk)
|
|
1177
|
+
chunk = df_sub.loc[:, mask]
|
|
1178
|
+
# When a chunk still has more rows than max_items, further
|
|
1179
|
+
# split by rows so per-subplot bar count stays bounded.
|
|
1180
|
+
if n_rows > max_items:
|
|
1181
|
+
n_row_chunks = (n_rows + max_items - 1) // max_items
|
|
1182
|
+
for ri in range(n_row_chunks):
|
|
1183
|
+
row_chunk = chunk.iloc[ri * max_items:(ri + 1) * max_items]
|
|
1184
|
+
sub_label = (
|
|
1185
|
+
f"{title}_{gi + 1}.{ri + 1}"
|
|
1186
|
+
if title else f"{gi + 1}.{ri + 1}"
|
|
1187
|
+
)
|
|
1188
|
+
_add(sub_label, row_chunk)
|
|
1189
|
+
else:
|
|
1190
|
+
chunk_label = f"{title}_{gi + 1}" if title else None
|
|
1191
|
+
_add(chunk_label, chunk)
|
|
1192
|
+
elif n_rows > max_items:
|
|
1193
|
+
for i in range(0, n_rows, max_items):
|
|
1194
|
+
chunk = df_sub.iloc[i:i + max_items]
|
|
1195
|
+
chunk_label = f"{title}_{i // max_items + 1}" if title else None
|
|
1196
|
+
_add(chunk_label, chunk)
|
|
1197
|
+
else:
|
|
1198
|
+
_add(title, df_sub)
|
|
1199
|
+
|
|
1200
|
+
if not effective_plots:
|
|
1201
|
+
return None
|
|
1202
|
+
|
|
1203
|
+
# Build shared color map. When the colored (legend) series IS the
|
|
1204
|
+
# scenario dim — comparison plots with scenario_rule 'g'/'s' — color it
|
|
1205
|
+
# from the scenarios section rather than the entity/category hint.
|
|
1206
|
+
shared_color_map = None
|
|
1207
|
+
color_bar_level: str | None = None
|
|
1208
|
+
colored_names = grouped_bar_level_names if grouped_bar_levels else stack_level_names
|
|
1209
|
+
use_scenario = (
|
|
1210
|
+
len(colored_names) == 1 and str(colored_names[0]) == 'scenario'
|
|
1211
|
+
)
|
|
1212
|
+
if (cfg.legend == 'shared' or cfg.color_category or cfg.color_entity_class
|
|
1213
|
+
or use_scenario) and (stack_levels or grouped_bar_levels):
|
|
1214
|
+
cat = None if use_scenario else cfg.color_category
|
|
1215
|
+
ent = None if use_scenario else cfg.color_entity_class
|
|
1216
|
+
all_labels: list[str] = []
|
|
1217
|
+
for _, df_sub in effective_plots:
|
|
1218
|
+
if grouped_bar_levels:
|
|
1219
|
+
if len(grouped_bar_level_names) == 1:
|
|
1220
|
+
items = df_sub.columns.get_level_values(
|
|
1221
|
+
grouped_bar_level_names[0]).unique().tolist()
|
|
1222
|
+
else:
|
|
1223
|
+
item_df = df_sub.columns.to_frame()[grouped_bar_level_names].drop_duplicates()
|
|
1224
|
+
items = [tuple(row) for row in item_df.values]
|
|
1225
|
+
else:
|
|
1226
|
+
if len(stack_level_names) == 1:
|
|
1227
|
+
if isinstance(df_sub.columns, pd.MultiIndex):
|
|
1228
|
+
items = df_sub.columns.get_level_values(
|
|
1229
|
+
stack_level_names[0]).unique().tolist()
|
|
1230
|
+
else:
|
|
1231
|
+
items = df_sub.columns.unique().tolist()
|
|
1232
|
+
else:
|
|
1233
|
+
stack_df = df_sub.columns.to_frame()[stack_level_names].drop_duplicates()
|
|
1234
|
+
items = [tuple(row) for row in stack_df.values]
|
|
1235
|
+
for item in items:
|
|
1236
|
+
label = _format_legend_labels([item])[0]
|
|
1237
|
+
if label not in all_labels:
|
|
1238
|
+
all_labels.append(label)
|
|
1239
|
+
template = load_color_template(color_path)
|
|
1240
|
+
# Listed labels first in file order; unlisted appended alphabetically.
|
|
1241
|
+
all_labels = order_labels_by_template(
|
|
1242
|
+
all_labels, template,
|
|
1243
|
+
category=cat, entity_class=ent, scenario=use_scenario,
|
|
1244
|
+
)
|
|
1245
|
+
shared_color_map = build_shared_color_map(
|
|
1246
|
+
all_labels, color_template=template,
|
|
1247
|
+
category=cat, entity_class=ent, scenario=use_scenario,
|
|
1248
|
+
)
|
|
1249
|
+
elif cfg.color_entity_class and not (stack_levels or grouped_bar_levels):
|
|
1250
|
+
# Simple bars: no role char designates a colored level, so resolve
|
|
1251
|
+
# the level carrying the entity class and color each bar by its value
|
|
1252
|
+
# there — a column (expand → per-bar; subplot → whole-subplot) or the
|
|
1253
|
+
# row index (a 'b'-role dim, e.g. the sum/total variant). Same
|
|
1254
|
+
# entities.<class> resolution + picker order as the line path; shared
|
|
1255
|
+
# with the live render path via build_simple_bar_color_map.
|
|
1256
|
+
shared_color_map, color_bar_level = build_simple_bar_color_map(
|
|
1257
|
+
df_fm, cfg.color_entity_class, load_color_template(color_path),
|
|
1258
|
+
)
|
|
1259
|
+
|
|
1260
|
+
# Compute layout
|
|
1261
|
+
layout = _compute_bar_layout(
|
|
1262
|
+
effective_plots, df_fm,
|
|
1263
|
+
expand_axis_levels, expand_axis_level_names,
|
|
1264
|
+
stack_levels, stack_level_names,
|
|
1265
|
+
grouped_bar_levels, grouped_bar_level_names,
|
|
1266
|
+
cfg.legend, cfg.subplots_per_row,
|
|
1267
|
+
cfg.base_length,
|
|
1268
|
+
)
|
|
1269
|
+
|
|
1270
|
+
# Split into file batches respecting max_subplots_per_file and, for
|
|
1271
|
+
# horizontal bars, max_items_per_subplot_column (total bar-label count
|
|
1272
|
+
# in any single grid column must not exceed the limit).
|
|
1273
|
+
spr = max(cfg.subplots_per_row, 1)
|
|
1274
|
+
max_per_file = cfg.max_subplots_per_file or len(effective_plots)
|
|
1275
|
+
col_limit = (
|
|
1276
|
+
cfg.max_items_per_subplot_column
|
|
1277
|
+
if cfg.bar_orientation == 'horizontal' else 0
|
|
1278
|
+
)
|
|
1279
|
+
|
|
1280
|
+
# Compute visual item counts per subplot (bar labels on the y-axis).
|
|
1281
|
+
# With expand groups, each group contributes its own set of labels,
|
|
1282
|
+
# so the count can be much larger than len(df_sub).
|
|
1283
|
+
def _count_visual_items(df_sub: pd.DataFrame) -> int:
|
|
1284
|
+
if not expand_axis_level_names:
|
|
1285
|
+
return max(len(df_sub), 1)
|
|
1286
|
+
if not isinstance(df_sub.columns, pd.MultiIndex):
|
|
1287
|
+
# Single-level columns: each unique value is its own expand
|
|
1288
|
+
# group, each contributing one row-set worth of bars.
|
|
1289
|
+
return max(len(df_sub) * len(df_sub.columns.unique()), 1)
|
|
1290
|
+
count = 0
|
|
1291
|
+
if len(expand_axis_level_names) == 1:
|
|
1292
|
+
groups = df_sub.columns.get_level_values(expand_axis_level_names[0]).unique()
|
|
1293
|
+
else:
|
|
1294
|
+
gf = df_sub.columns.to_frame()[expand_axis_level_names].drop_duplicates()
|
|
1295
|
+
groups = [tuple(r) for r in gf.values]
|
|
1296
|
+
for grp in groups:
|
|
1297
|
+
try:
|
|
1298
|
+
if len(expand_axis_level_names) == 1:
|
|
1299
|
+
df_g = df_sub.xs(grp, level=expand_axis_level_names[0], axis=1)
|
|
1300
|
+
else:
|
|
1301
|
+
df_g = df_sub.xs(grp, level=expand_axis_level_names, axis=1)
|
|
1302
|
+
except KeyError:
|
|
1303
|
+
continue
|
|
1304
|
+
if isinstance(df_g, pd.Series):
|
|
1305
|
+
df_g = df_g.to_frame()
|
|
1306
|
+
count += len(df_g)
|
|
1307
|
+
return max(count, 1)
|
|
1308
|
+
|
|
1309
|
+
visual_item_counts = [_count_visual_items(df_sub) for _, df_sub in effective_plots]
|
|
1310
|
+
|
|
1311
|
+
# Group subplots into grid rows so we never break mid-row.
|
|
1312
|
+
grid_rows: list[list[int]] = []
|
|
1313
|
+
for i in range(0, len(effective_plots), spr):
|
|
1314
|
+
grid_rows.append(list(range(i, min(i + spr, len(effective_plots)))))
|
|
1315
|
+
|
|
1316
|
+
raw_batches: list[list[tuple[str | None, pd.DataFrame]]] = []
|
|
1317
|
+
current_batch: list[tuple[str | None, pd.DataFrame]] = []
|
|
1318
|
+
col_counts = [0] * spr # cumulative bar-label count per grid column
|
|
1319
|
+
|
|
1320
|
+
for row_indices in grid_rows:
|
|
1321
|
+
# Check whether adding this row would breach either limit.
|
|
1322
|
+
would_exceed_subplots = len(current_batch) + len(row_indices) > max_per_file
|
|
1323
|
+
would_exceed_col = False
|
|
1324
|
+
if col_limit and current_batch:
|
|
1325
|
+
for j, idx in enumerate(row_indices):
|
|
1326
|
+
if col_counts[j] + visual_item_counts[idx] > col_limit:
|
|
1327
|
+
would_exceed_col = True
|
|
1328
|
+
break
|
|
1329
|
+
|
|
1330
|
+
if (would_exceed_subplots or would_exceed_col) and current_batch:
|
|
1331
|
+
raw_batches.append(current_batch)
|
|
1332
|
+
current_batch = []
|
|
1333
|
+
col_counts = [0] * spr
|
|
1334
|
+
|
|
1335
|
+
current_batch.extend([effective_plots[i] for i in row_indices])
|
|
1336
|
+
for j, idx in enumerate(row_indices):
|
|
1337
|
+
col_counts[j] += visual_item_counts[idx]
|
|
1338
|
+
|
|
1339
|
+
if current_batch:
|
|
1340
|
+
raw_batches.append(current_batch)
|
|
1341
|
+
|
|
1342
|
+
total_file_count = len(raw_batches)
|
|
1343
|
+
|
|
1344
|
+
# Encode effective_plot_specs: store row index labels AND column
|
|
1345
|
+
# selectors so that reconstruction filters both dimensions.
|
|
1346
|
+
# The column selector is stored as FULL df_fm tuples (with the
|
|
1347
|
+
# sub-level values reinserted) so reconstruction via
|
|
1348
|
+
# ``df_fm.columns.isin(...)`` matches exactly. Without this the
|
|
1349
|
+
# narrower tuples from ``xs`` never match and the fallback returns
|
|
1350
|
+
# the whole df_fm, flooding the subplot with columns from every
|
|
1351
|
+
# other subplot.
|
|
1352
|
+
effective_plot_specs: list[tuple[str | None, list]] = []
|
|
1353
|
+
for (title, df_sub), chunk_sub in zip(effective_plots, chunk_subs):
|
|
1354
|
+
selector = {
|
|
1355
|
+
'rows': _encode_row_selector(df_sub),
|
|
1356
|
+
'cols': _encode_column_selector(df_sub, df_fm, sub_levels, chunk_sub),
|
|
1357
|
+
}
|
|
1358
|
+
effective_plot_specs.append((title, selector))
|
|
1359
|
+
|
|
1360
|
+
# Build batch index lists
|
|
1361
|
+
file_batches: list[list[int]] = []
|
|
1362
|
+
offset = 0
|
|
1363
|
+
for batch in raw_batches:
|
|
1364
|
+
batch_size = len(batch)
|
|
1365
|
+
file_batches.append(list(range(offset, offset + batch_size)))
|
|
1366
|
+
offset += batch_size
|
|
1367
|
+
|
|
1368
|
+
# Serialize layout
|
|
1369
|
+
layout_params = {
|
|
1370
|
+
'bar_label_width': layout.bar_label_width,
|
|
1371
|
+
'total_label_width': layout.total_label_width,
|
|
1372
|
+
'legend_width': layout.legend_width,
|
|
1373
|
+
'legend_height': layout.legend_height,
|
|
1374
|
+
'base_bar_length': layout.base_bar_length,
|
|
1375
|
+
'value_axis_width': layout.value_axis_width,
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
return PlotPlan(
|
|
1379
|
+
chart_type='bar',
|
|
1380
|
+
plot_name=effective_plot_name,
|
|
1381
|
+
total_file_count=total_file_count,
|
|
1382
|
+
processed_df=df_fm,
|
|
1383
|
+
effective_plot_specs=effective_plot_specs,
|
|
1384
|
+
file_batches=file_batches,
|
|
1385
|
+
shared_color_map=shared_color_map,
|
|
1386
|
+
color_category=cfg.color_category,
|
|
1387
|
+
color_entity_class=cfg.color_entity_class,
|
|
1388
|
+
color_bar_level=color_bar_level,
|
|
1389
|
+
axis_bounds=axis_bounds,
|
|
1390
|
+
layout_type='bar',
|
|
1391
|
+
layout_params=layout_params,
|
|
1392
|
+
sub_levels=sub_levels,
|
|
1393
|
+
item_level_names=[],
|
|
1394
|
+
subplots_per_row=cfg.subplots_per_row,
|
|
1395
|
+
legend_position=cfg.legend,
|
|
1396
|
+
xlabel=cfg.xlabel,
|
|
1397
|
+
ylabel=cfg.ylabel,
|
|
1398
|
+
bar_orientation=cfg.bar_orientation,
|
|
1399
|
+
axis_tick_format=cfg.axis_tick_format,
|
|
1400
|
+
always_include_zero_in_axis=cfg.always_include_zero_in_axis,
|
|
1401
|
+
value_label=cfg.value_label,
|
|
1402
|
+
base_bar_length=cfg.base_length,
|
|
1403
|
+
skip_data_with_only_zeroes=cfg.skip_data_with_only_zeroes,
|
|
1404
|
+
stack_levels=stack_levels,
|
|
1405
|
+
stack_level_names=stack_level_names,
|
|
1406
|
+
expand_axis_levels=expand_axis_levels,
|
|
1407
|
+
expand_axis_level_names=expand_axis_level_names,
|
|
1408
|
+
grouped_bar_levels=grouped_bar_levels,
|
|
1409
|
+
grouped_bar_level_names=grouped_bar_level_names,
|
|
1410
|
+
)
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
# ---------------------------------------------------------------------------
|
|
1414
|
+
# compute_plot_plans_for_result
|
|
1415
|
+
# ---------------------------------------------------------------------------
|
|
1416
|
+
|
|
1417
|
+
def compute_plot_plans_for_result(
|
|
1418
|
+
df: pd.DataFrame,
|
|
1419
|
+
result_key: str,
|
|
1420
|
+
plot_settings: dict,
|
|
1421
|
+
output_dir: Path,
|
|
1422
|
+
plot_rows: tuple[int, int] = (0, 167),
|
|
1423
|
+
break_times: set[str] | None = None,
|
|
1424
|
+
active_settings: list[str] | None = None,
|
|
1425
|
+
period_weights=None,
|
|
1426
|
+
manifest_accumulator=None,
|
|
1427
|
+
manifest_scenario_name: str | None = None,
|
|
1428
|
+
color_path: Path | None = None,
|
|
1429
|
+
) -> list[tuple[str, str]]:
|
|
1430
|
+
"""Compute and save PlotPlans for all configs of a result_key.
|
|
1431
|
+
|
|
1432
|
+
Called after scenario runs to pre-compute plans.
|
|
1433
|
+
Returns list of (result_key, sub_config) pairs that produced valid plans.
|
|
1434
|
+
The *active_settings* parameter is accepted for backward compatibility
|
|
1435
|
+
but ignored — all configs in the YAML entry are processed so that the
|
|
1436
|
+
viewer can build a complete availability manifest.
|
|
1437
|
+
|
|
1438
|
+
When *manifest_accumulator* is provided (a
|
|
1439
|
+
:class:`~flextool.plot_outputs.shared_manifest.ManifestAccumulator`),
|
|
1440
|
+
each generated plan is also folded into a cross-scenario axis-bounds
|
|
1441
|
+
manifest under *manifest_scenario_name*. The caller is responsible
|
|
1442
|
+
for invoking ``accumulator.write()`` once the batch is done.
|
|
1443
|
+
"""
|
|
1444
|
+
from flextool.plot_outputs.config import PlotConfig, PLOT_FIELD_NAMES, _is_single_config
|
|
1445
|
+
from flextool.plot_outputs.orchestrator import (
|
|
1446
|
+
_apply_dimension_rules, _resolve_shared_axis_bounds, _process_file_member,
|
|
1447
|
+
reattach_file_level_for_color,
|
|
1448
|
+
)
|
|
1449
|
+
from flextool.plot_outputs.axis_helpers import _normalize_axis_bounds
|
|
1450
|
+
from flextool.plot_outputs.format_helpers import insert_timeline_breaks
|
|
1451
|
+
|
|
1452
|
+
entry = plot_settings.get(result_key)
|
|
1453
|
+
if not isinstance(entry, dict):
|
|
1454
|
+
return []
|
|
1455
|
+
|
|
1456
|
+
succeeded: list[tuple[str, str]] = []
|
|
1457
|
+
|
|
1458
|
+
# Collect all configs (not just active ones)
|
|
1459
|
+
chosen: list[tuple[str, dict]] = []
|
|
1460
|
+
if _is_single_config(entry):
|
|
1461
|
+
chosen.append(('default', entry))
|
|
1462
|
+
else:
|
|
1463
|
+
for name, sub in entry.items():
|
|
1464
|
+
if isinstance(sub, dict):
|
|
1465
|
+
chosen.append((name, sub))
|
|
1466
|
+
|
|
1467
|
+
for sub_config, raw_setting in chosen:
|
|
1468
|
+
# Parse into PlotConfig
|
|
1469
|
+
filtered = {k: v for k, v in raw_setting.items() if k in PLOT_FIELD_NAMES}
|
|
1470
|
+
if 'axis_scale_min_max' in filtered and 'axis_bounds' not in filtered:
|
|
1471
|
+
filtered['axis_bounds'] = filtered.pop('axis_scale_min_max')
|
|
1472
|
+
elif 'axis_scale_min_max' in filtered:
|
|
1473
|
+
del filtered['axis_scale_min_max']
|
|
1474
|
+
filtered.pop('variant', None)
|
|
1475
|
+
try:
|
|
1476
|
+
cfg = PlotConfig(**filtered)
|
|
1477
|
+
except TypeError:
|
|
1478
|
+
continue
|
|
1479
|
+
|
|
1480
|
+
if not cfg.map_dimensions_for_plots or len(cfg.map_dimensions_for_plots) < 2:
|
|
1481
|
+
continue
|
|
1482
|
+
|
|
1483
|
+
# Title: explicit plot_name > settings-group entry name > result key.
|
|
1484
|
+
# Mirrors the live-render path in orchestrator.py so disk plans don't
|
|
1485
|
+
# fall back to the raw parquet file name (result_key) as the title.
|
|
1486
|
+
plot_name = cfg.plot_name or raw_setting.get('_entry_name') or result_key
|
|
1487
|
+
|
|
1488
|
+
# Check availability using the FULL data range — a variant is
|
|
1489
|
+
# "available" if there's any non-zero data anywhere in the time
|
|
1490
|
+
# series, even if the current plot_rows window is all zeros.
|
|
1491
|
+
# This prevents hourly variants from being marked unavailable when
|
|
1492
|
+
# spikes happen outside the displayed window.
|
|
1493
|
+
full_range = (0, len(df))
|
|
1494
|
+
avail_result = _apply_dimension_rules(df, cfg, full_range, period_weights=period_weights)
|
|
1495
|
+
if avail_result is None:
|
|
1496
|
+
continue
|
|
1497
|
+
df_avail = avail_result[0]
|
|
1498
|
+
has_data = True
|
|
1499
|
+
if cfg.skip_data_with_only_zeroes:
|
|
1500
|
+
numeric = df_avail.select_dtypes(include='number')
|
|
1501
|
+
has_data = not numeric.empty and (numeric.abs() >= 1e-6).any().any()
|
|
1502
|
+
if not has_data:
|
|
1503
|
+
continue
|
|
1504
|
+
# Record availability regardless of whether plan generation succeeds
|
|
1505
|
+
succeeded.append((result_key, sub_config))
|
|
1506
|
+
|
|
1507
|
+
# Apply dimension rules. For time-series plans, use the full data
|
|
1508
|
+
# range so the plan can be rendered at any start/duration without
|
|
1509
|
+
# recomputing. For bar charts, plot_rows is irrelevant.
|
|
1510
|
+
dim_result = _apply_dimension_rules(df, cfg, full_range, period_weights=period_weights)
|
|
1511
|
+
if dim_result is None:
|
|
1512
|
+
continue
|
|
1513
|
+
df_processed, rules, chart_type, summed_dims, averaged_dims = dim_result
|
|
1514
|
+
|
|
1515
|
+
# Identify level roles
|
|
1516
|
+
col_rules = rules[df_processed.index.nlevels:]
|
|
1517
|
+
grouped_bar_levels = [i for i, c in enumerate(col_rules) if c == 'g']
|
|
1518
|
+
stack_levels = [i for i, c in enumerate(col_rules) if c == 's']
|
|
1519
|
+
expand_axis_levels = [i for i, c in enumerate(col_rules) if c == 'e']
|
|
1520
|
+
subplot_levels = [i for i, c in enumerate(col_rules) if c == 'u']
|
|
1521
|
+
line_levels = [i for i, c in enumerate(col_rules) if c == 'l']
|
|
1522
|
+
file_levels = [i for i, c in enumerate(col_rules) if c == 'f']
|
|
1523
|
+
|
|
1524
|
+
# Build plot title
|
|
1525
|
+
plot_title = plot_name
|
|
1526
|
+
if summed_dims:
|
|
1527
|
+
dim_str = "', '".join(str(d) for d in summed_dims)
|
|
1528
|
+
plot_title = f"{plot_title} ('{dim_str}' summed)"
|
|
1529
|
+
if averaged_dims:
|
|
1530
|
+
dim_str = "', '".join(str(d) for d in averaged_dims)
|
|
1531
|
+
plot_title = f"{plot_title} ('{dim_str}' averaged)"
|
|
1532
|
+
|
|
1533
|
+
# File members
|
|
1534
|
+
if file_levels:
|
|
1535
|
+
if len(file_levels) == 1:
|
|
1536
|
+
all_file_members = (
|
|
1537
|
+
df_processed.columns.get_level_values(file_levels[0]).unique().tolist()
|
|
1538
|
+
)
|
|
1539
|
+
else:
|
|
1540
|
+
fm_df = (
|
|
1541
|
+
df_processed.columns.to_frame().iloc[:, file_levels].drop_duplicates()
|
|
1542
|
+
)
|
|
1543
|
+
all_file_members = [tuple(row) for row in fm_df.values]
|
|
1544
|
+
else:
|
|
1545
|
+
all_file_members = [None]
|
|
1546
|
+
|
|
1547
|
+
# Resolve shared axis bounds
|
|
1548
|
+
axis_bounds = _normalize_axis_bounds(cfg.axis_bounds)
|
|
1549
|
+
axis_bounds = _resolve_shared_axis_bounds(
|
|
1550
|
+
df_processed, axis_bounds, stack_levels, subplot_levels,
|
|
1551
|
+
cfg.always_include_zero_in_axis,
|
|
1552
|
+
)
|
|
1553
|
+
|
|
1554
|
+
# Process each file member
|
|
1555
|
+
for file_member in all_file_members:
|
|
1556
|
+
result = _process_file_member(
|
|
1557
|
+
df_processed, file_member, file_levels, plot_title,
|
|
1558
|
+
grouped_bar_levels, stack_levels, expand_axis_levels,
|
|
1559
|
+
subplot_levels, line_levels,
|
|
1560
|
+
)
|
|
1561
|
+
if result is None:
|
|
1562
|
+
continue
|
|
1563
|
+
(df_fm, effective_plot_name, member_str,
|
|
1564
|
+
fm_grouped_bar_levels, fm_stack_levels, fm_expand_axis_levels,
|
|
1565
|
+
fm_subplot_levels, fm_line_levels) = result
|
|
1566
|
+
|
|
1567
|
+
# Simple bar colored by the file-split entity: restore the file
|
|
1568
|
+
# member's value so the whole file gets that group's color.
|
|
1569
|
+
if chart_type == 'bar' and cfg.color_entity_class:
|
|
1570
|
+
file_level_names = [df_processed.columns.names[i] for i in file_levels]
|
|
1571
|
+
df_fm = reattach_file_level_for_color(
|
|
1572
|
+
df_fm, file_member, file_level_names, cfg.color_entity_class)
|
|
1573
|
+
|
|
1574
|
+
# Apply skip_zeroes, multiply_by, timeline breaks
|
|
1575
|
+
if cfg.skip_data_with_only_zeroes:
|
|
1576
|
+
df_fm = df_fm.loc[:, (df_fm.abs() > 1e-6).any()]
|
|
1577
|
+
if chart_type == 'bar':
|
|
1578
|
+
df_fm = df_fm.loc[(df_fm.abs() > 1e-6).any(axis=1)]
|
|
1579
|
+
if df_fm.empty:
|
|
1580
|
+
continue
|
|
1581
|
+
|
|
1582
|
+
if cfg.multiply_by is not None:
|
|
1583
|
+
df_fm = df_fm * cfg.multiply_by
|
|
1584
|
+
|
|
1585
|
+
if chart_type == 'time' and break_times:
|
|
1586
|
+
df_fm = insert_timeline_breaks(df_fm, break_times)
|
|
1587
|
+
|
|
1588
|
+
# Compute plan based on chart type
|
|
1589
|
+
if chart_type == 'bar':
|
|
1590
|
+
plan = _compute_bar_plan(
|
|
1591
|
+
df_fm, effective_plot_name, cfg,
|
|
1592
|
+
fm_stack_levels, fm_expand_axis_levels,
|
|
1593
|
+
fm_subplot_levels, fm_grouped_bar_levels,
|
|
1594
|
+
axis_bounds,
|
|
1595
|
+
color_path=color_path,
|
|
1596
|
+
)
|
|
1597
|
+
elif chart_type == 'time':
|
|
1598
|
+
plan = _compute_time_plan(
|
|
1599
|
+
df_fm, effective_plot_name, cfg,
|
|
1600
|
+
fm_stack_levels, fm_subplot_levels, fm_line_levels,
|
|
1601
|
+
axis_bounds, plot_rows,
|
|
1602
|
+
color_path=color_path,
|
|
1603
|
+
)
|
|
1604
|
+
else:
|
|
1605
|
+
continue
|
|
1606
|
+
|
|
1607
|
+
if plan is None:
|
|
1608
|
+
continue
|
|
1609
|
+
|
|
1610
|
+
# Determine sub_config name for file member variants
|
|
1611
|
+
save_sub = sub_config
|
|
1612
|
+
if member_str:
|
|
1613
|
+
save_sub = f"{sub_config}__{member_str}"
|
|
1614
|
+
|
|
1615
|
+
save_plot_plan(plan, output_dir, result_key, save_sub)
|
|
1616
|
+
if manifest_accumulator is not None and manifest_scenario_name:
|
|
1617
|
+
try:
|
|
1618
|
+
manifest_accumulator.add_plan(
|
|
1619
|
+
result_key, save_sub, plan, manifest_scenario_name,
|
|
1620
|
+
)
|
|
1621
|
+
except Exception as exc:
|
|
1622
|
+
logger.warning(
|
|
1623
|
+
"Failed to accumulate axis bounds for %s/%s: %s",
|
|
1624
|
+
result_key, save_sub, exc,
|
|
1625
|
+
)
|
|
1626
|
+
# Record file-member-specific availability if different from base
|
|
1627
|
+
if save_sub != sub_config:
|
|
1628
|
+
succeeded.append((result_key, save_sub))
|
|
1629
|
+
|
|
1630
|
+
return succeeded
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def compute_live_plan(
|
|
1634
|
+
df: pd.DataFrame,
|
|
1635
|
+
cfg: 'PlotConfig',
|
|
1636
|
+
plot_name: str,
|
|
1637
|
+
break_times: set[str] | None = None,
|
|
1638
|
+
period_weights=None,
|
|
1639
|
+
color_path: Path | None = None,
|
|
1640
|
+
) -> PlotPlan | None:
|
|
1641
|
+
"""Compute a PlotPlan in memory without disk I/O.
|
|
1642
|
+
|
|
1643
|
+
Uses the full data range so time-series plans can be rendered at any
|
|
1644
|
+
start/duration via :func:`build_figure_from_plan` with *plot_rows*.
|
|
1645
|
+
"""
|
|
1646
|
+
from flextool.plot_outputs.orchestrator import (
|
|
1647
|
+
_apply_dimension_rules, _resolve_shared_axis_bounds, _process_file_member,
|
|
1648
|
+
reattach_file_level_for_color,
|
|
1649
|
+
)
|
|
1650
|
+
from flextool.plot_outputs.axis_helpers import _normalize_axis_bounds
|
|
1651
|
+
from flextool.plot_outputs.format_helpers import insert_timeline_breaks
|
|
1652
|
+
|
|
1653
|
+
full_range = (0, len(df))
|
|
1654
|
+
dim_result = _apply_dimension_rules(df, cfg, full_range, period_weights=period_weights)
|
|
1655
|
+
if dim_result is None:
|
|
1656
|
+
return None
|
|
1657
|
+
df_processed, rules, chart_type, summed_dims, averaged_dims = dim_result
|
|
1658
|
+
|
|
1659
|
+
col_rules = rules[df_processed.index.nlevels:]
|
|
1660
|
+
grouped_bar_levels = [i for i, c in enumerate(col_rules) if c == 'g']
|
|
1661
|
+
stack_levels = [i for i, c in enumerate(col_rules) if c == 's']
|
|
1662
|
+
expand_axis_levels = [i for i, c in enumerate(col_rules) if c == 'e']
|
|
1663
|
+
subplot_levels = [i for i, c in enumerate(col_rules) if c == 'u']
|
|
1664
|
+
line_levels = [i for i, c in enumerate(col_rules) if c == 'l']
|
|
1665
|
+
file_levels = [i for i, c in enumerate(col_rules) if c == 'f']
|
|
1666
|
+
|
|
1667
|
+
plot_title = plot_name
|
|
1668
|
+
if summed_dims:
|
|
1669
|
+
plot_title = f"{plot_title} ('{', '.join(str(d) for d in summed_dims)}' summed)"
|
|
1670
|
+
if averaged_dims:
|
|
1671
|
+
plot_title = f"{plot_title} ('{', '.join(str(d) for d in averaged_dims)}' averaged)"
|
|
1672
|
+
|
|
1673
|
+
# File members (take the first — live plans don't split across files by member)
|
|
1674
|
+
file_member = None
|
|
1675
|
+
if file_levels:
|
|
1676
|
+
if len(file_levels) == 1:
|
|
1677
|
+
members = df_processed.columns.get_level_values(file_levels[0]).unique().tolist()
|
|
1678
|
+
else:
|
|
1679
|
+
fm_df = df_processed.columns.to_frame().iloc[:, file_levels].drop_duplicates()
|
|
1680
|
+
members = [tuple(row) for row in fm_df.values]
|
|
1681
|
+
if members:
|
|
1682
|
+
file_member = members[0]
|
|
1683
|
+
|
|
1684
|
+
axis_bounds = _normalize_axis_bounds(cfg.axis_bounds)
|
|
1685
|
+
axis_bounds = _resolve_shared_axis_bounds(
|
|
1686
|
+
df_processed, axis_bounds, stack_levels, subplot_levels,
|
|
1687
|
+
cfg.always_include_zero_in_axis,
|
|
1688
|
+
)
|
|
1689
|
+
|
|
1690
|
+
result = _process_file_member(
|
|
1691
|
+
df_processed, file_member, file_levels, plot_title,
|
|
1692
|
+
grouped_bar_levels, stack_levels, expand_axis_levels,
|
|
1693
|
+
subplot_levels, line_levels,
|
|
1694
|
+
)
|
|
1695
|
+
if result is None:
|
|
1696
|
+
return None
|
|
1697
|
+
(df_fm, effective_plot_name, _member_str,
|
|
1698
|
+
fm_grouped_bar_levels, fm_stack_levels, fm_expand_axis_levels,
|
|
1699
|
+
fm_subplot_levels, fm_line_levels) = result
|
|
1700
|
+
|
|
1701
|
+
if chart_type == 'bar' and cfg.color_entity_class:
|
|
1702
|
+
file_level_names = [df_processed.columns.names[i] for i in file_levels]
|
|
1703
|
+
df_fm = reattach_file_level_for_color(
|
|
1704
|
+
df_fm, file_member, file_level_names, cfg.color_entity_class)
|
|
1705
|
+
|
|
1706
|
+
if cfg.skip_data_with_only_zeroes:
|
|
1707
|
+
df_fm = df_fm.loc[:, (df_fm.abs() > 1e-6).any()]
|
|
1708
|
+
if chart_type == 'bar':
|
|
1709
|
+
df_fm = df_fm.loc[(df_fm.abs() > 1e-6).any(axis=1)]
|
|
1710
|
+
if df_fm.empty:
|
|
1711
|
+
return None
|
|
1712
|
+
|
|
1713
|
+
if cfg.multiply_by is not None:
|
|
1714
|
+
df_fm = df_fm * cfg.multiply_by
|
|
1715
|
+
|
|
1716
|
+
if chart_type == 'time' and break_times:
|
|
1717
|
+
df_fm = insert_timeline_breaks(df_fm, break_times)
|
|
1718
|
+
|
|
1719
|
+
if chart_type == 'bar':
|
|
1720
|
+
return _compute_bar_plan(
|
|
1721
|
+
df_fm, effective_plot_name, cfg,
|
|
1722
|
+
fm_stack_levels, fm_expand_axis_levels,
|
|
1723
|
+
fm_subplot_levels, fm_grouped_bar_levels,
|
|
1724
|
+
axis_bounds,
|
|
1725
|
+
color_path=color_path,
|
|
1726
|
+
)
|
|
1727
|
+
elif chart_type == 'time':
|
|
1728
|
+
return _compute_time_plan(
|
|
1729
|
+
df_fm, effective_plot_name, cfg,
|
|
1730
|
+
fm_stack_levels, fm_subplot_levels, fm_line_levels,
|
|
1731
|
+
axis_bounds, full_range,
|
|
1732
|
+
color_path=color_path,
|
|
1733
|
+
)
|
|
1734
|
+
return None
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def rebuild_plan_color_map(
|
|
1738
|
+
plan: PlotPlan,
|
|
1739
|
+
color_path: Path | None = None,
|
|
1740
|
+
) -> dict[str, tuple] | None:
|
|
1741
|
+
"""Recompute *plan*'s ``shared_color_map`` from the current template.
|
|
1742
|
+
|
|
1743
|
+
This is the in-place recolor/reorder path used by the result viewer
|
|
1744
|
+
when the user edits the project's ``plot_settings.yaml``. It rebuilds
|
|
1745
|
+
ONLY the color map (new colors AND new file order) from the freshly
|
|
1746
|
+
loaded template, reusing the plan's stored ``color_category`` /
|
|
1747
|
+
``color_entity_class`` hints and its existing color-map label keys —
|
|
1748
|
+
without touching dimension rules, layout, or the processed DataFrame.
|
|
1749
|
+
|
|
1750
|
+
The label set is taken from the plan's current ``shared_color_map``
|
|
1751
|
+
keys: ``build_shared_color_map`` returns exactly one entry per input
|
|
1752
|
+
label, so those keys are precisely the label set the compute path
|
|
1753
|
+
produced. Running them back through ``order_labels_by_template`` +
|
|
1754
|
+
``build_shared_color_map`` with the same ``category`` / ``entity_class``
|
|
1755
|
+
arguments yields a result identical to a full ``compute_live_plan``
|
|
1756
|
+
recompute for a color/order-only template change.
|
|
1757
|
+
|
|
1758
|
+
Returns the new ordered ``shared_color_map`` (and assigns it to
|
|
1759
|
+
``plan.shared_color_map``). Returns ``None`` and leaves the plan
|
|
1760
|
+
unchanged when the plan has no ``shared_color_map`` to rebuild (e.g. a
|
|
1761
|
+
non-shared legend) — the caller falls back gracefully.
|
|
1762
|
+
"""
|
|
1763
|
+
from flextool.plot_outputs.color_template import (
|
|
1764
|
+
load_color_template,
|
|
1765
|
+
order_labels_by_template,
|
|
1766
|
+
)
|
|
1767
|
+
from flextool.plot_outputs.legend_helpers import build_shared_color_map
|
|
1768
|
+
|
|
1769
|
+
if not plan.shared_color_map:
|
|
1770
|
+
return None
|
|
1771
|
+
|
|
1772
|
+
labels = list(plan.shared_color_map.keys())
|
|
1773
|
+
template = load_color_template(color_path)
|
|
1774
|
+
ordered = order_labels_by_template(
|
|
1775
|
+
labels,
|
|
1776
|
+
template,
|
|
1777
|
+
category=plan.color_category,
|
|
1778
|
+
entity_class=plan.color_entity_class,
|
|
1779
|
+
)
|
|
1780
|
+
new_map = build_shared_color_map(
|
|
1781
|
+
ordered,
|
|
1782
|
+
color_template=template,
|
|
1783
|
+
category=plan.color_category,
|
|
1784
|
+
entity_class=plan.color_entity_class,
|
|
1785
|
+
)
|
|
1786
|
+
plan.shared_color_map = new_map
|
|
1787
|
+
return new_map
|