flextool 4.0.0__py3-none-any.whl

flextool/export_to_tabular/export_settings.yaml

@@ -0,0 +1,377 @@
+# Entity classes that need an Entity Alternative column (active_by_default=False).
+entity_alternative_classes:
+  - node
+  - unit
+  - connection
+
+# Parameters that must appear before the Entity Alternative column.
+# Ensures roundtrip compatibility with the position-based importer.
+pre_entity_alternative_params:
+  node: ["node_type"]
+
+# Column width settings (unit ≈ width of character '0' in Calibri 11, ~7px).
+minimum_parameter_column_width: 10
+width_non_parameter_columns: 22
+definition_column_width: 11
+index_column_width: 12
+
+# Default index column names per layout type (replaces 'x' from the DB).
+index_name_defaults:
+  periodic: "period"
+  timeseries: "time"
+  nested_periodic: "period"
+
+# Default index names for stochastic (nested Map) dimensions.
+# Applied when the DB index_name is empty or 'x'.
+stochastic_index_names:
+  - "forecast"
+  - "branch_time"
+  - "time"
+  - "is_realized"
+
+# Entity classes that define time structure.  Their Map parameters use timestep
+# names as indexes but should NOT be filtered out by the time-indexed check.
+time_structure_classes:
+  - timeline
+  - timeset
+
+# Merge two entity classes onto one sheet set, adding a direction column.
+merge_classes:
+  unit_node:
+    classes:
+      unit__inputNode: "input"
+      unit__outputNode: "output"
+    direction_column: "input_output"
+
+# Unpack a Map parameter's indexes as an extra row-identifier column.
+# Creates a separate sheet from the main entity class sheet.
+unpack_map_params:
+  unit_node_constraint:
+    source_classes: ["unit__inputNode", "unit__outputNode"]
+    direction_column: "input_output"
+    params: ["constraint_flow_coeff"]
+    index_column: "constraint"
+  connection_node_constraint:
+    source_classes: ["connection__node"]
+    params: ["constraint_flow_coeff"]
+    index_column: "constraint"
+  node_constraint:
+    source_classes: ["node"]
+    params: ["constraint_invested_capacity_coeff", "constraint_cumulative_pre_built_capacity_coeff", "constraint_state_coeff"]
+    index_column: "constraint"
+  unit_constraint:
+    source_classes: ["unit"]
+    params: ["constraint_invested_capacity_coeff", "constraint_cumulative_pre_built_capacity_coeff"]
+    index_column: "constraint"
+  connection_constraint:
+    source_classes: ["connection"]
+    params: ["constraint_invested_capacity_coeff", "constraint_cumulative_pre_built_capacity_coeff"]
+    index_column: "constraint"
+
+# Add columns from a related multi-dim entity class to a sheet.
+add_entity_elements:
+  connection:
+    from_class: "connection__node__node"
+    columns: ["left_node", "right_node"]
+
+# Split an entity class across multiple sheet groups with different param subsets.
+split_params:
+  solve:
+    solve:
+      params: ["solve_mode", "solver",
+               "contains_solves", "solver_precommand",
+               "solver_arguments", "new_stepduration",
+               "solver_mip_gap", "scaling",
+               "timeline_hole_multiplier",
+               "decomposition", "benders_tolerance",
+               "benders_max_iter", "benders_in_out_weight"]
+    solve_period:
+      params: ["period_timeset", "years_represented", "realized_periods",
+               "invest_periods", "realized_invest_periods", "fix_storage_periods"]
+    rolling_parameters:
+      params: ["rolling_solve_jump", "rolling_solve_horizon", "rolling_duration"]
+    solve_s_stochastic_branches:
+      params: ["stochastic_branches"]
+  model:
+    model:
+      params: ["inflation_rate",
+               "inflation_offset_investment", "inflation_offset_operations",
+               "max_flow_for_unconstrained_variables",
+               "small_number_threshold",
+               "output_horizon"]
+    model_solve_sequence:
+      params: ["solves"]
+    model_periods_available:
+      params: ["periods_available"]
+  timeline:
+    timeline:
+      params: ["timestep_duration"]
+  timeset:
+    timeset_c:
+      params: ["timeline"]
+    timeset_t:
+      params: ["timeset_duration", "timeset_weights"]
+    timeset_s:
+      # representative_period_weights is a rank-2 nested Map (schema type
+      # 2d_map) — route it to the multi-index stochastic writer so the Map
+      # value survives round-trip (the periodic writer would drop it).
+      layout: stochastic
+      params: ["representative_period_weights"]
+
+# Dimensions to drop from generated sheet names (always-present relationship dims).
+drop_dimensions: ["upDown"]
+
+# Rename generated base sheet names to match template conventions.
+rename_sheets:
+  constraint: "constraint_sense"
+
+# Sheet ordering
+sheet_order:
+  - navigate
+  - scenario
+  - model
+  - model_solve_sequence
+  - model_periods_available
+  - solve
+  - solve_period
+  - solve_period_period
+  - rolling_parameters
+  - timeline
+  - timeset_c
+  - timeset_t
+  - timeset_s
+  - node_c
+  - node_p
+  - node_t
+  - node_s_inflow
+  - node_s_availability
+  - commodity_c
+  - commodity_p
+  - commodity_node
+  - connection_c
+  - connection_p
+  - connection_t
+  - unit_c
+  - unit_p
+  - unit_t
+  - unit_s_availability
+  - unit_s_efficiency
+  - unit_s_min_load
+  - unit_node_c
+  - unit_node_p
+  - unit_node_t
+  - unit_node_s
+  - profile_t
+  - profile_s
+  - node_profile_c
+  - connection_profile_c
+  - unit_node_profile_c
+  - group_c
+  - group_p
+  - group_connection
+  - group_connection_node
+  - group_node
+  - group_unit
+  - group_unit_node
+  - reserve_connection_node_c
+  - reserve_group_c
+  - reserve_group_p
+  - reserve_group_t
+  - reserve_group_s
+  - reserve_unit_node_c
+  - solve_s_stochastic_branches
+  - constraint_sense_c
+  - unit_node_constraint_c
+  - connection_node_constraint_c
+  - node_constraint_c
+  - unit_constraint_c
+  - connection_constraint_c
+  - connection_node
+
+# Navigate sheet: groups of related sheets, each row lists sheet names
+# placed across columns A (Constants), B (Periodic), C (Timeseries), D (Stochastic).
+navigate_groups:
+  - name: "Configuration"
+    color: "D9D9D9"
+    rows:
+      - [scenario]
+      - [model]
+      - [model_solve_sequence]
+      - [model_periods_available]
+      - [solve, solve_period, solve_period_period, solve_s_stochastic_branches]
+      - [rolling_parameters]
+  - name: "Time"
+    color: "D9D9D9"
+    rows:
+      - [timeline]
+      - [timeset_c, timeset_t, "", timeset_s]
+  - name: "Core entities"
+    color: "D5C2EE"
+    rows:
+      - [node_c, node_p, node_t, node_s_inflow]
+      - ["", "", "", node_s_availability]
+      - [commodity_c, commodity_p]
+      - [commodity_node]
+      - [connection_c, connection_p, connection_t]
+      - [connection_node]
+      - [unit_c, unit_p, unit_t, unit_s_availability]
+      - ["", "", "", unit_s_efficiency]
+      - ["", "", "", unit_s_min_load]
+      - [unit_node_c, unit_node_p, unit_node_t, unit_node_s]
+  - name: "Profiles"
+    color: "C5E0B4"
+    rows:
+      - [profile_t, "", "", profile_s]
+      - [node_profile_c]
+      - [connection_profile_c]
+      - [unit_node_profile_c]
+  - name: "Groups"
+    color: "BDD7EE"
+    rows:
+      - [group_c, group_p]
+      - [group_connection]
+      - [group_connection_node]
+      - [group_node]
+      - [group_unit]
+      - [group_unit_node]
+  - name: "Reserves"
+    color: "F8CBAD"
+    rows:
+      - [reserve_connection_node_c, reserve_group_p, reserve_group_t, reserve_group_s]
+      - [reserve_group_c]
+      - [reserve_unit_node_c]
+  - name: "Constraints"
+    color: "FFE699"
+    rows:
+      - [constraint_sense_c]
+      - [unit_node_constraint_c]
+      - [connection_node_constraint_c]
+      - [node_constraint_c]
+      - [unit_constraint_c]
+      - [connection_constraint_c]
+
+# Sheets that require the --include-advanced flag.
+# Omitted by default; included when advanced features are needed or when the
+# database contains data that requires them.
+advanced_sheets:
+  - model_periods_available
+  - solve_period_period
+
+# Parameter groups required for a functioning FlexTool model.  The GUI
+# group picker highlights these and ships them pre-checked.  A user can
+# still produce a valid input file without them by combining data from
+# multiple input sources in Spine Toolbox.  Order here is the display
+# order in the picker (most foundational first).
+required_groups:
+  - timeline
+  - model
+  - solve_basics
+  - basics
+
+# When --groups is used, sheets listed here are kept even if all their
+# parameters were filtered out, provided one of the listed parent groups
+# is in the selection.  Use this for structural sheets whose presence is
+# needed to wire up a group's data even when the columns themselves live
+# in another group.
+#
+# Example:
+#   solve_basics:
+#     - solve_period
+always_include_with_groups: {}
+
+# INFO rows: optional information text shown at the top of specific sheets.
+# The reader ignores rows starting with "INFO:".
+info_rows:
+  model: "INFO: There can be only one model per scenario - best to just use same model name always."
+  model_solve_sequence: "INFO: FlexTool can define one or more solves to be executed in sequence. They will be executed in top to down order from the column the model uses."
+
+# Data type overrides for parameters where the DB type_list doesn't capture
+# the full convention.  Spine DB uses 2d-map with string "yes" values as a
+# workaround for map-of-arrays (which it doesn't support natively).
+data_type_overrides:
+  nested_periodic:
+    invest_periods: "string (2d-map)"
+    realized_invest_periods: "string (2d-map)"
+    realized_periods: "string (2d-map)"
+    fix_storage_periods: "string (2d-map)"
+
+# Parameters that are ONLY period-indexed (never time-indexed).
+# Used to filter them out of _t sheets.  Derived from parameter descriptions.
+# Parameters not listed here but having 1d_map in type_list are assumed to be
+# usable on both _p and _t sheets.
+period_only_params:
+  node:
+    - constraint_invested_capacity_coeff
+    - constraint_cumulative_pre_built_capacity_coeff
+    - constraint_state_coeff
+    - annual_flow
+    - cumulative_max_capacity
+    - cumulative_min_capacity
+    - existing
+    - fixed_cost
+    - interest_rate
+    - invest_cost
+    - invest_forced
+    - invest_max_period
+    - invest_min_period
+    - lifetime
+    - peak_inflow
+    - retire_max_period
+    - retire_min_period
+    - penalty_down
+    - penalty_up
+    - salvage_value
+    - self_discharge_loss
+    - storage_state_reference_price
+    - storage_state_reference_value
+  unit:
+    - constraint_invested_capacity_coeff
+    - constraint_cumulative_pre_built_capacity_coeff
+    - cumulative_max_capacity
+    - cumulative_min_capacity
+    - existing
+    - fixed_cost
+    - interest_rate
+    - invest_cost
+    - invest_max_period
+    - invest_min_period
+    - lifetime
+    - retire_max_period
+    - retire_min_period
+    - salvage_value
+  connection:
+    - constraint_invested_capacity_coeff
+    - constraint_cumulative_pre_built_capacity_coeff
+    - cumulative_max_capacity
+    - cumulative_min_capacity
+    - existing
+    - fixed_cost
+    - interest_rate
+    - invest_cost
+    - invest_max_period
+    - invest_min_period
+    - lifetime
+    - retire_max_period
+    - retire_min_period
+    - salvage_value
+  group:
+    - co2_max_period
+    - inertia_limit
+    - invest_max_period
+    - invest_min_period
+    - max_cumulative_flow
+    - max_instant_flow
+    - min_cumulative_flow
+    - min_instant_flow
+    - non_synchronous_limit
+    - penalty_capacity_margin
+    - penalty_inertia
+    - penalty_non_synchronous
+  solve:
+    - years_represented
+  unit__inputNode:
+    - constraint_flow_coeff
+  unit__outputNode:
+    - constraint_flow_coeff
+  connection__node:
+    - constraint_flow_coeff

flextool/export_to_tabular/export_to_excel.py

@@ -0,0 +1,227 @@
+"""Top-level orchestrator for exporting a FlexTool Spine DB to an Excel file."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from openpyxl import Workbook
+
+from flextool.export_to_tabular.db_reader import DatabaseContents, read_database
+from flextool.export_to_tabular.sheet_config import SheetSpec, build_sheet_specs, load_settings
+from flextool.export_to_tabular.excel_writer import (
+    IndexClassifier,
+    write_array_transposed_sheet_v2,
+    write_constant_sheet,
+    write_constant_sheet_v2,
+    write_link_sheet,
+    write_link_sheet_v2,
+    write_navigate_sheet,
+    write_nested_periodic_sheet,
+    write_nested_periodic_sheet_v2,
+    write_periodic_sheet,
+    write_periodic_sheet_v2,
+    write_scenario_sheet,
+    write_ladder_sheet_v2,
+    write_stochastic_sheet_v2,
+    write_timeseries_sheet,
+    write_timeseries_sheet_v2,
+    write_version_sheet,
+)
+import flextool.export_to_tabular.excel_writer as _ew
+
+
+def _build_tab_color_map(navigate_groups: list[dict[str, Any]]) -> dict[str, str]:
+    """Build a mapping from sheet name to tab color (hex RGB string) from navigate groups."""
+    color_map: dict[str, str] = {}
+    for group in navigate_groups:
+        color = group["color"]
+        for row in group.get("rows", []):
+            for sheet_name in row:
+                color_map[sheet_name] = color
+    return color_map
+
+
+def export_to_excel(
+    db_url: str,
+    output_path: str,
+    include_advanced: bool = False,
+    use_new_format: bool = True,
+    include_stochastics: bool = False,  # deprecated alias for include_advanced
+    include_groups: list[str] | None = None,
+) -> None:
+    """Export a FlexTool Spine DB to an Excel (.xlsx) file.
+
+    Args:
+        db_url: Spine DB URL (e.g., 'sqlite:///path/to/db.sqlite')
+        output_path: Output .xlsx file path
+        include_advanced: Whether to include advanced sheets (solve sequences,
+            periods_available, stochastic data).
+        use_new_format: If True, use the v2 self-describing format with embedded
+            metadata.  If False, use the original v1 format.
+        include_stochastics: Deprecated alias for include_advanced.
+        include_groups: When given, restrict output to parameters whose
+            ``parameter_group_name`` is in the list (e.g. ``['basics',
+            'model', 'timeline', 'solve_basics']``).  Parameters with no
+            group are dropped.  Sheets that lose all their columns are
+            dropped, unless listed under a selected group in
+            ``always_include_with_groups`` in ``export_settings.yaml``.
+    """
+    if include_stochastics:
+        include_advanced = True
+
+    # 1. Read all data from the database
+    print(f"Reading database: {db_url}")
+    db_contents: DatabaseContents = read_database(db_url)
+
+    # 1b. Build index classifier and entity alternatives index
+    _ew._index_classifier = IndexClassifier(db_contents)
+    _ew._build_entity_alts_index(db_contents)
+
+    # 2. Load settings and build ordered sheet specifications
+    print("Building sheet specifications...")
+    settings = load_settings()
+
+    # 2b. Load data type overrides and width settings
+    _ew._data_type_overrides = settings.get("data_type_overrides", {})
+    _ew._min_param_width = settings.get("minimum_parameter_column_width", 10)
+    _ew._non_param_width = settings.get("width_non_parameter_columns", 22)
+    _ew._def_col_width = settings.get("definition_column_width", 11)
+    _ew._index_col_width = settings.get("index_column_width", 12)
+    _ew._period_only_params = settings.get("period_only_params", {})
+    _ew._stochastic_index_names = settings.get("stochastic_index_names",
+                                                ["forecast", "branch_time", "time", "is_realized"])
+    _ew._time_structure_classes = set(settings.get("time_structure_classes", []))
+    specs: list[SheetSpec] = build_sheet_specs(
+        db_contents,
+        settings,
+        include_groups=set(include_groups) if include_groups else None,
+    )
+    navigate_groups: list[dict[str, Any]] = settings.get("navigate_groups", [])
+    tab_color_map = _build_tab_color_map(navigate_groups) if navigate_groups else {}
+
+    # Filter out advanced sheets unless requested OR they have data
+    advanced_sheets: set[str] = set(settings.get("advanced_sheets", []))
+    if not include_advanced:
+        # Pre-build a set of (class, param) pairs that have data for fast lookup
+        _data_keys: set[tuple[str, str]] = set()
+        for (cls, _bn, pn, _alt) in db_contents.parameter_values:
+            _data_keys.add((cls, pn))
+
+        def _has_data(spec: SheetSpec) -> bool:
+            for cls in spec.entity_classes:
+                for pn in spec.parameter_names:
+                    if (cls, pn) in _data_keys:
+                        return True
+            return False
+
+        specs = [s for s in specs
+                 if s.sheet_name not in advanced_sheets or _has_data(s)]
+
+    # Load info rows setting
+    _ew._info_rows = settings.get("info_rows", {})
+
+    # 3. Create the workbook
+    wb = Workbook()
+
+    # 4. Delete the default 'Sheet' worksheet
+    if "Sheet" in wb.sheetnames:
+        del wb["Sheet"]
+
+    # Track counts by layout for the summary
+    layout_counts: dict[str, int] = {}
+
+    # 5. Write each sheet
+    for spec in specs:
+        ws = wb.create_sheet(spec.sheet_name)
+        layout = spec.layout
+
+        if layout == "constant":
+            if use_new_format:
+                # Check if this is a pure-array sheet (model_solve_sequence etc.)
+                _is_array_sheet = (
+                    len(spec.parameter_names) == 1
+                    and _ew._get_param_data_type(
+                        spec.parameter_names[0], spec.entity_classes,
+                        db_contents, layout="constant",
+                    ) == "boolean (array)"
+                )
+                if _is_array_sheet:
+                    write_array_transposed_sheet_v2(ws, spec, db_contents)
+                else:
+                    write_constant_sheet_v2(ws, spec, db_contents)
+            else:
+                write_constant_sheet(ws, spec, db_contents)
+        elif layout == "periodic":
+            if use_new_format:
+                write_periodic_sheet_v2(ws, spec, db_contents)
+            else:
+                write_periodic_sheet(ws, spec, db_contents)
+        elif layout == "nested_periodic":
+            if use_new_format:
+                write_nested_periodic_sheet_v2(ws, spec, db_contents)
+            else:
+                write_nested_periodic_sheet(ws, spec, db_contents)
+        elif layout == "timeseries":
+            if use_new_format:
+                write_timeseries_sheet_v2(ws, spec, db_contents)
+            else:
+                write_timeseries_sheet(ws, spec, db_contents)
+        elif layout == "stochastic":
+            if use_new_format:
+                write_stochastic_sheet_v2(ws, spec, db_contents)
+            else:
+                # v1 has no stochastic writer — skip
+                del wb[spec.sheet_name]
+                continue
+        elif layout == "ladder":
+            if use_new_format:
+                write_ladder_sheet_v2(ws, spec, db_contents)
+            else:
+                # v1 has no ladder writer — skip
+                del wb[spec.sheet_name]
+                continue
+        elif layout == "link":
+            if use_new_format:
+                write_link_sheet_v2(ws, spec, db_contents)
+            else:
+                write_link_sheet(ws, spec, db_contents)
+        elif layout == "scenario":
+            write_scenario_sheet(ws, db_contents, include_stochastics=include_advanced)
+        elif layout == "version":
+            if use_new_format:
+                # In v2, version info is on the navigate sheet — skip separate sheet
+                del wb[spec.sheet_name]
+                continue
+            write_version_sheet(ws, db_contents.version)
+        elif layout == "navigate":
+            write_navigate_sheet(
+                ws, specs, navigate_groups=navigate_groups,
+                version=db_contents.version if use_new_format else None,
+            )
+        else:
+            print(f"  Warning: unknown layout '{layout}' for sheet '{spec.sheet_name}', skipping.")
+            continue
+
+        # Set tab color from navigate group config
+        if spec.sheet_name in tab_color_map:
+            ws.sheet_properties.tabColor = tab_color_map[spec.sheet_name]
+
+        layout_counts[layout] = layout_counts.get(layout, 0) + 1
+
+    # 6. Ensure the output directory exists and save
+    output_dir = Path(output_path).parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    wb.save(output_path)
+
+    # 7. Print summary
+    format_label = "v2 (self-describing)" if use_new_format else "v1 (original)"
+    total = len(specs)
+    print(f"\nExport complete: {output_path} [{format_label}]")
+    print(f"  Total sheets: {total}")
+    for layout_type, count in sorted(layout_counts.items()):
+        print(f"    {layout_type}: {count}")
+    if db_contents.version is not None:
+        version_display = int(db_contents.version) if db_contents.version == int(db_contents.version) else db_contents.version
+        print(f"  DB version: {version_display}")