flextool 4.0.0__py3-none-any.whl

flextool/representative_periods/scenario_stack.py

@@ -0,0 +1,195 @@
+"""Append an alternative onto a scenario's alternative stack.
+
+A FlexTool scenario is defined by an *ordered* list of alternatives via the
+``scenario_alternative`` relationship: each link carries a ``rank`` (integer,
+higher = applied later, so a higher-rank alternative overrides the values of
+lower-rank ones).  The representative-periods pre-processor writes its results
+into a fresh RP alternative, but that alternative only influences a solve once
+it is part of the scenario's alternative stack.
+
+:func:`add_alternative_to_scenario` appends the RP alternative at the TOP of a
+scenario's stack (the highest rank, so it wins over the baseline
+``period_timeset``), computing the next rank from the scenario's existing max.
+It mirrors the ``DatabaseMapping`` open/commit style of the RP writer
+(``preprocess.py::_write_results_to_db``) and is idempotent — an alternative
+already in the stack is left untouched.
+"""
+
+from __future__ import annotations
+
+from spinedb_api import DatabaseMapping
+
+
+def existing_alternative_names(db_url: str) -> set[str]:
+    """Return the set of alternative names already present in the database.
+
+    A read-only helper the GUI uses to de-duplicate a freshly derived RP
+    alternative name (append ``_2``, ``_3``, … when the base is taken) so a
+    repeated build never silently overwrites an earlier one.
+
+    Args:
+        db_url: Spine database URL (e.g. ``'sqlite:///path.sqlite'``).
+
+    Returns:
+        The names of every alternative in the (unfiltered) database.
+    """
+    with DatabaseMapping(db_url) as db:
+        return {alt["name"] for alt in db.get_alternative_items()}
+
+
+def dedup_alternative_name(base: str, taken: set[str]) -> str:
+    """Return *base*, or the first free ``base_2`` / ``base_3`` / … suffix.
+
+    Pure helper (no DB access) so the GUI can compute the de-duplicated name
+    against a cached name set both for the live CLI preview and for the launch,
+    keeping the two in lock-step.
+
+    Args:
+        base: The desired alternative name.
+        taken: Names already in use (case-sensitive).
+
+    Returns:
+        *base* if free, else ``f"{base}_{n}"`` for the smallest ``n >= 2`` that
+        is not in *taken*.
+    """
+    if base not in taken:
+        return base
+    n = 2
+    while f"{base}_{n}" in taken:
+        n += 1
+    return f"{base}_{n}"
+
+
+def add_alternative_to_scenario(
+    db_url: str,
+    scenario_name: str,
+    alternative_name: str,
+) -> None:
+    """Append *alternative_name* to *scenario_name*'s stack at the top rank.
+
+    The alternative is added with ``rank = max(existing ranks) + 1`` so it is
+    applied last and overrides the alternatives already in the scenario.
+
+    Idempotent: if *alternative_name* is already in the scenario's stack, the
+    function does nothing (no duplicate link, no rank churn, no error).
+
+    Args:
+        db_url: Spine database URL (e.g. ``'sqlite:///path.sqlite'``).
+        scenario_name: Name of the scenario whose stack to extend. Must exist.
+        alternative_name: Name of the alternative to append. Must exist.
+
+    Raises:
+        ValueError: If the scenario or the alternative does not exist.
+    """
+    with DatabaseMapping(db_url) as db:
+        # ``get_*_item`` returns an empty dict (falsy) when nothing matches.
+        if not db.get_scenario_item(name=scenario_name):
+            raise ValueError(f"Scenario '{scenario_name}' does not exist in the database.")
+        if not db.get_alternative_item(name=alternative_name):
+            raise ValueError(
+                f"Alternative '{alternative_name}' does not exist in the database."
+            )
+
+        existing = db.get_scenario_alternative_items(scenario_name=scenario_name)
+        # Idempotent: already in the stack (at any rank) → leave it untouched.
+        if any(sa["alternative_name"] == alternative_name for sa in existing):
+            return
+
+        next_rank = max((sa["rank"] for sa in existing), default=0) + 1
+        # ``add_scenario_alternative`` returns the added item and raises
+        # ``SpineDBAPIError`` on failure — let that surface rather than swallow.
+        db.add_scenario_alternative(
+            scenario_name=scenario_name,
+            alternative_name=alternative_name,
+            rank=next_rank,
+        )
+
+        db.commit_session(
+            f"Append alternative '{alternative_name}' to scenario '{scenario_name}'"
+        )
+
+
+def create_scenario_with_alternative(
+    db_url: str,
+    base_scenario_name: str,
+    new_scenario_name: str,
+    alternative_name: str,
+) -> None:
+    """Clone *base_scenario_name* into *new_scenario_name* + append *alternative_name*.
+
+    Creates *new_scenario_name* as a copy of *base_scenario_name*'s ordered
+    alternative stack (same alternatives, same ranks) and then appends
+    *alternative_name* at ``max(rank) + 1`` so the RP alternative overrides the
+    baseline ``period_timeset``. The base scenario is left completely
+    untouched, so the original and the representative-period runs can be
+    compared side by side.
+
+    Idempotent: if *new_scenario_name* already exists it is not recreated — the
+    function only ensures *alternative_name* is present at the top of its stack
+    (mirroring :func:`add_alternative_to_scenario`), so a repeated launch is a
+    no-op rather than an error.
+
+    Args:
+        db_url: Spine database URL (e.g. ``'sqlite:///path.sqlite'``).
+        base_scenario_name: Scenario to clone. Must exist.
+        new_scenario_name: Name of the scenario to create / extend.
+        alternative_name: Alternative to append on top. Must exist.
+
+    Raises:
+        ValueError: If the base scenario or the alternative does not exist.
+    """
+    with DatabaseMapping(db_url) as db:
+        if not db.get_scenario_item(name=base_scenario_name):
+            raise ValueError(
+                f"Scenario '{base_scenario_name}' does not exist in the database."
+            )
+        if not db.get_alternative_item(name=alternative_name):
+            raise ValueError(
+                f"Alternative '{alternative_name}' does not exist in the database."
+            )
+
+        # Create the new scenario the first time, cloning the base stack. On a
+        # repeat launch the scenario already exists, so only the top-rank append
+        # below runs (idempotent).
+        changed = False
+        if not db.get_scenario_item(name=new_scenario_name):
+            db.add_scenario(name=new_scenario_name)
+            for sa in db.get_scenario_alternative_items(
+                scenario_name=base_scenario_name
+            ):
+                db.add_scenario_alternative(
+                    scenario_name=new_scenario_name,
+                    alternative_name=sa["alternative_name"],
+                    rank=sa["rank"],
+                )
+            changed = True
+
+        existing = db.get_scenario_alternative_items(
+            scenario_name=new_scenario_name
+        )
+        if not any(
+            sa["alternative_name"] == alternative_name for sa in existing
+        ):
+            next_rank = max((sa["rank"] for sa in existing), default=0) + 1
+            db.add_scenario_alternative(
+                scenario_name=new_scenario_name,
+                alternative_name=alternative_name,
+                rank=next_rank,
+            )
+            changed = True
+
+        # Nothing to persist on a fully idempotent repeat call — committing an
+        # unchanged session raises ``NothingToCommit``.
+        if changed:
+            db.commit_session(
+                f"Create scenario '{new_scenario_name}' from "
+                f"'{base_scenario_name}' with alternative '{alternative_name}'"
+            )
+
+
+__all__ = [
+    "add_alternative_to_scenario",
+    "create_scenario_with_alternative",
+    "dedup_alternative_name",
+    "existing_alternative_names",
+]

flextool/representative_periods/weights.py

@@ -0,0 +1,124 @@
+"""Convex weight fitting and simplex projection.
+
+Implements the weight computation from the hull clustering paper:
+for each base period, find the convex combination of representative periods
+that best approximates it (minimum L2 error).
+"""
+
+import numpy as np
+
+
+def project_onto_simplex(v: np.ndarray) -> np.ndarray:
+    """Project vector v onto the probability simplex {w >= 0, sum(w) = 1}.
+
+    Uses Condat's algorithm (O(n) average case).
+
+    Args:
+        v: Input vector of length n.
+
+    Returns:
+        Projected vector on the simplex.
+    """
+    n = len(v)
+    if n == 0:
+        return v
+    if n == 1:
+        return np.array([1.0])
+
+    u = np.sort(v)[::-1]  # sort descending
+    cumsum = np.cumsum(u)
+    k_candidates = u + (1.0 - cumsum) / np.arange(1, n + 1)
+    K = np.max(np.where(k_candidates > 0)[0]) + 1 if np.any(k_candidates > 0) else 1
+    tau = (cumsum[K - 1] - 1.0) / K
+    return np.maximum(v - tau, 0.0)
+
+
+def fit_convex_weights(
+    R: np.ndarray,
+    c: np.ndarray,
+    max_iter: int = 200,
+    tol: float = 1e-8,
+    alpha: float | None = None,
+) -> np.ndarray:
+    """Find convex weights w that minimize ||R @ w - c||^2 s.t. w in simplex.
+
+    Uses projected gradient descent with simplex projection.
+
+    Args:
+        R: Matrix of representative period feature vectors, shape (n_features, n_rp).
+        c: Feature vector of the base period to approximate, shape (n_features,).
+        max_iter: Maximum PGD iterations.
+        tol: Convergence tolerance.
+        alpha: Learning rate. If None, computed from Lipschitz constant.
+
+    Returns:
+        Weight vector w of length n_rp, with w >= 0 and sum(w) = 1.
+    """
+    n_rp = R.shape[1]
+    if n_rp == 1:
+        return np.array([1.0])
+
+    # Precompute R^T R and R^T c for gradient computation
+    RtR = R.T @ R
+    Rtc = R.T @ c
+
+    # Learning rate from Lipschitz constant of gradient
+    if alpha is None:
+        L = np.linalg.norm(RtR, ord=2)
+        alpha = 1.0 / max(L, 1e-10)
+
+    # Initial guess via pseudoinverse
+    R_pinv = np.linalg.pinv(R)
+    w = project_onto_simplex(R_pinv @ c)
+
+    for _ in range(max_iter):
+        # Gradient: R^T (R w - c) = RtR w - Rtc
+        grad = RtR @ w - Rtc
+        w_prev = w.copy()
+        w = project_onto_simplex(w - alpha * grad)
+
+        if np.max(np.abs(w - w_prev)) < tol:
+            break
+
+    return w
+
+
+def distance_to_hull(R: np.ndarray, c: np.ndarray, **kwargs) -> tuple[float, np.ndarray]:
+    """Compute the L2 distance from point c to the convex hull of columns of R.
+
+    Args:
+        R: Matrix of representative period feature vectors, shape (n_features, n_rp).
+        c: Feature vector to measure distance from, shape (n_features,).
+
+    Returns:
+        (distance, weights): the L2 distance and the optimal weight vector.
+    """
+    w = fit_convex_weights(R, c, **kwargs)
+    residual = R @ w - c
+    dist = np.linalg.norm(residual)
+    return dist, w
+
+
+def compute_weight_matrix(
+    C: np.ndarray, rep_indices: list[int]
+) -> np.ndarray:
+    """Compute the full weight matrix W for all base periods.
+
+    Args:
+        C: Clustering matrix, shape (n_features, n_base_periods).
+        rep_indices: Indices of selected representative periods.
+
+    Returns:
+        Weight matrix W of shape (n_base_periods, n_rp),
+        where W[d, r] is the weight of representative period r for base period d.
+        Each row sums to 1 and all entries >= 0.
+    """
+    n_base = C.shape[1]
+    n_rp = len(rep_indices)
+    R = C[:, rep_indices]
+    W = np.zeros((n_base, n_rp))
+
+    for d in range(n_base):
+        W[d, :] = fit_convex_weights(R, C[:, d])
+
+    return W

flextool/scenario_comparison/__init__.py

@@ -0,0 +1,13 @@
+"""Cross-scenario analysis module.
+
+Navigation:
+- data_models.py      : TimeSeriesResults, DispatchMappings — start here to understand data shapes
+- db_reader.py        : Load parquet files from scenario folders → TimeSeriesResults
+- dispatch_mappings.py: Load dispatch mapping parquet files → DispatchMappings
+- config_builder.py   : Discover dispatch entity/scenario names + assign palette colors
+- dispatch_data.py    : Prepare per-scenario dispatch DataFrames for plotting
+- dispatch_plots.py   : Render stacked area dispatch plots
+- orchestrator.py     : Top-level run() function tying all pieces together
+"""
+from flextool.scenario_comparison.db_reader import get_scenario_results
+__all__ = ['get_scenario_results']

flextool/scenario_comparison/config_builder.py

@@ -0,0 +1,158 @@
+"""Discover dispatch entity / scenario names and assign palette colors.
+
+Reads dispatch mappings to enumerate the group / unit / connection / scenario
+names present in a run, and assigns default palette colors.  These feed the
+additive seeding of the project ``plot_settings.yaml`` (the durable colors
+file the renderers read); the legacy ``config.yaml`` system has been removed.
+"""
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+
+from flextool.scenario_comparison.constants import DEFAULT_SPECIAL_COLORS
+from flextool.scenario_comparison.data_models import DispatchMappings
+
+
+def discover_dispatch_entities(
+    mappings: DispatchMappings,
+    scenarios: list[str],
+) -> dict[str, list[str]]:
+    """Discover and classify dispatch entity / scenario names from mappings.
+
+    Reads the dispatch-mapping fields and classifies each discovered name
+    into the entity class the project ``plot_settings.yaml`` expects:
+
+    * ``nodeGroup``   — the dispatch ``group`` name(s) being plotted (the
+      group__node collection that is the plot subject);
+    * ``flowGroup``   — processGroup / unitGroup / connectionGroup aggregate
+      names (``group_aggregate`` of ``processGroup_Unit_to_group`` /
+      ``processGroup_Group_to_unit`` / ``processGroup_Connection``) that
+      stack as items in the dispatch plot;
+    * ``unit``        — individual (not-aggregated) unit names, taken as the
+      bare ``unit`` / ``process`` entity of the unit member and
+      ``not_in_aggregate`` unit fields;
+    * ``connection``  — individual (not-aggregated) connection names, taken
+      as the bare ``process`` / ``connection`` entity of the connection
+      member and ``not_in_aggregate`` connection fields;
+    * ``scenarios``   — the run's scenario names.
+
+    The *bare* entity name is recorded (the unit / connection name), matching
+    what the dispatch color resolver looks up — never the ``(process, node)``
+    composite string.  Nodes are deliberately not discovered (node colors are
+    dataset-coupled and are not seeded into the portable settings file).
+
+    Returns a mapping ``{class -> sorted list of names}`` with the keys
+    ``nodeGroup``, ``flowGroup``, ``unit``, ``connection`` and ``scenarios``;
+    empty lists when a class has no members.
+    """
+    node_groups: set[str] = set()
+    flow_groups: set[str] = set()
+    units: set[str] = set()
+    connections: set[str] = set()
+
+    # --- nodeGroups: the dispatch group(s) being plotted (group__node). ---
+    dispatch_groups_df = mappings.dispatch_groups
+    if dispatch_groups_df is not None and not dispatch_groups_df.empty:
+        if 'group' in dispatch_groups_df.columns:
+            node_groups.update(str(g) for g in dispatch_groups_df['group'].unique())
+
+    # --- flowGroups: the processGroup aggregate names (unit/connection
+    #     aggregates that stack as items in the dispatch plot). ---
+    for pg_attr in (
+        'processGroup_Unit_to_group',
+        'processGroup_Group_to_unit',
+        'processGroup_Connection',
+    ):
+        pg_df = getattr(mappings, pg_attr, None)
+        if pg_df is not None and not pg_df.empty and 'group_aggregate' in pg_df.columns:
+            flow_groups.update(str(g) for g in pg_df['group_aggregate'].unique())
+
+    # --- Individual unit names (bare entity, not the composite) ---
+    # processGroup_*_members carry the units/connections that DO aggregate but
+    # are still useful per-entity colors when shown individually; the
+    # not_in_aggregate_* fields carry the units/connections shown un-aggregated.
+    for unit_attr in (
+        'processGroup_unit_to_node_members',
+        'processGroup_node_to_unit_members',
+        'not_in_aggregate_unit_to_node',
+        'not_in_aggregate_node_to_unit',
+    ):
+        u_df = getattr(mappings, unit_attr, None)
+        if u_df is None or u_df.empty:
+            continue
+        col = 'unit' if 'unit' in u_df.columns else (
+            'process' if 'process' in u_df.columns else None
+        )
+        if col is not None:
+            units.update(str(u) for u in u_df[col].unique())
+
+    # --- Individual connection names (bare entity) ---
+    for conn_attr in (
+        'processGroup_connection_to_node_members',
+        'processGroup_node_to_connection_members',
+        'not_in_aggregate_connection_to_node',
+        'not_in_aggregate_node_to_connection',
+    ):
+        c_df = getattr(mappings, conn_attr, None)
+        if c_df is None or c_df.empty:
+            continue
+        col = 'connection' if 'connection' in c_df.columns else (
+            'process' if 'process' in c_df.columns else None
+        )
+        if col is not None:
+            connections.update(str(c) for c in c_df[col].unique())
+
+    na_conn_df = mappings.not_in_aggregate_connection
+    if na_conn_df is not None and not na_conn_df.empty:
+        col = 'connection' if 'connection' in na_conn_df.columns else (
+            'process' if 'process' in na_conn_df.columns else None
+        )
+        if col is not None:
+            connections.update(str(c) for c in na_conn_df[col].unique())
+
+    # A name discovered as both a group aggregate and an individual entity is
+    # a group (the aggregate is the durable, project-portable color).
+    all_groups = node_groups | flow_groups
+    units -= all_groups
+    connections -= all_groups
+    # Guard against a name landing in both individual classes.
+    connections -= units
+
+    scenario_names = [str(s) for s in scenarios if s]
+
+    return {
+        'nodeGroup': sorted(node_groups),
+        'flowGroup': sorted(flow_groups),
+        'unit': sorted(units),
+        'connection': sorted(connections),
+        'scenarios': sorted(dict.fromkeys(scenario_names)),
+    }
+
+
+def assign_palette_colors(
+    names: list[str],
+    start_index: int = 0,
+    use_tab10: bool = False,
+) -> dict[str, str]:
+    """Assign default palette colors to *names*, in order.
+
+    Special-token names get their fixed ``DEFAULT_SPECIAL_COLORS`` value;
+    everything else cycles the tab20 (or tab10 for scenarios) matplotlib
+    palette starting at *start_index* so the assigned colors are stable and
+    visually sensible.
+
+    Returns an ordered ``{name -> '#RRGGBB'}`` mapping for *names*.
+    """
+    palette = plt.cm.tab10(np.linspace(0, 1, 10)) if use_tab10 else \
+        plt.cm.tab20(np.linspace(0, 1, 20))
+    span = 10 if use_tab10 else 20
+    out: dict[str, str] = {}
+    idx = start_index
+    for name in names:
+        if name in DEFAULT_SPECIAL_COLORS:
+            out[name] = DEFAULT_SPECIAL_COLORS[name]
+            continue
+        out[name] = matplotlib.colors.rgb2hex(palette[idx % span])
+        idx += 1
+    return out

flextool/scenario_comparison/constants.py

@@ -0,0 +1,20 @@
+"""Color and column-name constants for dispatch and summary plots."""
+
+# Default color mapping for special columns
+DEFAULT_SPECIAL_COLORS = {
+    # Positive special columns (at top of legend/plot)
+    'LossOfLoad': 'crimson',
+    'Discharge': 'aqua',
+    'Import': 'indigo',
+    # Negative special columns (at bottom of legend/plot)
+    'Charge': 'lime',
+    'Export': 'purple',
+    'internal_losses': 'darkgray',
+}
+
+# Special columns that should be POSITIVE (at top of stacked plot, top of legend)
+POSITIVE_SPECIAL = ['LossOfLoad', 'Discharge', 'Import']
+# Special columns that should be NEGATIVE (at bottom of stacked plot, bottom of legend)
+NEGATIVE_SPECIAL = ['Charge', 'Export', 'internal_losses']
+# Columns plotted as lines, not stacked areas
+LINE_COLUMNS = ['Curtailed', 'Demand']