flextool 4.0.0__py3-none-any.whl

flextool/process_outputs/drop_levels.py

@@ -0,0 +1,208 @@
+from types import SimpleNamespace
+
+import pandas as pd
+
+
+# Variables: droplevel('solve') only
+_V_DROP = [
+    'flow', 'ramp', 'reserve', 'state',
+    'online_linear', 'startup_linear', 'shutdown_linear',
+    'online_integer', 'startup_integer', 'shutdown_integer',
+    'q_state_up', 'q_state_down', 'q_reserve',
+    'q_inertia', 'q_non_synchronous', 'q_state_up_group', 'q_capacity_margin',
+    'invest', 'divest',
+    'dual_invest_connection', 'dual_invest_node', 'dual_invest_unit',
+    'dual_maxInvest_period', 'dual_maxCumulative',
+    'dual_maxInvestGroup_period', 'dual_maxInvestGroup_total', 'dual_maxInvestGroup_cumulative',
+    # Investment-floor (min-side) duals — all keep a ('solve', 'period')
+    # index (the min invest-total constraint, unlike max-total, retains its
+    # period axis), so every family drops the solve level here rather than
+    # collapsing to solve-only.
+    'dual_minInvest_period', 'dual_minInvest_total', 'dual_minCumulative',
+    'dual_minInvestGroup_period', 'dual_minInvestGroup_total', 'dual_minInvestGroup_cumulative',
+    'dual_co2_max_period',
+]
+
+# Variables with a solve-only row index — must collapse to a single
+# row (last solve wins) instead of droplevel.  Same pattern as
+# ``dual_co2_max_total`` below.
+_V_SOLVE_ONLY = [
+    'dual_maxInvest_total',
+]
+
+# Parameters: droplevel('solve') + deduplicate with keep='last'.
+#
+# Use this bucket for params whose per-roll values represent successive
+# states of the same (entity, period) (e.g. ``entity_all_existing``,
+# which is a chain-sum that gets updated as later rolls realise their
+# investments — the LAST roll covering a given period carries the
+# post-decision value).
+_PAR_DROP = [
+    'step_duration', 'timestep_weight', 'flow_min', 'flow_max',
+    'process_availability', 'process_source_sink_varCost',
+    'process_slope', 'process_section',
+    'node_self_discharge_loss', 'node_penalty_up', 'node_penalty_down',
+    'node_inflow', 'commodity_price', 'group_co2_price',
+    'reserve_upDown_group_reservation', 'profile',
+    'entity_annual_discounted', 'entity_annual_divest_discounted',
+    'inflation_factor_investment_yearly',
+    # inflation_factor_operations_yearly evolves across rolls: each step
+    # carries period-d's "active" factor (4.54 ≈ Σ(1+r)^y × p_years[d,y])
+    # in its FIRST row and a forward-looking discount (3.73) in its
+    # SECOND row.  ``keep='last'`` therefore picks the value from the
+    # step that COMMITS period d (where d is the realized invest/dispatch
+    # period), matching what the LP objective uses.  ``keep='first'``
+    # would lock in the earlier step's forward-looking discount for every
+    # period after the first, leaving costs_discounted off by the
+    # active/forward ratio (~1.18 for the multi_year fixture).
+    'inflation_factor_operations_yearly',
+    'group_penalty_capacity_margin', 'group_capacity_margin',
+    # entity_all_existing carries the cumulative existing-capacity
+    # chain (existing + Σ prior invest − Σ prior divest).  The last roll
+    # covering a given period writes the post-decision value; the
+    # earlier rolls' rows for the same period carry the pre-decision
+    # value.  keep='last' picks the post-decision row to match v3.32.0.
+    'entity_all_existing',
+    # entity_pre_existing is the static baseline before any invest
+    # decision.  In practice constant across rolls for a given
+    # (entity, period), so either keep choice yields the same value —
+    # moved here for symmetry with entity_all_existing per the
+    # 2026-05-14 investigation into the unit_capacity__d.csv lag.
+    'entity_pre_existing',
+    # NOTE: complete_period_share_of_year is intentionally NOT in this
+    # bucket — it needs an order-INDEPENDENT max-per-period reduction,
+    # done explicitly in drop_levels() below.  See the comment there.
+]
+
+# Parameters: droplevel('solve') + deduplicate with keep='first'.
+#
+# Use this bucket ONLY for params that are invariant across rolls (so
+# either keep choice would yield the same result).  Do NOT use this
+# bucket for params whose value evolves across rolls — see _PAR_DROP
+# above and the comment there.
+_PAR_DEDUP = [
+    'years_from_start_d', 'years_represented_d',
+    'entity_max_units',
+    'process_startup_cost',
+    'entity_fixed_cost', 'entity_lifetime_fixed_cost', 'entity_lifetime_fixed_cost_divest',
+    'node_annual_flow',
+    'group_penalty_inertia', 'group_penalty_non_synchronous',
+    'group_inertia_limit',
+    'node_capacity_for_scaling', 'group_capacity_for_scaling',
+]
+
+
+def _reduce_share_per_period(csoy: pd.Series) -> pd.Series:
+    """Collapse ``complete_period_share_of_year`` to one value per period.
+
+    ``complete_period_share_of_year`` is the per-period annualisation
+    denominator for DISPATCH outputs (energy/cost = Σ power·dur ÷ share).
+    It is NOT invariant across sub-solves: an invest sub-solve whose
+    period samples a representative window (e.g. 3×24 h) carries
+    ``share = 72/8760 ≈ 0.0082``, while the dispatch sub-solve that
+    realises the same period over the full year carries ``share = 1``.
+
+    BOTH sub-solves persist a row for the realised period, so a plain
+    ``keep='first'/'last'`` dedup is **order-dependent** — and the multi-
+    solve union order is NOT guaranteed: when the ``_solve_order.txt``
+    creation manifest is absent, ``union_realized_slice`` falls back to
+    filesystem glob order, which can place the invest row last and pick
+    the 0.0082 window share, inflating every period total by
+    ``8760/(sampled hours)`` (≈121.7× for a 3-day window).  This was the
+    cause of an intermittent multi-solve period/aggregate over-count that
+    appeared on some scenarios/runs but not others.
+
+    The dispatch sub-solve covers the full period, so its share is always
+    ≥ any sampling sub-solve's for the same period.  Reducing by **max per
+    period** therefore deterministically selects the realising-dispatch
+    share regardless of union order, and is byte-identical to the correct
+    (manifest-present) ``keep='last'`` runs.  ``sort=False`` preserves the
+    first-appearance period order those runs produced.
+    """
+    csoy = csoy.droplevel('solve')
+    return csoy.groupby(level=0, sort=False).max()
+
+
+def drop_levels(par: SimpleNamespace, s: SimpleNamespace, v: SimpleNamespace):
+    """Strip the 'solve' level from all time-indexed variables, parameters and sets.
+
+    Must be called before any post-processing calculations.
+    Returns (par, s, v) with the solve level removed in-place.
+    """
+    for attr in _V_DROP:
+        obj = getattr(v, attr)
+        obj = obj.droplevel('solve')
+        # For rolling scenarios, keep only last solve's value per (period, time)
+        if obj.index.duplicated().any():
+            obj = obj[~obj.index.duplicated(keep='last')]
+        setattr(v, attr, obj)
+
+    # Solve-only indexes (``dual_co2_max_total`` historically, plus
+    # ``dual_maxInvest_total`` which became solve-only when its reader
+    # arity was fixed): collapse to a single row (last solve wins).
+    for attr in [*_V_SOLVE_ONLY, 'dual_co2_max_total']:
+        obj = getattr(v, attr)
+        if not obj.empty:
+            last = obj.iloc[[-1]].copy()
+            last.index = pd.RangeIndex(1)
+            setattr(v, attr, last)
+        else:
+            setattr(v, attr, obj.reset_index(drop=True))
+
+    # v_angle: drop solve level only when non-empty (may be empty when no DC PF nodes)
+    if not v.angle.empty:
+        v.angle = v.angle.droplevel('solve')
+        if v.angle.index.duplicated().any():
+            v.angle = v.angle[~v.angle.index.duplicated(keep='last')]
+
+    for attr in _PAR_DROP:
+        obj = getattr(par, attr)
+        obj = obj.droplevel('solve')
+        if obj.index.duplicated().any():
+            obj = obj[~obj.index.duplicated(keep='last')]
+        setattr(par, attr, obj)
+
+    for attr in _PAR_DEDUP:
+        obj = getattr(par, attr)
+        obj = obj.droplevel('solve')
+        setattr(par, attr, obj[~obj.index.duplicated(keep='first')])
+
+    # complete_period_share_of_year — order-independent max-per-period
+    # reduction (deterministic across multi-solve union order).  See
+    # _reduce_share_per_period for the full rationale.
+    par.complete_period_share_of_year = _reduce_share_per_period(
+        par.complete_period_share_of_year
+    )
+
+    # Sets have varied special handling so are done individually
+    s.solve_period = s.d_realized_period
+    # Save per-timestep solve mapping before dropping (for correct re-join in CSV output)
+    # For rolling scenarios, keep only the last solve's mapping per (period, time)
+    s.solve_period_time = s.dt_realize_dispatch
+    if s.solve_period_time.droplevel('solve').duplicated().any():
+        s.solve_period_time = s.solve_period_time[~s.solve_period_time.droplevel('solve').duplicated(keep='last')]
+    s.period = s.period.droplevel('solve')
+    s.period__time_first = s.period__time_first.droplevel('solve')
+    s.period_first_of_solve = s.period_first_of_solve.droplevel('solve')
+    s.period_in_use = s.period_in_use.droplevel('solve').unique()
+    s.d_realize_dispatch_or_invest = s.d_realize_dispatch_or_invest.droplevel('solve').unique()
+    s.d_realize_invest = s.d_realize_invest.droplevel('solve').unique()
+    s.d_realized_period = s.d_realized_period.droplevel('solve').unique()
+    s.dt_realize_dispatch = s.dt_realize_dispatch.droplevel('solve').unique()
+    s.dt = s.dt.droplevel('solve').unique()
+    s.dt_fix_storage_timesteps = s.dt_fix_storage_timesteps.droplevel('solve')
+    # For rolling scenarios, dtt/dtttdt have overlapping timesteps from different solves.
+    # Filter to only realized (period, time) pairs, keeping the last solve's mapping.
+    s.dtt = s.dtt.droplevel('solve')
+    dtt_pt = s.dtt.droplevel('t_previous')
+    s.dtt = s.dtt[dtt_pt.isin(s.dt_realize_dispatch) & ~dtt_pt.duplicated(keep='last')]
+    s.dtttdt = s.dtttdt.droplevel('solve')
+    dtttdt_pt = s.dtttdt.droplevel(['t_previous', 't_previous_within_timeset', 'd_previous', 't_previous_within_solve'])
+    s.dtttdt = s.dtttdt[dtttdt_pt.isin(s.dt_realize_dispatch) & ~dtttdt_pt.duplicated(keep='last')]
+    s.ed_invest = s.ed_invest.droplevel('solve').unique().join(s.d_realize_invest, how='inner')
+    s.edd_invest = s.edd_invest.droplevel('solve').unique()
+    s.edd_invest.names = ['entity', 'period_invest', 'period']
+    s.edd_invest = s.edd_invest.join(s.d_realize_invest, how='inner')
+    s.ed_divest = s.ed_divest.droplevel('solve').unique().join(s.d_realize_invest, how='inner')
+
+    return par, s, v