flextool 4.0.0__py3-none-any.whl

flextool/process_outputs/out_flowgroup.py

@@ -0,0 +1,189 @@
+"""Output functions for flow group results.
+
+Groups that have ``output_flowGroup_indicators = yes`` appear in
+``s.flowGroupIndicators``.  Their member flows are listed in
+``s.group_process_node`` — each tuple ``(group, process, node)`` identifies
+one unit-flow or connection-flow leg that should be summed into the group.
+
+The metrics emitted here are intentionally minimal (a stub); richer
+direction-aware or technology-aware metrics are future work.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+
+def _member_legs(s, g: str) -> pd.MultiIndex:
+    """Return the ``(process, node)`` legs belonging to group ``g``.
+
+    ``s.group_process_node`` is a ``(group, process, node)`` MultiIndex;
+    filtering by the ``group`` level gives the flows aggregated into this
+    group.  The returned index drops the ``group`` level so callers can
+    match against ``r.flow_dt``/``r.from_conn``/``r.to_conn`` columns
+    (which are ``(process, node)`` after de-duplication).
+    """
+    gpn = s.group_process_node
+    mask = gpn.get_level_values('group') == g
+    return gpn[mask].droplevel('group').unique()
+
+
+def flowGroup_indicators(par, s, v, r, debug):
+    """Flow-group indicator results by period and a signed timewise series.
+
+    For each group in ``s.flowGroupIndicators`` two quantities are built per
+    timestep:
+
+    * ``magnitude_dt`` — the UNSIGNED sum of all member unit- and
+      connection-flow magnitudes (``.abs()``).  This is weighted by step
+      duration to get energy per period and divided by period hours for the
+      average power.  This path is unchanged (byte-identical).
+    * ``net_dt`` — the SIGNED net flow into the group's nodes (into-node
+      ``+``, out-of-node ``−``; matches ``calc_group_flows``/
+      ``calc_connections``).  A unit leg whose ``node`` is the flow's *sink*
+      counts ``+`` (delivery into the node); whose ``node`` is the *source*
+      counts ``−`` (withdrawal).  Connection ``from_conn`` (into node) counts
+      ``+`` and ``to_conn`` (out of node) counts ``−``.  No ``.abs()`` and no
+      step-duration weighting — it is MW per timestep.
+
+    Returns a list of two ``(frame, name)`` tuples:
+
+    * ``flowGroup_gd_p`` — keyed ``(group, period)`` with columns
+      ``cumulative_flow`` [MWh] and ``average_flow`` [MW].  A group with no
+      matched member flows gets zeros rather than being dropped.
+    * ``flowGroup_gd_t`` — keyed ``(group, period, time)`` with the single
+      column ``net_flow`` [MW], the signed per-timestep net flow.
+    """
+    results: list[tuple[pd.DataFrame, str]] = []
+
+    if not list(s.flowGroupIndicators) or s.dt_realize_dispatch.empty:
+        return results
+
+    step_dur = par.step_duration  # Series indexed by (period, time), MW→MWh weight
+    dt_index = s.dt_realize_dispatch
+
+    # Pre-compute lookups of member columns.  ``r.flow_dt`` is keyed by
+    # (process, source, sink) — a unit flow touches a group's (process,
+    # node) membership whenever the node is the source *or* the sink.
+    # ``r.from_conn``/``r.to_conn`` are already keyed by (process, node),
+    # so membership is a direct column pick.
+    flow_cols = r.flow_dt.columns
+    flow_sink_pairs = None
+    flow_source_pairs = None
+    if not r.flow_dt.empty:
+        flow_sink_pairs = pd.MultiIndex.from_arrays(
+            [flow_cols.get_level_values('process'), flow_cols.get_level_values('sink')],
+            names=['process', 'node'],
+        )
+        flow_source_pairs = pd.MultiIndex.from_arrays(
+            [flow_cols.get_level_values('process'), flow_cols.get_level_values('source')],
+            names=['process', 'node'],
+        )
+
+    rows: list[dict] = []
+    net_rows: list[dict] = []
+    # Period hours = share of year * 8760 (same convention used by
+    # out_flows.unit_online_and_startup for period-hour normalization).
+    period_hours = par.complete_period_share_of_year.mul(8760.0)
+
+    for g in s.flowGroupIndicators:
+        legs = _member_legs(s, g)  # MultiIndex[(process, node)]
+
+        # Accumulate |flow| (MW) at each timestep (UNSIGNED magnitude path).
+        magnitude_dt = pd.Series(0.0, index=dt_index)
+        # Accumulate signed net flow (MW) into the group's nodes at each
+        # timestep (into-node +, out-of-node −).
+        net_dt = pd.Series(0.0, index=dt_index)
+
+        if len(legs) and flow_sink_pairs is not None:
+            # Unit flows — pick every (process, source, sink) column whose
+            # (process, sink) or (process, source) is a member leg.
+            sink_hits = flow_sink_pairs.isin(legs)
+            source_hits = flow_source_pairs.isin(legs)
+            is_unit = flow_cols.get_level_values('process').isin(s.process_unit)
+            unit_mask = is_unit & (sink_hits | source_hits)
+            if unit_mask.any():
+                magnitude_dt = magnitude_dt.add(
+                    r.flow_dt.loc[:, unit_mask].abs().sum(axis=1), fill_value=0.0,
+                )
+
+            # Signed: node==sink (unit delivers into the node) counts +,
+            # node==source (unit withdraws from the node) counts −.
+            unit_into = is_unit & sink_hits
+            unit_out = is_unit & source_hits
+            if unit_into.any():
+                net_dt = net_dt.add(
+                    r.flow_dt.loc[:, unit_into].sum(axis=1), fill_value=0.0,
+                )
+            if unit_out.any():
+                net_dt = net_dt.sub(
+                    r.flow_dt.loc[:, unit_out].sum(axis=1), fill_value=0.0,
+                )
+
+            # Connection flows — from_conn and to_conn are already keyed by
+            # (process, node).  Pick every column whose key is a member leg.
+            if not r.from_conn.empty:
+                conn_legs_from = r.from_conn.columns.intersection(legs)
+                if len(conn_legs_from):
+                    magnitude_dt = magnitude_dt.add(
+                        r.from_conn[conn_legs_from].abs().sum(axis=1), fill_value=0.0,
+                    )
+                    # from_conn flows into the node → +.
+                    net_dt = net_dt.add(
+                        r.from_conn[conn_legs_from].sum(axis=1), fill_value=0.0,
+                    )
+            if not r.to_conn.empty:
+                conn_legs_to = r.to_conn.columns.intersection(legs)
+                if len(conn_legs_to):
+                    magnitude_dt = magnitude_dt.add(
+                        r.to_conn[conn_legs_to].abs().sum(axis=1), fill_value=0.0,
+                    )
+                    # to_conn flows out of the node → −.
+                    net_dt = net_dt.sub(
+                        r.to_conn[conn_legs_to].sum(axis=1), fill_value=0.0,
+                    )
+
+        # Weight MW → MWh per step, then sum to period level.
+        energy_dt = magnitude_dt.mul(step_dur, fill_value=0.0)
+        cumulative_d = energy_dt.groupby(level='period').sum()
+        # Reindex to the full realized-period set so groups with no matched
+        # flows still produce a row per realized period.
+        cumulative_d = cumulative_d.reindex(s.d_realized_period, fill_value=0.0)
+
+        hours_d = period_hours.reindex(cumulative_d.index)
+        average_d = cumulative_d.div(hours_d.where(hours_d != 0, pd.NA))
+        average_d = average_d.fillna(0.0)
+
+        for period in cumulative_d.index:
+            rows.append({
+                'group': g,
+                'period': period,
+                'cumulative_flow': float(cumulative_d.loc[period]),
+                'average_flow': float(average_d.loc[period]),
+            })
+
+        # Signed per-timestep net flow (MW) — one row per realized (period,
+        # time).  No step-duration weighting; this is instantaneous power.
+        for (period, time), value in net_dt.items():
+            net_rows.append({
+                'group': g,
+                'period': period,
+                'time': time,
+                'net_flow': float(value),
+            })
+
+    if not rows:
+        return results
+
+    frame = pd.DataFrame(rows).set_index(['group', 'period'])[
+        ['cumulative_flow', 'average_flow']
+    ]
+    frame.columns.name = 'parameter'
+    results.append((frame, 'flowGroup_gd_p'))
+
+    if net_rows:
+        net_frame = pd.DataFrame(net_rows).set_index(
+            ['group', 'period', 'time'])[['net_flow']]
+        net_frame.columns.name = 'parameter'
+        results.append((net_frame, 'flowGroup_gd_t'))
+
+    return results

flextool/process_outputs/out_flows.py

@@ -0,0 +1,301 @@
+import pandas as pd
+
+from flextool.process_outputs._annualize import annualize_dt_to_d
+
+
+# Threshold below which LP residuals on flow variables are treated as exact
+# zero.  HiGHS's primal feasibility tolerance is 1e-8 (see HiGHS options),
+# so any output below 1e-9 is conclusively below solver precision and
+# represents zero.  Without this clip the wide-format flow CSVs round-trip
+# through ``pd.read_csv`` as ``float64`` while integer-zero goldens load
+# as ``int64``, causing ``pd.testing.assert_frame_equal`` to fail on dtype
+# even when values match under ``round_for_comparison``'s 4-decimal
+# threshold.
+#
+# Apply ONLY to the wide output frames destined for CSV write — never to
+# ``r.flow_dt`` itself, because curtailment derives ``potentialVRE - flow``
+# and small physically-meaningful curtailment values could otherwise be
+# masked.
+_FLOW_RESIDUAL_TOLERANCE: float = 1e-9
+
+
+def _clip_flow_residuals(df: pd.DataFrame) -> pd.DataFrame:
+    """Snap sub-femto LP residuals to exactly zero.
+
+    Returns a frame where every cell with ``|x| < 1e-9`` is replaced with
+    ``0.0``.  Empty / all-NaN frames are returned unchanged.
+    """
+    if df.empty:
+        return df
+    return df.mask(df.abs() < _FLOW_RESIDUAL_TOLERANCE, 0.0)
+
+
+def _indirect_units(s) -> set:
+    """Return set of units classified as indirect (2-arc conversion form).
+
+    An indirect unit has BOTH a row where ``source == p`` and a row where
+    ``sink == p`` in ``process_source_sink`` — i.e. the topology was split
+    into separate input-side and output-side arcs.  Direct-method and
+    profile/noConversion units have at most one of those, never both.
+
+    This mirrors v3.32.0's ordering convention for the wide-pivot output
+    CSVs (``unit__outputNode__dt.csv`` etc.): direct-method units listed
+    first (in alphabetical order), then indirect units.  v3.32.0 inherited
+    the partition from the legacy ``write_process_arc_unions`` union order
+    (process_arc_unions.py L248-263) which placed ``*_direct.csv`` sets
+    ahead of the indirect/noConversion ones.
+    """
+    if s.process_source_sink is None or len(s.process_source_sink) == 0:
+        return set()
+    procs = s.process_source_sink.get_level_values('process')
+    srcs = s.process_source_sink.get_level_values('source')
+    snks = s.process_source_sink.get_level_values('sink')
+    as_source = {p for p, src in zip(procs, srcs) if src == p}
+    as_sink = {p for p, snk in zip(procs, snks) if snk == p}
+    return as_source & as_sink
+
+
+def _sort_unit_node_columns(df: pd.DataFrame, indirect: set) -> pd.DataFrame:
+    """Reorder a (unit, node) MultiIndex column frame to match v3.32.0.
+
+    Partition: direct units first, indirect units last.  Alphabetical by
+    (unit, node) within each partition.
+    """
+    if df.empty or df.columns.empty:
+        return df
+    sorted_cols = sorted(
+        df.columns,
+        key=lambda c: (c[0] in indirect, c[0], c[1]),
+    )
+    return df.reindex(columns=pd.MultiIndex.from_tuples(
+        sorted_cols, names=df.columns.names,
+    ))
+
+
+def unit_outputNode(par, s, v, r, debug):
+    """Unit output node flow for periods and time"""
+
+    results = []
+
+    if r.flow_dt.empty:
+        return results
+
+    # Calculate timestep-level results first
+    result_multi_dt = pd.DataFrame(index=s.dt_realize_dispatch, columns=pd.MultiIndex.from_tuples([], names=['unit', 'node']))
+
+    # Filter columns: unit processes that have sinks
+    unit_sink_cols = r.flow_dt.columns[
+        r.flow_dt.columns.get_level_values(0).isin(s.process_unit) &
+        r.flow_dt.columns.to_series().apply(lambda col: (col[0], col[2]) in s.process_sink)
+    ]
+
+    for col in unit_sink_cols:
+        u, source, sink = col
+        result_multi_dt[(u, sink)] = r.flow_dt[col]
+
+    # Reorder columns to match v3.32.0's direct-then-indirect / alphabetical
+    # ordering — replaces the upstream ``flow_dt.columns`` order which is
+    # non-deterministic at HEAD (polars ``unique()`` after sort).
+    indirect = _indirect_units(s)
+    result_multi_dt = _sort_unit_node_columns(result_multi_dt, indirect)
+
+    # Snap sub-femto HiGHS LP residuals to exact zero before emitting so the
+    # CSV round-trip preserves the int dtype of zero-valued columns.
+    result_multi_dt = _clip_flow_residuals(result_multi_dt)
+
+    # Return timestep results
+    results.append((result_multi_dt, 'unit_outputNode_dt_ee'))
+
+    # Aggregate to period level.  v_flow is in MW (MWh/h) — multiply by
+    # step_duration to get MWh per step, weight each (d, t) by
+    # par.timestep_weight (=1.0 with no/uniform timeset_weights →
+    # byte-identical), then sum and annualize.
+    result_multi_d = annualize_dt_to_d(
+        result_multi_dt, par.timestep_weight,
+        par.complete_period_share_of_year, par.step_duration,
+    )
+    result_multi_d = _clip_flow_residuals(result_multi_d)
+
+    # Return period results
+    results.append((result_multi_d, 'unit_outputNode_d_ee'))
+
+    return results
+
+
+def unit_inputNode(par, s, v, r, debug):
+    """Unit input node flow for periods and time"""
+
+    results = []
+
+    if r.flow_dt.empty:
+        return results
+
+    # Calculate timestep-level results first
+    result_multi_dt = pd.DataFrame(index=s.dt_realize_dispatch, columns=pd.MultiIndex.from_tuples([], names=['unit', 'node']))
+
+    # Filter columns: unit processes that have sources
+    unit_source_cols = r.flow_dt.columns[
+        r.flow_dt.columns.get_level_values(0).isin(s.process_unit) &
+        r.flow_dt.columns.to_series().apply(lambda col: (col[0], col[1]) in s.process_source)
+    ]
+
+    for col in unit_source_cols:
+        u, source, sink = col
+        result_multi_dt[(u, source)] = -r.flow_dt[col]
+
+    # Reorder columns to match v3.32.0's direct-then-indirect / alphabetical
+    # ordering (see ``unit_outputNode`` for rationale).
+    indirect = _indirect_units(s)
+    result_multi_dt = _sort_unit_node_columns(result_multi_dt, indirect)
+
+    # Snap sub-femto HiGHS LP residuals to exact zero (see ``unit_outputNode``).
+    result_multi_dt = _clip_flow_residuals(result_multi_dt)
+
+    # Return timestep results
+    results.append((result_multi_dt, 'unit_inputNode_dt_ee'))
+
+    # Aggregate to period level.  v_flow is MW — multiply by step_duration
+    # to get MWh per step, weight each (d, t) by par.timestep_weight (=1.0
+    # with no/uniform timeset_weights → byte-identical), then sum/annualize.
+    result_multi_d = annualize_dt_to_d(
+        result_multi_dt, par.timestep_weight,
+        par.complete_period_share_of_year, par.step_duration,
+    )
+    result_multi_d = _clip_flow_residuals(result_multi_d)
+
+    # Return period results
+    results.append((result_multi_d, 'unit_inputNode_d_ee'))
+
+    return results
+
+
+def unit_cf_outputNode(par, s, v, r, debug):
+    """Unit capacity factors by output node for periods"""
+    complete_hours = par.complete_period_share_of_year * 8760
+    unit_cols = r.process_sink_flow_d.columns[r.process_sink_flow_d.columns.get_level_values(0).isin(s.process_unit)]
+    unit_capacity = r.entity_all_capacity[unit_cols.droplevel(1).unique()].rename_axis('process', axis=1)
+    unit_capacity.columns = unit_capacity.columns.get_level_values(0)
+    results = r.process_sink_flow_d[unit_cols].div(unit_capacity, level=0).div(complete_hours, axis=0)
+    # Capacity factor is undefined for units with no capacity (e.g. an un-built
+    # investment candidate, capacity == 0).  Dividing a tiny LP residual flow by
+    # zero capacity yields +/-inf; collapse those to NaN so downstream reductions
+    # (plot sums/means) skip them instead of summing +inf and -inf into a
+    # spurious NaN (RuntimeWarning: invalid value encountered in reduce).
+    results = results.replace([float('inf'), float('-inf')], float('nan'))
+    results.columns.names = ['unit', 'sink']
+    return results, 'unit_outputs_cf_d_ee'
+
+
+def unit_cf_inputNode(par, s, v, r, debug):
+    """Unit capacity factors by input node for periods"""
+    # !!! This should account for efficiency losses in direct conversion units (but it does not)
+    complete_hours = par.complete_period_share_of_year * 8760
+    unit_source = r.process_source_flow_d.columns[r.process_source_flow_d.columns.get_level_values(0).isin(s.process_unit)]
+    unit_capacity = r.entity_all_capacity[unit_source.droplevel(1).unique()].rename_axis('process', axis=1)
+    unit_capacity.columns = unit_capacity.columns.get_level_values(0)
+    results = r.process_source_flow_d[unit_source].div(unit_capacity, level=0).div(complete_hours, axis=0)
+    # CF undefined for zero-capacity units -> NaN (see unit_cf_outputNode).
+    results = results.replace([float('inf'), float('-inf')], float('nan'))
+    results.columns.names = ['unit', 'source']
+    return results, 'unit_inputs_cf_d_ee'
+
+
+def unit_VRE_curtailment_and_potential(par, s, v, r, debug):
+    """Unit VRE curtailment and potential for both periods and timesteps"""
+
+    results = []
+    vre_processes = s.process_VRE.unique()
+
+    # Timestep-level curtailment (absolute values) - calculate first
+    if not r.flow_dt.empty and not r.potentialVREgen_dt.empty:
+        curtail_dt = pd.DataFrame(index=s.dt_realize_dispatch, columns=pd.MultiIndex.from_tuples([], names=['unit', 'sink']))
+        potential_dt = pd.DataFrame(index=s.dt_realize_dispatch, columns=pd.MultiIndex.from_tuples([], names=['unit', 'sink']))
+
+        for col in r.flow_dt.columns:
+            u, source, sink = col
+            if u in vre_processes and (u, sink) in s.process_sink and (u, sink) in r.potentialVREgen_dt.columns:
+                curtail_dt[u, sink] = r.potentialVREgen_dt[(u, sink)] - r.flow_dt[col]
+                potential_dt[u, sink] = r.potentialVREgen_dt[(u, sink)]
+
+        results.append((curtail_dt, 'unit_curtailment_outputNode_dt_ee'))
+        results.append((potential_dt, 'unit_VRE_potential_outputNode_dt_ee'))
+
+        # Calculate curtailment share at timestep level
+        curtail_share_dt = (curtail_dt / potential_dt).where(potential_dt != 0, 0)
+        results.append((curtail_share_dt, 'unit_curtailment_share_outputNode_dt_ee'))
+
+        # Aggregate to period level.  curtail_dt / potential_dt are MW —
+        # multiply by step_duration to get MWh per step, then weight each
+        # (d, t) by par.timestep_weight (=1.0 with no/uniform timeset_weights
+        # → byte-identical) before summing.  Both numerator and denominator
+        # carry the same weight, so the curtailment-share ratio below is
+        # unchanged; only the absolute MWh outputs move.
+        curtail_period = (
+            curtail_dt.mul(par.step_duration, axis=0)
+            .mul(par.timestep_weight, axis=0)
+            .groupby(level='period').sum()
+        )
+        potential_period = (
+            potential_dt.mul(par.step_duration, axis=0)
+            .mul(par.timestep_weight, axis=0)
+            .groupby(level='period').sum()
+        )
+
+        # Calculate curtailment share at period level (ratio — step_duration cancels)
+        curtail_share_period = (curtail_period / potential_period).where(potential_period != 0, 0)
+
+        results.append((curtail_period, 'unit_curtailment_outputNode_d_ee'))
+        results.append((curtail_share_period, 'unit_curtailment_share_outputNode_d_ee'))
+        results.append((potential_period, 'unit_VRE_potential_outputNode_d_ee'))
+
+    return results
+
+
+def unit_ramps(par, s, v, r, debug):
+    """Unit ramps by input and output nodes for timesteps"""
+    results = []
+    if r.ramp_dtt.empty:
+        return results
+
+    # Output node ramps
+    process_sink_ramp_output = s.process_sink[s.process_sink.get_level_values(0).isin(s.process_unit)]
+    pss_ramp_output = r.ramp_dtt.columns[r.ramp_dtt.columns.droplevel(1).isin(process_sink_ramp_output)]
+    ramp_output = r.ramp_dtt[pss_ramp_output].droplevel('t_previous')
+    ramp_output.columns = ramp_output.columns.droplevel(1)  # Remove 'source' from (process, source, sink) to get (unit, source)
+    ramp_output.columns.names = ['unit', 'sink']
+    results.append((ramp_output, 'unit_ramp_outputs_dt_ee'))
+
+    # Input node ramps
+    process_source_ramp_input = s.process_source[s.process_source.get_level_values(0).isin(s.process_unit)]
+    pss_ramp_input = r.ramp_dtt.columns[r.ramp_dtt.columns.droplevel(2).isin(process_source_ramp_input)]
+    ramp_input = r.ramp_dtt[pss_ramp_input].droplevel('t_previous')
+    ramp_input.columns = ramp_input.columns.droplevel(2)  # Remove 'sink' from (process, source, sink) to get (unit, source)
+    ramp_input.columns.names = ['unit', 'source']
+    results.append((ramp_input, 'unit_ramp_inputs_dt_ee'))
+
+    return results
+
+
+def unit_online_and_startup(par, s, v, r, debug):
+    """Unit online status and startups for timesteps and periods"""
+    results = []
+
+    # 1. Online units dt
+    online_units_dt = r.process_online_dt[s.process_unit.intersection(s.process_online)]
+    results.append((online_units_dt, 'unit_online_dt_e'))
+
+    # 2. Average online status at period level (weighted by step_duration)
+    complete_hours = par.complete_period_share_of_year * 8760
+    online_units_d = online_units_dt.mul(par.step_duration, axis=0).groupby('period').sum().div(complete_hours, axis=0)
+    results.append((online_units_d, 'unit_online_average_d_e'))
+
+    # 3. Startups annualized to period level.  Raw startups only cover modeled
+    # timesteps; weight each by p_timestep_weight and divide by the period share
+    # of a year so the value is an annual count (same convention as flows).
+    units_online = s.process_unit.intersection(s.process_online)
+    startup_units_d = annualize_dt_to_d(
+        r.process_startup_dt[units_online], par.timestep_weight,
+        par.complete_period_share_of_year)
+    results.append((startup_units_d, 'unit_startup_d_e'))
+
+    return results