flextool 4.2.0__tar.gz → 4.2.2__tar.gz

{flextool-4.2.0 → flextool-4.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flextool
-Version: 4.2.0
+Version: 4.2.2
 Summary: IRENA FlexTool is an energy and power systems model for understanding the role of variable power generation in future energy systems.
 Author: IRENA FlexTool contributors
 License: © International Renewable Energy Agency 2018-2022
@@ -39,9 +39,9 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: polar-high>=3.12.2
+Requires-Dist: polar-high>=3.12.3
 Requires-Dist: highspy<1.16,>=1.15.1
-Requires-Dist: polars<2,>=1.40
+Requires-Dist: polars<3,>=1.40
 Requires-Dist: pandas<4,>=2.2.3
 Requires-Dist: spinedb-api<0.37,>=0.36.5
 Requires-Dist: openpyxl
@@ -60,7 +60,7 @@ Provides-Extra: viewer
 Requires-Dist: tsdownsample>=0.1.3; extra == "viewer"
 Requires-Dist: networkx>=3.0; extra == "viewer"
 Provides-Extra: compat
-Requires-Dist: polars[rtcompat]<2,>=1.40; extra == "compat"
+Requires-Dist: polars[rtcompat]<3,>=1.40; extra == "compat"
 Provides-Extra: engine-polars
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"

{flextool-4.2.0 → flextool-4.2.2}/flextool/engine/input/_timeline.py

@@ -62,6 +62,7 @@ from flextool.engine.input._solve_state import (
     ActiveTimeEntry,
     FlexToolConfigError,
 )
+from flextool.engine.preprocess.emit._base_steps import NEW_STEP_BASE_STEPS, NEW_STEP_BASE_STEPS_SCHEMA
 from flextool.engine.preprocess._provider_io import (
     _provider_key,
 )
@@ -390,12 +391,18 @@ class TimelineConfig:
                     f"{ts} (representative_period_weights)": [str(b) for b in self.rp_weights[ts]]
                     for ts in tl_timesets if self.rp_weights.get(ts)
                 }
+                # The fit of the new step is checked only for solves the model
+                # runs: an unused solve must not stop the run.
+                used_by = [step_solve_in_model_of[ts] for ts in tl_timesets if ts in step_solve_in_model_of]
                 chunks, block_chunks = _aggregate_timeline_pieces(
                     old_steps,
                     {ts: raw_durations[ts] for ts in tl_timesets},
                     step_duration,
                     logger,
                     extra_cuts=base_cuts,
+                    solve_name=used_by[0] if used_by else first_solve,
+                    timeline_name=timeline_name,
+                    check_fit=bool(used_by),
                 )
                 # Guard: every base start on the original timeline must start a step
                 # of the aggregated one, else the rp base chain would sort it last.
@@ -668,18 +675,24 @@ class TimelineConfig:
 
         Aggregation rules (lines 405-422 of the flextool source):
 
+        Each new step combines the time steps of the original timeline
+        that it covers (any step length; see ``_base_steps_of_new_steps``):
+        the rows of those steps are read and combined into one row.
+
         * ``pt_node_inflow.csv`` and ``pbt_node_inflow.csv`` →
-          ``"sum"`` (inflow energies sum across coarsened steps).
+          ``"sum"`` (inflow energies per step sum across coarsened steps).
         * Every other ``pt_*.csv`` / ``pbt_*.csv``, plus ``pdt_process.csv`` / ``pdt_node.csv`` (the
           genuine period × time maps — the group key includes their leading ``period`` column, found by
-          name like every other column before ``time``) → ``"average"``.
+          name like every other column before ``time``) → ``"average"``, weighted by the durations of
+          the combined steps (the plain mean when they are equally long), rounded to 6 decimals.
         * Rows with parameter ``"storage_state_reference_value"`` are
           ALWAYS bypassed (line 474) — they're per-step state targets
           rather than time-integrated quantities.
 
         After file aggregation, ``input/p_node``'s ``inflow`` rows
         are appended to ``solve_data/pt_node_inflow`` as one entry
-        per new timestep, scaled by the new step's duration.
+        per new timestep, times the number of time steps the new step
+        combines (a constant inflow is energy per time step).
 
         Args:
             solve: Active solve name.
@@ -722,6 +735,7 @@ class TimelineConfig:
             solve in self.new_step_durations
             and self.new_step_durations[solve] is not None
         )
+        base_steps_key = _provider_key(wf / "solve_data" / NEW_STEP_BASE_STEPS)
         if not create:
             for timeseries in timeseries_map:
                 src_path = wf / "input" / timeseries
@@ -732,6 +746,13 @@ class TimelineConfig:
                 if frame is None:
                     continue
                 provider.put(dst_key, frame)
+            # Every solve step is one original step (an earlier coarsened
+            # solve's rows must not reach this solve).
+            provider.put(base_steps_key, pl.DataFrame(schema=NEW_STEP_BASE_STEPS_SCHEMA))
+            # No full-timeline inflow energy: scale_to_annual_flow uses the
+            # solve's own period share (not an earlier coarsened solve's).
+            provider.put(_provider_key(wf / "solve_data" / "node_full_timeline_flow.csv"),
+                         pl.DataFrame(schema={"node": pl.Utf8, "value": pl.Float64}))
             return
 
         # All timesets used by *solve* must resolve to the same
@@ -751,14 +772,30 @@ class TimelineConfig:
                     logger.error(message)
                     raise FlexToolConfigError(message)
                 timelines_list.append(timeline)
-        # Pre-build timeline duration lookup for O(1) access per row.
-        timeline_duration_lookup: dict[str, int] = {}
+        # Durations of the original time steps each new step combines
+        # (O(1) access per row); a timeline that is not an aggregated
+        # one keeps one step per step.
+        base_durations: dict[str, list[float]] = {}
+        # (original step, the new step that combines it), in timeline order.
+        base_step_rows: list[tuple[str, str]] = []
         for timeline in timelines_list:
             new_timeline = self.timelines[timeline]
-            for timeline_row in new_timeline:
-                timeline_duration_lookup[timeline_row[0]] = int(
-                    float(timeline_row[1])
-                )
+            original = self.original_timeline.get(timeline)
+            if original is not None and original in self.timelines:
+                base_durations.update(_base_steps_of_new_steps(
+                    self.timelines[original], new_timeline, logger,
+                    solve_name=solve, timeline_name=timeline,
+                ))
+                base_step_rows.extend(_new_step_of_base_steps(self.timelines[original], new_timeline))
+            else:
+                for timeline_row in new_timeline:
+                    base_durations[timeline_row[0]] = [float(timeline_row[1])]
+                    base_step_rows.append((timeline_row[0], timeline_row[0]))
+        # The scalings read how many original steps each solve step combines
+        # (inflow and peak_inflow are energy per original step).
+        provider.put(base_steps_key, pl.DataFrame(
+            [(t, s, float(len(base_durations[s]))) for t, s in base_step_rows],
+            schema=NEW_STEP_BASE_STEPS_SCHEMA, orient="row"))
 
         # node-name → constant-inflow value (used to append per-step
         # contributions to pt_node_inflow below).
@@ -799,10 +836,8 @@ class TimelineConfig:
             try:
                 while True:
                     datain = next(rows_iter)
-                    timeline_step_duration = (
-                        timeline_duration_lookup.get(datain[time_index])
-                    )
-                    if timeline_step_duration is not None:
+                    step_base_durations = base_durations.get(datain[time_index])
+                    if step_base_durations is not None:
                         values: list[float] = []
                         params = list(datain[0:time_index])
                         row = list(datain[0 : time_index + 1])
@@ -811,7 +846,7 @@ class TimelineConfig:
                             time_index >= 2
                             and datain[1] != "storage_state_reference_value"
                         ) or time_index < 2:
-                            for _i in range(timeline_step_duration - 1):
+                            for _i in range(len(step_base_durations) - 1):
                                 try:
                                     datain = next(rows_iter)
                                 except StopIteration:
@@ -832,9 +867,17 @@ class TimelineConfig:
                                 )
 
                         if timeseries_map[timeseries] == "average":
-                            out_value = round(
-                                sum(values) / len(values), 6
-                            )
+                            ds = step_base_durations[: len(values)]
+                            if all(d == ds[0] for d in ds):
+                                out_value = round(
+                                    sum(values) / len(values), 6
+                                )
+                            else:
+                                # Rates per hour (profile, limits, prices):
+                                # the new step keeps their time integral.
+                                out_value = round(
+                                    sum(v * d for v, d in zip(values, ds)) / sum(ds), 6
+                                )
                         else:
                             out_value = sum(values)
                         row.append(out_value)
@@ -884,15 +927,16 @@ class TimelineConfig:
                 pass
 
             # Append per-step inflow contributions from p_node to
-            # pt_node_inflow only; scaled by each new step's duration.
+            # pt_node_inflow only; times the number of time steps each new
+            # step combines (a constant inflow is energy per time step).
             if timeseries == "pt_node_inflow.csv" and node__inflow:
                 for timeline in timelines_list:
                     new_timeline = self.timelines[timeline]
                     for node__value in node__inflow:
                         for timeline_row in new_timeline:
-                            tl_step_duration = int(float(timeline_row[1]))
                             value = (
-                                float(node__value[1]) * tl_step_duration
+                                float(node__value[1])
+                                * len(base_durations[timeline_row[0]])
                             )
                             out_rows.append(
                                 [node__value[0], timeline_row[0], value]
@@ -962,9 +1006,10 @@ class TimelineConfig:
             for n_name in dict.fromkeys(
                 list(ts_sum) + list(const_for_node)
             ):
+                # A constant inflow is energy per time step: once per step.
                 energy = (
                     ts_sum.get(n_name, 0.0)
-                    + const_for_node.get(n_name, 0.0) * total_hours
+                    + const_for_node.get(n_name, 0.0) * len(base_steps)
                 )
                 flow_rows.append([n_name, energy / duration_in_years])
             flow_path = wf / "solve_data" / "node_full_timeline_flow.csv"
@@ -992,12 +1037,59 @@ def _solves_in_model(solve_config: "SolveConfig") -> set[str]:
     return out
 
 
+def _base_steps_of_new_steps(
+    old_steps: list, new_steps: list, logger: logging.Logger, *, solve_name: str, timeline_name: str,
+) -> dict[str, list[float]]:
+    """``{new step: [durations of the original steps it combines]}``.
+
+    The new steps (chunks of :func:`_aggregate_timeline_pieces`) partition
+    the whole original timeline from its first step, each labelled by its
+    first original step, so every original step belongs to the latest new
+    step label seen on the walk.
+    """
+    labels = {step[0] for step in new_steps}
+    out: dict[str, list[float]] = {}
+    current: list[float] | None = None
+    for name, duration in old_steps:
+        if name in labels:
+            current = out.setdefault(name, [])
+        if current is not None:
+            current.append(float(duration))
+    missing = [step[0] for step in new_steps if step[0] not in out]
+    if missing:
+        message = (
+            f"Solve '{solve_name}': time steps {missing[:5]} of the aggregated timeline '{timeline_name}' are not "
+            f"time steps of its original timeline."
+        )
+        logger.error(message)
+        raise FlexToolConfigError(message)
+    return out
+
+
+def _new_step_of_base_steps(old_steps: list, new_steps: list) -> list[tuple[str, str]]:
+    """``[(original step, the new step that combines it)]`` in timeline order: the walk of
+    :func:`_base_steps_of_new_steps` (every original step belongs to the latest new step label seen)."""
+    labels = {step[0] for step in new_steps}
+    out: list[tuple[str, str]] = []
+    current: str | None = None
+    for name, _duration in old_steps:
+        if name in labels:
+            current = name
+        if current is not None:
+            out.append((name, current))
+    return out
+
+
 def _aggregate_timeline_pieces(
     old_steps: list,
     timeset_blocks: dict[str, list],
     step_duration: float,
     logger: logging.Logger,
     extra_cuts: dict[str, list[str]] | None = None,
+    *,
+    solve_name: str,
+    timeline_name: str,
+    check_fit: bool = True,
 ) -> tuple[list[tuple[str, str]], dict[str, list[tuple[str, int]]]]:
     """Aggregate a whole timeline to ``step_duration`` hours, piece by piece.
 
@@ -1005,12 +1097,23 @@ def _aggregate_timeline_pieces(
     the runs between the block starts and block ends of ``timeset_blocks``
     (``{timeset: [(start, count), ...]}``) and the steps in ``extra_cuts``
     (``{owner: [step, ...]}``, e.g. representative base-period starts).  Each piece is aggregated from its
-    first step: a chunk closes once it holds ``step_duration`` hours or more.
+    first step: a chunk closes as soon as its steps add up to ``step_duration`` hours (within a relative
+    tolerance of 1e-4, so steps entered with rounded fractions still fit), and a chunk always holds at least
+    one step.  The steps may have any length (0.5 h, 2 h, mixed).  The
+    tolerance is capped at half the shortest step, so a long new step still
+    closes at the right step.
     Returns the chunks ``[(first step, str(hours)), ...]`` in timeline order and
     per timeset ``[(block start, number of chunks in the block), ...]`` (a block
     whose start is not on the timeline is kept as given; ``get_active_time``
     skips it).  Logs a warning when a block ends with a shorter chunk, naming
     the other timeset when its boundary is what cut the chunk short.
+
+    Raises :class:`FlexToolConfigError` (only with ``check_fit``, i.e. for a
+    solve the model runs) when a chunk that starts inside a block
+    would be longer than ``step_duration``: the new step is not a whole multiple
+    of the timeline's steps, or (steps of different lengths) it would end inside
+    a step.  A gap chunk (outside every block, never active) may be longer, e.g.
+    one long filler step between two blocks.
     """
     index_of = {step[0]: i for i, step in enumerate(old_steps)}
     n_steps = len(old_steps)
@@ -1033,6 +1136,12 @@ def _aggregate_timeline_pieces(
             if label in index_of:
                 cut_owners.setdefault(index_of[label], set()).add(owner)
     cuts = sorted(c for c in cut_owners if c < n_steps) + [n_steps]
+    block_ranges = [(rng[1], rng[2]) for rngs in ranges.values() for rng in rngs if rng is not None]
+    # Relative to the new step (rounded fractions like 0.333333 h still add
+    # up), but below half a base step, so a long new step never closes one
+    # base step early.
+    positive = [float(step[1]) for step in old_steps if float(step[1]) > 0]
+    tol = min(1e-4 * step_duration, 0.5 * min(positive)) if positive else 0.0
 
     chunks: list[tuple[str, str]] = []
     chunk_first: list[int] = []
@@ -1042,12 +1151,33 @@ def _aggregate_timeline_pieces(
         while i < piece_end:
             first = i
             hours = 0.0
-            while i < piece_end and (hours < step_duration or i == first):
+            while i < piece_end and (hours < step_duration - tol or i == first):
                 hours += float(old_steps[i][1])
                 i += 1
-            chunks.append((old_steps[first][0], str(hours)))
+            label = old_steps[first][0]
+            if (check_fit and hours > step_duration + tol
+                    and any(b_first <= first < b_end for b_first, b_end in block_ranges)):
+                ds = [float(step[1]) for step in old_steps[first:i]]
+                if all(d == ds[0] for d in ds):
+                    message = (
+                        f"Solve '{solve_name}': new_stepduration {step_duration:g} h is not a whole multiple of "
+                        f"the {ds[0]:g} h time steps of timeline '{timeline_name}' (the new time step starting "
+                        f"at '{label}' would be {hours:g} h long). Give a new_stepduration that is a whole "
+                        f"multiple of the timeline's time steps."
+                    )
+                else:
+                    message = (
+                        f"Solve '{solve_name}': new_stepduration {step_duration:g} h does not fit the time steps "
+                        f"of timeline '{timeline_name}': the new time step starting at '{label}' would end inside "
+                        f"base time step '{old_steps[i - 1][0]}' ({hours:g} h in total). On a timeline with time "
+                        f"steps of different lengths, every new time step must end where a time step of the "
+                        f"timeline ends."
+                    )
+                logger.error(message)
+                raise FlexToolConfigError(message)
+            chunks.append((label, str(hours)))
             chunk_first.append(first)
-            if hours < step_duration:
+            if hours < step_duration - tol:
                 short_chunks.append((first, i, hours))
 
     block_chunks: dict[str, list[tuple[str, int]]] = {}

flextool-4.2.2/flextool/engine/preprocess/emit/_base_steps.py

@@ -0,0 +1,74 @@
+"""The original time steps behind each time step of a solve with ``new_stepduration``.
+
+``TimelineConfig.create_averaged_timeseries`` writes ``solve_data/new_step_base_steps``: one row per original time
+step of the coarsened timeline, ``time`` (the original step), ``new_step`` (the solve's time step that combines it;
+its label is its first original step) and ``value`` (how many original steps that new step combines).  A solve
+without ``new_stepduration`` gets an empty frame, so a later uncoarsened solve never reads an earlier solve's
+rows.  An absent or empty frame means every solve step is one original step.
+
+``inflow`` and ``peak_inflow`` are energy per original time step; a coarsened solve step carries the sum of its
+original steps, so the inflow scaling divides by (or multiplies with) the number of steps it combines.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import polars as pl
+
+from flextool.engine.preprocess._provider_io import _provider_key
+
+NEW_STEP_BASE_STEPS = "new_step_base_steps.csv"
+NEW_STEP_BASE_STEPS_SCHEMA = {"time": pl.Utf8, "new_step": pl.Utf8, "value": pl.Float64}
+
+
+def read_new_step_base_steps(solve_data_dir: Path, *, provider) -> tuple[dict[str, float], frozenset[str]]:
+    """``({solve step: number of original steps it combines}, original steps absorbed into another step)``.
+
+    Both are empty when the solve has no ``new_stepduration``.  The absorbed steps are the original steps that are
+    not a solve step themselves (their energy is in the solve step that combines them).
+    """
+    df = provider.get(_provider_key(solve_data_dir / NEW_STEP_BASE_STEPS)) if provider is not None else None
+    if df is None or df.height == 0:
+        return {}, frozenset()
+    df = df.select(pl.col("time").cast(pl.Utf8), pl.col("new_step").cast(pl.Utf8),
+                   pl.col("value").cast(pl.Float64))
+    n_of = {t: float(v) for t, s, v in df.iter_rows() if t == s}
+    absorbed = frozenset(t for t, s, _v in df.iter_rows() if t != s)
+    return n_of, absorbed
+
+
+def base_steps_of_solve_steps(solve_data_dir: Path, *, provider) -> dict[str, tuple[str, ...]]:
+    """``{solve step: the original steps it combines, in timeline order}``; empty without ``new_stepduration``
+    (then every solve step is its own original step)."""
+    df = provider.get(_provider_key(solve_data_dir / NEW_STEP_BASE_STEPS)) if provider is not None else None
+    if df is None or df.height == 0:
+        return {}
+    out: dict[str, list[str]] = {}
+    for t, s in df.select(pl.col("time").cast(pl.Utf8), pl.col("new_step").cast(pl.Utf8)).iter_rows():
+        out.setdefault(s, []).append(t)
+    return {s: tuple(ts) for s, ts in out.items()}
+
+
+def complete_steps_by_period(solve_data_dir: Path, periods: list[str], *, provider) -> list[tuple[str, str]]:
+    """``[(period, complete step)]`` for every period in ``periods``: the steps of ``steps_complete_solve`` of the
+    periods ``d2`` with ``(d2, period)`` in ``period__branch`` — the mapping ``complete_hours_in_period`` sums the
+    hours of.  A stochastic branch period has no complete steps of its own and gets its parent's; a period without a
+    ``period__branch`` row keeps its own steps.  When every period maps to itself the ``steps_complete_solve`` rows
+    are returned as they are (same order)."""
+    def _pairs(stem: str) -> list[tuple[str, str]]:
+        df = provider.get(_provider_key(solve_data_dir / stem)) if provider is not None else None
+        if df is None or df.height == 0 or df.width < 2:
+            return []
+        return [(str(a), str(b)) for a, b in df.select(df.columns[:2]).iter_rows()
+                if a is not None and b is not None and str(a) and str(b)]
+
+    complete = _pairs("steps_complete_solve.csv")
+    parents_of: dict[str, list[str]] = {}
+    for d2, d in _pairs("period__branch.csv"):
+        parents_of.setdefault(d, []).append(d2)
+    if all(parents_of.get(d, [d]) == [d] for d in periods):
+        return complete
+    steps_of: dict[str, list[str]] = {}
+    for d, t in complete:
+        steps_of.setdefault(d, []).append(t)
+    return [(d, t) for d in periods for d2 in parents_of.get(d, [d]) for t in steps_of.get(d2, ())]