ClimateGraph 0.4.0__py3-none-any.whl

ClimateGraph/utils/general_utils.py

@@ -0,0 +1,259 @@
+"""Enums, path expansion and time parsing.
+
+The leaf utilities: how the control file's string values become real objects,
+and how paths and dates are interpreted.
+"""
+
+import datetime
+import glob
+import logging
+import re
+from enum import Enum
+from pathlib import Path
+
+import cartopy.crs as ccrs
+import numpy as np
+import pandas as pd
+from dateutil import parser
+
+log = logging.getLogger(__name__)
+
+CRS_TYPES = {"platecarree": ccrs.PlateCarree}
+TIME_INTERVAL_FORMAT = r"^(.+?)\s*(?:-|to)\s*(.+)$"  # Accepts "date - date" or "date to date" and a date should be in dayfirst format.
+
+
+class TimestepEnum(str, Enum):
+    """Enum used for timestep handling. Keeps consistent timestep values."""
+
+    business_day = "B"
+    daily = "D"
+    weekly = "W"
+    monthly = "ME"
+    quarterly = "Q"
+    yearly = "Y"
+    hourly = "h"
+    minutely = "min"
+    secondly = "s"
+    milliseconds = "ms"
+    microseconds = "us"
+    nanoseconds = "ns"
+
+
+class TimeBucketEnum(str, Enum):
+    """Enum used for timestep handling. Keeps consistent timestep values."""
+
+    minute = "minute"
+    hour = "hour"
+    day = "day"
+    season = "season"
+    weekly = "week"
+    monthly = "quarter"
+
+
+class ReductionMethodEnum(str, Enum):
+    """Enum used for Reduction Method handling. Keeps consistent Reduction methods values."""
+
+    def __new__(cls, value, func):
+        obj = str.__new__(cls, value)
+        obj._value_ = value
+        obj.func = func
+        return obj
+
+    mean = ("mean", np.nanmean)
+    min = ("min", np.nanmin)
+    max = ("max", np.nanmax)
+
+
+class CRSEnum(str, Enum):
+    """Enum used for Coordinate Reference System handling. Keeps consistent CRS values."""
+
+    def __new__(cls, value, crs):
+        obj = str.__new__(cls, value)
+        obj._value_ = value
+        obj.crs = crs
+        return obj
+
+    platecarree = ("platecarree", ccrs.PlateCarree)
+
+
+def manage_path(
+    paths: str | Path | list[str] | list[Path],
+    sort: bool = False,
+) -> list[Path]:
+    """Handles paths, including lists of paths and glob patterns (``*``, ``?``).
+
+    Parameters
+    ----------
+    paths : str | Path | List[str] | List[Path]
+        Path or list of paths that compose the data object.
+    sort : bool, optional
+        When True, return paths in lexicographic order. If the post-sort
+        order differs from the input order, log a warning — callers that
+        concat in input order (e.g. the Reader unsafe path) will get a
+        silently non-monotonic time axis if filenames don't embed a
+        sortable timestamp. Default False keeps the historical
+        glob/input order.
+
+    Returns
+    -------
+    List[Path]
+        List of existing pathlib.Path's created from the input paths.
+    """
+    if isinstance(paths, (str, Path)):
+        paths = [paths]
+
+    result: list[Path] = []
+
+    for raw in paths:
+        p = Path(raw) if isinstance(raw, str) else raw
+        p = p.resolve() if not p.is_absolute() else p
+
+        matches = glob.glob(str(p))
+        if not matches:
+            log.debug(f"No files match pattern: {raw}")
+        result.extend(Path(m).resolve() for m in matches if Path(m).exists())
+
+    if not result:
+        log.debug(f"No files exist for paths: {paths}")
+
+    if sort:
+        ordered = sorted(result)
+        if ordered != result:
+            log.warning(
+                "manage_path: input paths were not in lexicographic order; "
+                "sorted automatically. Confirm filenames embed a sortable "
+                "timestamp or callers that concat in input order will produce "
+                "a non-monotonic time axis. First few: %s",
+                [p.name for p in ordered[:3]],
+            )
+        result = ordered
+
+    return result
+
+
+# Coarsest-to-finest. Only resolutions coarser than "hour" expand to a full bucket
+_RESOLUTION_ORDER = ("year", "month", "day", "hour", "minute", "second")
+COARSE_OFFSETS = {
+    "day": pd.Timedelta(days=1),
+    "month": pd.DateOffset(months=1),
+    "year": pd.DateOffset(years=1),
+}
+
+
+def _parse_with_resolution(token: str) -> tuple[pd.Timestamp, str]:
+    """Parse a date token and detect its resolution.
+
+    Parameters
+    ----------
+    token : str
+        A single date/datetime in dayfirst format. With hours or without
+
+    Returns
+    -------
+    tuple[pd.Timestamp, str]
+        The floored timestamp and its resolution (one of ``_RESOLUTION_ORDER``).
+
+    Raises
+    ------
+    ValueError
+        If the token isn't a parseable date.
+    """
+    floored = parser.parse(token, dayfirst=True, default=datetime.datetime(1999, 1, 1))
+    probe = parser.parse(
+        token, dayfirst=True, default=datetime.datetime(2002, 7, 8, 9, 10, 11)
+    )
+
+    resolution = "year"
+    for field in _RESOLUTION_ORDER:
+        if getattr(floored, field) == getattr(probe, field):
+            resolution = field
+    return pd.Timestamp(floored), resolution
+
+
+def _bucket_end(value: pd.Timestamp, resolution: str) -> pd.Timestamp:
+    """End of the bucket ``value`` falls in, for its resolution.
+
+    For coarse resolutions (day/month/year) returns the last representable
+    instant of the bucket (start of the next bucket minus 1 ns), so an inclusive
+    ``slice`` covers the whole day/month/year without spilling into the next.
+    For hour-or-finer resolutions the value is an exact point and is returned
+    unchanged.
+    """
+    offset = COARSE_OFFSETS.get(resolution)
+    if offset is None:
+        return value
+    return value + offset - pd.Timedelta(1, "ns")
+
+
+def manage_time_interval(
+    time_interval: str,
+) -> tuple[pd.Timestamp, pd.Timestamp]:
+    """Turn a time-interval string into (start, end).
+
+    Accepts either a single date or a "start - (or to) end" range. Each endpoint is treated
+    as an *interval covering its own resolution* when that resolution is coarser
+    than hourly
+
+    If the two endpoints have different resolutions, a warning is logged and each
+    is expanded on its own bucket (best-effort).
+
+    Parameters
+    ----------
+    time_interval : str
+        A single date or a ``"start [-|to] end"`` range in dayfirst format.
+
+    Returns
+    -------
+    tuple[pd.Timestamp, pd.Timestamp]
+        Start (bucket start of the first endpoint) and end (bucket end of the
+        last endpoint).
+
+    Raises
+    ------
+    ValueError
+        If an endpoint isn't a parseable date.
+    """
+    if time_interval is None:
+        return None, None
+    time_interval = time_interval.strip()
+
+    # A separator splits a range; otherwise the whole string is a single date
+    # that spans its own bucket (start_str == end_str).
+    if (match := re.match(TIME_INTERVAL_FORMAT, time_interval)) is not None:
+        start_str, end_str = match.group(1), match.group(2)
+    else:
+        start_str = end_str = time_interval
+
+    start_val, start_res = _parse_with_resolution(start_str)
+    end_val, end_res = _parse_with_resolution(end_str)
+
+    if start_res != end_res:
+        log.warning(
+            "time_interval %r endpoints have different temporal resolutions "
+            "(%s vs %s); expanding each on its own bucket.",
+            time_interval,
+            start_res,
+            end_res,
+        )
+
+    return start_val, _bucket_end(end_val, end_res)
+
+
+def normalize_time(time: str | list[str] | None) -> list[str | None]:
+    """Coerce a plot's ``time`` field into a list of entries to
+    iterate over. A single date/interval/None becomes a one-element list; a
+    list passes through unchanged, fanning the plot out into one output per entry.
+
+    Parameters
+    ----------
+    time : str | list[str] | None
+        A single date/interval string, a list of them, or None.
+
+    Returns
+    -------
+    list[str | None]
+        Entries to iterate over, each independently valid for manage_time_interval.
+    """
+    if isinstance(time, list):
+        return time
+    return [time]

ClimateGraph/utils/parser.py

@@ -0,0 +1,269 @@
+"""Control file reading and object construction.
+
+Reads a control file in any supported format, validates it against the Pydantic
+schema in :mod:`ClimateGraph.utils.control_model`, and builds the three
+registries the run works from: the datasets, the domains and the plots.
+
+"""
+
+import json
+import logging
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from ClimateGraph.data import Data
+from ClimateGraph.data.point_surface import PointSurface
+from ClimateGraph.domain import Domain
+from ClimateGraph.domain.domains import AttributeConfig, PointsConfig
+from ClimateGraph.plot import Plot
+from ClimateGraph.utils.control_model import ControlFile
+
+log = logging.getLogger(__name__)
+
+FILE_READERS = {".json": json.load, ".yaml": yaml.safe_load, ".yml": yaml.safe_load}
+
+
+def _expansion_entries(name: str, model) -> list[tuple[str, object]] | None:
+    """Compute the per-value expansions for a ``one_for_each`` domain.
+
+    Returns ``[(expanded_name, expanded_model), ...]``, or ``None`` when the model
+    can't be expanded (so the caller leaves it as-is). Handles the two domain types
+    that carry ``one_for_each``:
+
+    - ``AttributeConfig``: one domain per ``field_value`` (needs a list), named
+      ``{name}__{value}``.
+    - ``PointsConfig``: one domain per named point, named by the point; each keeps
+      the full point set and gets a ``select_name`` so they share one resample
+      target and just ``.sel`` their site.
+    """
+    if isinstance(model, PointsConfig):
+        return [
+            (
+                pname,
+                model.model_copy(update={"select_name": pname, "one_for_each": False}),
+            )
+            for pname, _lat, _lon in model.resolve_points()
+        ]
+    if isinstance(model, AttributeConfig):
+        if not isinstance(model.field_value, list):
+            return None
+        return [
+            (
+                f"{name}__{value}",
+                model.model_copy(update={"field_value": value, "one_for_each": False}),
+            )
+            for value in model.field_value
+        ]
+    return None
+
+
+def _points_target(model: PointsConfig, cache: dict) -> PointSurface:
+    """Build (or reuse) the in-memory PointSurface target for a Points domain.
+
+    Memoized on the resolved points so all fan-out expansions of one block share a
+    single target object — giving them one resample-cache entry (keyed by the
+    target's unique name) instead of recomputing the projection per point. A new
+    distinct point set gets a fresh unique name so different geometries never
+    collide in the resample cache.
+    """
+    pts = tuple(model.resolve_points())
+    target = cache.get(pts)
+    if target is None:
+        names = [p[0] for p in pts]
+        lats = [p[1] for p in pts]
+        lons = [p[2] for p in pts]
+        target = PointSurface.from_points(
+            f"__points_target_{len(cache)}", names, lats, lons
+        )
+        cache[pts] = target
+    return target
+
+
+def _expand_domains(domain_models: dict) -> tuple[dict, dict[str, list[str]]]:
+    """Expand domain configs flagged ``one_for_each`` into one config per value.
+
+    Returns
+    -------
+    tuple[dict, dict[str, list[str]]]
+        The expanded {name: domain_model} dict (un-flagged domains pass through
+        unchanged), and a {original_name: [expanded_names]} rewrite map used to
+        resolve plot ``domains:`` references that still point at the original name.
+    """
+    expanded = {}
+    rewrite_map = {}
+    for name, model in domain_models.items():
+        if not getattr(model, "one_for_each", False):
+            expanded[name] = model
+            continue
+
+        entries = _expansion_entries(name, model)
+        if not entries:
+            expanded[name] = model
+            log.debug(
+                f"Attempted expansion of domain {name} but there was no expansible "
+                "value. Left as is."
+            )
+            continue
+
+        generated_names = []
+        for new_name, new_model in entries:
+            expanded[new_name] = new_model
+            generated_names.append(new_name)
+        rewrite_map[name] = generated_names
+
+    return expanded, rewrite_map
+
+
+class Parser:
+    """Parser class, used for managing input configuration files for ClimateGraph"""
+
+    @staticmethod
+    def parse_control(control_path: Path):
+        """Validate a control file and build the objects it describes.
+
+        Parameters
+        ----------
+        control_path : Path
+            Path to configuration file.
+
+        Returns
+        -------
+        AnalysisModel, Dict[str, Data], Dict[str, Plot], Dict[str,Domain]
+            analysis, data, plots and domains specified for ClimateGraph execution.
+
+        Raises
+        ------
+        ValueError
+            Configuration file doesn't meet the ControlFile pydantic model.
+        """
+        control_dict = Parser.read_control(control_path)
+
+        try:
+            valid = ControlFile.model_validate(control_dict)
+        except ValidationError as err:
+            error_str = ""
+            for e in err.errors():
+                error_str += f"Error {e['msg']}. Input given:\n{e['input']}\n    See more info at: {e['url']}\n"
+            raise ValueError(
+                f"Configuration file {control_path} doesn't meet the input structure.\n\n{error_str}\n\nFix this errors before retrying..."
+            ) from err
+
+        analysis = valid.analysis.model_dump()
+        data = dict()
+        plots = dict()
+        domains = dict()
+
+        # Default download cache for readers that fetch remote paths.
+        # Per-data `cache_dir` in YAML overrides this; local-only readers
+        # never look at it.
+        default_cache_dir = analysis["output_path"] / ".cache"
+
+        for data_name, data_model in valid.data.items():
+            _name = data_name
+            _topology, _reader, _path, _vars, _crs = (
+                data_model.topology,
+                data_model.reader,
+                data_model.path,
+                data_model.vars,
+                data_model.crs,
+            )
+            # Only necessary for dict, but not REALLY sure how necessary it is
+            if isinstance(_vars, dict):
+                _vars = {
+                    var: var_model.model_dump() for var, var_model in _vars.items()
+                }
+
+            # model_extra is reader-specific kwargs (rename overrides,
+            # vertical_level for Chimere, etc.). Lifecycle settings
+            # (load_mode, cache_dir) are declared fields — we add them
+            # in explicitly so Data.load_obj has the full picture
+            # without Data growing new parameters.
+            reader_kwargs = dict(data_model.model_extra or {})
+            reader_kwargs["load_mode"] = data_model.load_mode
+            reader_kwargs["cache_dir"] = data_model.cache_dir or default_cache_dir
+            reader_kwargs["save_to"] = data_model.save_to
+
+            data_instance = Data.create(
+                _name, _topology, _reader, _path, _vars, _crs, reader_kwargs
+            )
+
+            data[_name] = data_instance
+
+        rewrite_map = {}
+        if valid.domains:
+            expanded_domains, rewrite_map = _expand_domains(valid.domains)
+            # Points domains build their own in-memory target from the point set;
+            # memoize by the resolved points so every fan-out expansion (and the
+            # grouped case) shares one target -> one resample cache entry.
+            points_targets: dict[tuple, PointSurface] = {}
+            for domain_name, domain_model in expanded_domains.items():
+                _type = domain_model.type
+                # Optional resample pre-step: resolve the target geometry so the
+                # domain can reproject onto it.
+                target_data = None
+                if isinstance(domain_model, PointsConfig):
+                    target_data = _points_target(domain_model, points_targets)
+                elif getattr(domain_model, "resample_to", None):
+                    target_data = data.get(domain_model.resample_to)
+                    if target_data is None:
+                        raise ValueError(
+                            f"Domain '{domain_name}' resample_to references unknown "
+                            f"dataset '{domain_model.resample_to}'."
+                        )
+                domain_instance = Domain.create(
+                    domain_name, _type, domain_model, target_data=target_data
+                )
+                domains[domain_name] = domain_instance
+        if valid.plots:
+            for plot_name, plot_model in valid.plots.items():
+                if rewrite_map and getattr(plot_model, "domains", None):
+                    resolved = []
+                    for ref in plot_model.domains:
+                        resolved.extend(rewrite_map.get(ref, [ref]))
+                    plot_model.domains = resolved
+
+                _type = plot_model.type
+                plot_instance = Plot.create(
+                    plot_name,
+                    _type,
+                    plot_model,
+                    data,
+                    domains,
+                    output_path=analysis["output_path"],
+                )
+                plots[plot_name] = plot_instance
+
+        return analysis, data, plots, domains
+
+    @staticmethod
+    def read_control(control_path: Path):
+        """Read a control file into a python dictionary. Supports ``.json``, ``.yml`` and ``.yaml``.
+
+        Parameters
+        ----------
+        control_path : Path
+            Path to configuration file.
+
+        Returns
+        -------
+        Dict
+            Mapping with configuration arguments.
+
+        Raises
+        ------
+        FileNotFoundError
+            Path doesn't point to any file.
+        ValueError
+            File type not supported.
+        """
+        control_path = control_path.resolve()
+        if not (control_path.exists() and control_path.is_file()):
+            raise FileNotFoundError(f"File {control_path} not found.")
+        if (reader := FILE_READERS.get(control_path.suffix)) is None:
+            raise ValueError(f"File type {control_path.suffix} not supported.")
+
+        with open(control_path) as fp:
+            control_dict = reader(fp)
+        return control_dict

ClimateGraph/utils/registry.py

@@ -0,0 +1,81 @@
+"""Self-registering class registries.
+
+Lets an abstract base class keep a registry of its subclasses, so defining a
+subclass is enough to make it usable from a control file.
+"""
+
+from typing import Annotated, ClassVar, Union
+
+from pydantic import BaseModel, Field
+
+
+class RegistryMixin:
+    """Mixin that provides a self-registering class registry.
+
+    Each root ABC that inherits this mixin must declare its own:
+        registry: ClassVar[dict[str, type]] = {}
+
+    Concrete subclasses are registered automatically via __init_subclass__
+    under their lowercased class name and any names listed in their ``aliases``
+    class variable.  Only aliases declared directly on the subclass are
+    registered (not inherited ones), matching the Reader convention.
+
+    The mixin intentionally does NOT define ``registry`` itself — each root ABC
+    owns a separate dict so their namespaces never collide.
+    """
+
+    aliases: ClassVar[list[str]] = []
+    config: ClassVar[type[BaseModel] | None] = None
+
+    def __init_subclass__(cls, **kwargs: object) -> None:
+        super().__init_subclass__(**kwargs)
+
+        # Walk the MRO (skipping cls itself and RegistryMixin) to find the first
+        # ancestor that declares its own ``registry`` dict.  That ancestor is the
+        # root ABC for this family and owns the shared registry.
+        owner = None
+        for ancestor in cls.__mro__[1:]:
+            if ancestor is RegistryMixin:
+                continue
+            if "registry" in ancestor.__dict__:
+                owner = ancestor
+                break
+
+        if owner is None:
+            return
+
+        owner.registry[cls.__name__.lower()] = cls
+        for alias in cls.__dict__.get("aliases", []):
+            owner.registry[alias.lower()] = cls
+
+    @classmethod
+    def get_class(cls, name: str) -> type:
+        """Look up a registered subclass by name (case-insensitive)."""
+        try:
+            return cls.registry[name.lower()]
+        except KeyError as err:
+            raise ValueError(
+                f"No {cls.__name__} named '{name}'. "
+                f"Options: {list(cls.registry)} (case-insensitive)."
+            ) from err
+
+    @classmethod
+    def check_class(cls, name: str) -> bool:
+        """Return True if *name* maps to a registered subclass."""
+        return name.lower() in cls.registry
+
+    @classmethod
+    def build_config_union(cls) -> Annotated:
+        """Build a Pydantic discriminated-union over all registered configs.
+
+        Only meaningful for ABCs whose subclasses carry a ``config`` class
+        attribute (Plot, Domain).  ``Data`` never calls this.
+        """
+        configs = [
+            c.config
+            for c in cls.registry.values()
+            if getattr(c, "config", None) is not None
+        ]
+        # Union[tuple(configs)] unpacks at runtime — do not let Ruff rewrite this;
+        # the UP007 rewrite strips Union and breaks Pydantic's discriminator.
+        return Annotated[Union[tuple(configs)], Field(discriminator="type")]  # noqa: UP007