meteostat 1.7.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/core/providers.py

@@ -0,0 +1,184 @@
+"""
+Provider Service
+
+The Provider Service provides methods to interact with data providers.
+"""
+
+from datetime import datetime
+from importlib import import_module
+from statistics import fmean
+from typing import List, Optional, TypeGuard, cast
+
+import pandas as pd
+
+from meteostat.core.logger import logger
+from meteostat.enumerations import Granularity, Priority, Provider
+from meteostat.providers.index import DEFAULT_PROVIDERS
+from meteostat.typing import (
+    ProviderRequest,
+    ProviderSpec,
+    Station,
+    Request,
+)
+
+
+class ProviderService:
+    """
+    Provider Service
+    """
+
+    _providers: List[ProviderSpec]
+
+    def __init__(self, providers: List[ProviderSpec]) -> None:
+        self._providers = providers
+
+    @property
+    def providers(self) -> List[ProviderSpec]:
+        """
+        Get supported providers
+        """
+        return self._providers
+
+    def register(self, provider: ProviderSpec) -> None:
+        """
+        Register a provider
+        """
+        self._providers.append(provider)
+
+    def get_provider(self, provider_id: Provider | str) -> Optional[ProviderSpec]:
+        """
+        Get provider by ID
+        """
+        return next(
+            (provider for provider in self._providers if provider.id == provider_id),
+            None,
+        )
+
+    def _get_provider_priority(self, provider_id: Provider | str) -> int:
+        """
+        Get priority of a provider by its ID
+        """
+        baselines = {
+            Granularity.HOURLY: 0,
+            Granularity.DAILY: 100,
+            Granularity.MONTHLY: 200,
+            Granularity.NORMALS: 300,
+        }
+
+        provider = self.get_provider(provider_id)
+
+        if not provider:
+            return Priority.NONE
+
+        baseline = baselines[provider.granularity]
+
+        return int(provider.priority + baseline)
+
+    def get_source_priority(self, source: str) -> float:
+        """
+        Get priority of a source string
+        """
+        provider_ids = source.split(" ")
+
+        if len(provider_ids) == 1:
+            return self._get_provider_priority(provider_ids[0])
+
+        priorities = [
+            self._get_provider_priority(provider) for provider in provider_ids
+        ]
+
+        return fmean(priorities)
+
+    def filter_providers(self, query: Request, station: Station) -> List[Provider]:
+        """
+        Get a filtered list of providers
+        """
+
+        def _filter(provider_id: Provider) -> TypeGuard[Provider]:
+            provider = self.get_provider(provider_id)
+
+            if provider is None:
+                return False
+
+            # Filter out providers with diverging granularities
+            if provider.granularity is not query.granularity:
+                logger.error(
+                    "Provider '%s' does not support granularity '%s'",
+                    provider_id,
+                    query.granularity,
+                )
+                return False
+
+            # Filter out providers with no overlap in parameters
+            if set(provider.parameters).isdisjoint(query.parameters):
+                logger.info(
+                    "Provider '%s' does not support any requested parameter",
+                    provider_id,
+                )
+                return False
+
+            # Filter out providers which do not serve the station's country
+            if provider.countries and station.country not in provider.countries:
+                logger.info(
+                    "Skipping provider '%s' as it does not serve the station's country ('%s')",
+                    provider_id,
+                    station.country,
+                )
+                return False
+
+            # Filter out providers which stopped providing data before the request's start date
+            if query.end and query.end < datetime.combine(
+                provider.start, datetime.min.time()
+            ):
+                logger.info(
+                    "Skipping provider '%s' as it stopped providing data before request start",
+                    provider_id,
+                )
+                return False
+
+            # Filter out providers which only started providing data after the request's end date
+            if (
+                provider.end is not None
+                and query.start is not None
+                and query.start > datetime.combine(provider.end, datetime.max.time())
+            ):
+                logger.info(
+                    "Skipping provider '%s' as it only started providing data after request end",
+                    provider_id,
+                )
+                return False
+
+            return True
+
+        return list(filter(_filter, query.providers))
+
+    def fetch_data(
+        self, provider_id: Provider, req: Request, station: Station
+    ) -> Optional[pd.DataFrame]:
+        """
+        Fetch data from a given provider
+        """
+        provider = self.get_provider(provider_id)
+
+        if not provider:
+            return None
+
+        query = ProviderRequest(
+            station=station,
+            start=req.start
+            or (
+                datetime.combine(provider.start, datetime.min.time())
+                if provider.start
+                else None
+            ),
+            end=req.end or (provider.end or datetime.now()),
+            parameters=req.parameters,
+        )
+
+        module = import_module(cast(str, provider.module))
+        df = module.fetch(query)
+
+        return df
+
+
+provider_service = ProviderService(providers=DEFAULT_PROVIDERS)

meteostat/core/schema.py

@@ -0,0 +1,170 @@
+"""
+Schema Service
+
+The Schema Service provides methods to clean and format
+DataFrames based on a set of parameters.
+"""
+
+from copy import copy
+from inspect import isfunction
+from typing import Callable, List
+
+import pandas as pd
+
+from meteostat.core.logger import logger
+from meteostat.core.parameters import parameter_service
+from meteostat.core.validator import Validator
+from meteostat.enumerations import Granularity, Parameter, UnitSystem
+from meteostat.utils.conversions import CONVERSION_MAPPINGS, to_condition, to_direction
+
+
+class SchemaService:
+    """
+    Schema service
+    """
+
+    @staticmethod
+    def _apply_validator(
+        validator: Validator | Callable, df: pd.DataFrame, col: str
+    ) -> pd.Series:
+        """
+        Apply a validator
+        """
+        if isfunction(validator):
+            v: Validator = validator()  # type: ignore
+        else:
+            v = validator
+
+        if not isinstance(v, Validator):
+            return pd.Series(data=True, index=df.index, dtype=bool)
+
+        if v.ignore_na:
+            result = pd.Series(data=True, index=df.index, dtype=bool)
+            test_result = v.test(
+                df.loc[df[col].notnull()][col],
+                df.loc[df[col].notnull()],
+                col,
+            )
+            if isinstance(test_result, pd.Series):
+                result.update(test_result)
+            return result.astype(bool)
+
+        test_result = v.test(df[col], df, col)
+        if isinstance(test_result, bool):
+            return pd.Series(data=test_result, index=df.index, dtype=bool)
+        return test_result
+
+    @staticmethod
+    def purge(df: pd.DataFrame, parameters: List[Parameter]) -> pd.DataFrame:
+        """
+        Remove DataFrame columns which are not a known parameter
+        """
+        columns = [parameter for parameter in parameters if parameter in df.columns]
+        return df[columns]
+
+    @staticmethod
+    def fill(df: pd.DataFrame, parameters: List[Parameter]) -> pd.DataFrame:
+        """
+        Add missing schema columns to DataFrame
+        """
+        for parameter_id in parameters:
+            if parameter_id not in df:
+                df[parameter_id] = None
+
+        return df
+
+    @staticmethod
+    def format(df: pd.DataFrame, granularity: Granularity) -> pd.DataFrame:
+        """
+        Set data types and round values
+        """
+        temp = copy(df)
+
+        for col in df.columns:
+            parameter = parameter_service.get_parameter(col, granularity)
+
+            if not parameter:
+                logger.warning(
+                    "Column %s is not a valid column name and won't be formatted", col
+                )
+                continue
+
+            if "int" in str(parameter.dtype).lower():
+                temp[col] = pd.to_numeric(temp[col]).round(0)
+
+            temp[col] = temp[col].astype(parameter.dtype, errors="ignore")
+
+            if "float" in str(parameter.dtype).lower():
+                temp[col] = temp[col].round(1)
+
+        return temp
+
+    @classmethod
+    def clean(
+        cls, df: pd.DataFrame, granularity: Granularity, fill=None
+    ) -> pd.DataFrame:
+        """
+        Remove invalid data from a DataFrame
+        """
+        temp = copy(df)
+
+        for col in temp.columns:
+            if "_source" in col:
+                continue
+
+            parameter = parameter_service.get_parameter(col, granularity)
+
+            if not parameter:
+                logger.warning(
+                    "Column %s is not a valid column name and won't be cleaned", col
+                )
+                continue
+
+            for validator in parameter.validators:
+                test = cls._apply_validator(validator, temp, col)
+                temp.loc[~test, col] = fill
+
+        return temp
+
+    def humanize(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Convert wind direction and condition codes to human-readable values
+        """
+        temp = copy(df)
+
+        if Parameter.WDIR in temp.columns:
+            temp[Parameter.WDIR] = temp[Parameter.WDIR].apply(to_direction)
+
+        if Parameter.COCO in temp.columns:
+            temp[Parameter.COCO] = temp[Parameter.COCO].apply(to_condition)
+
+        return temp
+
+    @classmethod
+    def convert(cls, df, granularity: Granularity, units: UnitSystem) -> pd.DataFrame:
+        """
+        Convert units in a DataFrame
+        """
+        temp = copy(df)
+
+        for col in temp.columns:
+            if "_source" in col:
+                continue
+
+            parameter = parameter_service.get_parameter(col, granularity)
+
+            if not parameter:
+                logger.warning(
+                    "Column %s is not a valid column name and won't be converted", col
+                )
+                continue
+
+            if parameter.unit in CONVERSION_MAPPINGS:
+                if units in CONVERSION_MAPPINGS[parameter.unit]:
+                    conversion_func = CONVERSION_MAPPINGS[parameter.unit][units]
+                    temp[col] = temp[col].apply(conversion_func)
+
+        return temp
+
+
+schema_service = SchemaService()

meteostat/core/validator.py

@@ -0,0 +1,38 @@
+"""
+Schema Column Validator
+
+This class is used to validate DataFrame columns based
+on a parameter specification.
+"""
+
+from inspect import signature
+from typing import Callable
+
+from pandas import DataFrame, Series
+
+
+class Validator:
+    """
+    Schema Column Validator
+    """
+
+    func: Callable
+    ignore_na = False
+    is_relational = False
+
+    def __init__(self, func: Callable, ignore_na=False, is_relational=False):
+        self.func = func
+        self.ignore_na = ignore_na
+        self.is_relational = is_relational
+
+    def test(self, series: Series, df: DataFrame, column: str) -> bool | Series:
+        """
+        Run validator
+
+        Returns a bool series:
+        True -> Check passed
+        False -> Check failed
+        """
+        arg_count = len((signature(self.func)).parameters)
+        args = [series, df, column]
+        return self.func(*args[0:arg_count])

meteostat/enumerations.py

@@ -0,0 +1,149 @@
+"""
+Meteostat Enumerations
+"""
+
+from enum import StrEnum, IntEnum
+
+
+class Granularity(StrEnum):
+    """
+    The different levels of time series granularity
+    """
+
+    HOURLY = "hourly"
+    DAILY = "daily"
+    MONTHLY = "monthly"
+    NORMALS = "normals"
+
+
+class Frequency(StrEnum):
+    """
+    The different levels of time series frequency
+    """
+
+    HOURLY = "h"
+    DAILY = "D"
+    MONTHLY = "MS"
+
+
+class Parameter(StrEnum):
+    """
+    The different meteorological parameters supported by Meteostat
+    """
+
+    TEMP = "temp"  # Air temperature (aggregation: mean)
+    TMIN = "tmin"  # Daily minimum air temperature (aggregation: mean)
+    TMAX = "tmax"  # Daily maximum air temperature (aggregation: mean)
+    TXMN = "txmn"  # Extreme minimum air temperature (aggregation: min)
+    TXMX = "txmx"  # Extreme maximum air temperature (aggregation: max)
+    DWPT = "dwpt"  # Dew point (aggregation: mean)
+    PRCP = "prcp"  # Precipitation (aggregation: sum)
+    PDAY = "pday"  # Days with precipitation equal to or greater than 1 millimeter (aggregation: sum)
+    WDIR = "wdir"  # Wind direction at observation time
+    WSPD = "wspd"  # Wind speed (aggregation: mean)
+    WPGT = "wpgt"  # Peak wind gust (aggregation: max)
+    RHUM = "rhum"  # Relative humidity (aggregation: mean)
+    PRES = "pres"  # Air pressure at MSL (aggregation: mean)
+    SNWD = "snwd"  # Snow depth on ground
+    SNOW = "snow"  # Snowfall (aggregation: sum)
+    TSUN = "tsun"  # Sunshine duration (aggregation: sum)
+    SGHI = "sghi"  # TBD
+    SDNI = "sdni"  # TBD
+    SDHI = "sdhi"  # TBD
+    CLDC = "cldc"  # Cloud cover (aggregation: mean)
+    VSBY = "vsby"  # Visibility (aggregation: mean)
+    COCO = "coco"  # Weather condition code at time of observation
+
+
+class Unit(StrEnum):
+    """
+    Data Units
+    """
+
+    CELSIUS = "°C"
+    FAHRENHEIT = "°F"
+    KELVIN = "K"
+    PERCENTAGE = "%"
+    HPA = "hPa"
+    MILLIMETERS = "mm"
+    CENTIMETERS = "cm"
+    METERS = "m"
+    KMH = "km/h"
+    DEGREES = "°"
+    MINUTES = "min"
+    OKTAS = "okta"
+
+
+class UnitSystem(StrEnum):
+    """
+    Unit Systems
+    """
+
+    SI = "si"
+    METRIC = "metric"
+    IMPERIAL = "imperial"
+
+
+class Provider(StrEnum):
+    """
+    Providers supported by Meteostat
+    """
+
+    ISD_LITE = "isd_lite"
+    METAR = "metar"
+    GHCND = "ghcnd"
+    CLIMAT = "climat"
+    DWD_HOURLY = "dwd_hourly"
+    DWD_POI = "dwd_poi"
+    DWD_MOSMIX = "dwd_mosmix"
+    DWD_DAILY = "dwd_daily"
+    DWD_MONTHLY = "dwd_monthly"
+    ECCC_HOURLY = "eccc_hourly"
+    ECCC_DAILY = "eccc_daily"
+    ECCC_MONTHLY = "eccc_monthly"
+    METNO_FORECAST = "metno_forecast"
+
+    HOURLY = "hourly"
+    DAILY = "daily"
+    DAILY_DERIVED = "daily_derived"
+    MONTHLY = "monthly"
+    MONTHLY_DERIVED = "monthly_derived"
+
+    SYNOP = "synop"
+    METAR_LEGACY = "metar_legacy"
+    MODEL = "model"
+
+
+class Priority(IntEnum):
+    """
+    Provider priorities
+    """
+
+    HIGHEST = 25
+    HIGH = 20
+    MEDIUM = 15
+    LOW = 10
+    LOWEST = 5
+    NONE = 0
+
+
+class Grade(IntEnum):
+    """
+    Provider quality grades
+    """
+
+    RECORD = 4
+    OBSERVATION = 3
+    ANALYSIS = 2
+    FORECAST = 1
+
+
+class TTL(IntEnum):
+    """
+    Cache TTLs
+    """
+
+    HOUR = 60 * 60
+    DAY = 60 * 60 * 24
+    WEEK = 60 * 60 * 24 * 7
+    MONTH = 60 * 60 * 24 * 30

meteostat/interpolation/idw.py

@@ -0,0 +1,120 @@
+"""
+Inverse Distance Weighting (IDW) Interpolation
+
+Implements IDW interpolation for spatial weather data with support for
+elevation-weighted distance calculations.
+"""
+
+from typing import Callable
+
+import numpy as np
+import pandas as pd
+
+from meteostat.api.point import Point
+from meteostat.api.timeseries import TimeSeries
+
+
+def inverse_distance_weighting(
+    power: float = 2.0,
+) -> Callable:
+    """
+    Interpolate values using Inverse Distance Weighting (IDW).
+
+    This method calculates interpolated values as a weighted average of nearby
+    stations, where weights decrease with distance. Optionally incorporates
+    elevation differences in the distance calculation.
+
+    Parameters
+    ----------
+    power : float, optional
+        Power parameter for IDW (default: 2.0). Higher values give more
+        weight to closer stations.
+    elevation_weight : float, optional
+        Weight for elevation difference in distance calculation (default: 0.1).
+        The effective distance is calculated as:
+        sqrt(horizontal_distance^2 + (elevation_diff * elevation_weight)^2)
+
+    Notes
+    -----
+    - If elevation data is missing for either the point or stations, only
+    horizontal distance is used.
+    - Stations with zero effective distance get weight of 1.0, all others get 0.
+    - All numeric columns except location-related ones are interpolated.
+    """
+
+    def _get_df(
+        df: pd.DataFrame,
+        ts: TimeSeries,
+        point: Point,
+    ) -> pd.DataFrame:
+        # Group by time to interpolate each timestamp separately
+        grouped = df.groupby(pd.Grouper(level="time", freq=ts.freq))
+
+        # List to store interpolated results for each time period
+        interpolated_results = []
+
+        for time_idx, group in grouped:
+            if group.empty:
+                continue
+
+            effective_distance = group["effective_distance"]
+
+            # Calculate weights using IDW formula: w = 1 / d^p
+            # Handle zero distance case (station at exact location)
+            min_distance = effective_distance.min()
+            if min_distance == 0:
+                # If any station is at the exact location, use only that station
+                weights = (effective_distance == 0).astype(float)
+            else:
+                # Standard IDW weights
+                weights = 1.0 / (effective_distance**power)
+
+            # Normalize weights so they sum to 1
+            weights = weights / weights.sum()
+
+            # Get numeric columns to interpolate (exclude location-related columns)
+            location_cols = ["latitude", "longitude", "elevation", "distance"]
+            numeric_cols = [
+                col
+                for col in group.columns
+                if col not in location_cols
+                and pd.api.types.is_numeric_dtype(group[col])
+            ]
+
+            # Calculate weighted average for each numeric column
+            interpolated_row = {}
+            for col in numeric_cols:
+                # Only use non-NaN values for interpolation
+                valid_mask = group[col].notna()
+                if valid_mask.any():
+                    valid_values = group.loc[valid_mask, col]
+                    valid_weights = weights[valid_mask]
+                    # Re-normalize weights for valid values only
+                    valid_weights = valid_weights / valid_weights.sum()
+                    interpolated_row[col] = (valid_values * valid_weights).sum()
+                else:
+                    # If all values are NaN, result is NaN
+                    interpolated_row[col] = np.nan
+
+            # Add location information from the point
+            interpolated_row["latitude"] = point.latitude
+            interpolated_row["longitude"] = point.longitude
+            if point.elevation is not None:
+                interpolated_row["elevation"] = point.elevation
+            interpolated_row["distance"] = 0  # Distance from point to itself
+
+            # Create a DataFrame row with the time index
+            result_df = pd.DataFrame(
+                [interpolated_row], index=pd.DatetimeIndex([time_idx])
+            )
+            result_df.index.name = "time"
+            interpolated_results.append(result_df)
+
+        # Combine all time periods
+        if interpolated_results:
+            result = pd.concat(interpolated_results)
+            return result
+        else:
+            return pd.DataFrame()
+
+    return _get_df