meteostat 1.7.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/providers/meteostat/daily.py

@@ -0,0 +1,65 @@
+"""
+The code is licensed under the MIT license.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.providers.meteostat.shared import filter_model_data, handle_exceptions
+from meteostat.typing import ProviderRequest
+from meteostat.core.cache import cache_service
+from meteostat.core.config import config
+from meteostat.utils.data import reshape_by_source
+
+ENDPOINT = config.daily_endpoint
+
+
+def get_ttl(_station: str, year: int) -> int:
+    """
+    Get TTL based on year
+
+    Current + previous year = one day
+    Else = 30 days
+    """
+    current_year = datetime.now().year
+    return 60 * 60 * 24 if current_year - year < 2 else 60 * 60 * 24 * 30
+
+
+@cache_service.cache(get_ttl, "pickle")
+@handle_exceptions
+def get_df(station: str, year: int) -> Optional[pd.DataFrame]:
+    """
+    Get CSV file from Meteostat and convert to DataFrame
+    """
+    file_url = ENDPOINT.format(station=station, year=str(year))
+
+    df = pd.read_csv(file_url, sep=",", compression="gzip")
+
+    time_cols = df.columns[0:3]
+    df["time"] = pd.to_datetime(df[time_cols])
+    df = df.drop(time_cols, axis=1).set_index("time")
+
+    return reshape_by_source(df)
+
+
+@filter_model_data
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """
+    Fetch daily weather data from Meteostat's central data repository
+    """
+    if req.start is None or req.end is None:
+        return None
+
+    # Get a list of relevant years
+    years = range(req.start.year, req.end.year + 1)
+    # Get list of annual DataFrames
+    df_yearly = [get_df(req.station.id, year) for year in years]
+    # Concatenate into a single DataFrame
+    df = (
+        pd.concat(df_yearly)
+        if len(df_yearly) and not all(d is None for d in df_yearly)
+        else None
+    )
+    return df

meteostat/providers/meteostat/daily_derived.py

@@ -0,0 +1,110 @@
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from meteostat.enumerations import Parameter, Provider
+from meteostat.api.hourly import hourly
+from meteostat.typing import ProviderRequest
+from meteostat.utils.data import aggregate_sources, reshape_by_source
+
+
+def daily_mean(group: pd.Series):
+    """
+    Calculate the daily mean from a series of hourly data
+    """
+    if group.isna().sum() > 3:
+        return np.nan
+    return group.interpolate(axis=0).mean()
+
+
+def daily_min(group: pd.Series):
+    """
+    Calculate the daily minimum from a series of hourly data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.min()
+
+
+def daily_max(group: pd.Series):
+    """
+    Calculate the daily maximum from a series of hourly data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.max()
+
+
+def daily_sum(group: pd.Series):
+    """
+    Calculate the daily sum from a series of hourly data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.sum()
+
+
+# Available parameters with source column and aggregation method
+PARAMETER_AGGS = {
+    Parameter.TEMP: (Parameter.TEMP, daily_mean),
+    Parameter.TMIN: (Parameter.TEMP, daily_min),
+    Parameter.TMAX: (Parameter.TEMP, daily_max),
+    Parameter.RHUM: (Parameter.RHUM, daily_mean),
+    Parameter.DWPT: (Parameter.DWPT, daily_mean),
+    Parameter.PRCP: (Parameter.PRCP, daily_sum),
+    Parameter.SNWD: (Parameter.SNWD, daily_max),
+    Parameter.WSPD: (Parameter.WSPD, daily_mean),
+    Parameter.WPGT: (Parameter.WPGT, daily_max),
+    Parameter.PRES: (Parameter.PRES, daily_mean),
+    Parameter.TSUN: (Parameter.TSUN, daily_sum),
+    Parameter.CLDC: (Parameter.CLDC, daily_mean),
+}
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """
+    Fetch hourly weather data from Meteostat's central data
+    repository and aggregate to daily granularity
+    """
+    # Get all source columns
+    source_cols = list(dict.fromkeys([PARAMETER_AGGS[p][0] for p in req.parameters]))
+
+    # Get hourly DataFrame
+    ts_hourly = hourly(
+        req.station.id,
+        req.start,
+        req.end,
+        parameters=source_cols,
+        providers=[Provider.HOURLY],
+        timezone=req.station.timezone,
+    )
+
+    df_hourly = ts_hourly.fetch(fill=True, sources=True)
+
+    # If no hourly data is available, exit
+    if df_hourly is None:
+        return None
+
+    # Create daily aggregations
+    df = pd.DataFrame()
+    for parameter in req.parameters:
+        [hourly_param_name, agg_func] = PARAMETER_AGGS[parameter]
+        df[parameter] = (
+            df_hourly[hourly_param_name]
+            .groupby(pd.Grouper(level="time", freq="1D"))
+            .agg(agg_func)
+        )
+        df[f"{parameter}_source"] = (
+            df_hourly[f"{hourly_param_name}_source"]
+            .groupby(pd.Grouper(level="time", freq="1D"))
+            .agg(aggregate_sources)
+        )
+
+    # Adjust DataFrame and add index
+    df = df.round(1)
+    df.index = pd.to_datetime(df.index.astype("datetime64").dt.date)  # type: ignore[union-attr]
+    df.index.name = "time"
+
+    # Return reshaped DataFrame
+    return reshape_by_source(df)

meteostat/providers/meteostat/hourly.py

@@ -0,0 +1,66 @@
+"""
+The code is licensed under the MIT license.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.providers.meteostat.shared import filter_model_data, handle_exceptions
+from meteostat.typing import ProviderRequest
+from meteostat.core.config import config
+from meteostat.core.cache import cache_service
+from meteostat.utils.data import reshape_by_source
+
+
+ENDPOINT = config.hourly_endpoint
+
+
+def get_ttl(_station: str, year: int) -> int:
+    """
+    Get TTL based on year
+
+    Current + previous year = one day
+    Else = 30 days
+    """
+    current_year = datetime.now().year
+    return 60 * 60 * 24 if current_year - year < 2 else 60 * 60 * 24 * 30
+
+
+@cache_service.cache(get_ttl, "pickle")
+@handle_exceptions
+def get_df(station: str, year: int) -> Optional[pd.DataFrame]:
+    """
+    Get CSV file from Meteostat and convert to DataFrame
+    """
+    file_url = ENDPOINT.format(station=station, year=str(year))
+
+    df = pd.read_csv(file_url, sep=",", compression="gzip")
+
+    time_cols = df.columns[0:4]
+    df["time"] = pd.to_datetime(df[time_cols])
+    df = df.drop(time_cols, axis=1).set_index("time")
+
+    return reshape_by_source(df)
+
+
+@filter_model_data
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """
+    Fetch hourly weather data from Meteostat's central data repository
+    """
+    if req.start is None or req.end is None:
+        return None
+
+    # Get a list of relevant years
+    years = range(req.start.year, req.end.year + 1)
+    # Get list of annual DataFrames
+    df_yearly = [get_df(req.station.id, year) for year in years]
+    # Concatenate into a single DataFrame
+    df = (
+        pd.concat(df_yearly)
+        if len(df_yearly) and not all(d is None for d in df_yearly)
+        else None
+    )
+    return df

meteostat/providers/meteostat/monthly.py

@@ -0,0 +1,45 @@
+"""
+The code is licensed under the MIT license.
+"""
+
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.core.config import config
+from meteostat.enumerations import TTL
+from meteostat.providers.meteostat.shared import filter_model_data, handle_exceptions
+from meteostat.typing import ProviderRequest
+from meteostat.core.cache import cache_service
+from meteostat.utils.data import reshape_by_source
+
+ENDPOINT = config.monthly_endpoint
+
+
+@cache_service.cache(TTL.MONTH, "pickle")
+@handle_exceptions
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    """
+    Get CSV file from Meteostat and convert to DataFrame
+    """
+    file_url = ENDPOINT.format(station=station)
+
+    df = pd.read_csv(file_url, sep=",", compression="gzip")
+
+    time_cols = df.columns[0:2]
+    df["date"] = (
+        df["year"].astype(str) + "-" + df["month"].astype(str).str.zfill(2) + "-01"
+    )
+    df["time"] = pd.to_datetime(df["date"])
+    df = df.drop(time_cols, axis=1).drop("date", axis=1).set_index("time")
+
+    return reshape_by_source(df)
+
+
+@filter_model_data
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """
+    Fetch monthly weather data from Meteostat's central data repository
+    """
+    df = get_df(req.station.id)
+    return df

meteostat/providers/meteostat/monthly_derived.py

@@ -0,0 +1,106 @@
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from meteostat.enumerations import Parameter, Provider
+from meteostat.api.daily import daily
+from meteostat.typing import ProviderRequest
+from meteostat.utils.data import aggregate_sources, reshape_by_source
+from meteostat.utils.parsers import parse_month
+
+
+def monthly_mean(group: pd.Series):
+    """
+    Calculate the monthly mean from a series of daily data
+    """
+    if group.isna().sum() > 3:
+        return np.nan
+    return group.interpolate(axis=0).mean()
+
+
+def monthly_sum(group: pd.Series):
+    """
+    Calculate the monthly sum from a series of daily data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.sum()
+
+
+def monthly_min(group: pd.Series):
+    """
+    Calculate the absolute minimum from a series of daily data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.interpolate(axis=0).min()
+
+
+def monthly_max(group: pd.Series):
+    """
+    Calculate the absolute maximum from a series of daily data
+    """
+    if group.isna().sum() > 0:
+        return np.nan
+    return group.interpolate(axis=0).max()
+
+
+# Available parameters with source column and aggregation method
+PARAMETER_AGGS = {
+    Parameter.TEMP: (Parameter.TEMP, monthly_mean),
+    Parameter.TMIN: (Parameter.TMIN, monthly_mean),
+    Parameter.TMAX: (Parameter.TMAX, monthly_mean),
+    Parameter.TXMN: (Parameter.TMIN, monthly_min),
+    Parameter.TXMX: (Parameter.TMAX, monthly_max),
+    Parameter.PRCP: (Parameter.PRCP, monthly_sum),
+    Parameter.PRES: (Parameter.PRES, monthly_mean),
+    Parameter.TSUN: (Parameter.TSUN, monthly_sum),
+}
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """
+    Fetch daily weather data from Meteostat's central data
+    repository and aggregate to monthly granularity
+    """
+    # Get all source columns
+    source_cols = list(dict.fromkeys([PARAMETER_AGGS[p][0] for p in req.parameters]))
+
+    # Get daily DataFrame
+    ts_daily = daily(
+        req.station.id,
+        parse_month(req.start),
+        parse_month(req.end, is_end=True),
+        parameters=source_cols,
+        providers=[Provider.DAILY],
+    )
+
+    df_daily = ts_daily.fetch(fill=True, sources=True)
+
+    # If no daily data is available, exit
+    if df_daily is None:
+        return None
+
+    # Create monthly aggregations
+    df = pd.DataFrame()
+    for parameter in req.parameters:
+        [daily_param_name, agg_func] = PARAMETER_AGGS[parameter]
+        df[parameter] = (
+            df_daily[daily_param_name]
+            .groupby(pd.Grouper(level="time", freq="MS"))
+            .agg(agg_func)
+        )
+        df[f"{parameter}_source"] = (
+            df_daily[f"{daily_param_name}_source"]
+            .groupby(pd.Grouper(level="time", freq="MS"))
+            .agg(aggregate_sources)
+        )
+
+    # Adjust DataFrame and add index
+    df = df.round(1)
+    df.index = pd.to_datetime(df.index.astype("datetime64").dt.date)  # type: ignore[union-attr]
+    df.index.name = "time"
+
+    # Return final DataFrame
+    return reshape_by_source(df)

meteostat/providers/meteostat/shared.py

@@ -0,0 +1,93 @@
+import functools
+from urllib.error import HTTPError
+from typing import Optional, Callable, TypeVar
+
+import pandas as pd
+
+from meteostat.core.config import config
+from meteostat.core.logger import logger
+from meteostat.core.providers import provider_service
+from meteostat.enumerations import Grade
+
+T = TypeVar("T")
+
+
+def _get_station_year_info(args: tuple) -> str:
+    """
+    Get station and year info
+    """
+    station = args[0] if len(args) > 0 else None
+    year = args[1] if len(args) > 1 else None
+
+    info = "no station or year info"
+
+    if station is not None:
+        info = f'station "{station}"'
+    if year is not None:
+        info += f' and year "{year}"'
+
+    return info
+
+
+def handle_exceptions(func: Callable[..., Optional[T]]) -> Callable[..., Optional[T]]:
+    """
+    Decorator to handle exceptions during data fetching
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs) -> Optional[T]:
+        try:
+            return func(*args, **kwargs)
+        except HTTPError as error:
+            if error.code == 404:
+                logger.info(
+                    f"Data file for {_get_station_year_info(args)} was not found"
+                )
+            else:
+                logger.warning(
+                    f"HTTP error while loading data file for {_get_station_year_info(args)} (status: {error.code})",
+                    exc_info=True,
+                )
+        except Exception:
+            logger.warning(
+                f"Could not load data file for {_get_station_year_info(args)}",
+                exc_info=True,
+            )
+        return None
+
+    return wrapper
+
+
+def filter_model_data(func: Callable[..., Optional[T]]) -> Callable[..., Optional[T]]:
+    """
+    Decorator to filter out model/forecast data based on configuration
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs) -> Optional[T]:
+        result = func(*args, **kwargs)
+
+        if (
+            not config.include_model_data
+            and isinstance(result, pd.DataFrame)
+            and result is not None
+        ):
+            logger.debug("Filtering out model/forecast data")
+
+            excluded_providers = [
+                provider.id
+                for provider in provider_service.providers
+                if provider.grade
+                in (
+                    Grade.FORECAST,
+                    Grade.ANALYSIS,
+                )
+            ]
+
+            mask = result.index.get_level_values("source").isin(excluded_providers)
+
+            result = result[~mask]
+
+        return result
+
+    return wrapper

meteostat/providers/metno/forecast.py

@@ -0,0 +1,186 @@
+from typing import Optional, Union
+from urllib.error import HTTPError
+
+import pandas as pd
+
+from meteostat.core.config import config
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.logger import logger
+from meteostat.core.network import network_service
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import percentage_to_okta
+from meteostat.core.cache import cache_service
+
+
+ENDPOINT = config.metno_forecast_endpoint
+USER_AGENT = config.metno_user_agent
+CONDICODES = {
+    "clearsky": 1,
+    "cloudy": 3,
+    "fair": 2,
+    "fog": 5,
+    "heavyrain": 9,
+    "heavyrainandthunder": 26,
+    "heavyrainshowers": 18,
+    "heavyrainshowersandthunder": 26,
+    "heavysleet": 13,
+    "heavysleetandthunder": 26,
+    "heavysleetshowers": 20,
+    "heavysleetshowersandthunder": 26,
+    "heavysnow": 16,
+    "heavysnowandthunder": 26,
+    "heavysnowshowers": 22,
+    "heavysnowshowersandthunder": 26,
+    "lightrain": 7,
+    "lightrainandthunder": 25,
+    "lightrainshowers": 17,
+    "lightrainshowersandthunder": 25,
+    "lightsleet": 12,
+    "lightsleetandthunder": 25,
+    "lightsleetshowers": 19,
+    "lightsnow": 14,
+    "lightsnowandthunder": 25,
+    "lightsnowshowers": 21,
+    "lightssleetshowersandthunder": 25,
+    "lightssnowshowersandthunder": 25,
+    "partlycloudy": 3,
+    "rain": 8,
+    "rainandthunder": 25,
+    "rainshowers": 17,
+    "rainshowersandthunder": 25,
+    "sleet": 12,
+    "sleetandthunder": 25,
+    "sleetshowers": 19,
+    "sleetshowersandthunder": 25,
+    "snow": 15,
+    "snowandthunder": 25,
+    "snowshowers": 21,
+    "snowshowersandthunder": 25,
+}
+
+
+def get_condicode(code: str) -> Union[int, None]:
+    """
+    Map Met.no symbol codes to Meteostat condition codes
+
+    Documentation: https://api.met.no/weatherapi/weathericon/2.0/documentation
+    """
+    return CONDICODES.get(str(code).split("_")[0], None)
+
+
+def safe_get(data, keys, default=None, transform=lambda x: x):
+    """
+    Safely get a nested value from a dictionary, with an optional transformation.
+
+    :param data: The dictionary to get the value from.
+    :param keys: A list of keys to navigate through the dictionary.
+    :param default: The default value to return if any key is missing.
+    :param transform: A function to apply to the retrieved value if found.
+    :return: The retrieved value or the default.
+    """
+    for key in keys:
+        data = data.get(key)
+        if data is None:
+            return default
+    return transform(data)
+
+
+def map_data(record):
+    """
+    Map Met.no JSON data to Meteostat column names
+    """
+    details_instant = ["data", "instant", "details"]
+    details_next_1_hour = ["data", "next_1_hours", "details"]
+
+    return {
+        "time": record["time"],
+        Parameter.TEMP: safe_get(record, details_instant + ["air_temperature"]),
+        Parameter.CLDC: safe_get(
+            record,
+            details_instant + ["cloud_area_fraction"],
+            transform=percentage_to_okta,
+        ),
+        Parameter.RHUM: safe_get(record, details_instant + ["relative_humidity"]),
+        Parameter.PRCP: safe_get(
+            record, details_next_1_hour + ["precipitation_amount"]
+        ),
+        Parameter.WSPD: safe_get(
+            record, details_instant + ["wind_speed"], transform=lambda x: x * 3.6
+        ),
+        Parameter.WPGT: safe_get(
+            record,
+            details_instant + ["wind_speed_of_gust"],
+            transform=lambda x: x * 3.6,
+        ),
+        Parameter.WDIR: safe_get(
+            record,
+            details_instant + ["wind_from_direction"],
+            transform=lambda x: int(round(x)),
+        ),
+        Parameter.PRES: safe_get(
+            record, details_instant + ["air_pressure_at_sea_level"]
+        ),
+        Parameter.COCO: safe_get(
+            record,
+            ["data", "next_1_hours", "summary", "symbol_code"],
+            transform=get_condicode,
+        ),
+    }
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(latitude: float, longitude: float, elevation: int) -> Optional[pd.DataFrame]:
+    file_url = ENDPOINT.format(
+        latitude=latitude,
+        longitude=longitude,
+        elevation=elevation,
+    )
+
+    headers = {"User-Agent": USER_AGENT}
+
+    try:
+        response = network_service.get(file_url, headers=headers)
+
+        # Raise an exception if the request was unsuccessful
+        response.raise_for_status()
+
+        # Parse the JSON content into a DataFrame
+        data = response.json()
+
+        # Create DataFrame
+        df = pd.DataFrame(map(map_data, data["properties"]["timeseries"]))
+
+        # Handle time column & set index
+        df["time"] = pd.to_datetime(df["time"])
+        df = df.set_index(["time"])
+
+        # Remove the UTC timezone from the time index
+        df.index = df.index.tz_localize(None)  # type: ignore[union-attr]
+
+        # Shift prcp and coco columns by 1 (as they refer to the next hour)
+        df["prcp"] = df["prcp"].shift(1)
+        df["coco"] = df["coco"].shift(1)
+
+        return df
+
+    except HTTPError as error:
+        logger.warning(
+            f"Couldn't load weather forecast from met.no (status: {error.status})"
+        )
+
+    except Exception as error:
+        logger.error(error, exc_info=True)
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if not USER_AGENT:
+        logger.warning(
+            "MET Norway requires a unique user agent as per their terms of service. Please use config to specify your user agent. For now, this provider is skipped."
+        )
+        return None
+
+    return get_df(
+        req.station.latitude,
+        req.station.longitude,
+        req.station.elevation,
+    )