meteostat 1.7.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

meteostat/parameters.py

@@ -0,0 +1,354 @@
+"""
+Meteostat Parameter Definitions
+"""
+
+from meteostat.enumerations import Granularity, Parameter, Unit
+from meteostat.typing import ParameterSpec
+from meteostat.utils.validators import maximum, minimum
+
+
+DEFAULT_PARAMETERS = [
+    ParameterSpec(
+        id=Parameter.TEMP,
+        name="Air Temperature",
+        granularity=Granularity.HOURLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TEMP,
+        name="Mean Air Temperature",
+        granularity=Granularity.DAILY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TEMP,
+        name="Mean Air Temperature",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TEMP,
+        name="Mean Air Temperature",
+        granularity=Granularity.NORMALS,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMIN,
+        name="Minimum Air Temperature",
+        granularity=Granularity.DAILY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMIN,
+        name="Mean Daily Minimum Air Temperature",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMIN,
+        name="Mean Daily Minimum Air Temperature",
+        granularity=Granularity.NORMALS,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMAX,
+        name="Maximum Air Temperature",
+        granularity=Granularity.DAILY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMAX,
+        name="Mean Daily Maximum Air Temperature",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TMAX,
+        name="Mean Daily Maximum Air Temperature",
+        granularity=Granularity.NORMALS,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TXMN,
+        name="Absolute Minimum Air Temperature",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TXMN,
+        name="Absolute Minimum Air Temperature",
+        granularity=Granularity.NORMALS,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TXMX,
+        name="Absolute Maximum Air Temperature",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.TXMX,
+        name="Absolute Maximum Air Temperature",
+        granularity=Granularity.NORMALS,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.DWPT,
+        name="Dew Point",
+        granularity=Granularity.HOURLY,
+        unit=Unit.CELSIUS,
+        dtype="Float64",
+        validators=[minimum(-100), maximum(65)],
+    ),
+    ParameterSpec(
+        id=Parameter.RHUM,
+        name="Relative Humidity",
+        granularity=Granularity.HOURLY,
+        unit=Unit.PERCENTAGE,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(100)],
+    ),
+    ParameterSpec(
+        id=Parameter.RHUM,
+        name="Relative Humidity",
+        granularity=Granularity.DAILY,
+        unit=Unit.PERCENTAGE,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(100)],
+    ),
+    ParameterSpec(
+        id=Parameter.RHUM,
+        name="Relative Humidity",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.PERCENTAGE,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(100)],
+    ),
+    ParameterSpec(
+        id=Parameter.RHUM,
+        name="Relative Humidity",
+        granularity=Granularity.NORMALS,
+        unit=Unit.PERCENTAGE,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(100)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRCP,
+        name="Total Precipitation",
+        granularity=Granularity.HOURLY,
+        unit=Unit.MILLIMETERS,
+        dtype="Float64",
+        validators=[minimum(0), maximum(350)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRCP,
+        name="Total Precipitation",
+        granularity=Granularity.DAILY,
+        unit=Unit.MILLIMETERS,
+        dtype="Float64",
+        validators=[minimum(0), maximum(2000)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRCP,
+        name="Total Precipitation",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.MILLIMETERS,
+        dtype="Float64",
+        validators=[minimum(0), maximum(10000)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRCP,
+        name="Total Precipitation",
+        granularity=Granularity.NORMALS,
+        unit=Unit.MILLIMETERS,
+        dtype="Float64",
+        validators=[minimum(0), maximum(10000)],
+    ),
+    ParameterSpec(
+        id=Parameter.SNOW,
+        name="Snowfall",
+        granularity=Granularity.HOURLY,
+        unit=Unit.CENTIMETERS,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(10000)],
+    ),
+    ParameterSpec(
+        id=Parameter.SNWD,
+        name="Snow Depth",
+        granularity=Granularity.HOURLY,
+        unit=Unit.CENTIMETERS,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(1200)],
+    ),
+    ParameterSpec(
+        id=Parameter.SNWD,
+        name="Snow Depth",
+        granularity=Granularity.DAILY,
+        unit=Unit.CENTIMETERS,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(1200)],
+    ),
+    ParameterSpec(
+        id=Parameter.WDIR,
+        name="Wind Direction",
+        granularity=Granularity.HOURLY,
+        unit=Unit.DEGREES,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(360)],
+    ),
+    ParameterSpec(
+        id=Parameter.WSPD,
+        name="Wind Speed",
+        granularity=Granularity.HOURLY,
+        unit=Unit.KMH,
+        dtype="Float64",
+        validators=[minimum(0), maximum(250)],
+    ),
+    ParameterSpec(
+        id=Parameter.WSPD,
+        name="Average Wind Speed",
+        granularity=Granularity.DAILY,
+        unit=Unit.KMH,
+        dtype="Float64",
+        validators=[minimum(0), maximum(150)],
+    ),
+    ParameterSpec(
+        id=Parameter.WPGT,
+        name="Peak Wind Gust",
+        granularity=Granularity.HOURLY,
+        unit=Unit.KMH,
+        dtype="Float64",
+        validators=[minimum(0), maximum(500)],
+    ),
+    ParameterSpec(
+        id=Parameter.WPGT,
+        name="Peak Wind Gust",
+        granularity=Granularity.DAILY,
+        unit=Unit.KMH,
+        dtype="Float64",
+        validators=[minimum(0), maximum(500)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRES,
+        name="Air Pressure (MSL)",
+        granularity=Granularity.HOURLY,
+        unit=Unit.HPA,
+        dtype="Float64",
+        validators=[minimum(850), maximum(1090)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRES,
+        name="Average Air Pressure (MSL)",
+        granularity=Granularity.DAILY,
+        unit=Unit.HPA,
+        dtype="Float64",
+        validators=[minimum(850), maximum(1090)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRES,
+        name="Average Air Pressure (MSL)",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.HPA,
+        dtype="Float64",
+        validators=[minimum(850), maximum(1090)],
+    ),
+    ParameterSpec(
+        id=Parameter.PRES,
+        name="Average Air Pressure (MSL)",
+        granularity=Granularity.NORMALS,
+        unit=Unit.HPA,
+        dtype="Float64",
+        validators=[minimum(850), maximum(1090)],
+    ),
+    ParameterSpec(
+        id=Parameter.TSUN,
+        name="Sunshine Duration",
+        granularity=Granularity.HOURLY,
+        unit=Unit.MINUTES,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(60)],
+    ),
+    ParameterSpec(
+        id=Parameter.TSUN,
+        name="Sunshine Duration",
+        granularity=Granularity.DAILY,
+        unit=Unit.MINUTES,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(1440)],
+    ),
+    ParameterSpec(
+        id=Parameter.TSUN,
+        name="Sunshine Duration",
+        granularity=Granularity.MONTHLY,
+        unit=Unit.MINUTES,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(44640)],
+    ),
+    ParameterSpec(
+        id=Parameter.TSUN,
+        name="Sunshine Duration",
+        granularity=Granularity.NORMALS,
+        unit=Unit.MINUTES,
+        dtype="UInt16",
+        validators=[minimum(0), maximum(44640)],
+    ),
+    ParameterSpec(
+        id=Parameter.CLDC,
+        name="Cloud Cover",
+        granularity=Granularity.HOURLY,
+        unit=Unit.OKTAS,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(8)],
+    ),
+    ParameterSpec(
+        id=Parameter.CLDC,
+        name="Average Cloud Cover",
+        granularity=Granularity.DAILY,
+        unit=Unit.OKTAS,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(8)],
+    ),
+    ParameterSpec(
+        id=Parameter.VSBY,
+        name="Visibility",
+        granularity=Granularity.HOURLY,
+        unit=Unit.METERS,
+        dtype="UInt16",
+        validators=[minimum(0)],
+    ),
+    ParameterSpec(
+        id=Parameter.COCO,
+        name="Weather Condition Code",
+        granularity=Granularity.HOURLY,
+        dtype="UInt8",
+        validators=[minimum(0), maximum(27)],
+    ),
+]

meteostat/providers/dwd/climat.py

@@ -0,0 +1,166 @@
+"""
+DWD Global CLIMAT Data
+"""
+
+from datetime import datetime
+from ftplib import FTP
+from io import BytesIO
+from typing import List, Optional
+
+import pandas as pd
+
+from meteostat.core.logger import logger
+from meteostat.api.config import config
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest, Station
+from meteostat.core.cache import cache_service
+from meteostat.providers.dwd.shared import get_ftp_connection
+
+# Constants
+BASE_DIR = "/climate_environment/CDC/observations_global/CLIMAT/monthly/qc/"
+
+# Monthly column stubnames mapping template
+MONTHS_MAP = {
+    "jan": 1,
+    "feb": 2,
+    "mrz": 3,
+    "apr": 4,
+    "mai": 5,
+    "jun": 6,
+    "jul": 7,
+    "aug": 8,
+    "sep": 9,
+    "okt": 10,
+    "nov": 11,
+    "dez": 12,
+}
+
+# Parameter directory configurations
+PARAMETERS = [
+    ("precipitation_total", Parameter.PRCP),
+    ("air_temperature_mean", Parameter.TEMP),
+    ("air_temperature_mean_of_daily_max", Parameter.TMAX),
+    ("air_temperature_mean_of_daily_min", Parameter.TMIN),
+    ("air_temperature_absolute_max", Parameter.TXMX),
+    ("air_temperature_absolute_min", Parameter.TXMN),
+    ("mean_sea_level_pressure", Parameter.PRES),
+    ("sunshine_duration", Parameter.TSUN),
+]
+
+# Precomputed parameter configs with stubnames
+PARAMETER_CONFIGS = {
+    param: {
+        "dir": dir_name,
+        "stubnames": {
+            "jahr": "year",
+            **{
+                month: f"{param}{i + 1}"
+                for i, (month, _) in enumerate(MONTHS_MAP.items())
+            },
+        },
+    }
+    for dir_name, param in PARAMETERS
+}
+
+
+def find_file(ftp: FTP, mode: str, directory: str, search_term: str) -> Optional[str]:
+    """
+    Find a file in the FTP directory matching a pattern.
+    """
+    try:
+        ftp.cwd(f"{BASE_DIR}{directory}/{mode}")
+        matches = [f for f in ftp.nlst() if search_term in f]
+        return sorted(matches)[-1] if matches else None
+    except Exception:
+        logger.debug("Error while searching for file", exc_info=True)
+        return None
+
+
+@cache_service.cache(TTL.WEEK, "pickle")
+def get_df(parameter: str, mode: str, station_code: str) -> Optional[pd.DataFrame]:
+    """
+    Download and parse a CLIMAT dataset from DWD FTP.
+    """
+    param_config = PARAMETER_CONFIGS.get(parameter)
+    if not param_config:
+        logger.debug(f"Unknown parameter '{parameter}'")
+        return None
+
+    ftp = get_ftp_connection()
+    search_term = station_code if mode == "recent" else f"{station_code}_"
+    remote_file = find_file(ftp, mode, param_config["dir"], search_term)
+
+    if not remote_file:
+        logger.debug(
+            f"No file found for parameter '{parameter}', mode '{mode}', station '{station_code}'"
+        )
+        return None
+
+    buffer = BytesIO()
+    ftp.retrbinary(f"RETR {remote_file}", buffer.write)
+    ftp.close()
+
+    buffer.seek(0)
+    df = pd.read_csv(buffer, sep=";").rename(columns=lambda col: col.strip().lower())
+    df.rename(columns=param_config["stubnames"], inplace=True)
+
+    # Convert wide to long format
+    df = pd.wide_to_long(
+        df, stubnames=parameter, i="year", j="month", sep="", suffix="\\d+"
+    ).reset_index()
+
+    if parameter == Parameter.TSUN:
+        df[Parameter.TSUN] *= 60  # convert hours to minutes
+
+    # Create datetime index
+    df["time"] = pd.to_datetime(
+        df["year"].astype(str) + "-" + df["month"].astype(str).str.zfill(2) + "-01"
+    )
+
+    return df.drop(columns=["year", "month"]).set_index("time")
+
+
+def get_parameter(
+    parameter: str, modes: List[str], station: Station
+) -> Optional[pd.DataFrame]:
+    """
+    Fetch and merge data for a parameter over multiple modes (e.g., recent, historical).
+    """
+    try:
+        station_code = station.identifiers.get("wmo")
+        if not station_code:
+            return None
+
+        datasets = [get_df(parameter, mode, station_code) for mode in modes]
+        datasets = [df for df in datasets if df is not None]
+
+        if not datasets:
+            return None
+
+        return pd.concat(datasets).loc[lambda df: ~df.index.duplicated(keep="first")]
+    except Exception as e:
+        logger.warning(
+            f"Failed to fetch data for parameter '{parameter}': {e}", exc_info=True
+        )
+        return None
+
+
+def fetch(req: ProviderRequest) -> pd.DataFrame:
+    """
+    Entry point to fetch all requested parameters for a station query.
+    """
+    station_code = req.station.identifiers.get("wmo")
+    if not station_code:
+        return pd.DataFrame()
+
+    modes = ["historical"]
+    if (datetime.now() - req.end).days < 5 * 365:
+        modes.append("recent")
+
+    data_frames = [
+        get_parameter(param.value, config.dwd_climat_modes or modes, req.station)
+        for param in req.parameters
+        if param in PARAMETER_CONFIGS
+    ]
+
+    return pd.concat([df for df in data_frames if df is not None], axis=1)

meteostat/providers/dwd/daily.py

@@ -0,0 +1,144 @@
+"""
+DWD national daily data import routine
+
+Get daily data for weather stations in Germany.
+
+The code is licensed under the MIT license.
+"""
+
+from datetime import datetime
+from ftplib import FTP
+from io import BytesIO
+from typing import Optional
+from zipfile import ZipFile
+
+import pandas as pd
+
+from meteostat.api.config import config
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.core.cache import cache_service
+from meteostat.utils.conversions import ms_to_kmh, pres_to_msl
+from meteostat.providers.dwd.shared import get_ftp_connection
+
+BASE_DIR = "/climate_environment/CDC/observations_germany/climate/daily/kl/"
+USECOLS = [1, 3, 4, 6, 8, 9, 10, 12, 13, 14, 15, 16]  # CSV cols which should be read
+NAMES = {
+    "FX": Parameter.WPGT,
+    "FM": Parameter.WSPD,
+    "RSK": Parameter.PRCP,
+    "SDK": Parameter.TSUN,
+    "SHK_TAG": Parameter.SNWD,
+    "NM": Parameter.CLDC,
+    "PM": Parameter.PRES,
+    "TMK": Parameter.TEMP,
+    "UPM": Parameter.RHUM,
+    "TXK": Parameter.TMAX,
+    "TNK": Parameter.TMIN,
+}
+
+
+def find_file(ftp: FTP, mode: str, needle: str):
+    """
+    Find file in directory
+    """
+    match = None
+
+    try:
+        ftp.cwd(BASE_DIR + mode)
+        files = ftp.nlst()
+        matching = [f for f in files if needle in f]
+        match = matching[0]
+    except BaseException:
+        pass
+
+    return match
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(station: str, elevation: int, mode: str) -> Optional[pd.DataFrame]:
+    """
+    Get a file from DWD FTP server and convert to DataFrame
+    """
+    ftp = get_ftp_connection()
+    remote_file = find_file(ftp, mode, f"_{station}_")
+
+    if remote_file is None:
+        return None
+
+    buffer = BytesIO()
+    ftp.retrbinary("RETR " + remote_file, buffer.write)
+
+    ftp.close()
+
+    # Unzip file
+    with ZipFile(buffer, "r") as zipped:
+        filelist = zipped.namelist()
+        raw = None
+        for file in filelist:
+            if file[:7] == "produkt":
+                with zipped.open(file, "r") as reader:
+                    raw = BytesIO(reader.read())
+
+    # Convert raw data to DataFrame
+    df: pd.DataFrame = pd.read_csv(  # type: ignore
+        raw,
+        sep=r"\s*;\s*",
+        date_format="%Y%m%d",
+        na_values=["-999", -999],
+        usecols=USECOLS,
+        engine="python",
+    )
+
+    # Rename columns
+    df = df.rename(columns=lambda x: x.strip())
+    df = df.rename(columns=NAMES)
+
+    # Parse date column (first column contains the date)
+    df["time"] = pd.to_datetime(df.iloc[:, 0], format="%Y%m%d")
+    df = df.drop(df.columns[0], axis=1)
+
+    # Convert data
+    df[Parameter.SNWD] = df[Parameter.SNWD] * 10
+    df[Parameter.WPGT] = df[Parameter.WPGT].apply(ms_to_kmh)
+    df[Parameter.WSPD] = df[Parameter.WSPD].apply(ms_to_kmh)
+    df[Parameter.TSUN] = df[Parameter.TSUN] * 60
+    df[Parameter.PRES] = df.apply(lambda row: pres_to_msl(row, elevation), axis=1)
+
+    # Set index
+    df = df.set_index("time")
+
+    # Round decimals
+    df = df.round(1)
+
+    return df
+
+
+def fetch(req: ProviderRequest):
+    if "national" not in req.station.identifiers:
+        return pd.DataFrame()
+
+    # Check which modes to consider for data fetching
+    #
+    # The dataset is divided into a versioned part with completed quality check ("historical"),
+    # and a part for which the quality check has not yet been completed ("recent").
+    #
+    # There is no definite answer as to when the quality check is completed. We're assuming a
+    # period of 3 years here. If the end date of the query is within this period, we will also
+    # consider the "recent" mode.
+    modes = ["historical"]
+    if abs((req.end - datetime.now()).days) < 3 * 365:
+        modes.append("recent")
+
+    data = [
+        get_df(
+            req.station.identifiers["national"],
+            req.station.elevation,
+            mode,
+        )
+        for mode in config.dwd_daily_modes or modes
+    ]
+
+    df = pd.concat(data)
+
+    return df.loc[~df.index.duplicated(keep="first")]