meteostat 1.7.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

meteostat/providers/dwd/hourly.py

@@ -0,0 +1,218 @@
+"""
+DWD national hourly data import routine
+
+Get hourly data for weather stations in Germany.
+
+The code is licensed under the MIT license.
+"""
+
+from datetime import datetime
+from ftplib import FTP
+from io import BytesIO
+from typing import Callable, Dict, List, NotRequired, Optional, TypedDict
+from zipfile import ZipFile
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.logger import logger
+from meteostat.typing import ProviderRequest, Station
+from meteostat.core.cache import cache_service
+from meteostat.api.config import config
+from meteostat.utils.conversions import ms_to_kmh
+from meteostat.providers.dwd.shared import get_condicode
+from meteostat.providers.dwd.shared import get_ftp_connection
+
+
+class ParameterDefinition(TypedDict):
+    dir: str
+    usecols: List[int]
+    names: Dict[str, str]
+    convert: NotRequired[Dict[str, Callable]]
+    encoding: NotRequired[str]
+    historical_only: NotRequired[bool]
+
+
+BASE_DIR = "/climate_environment/CDC/observations_germany/climate/hourly/"
+PARAMETERS: List[ParameterDefinition] = [
+    {
+        "dir": "precipitation",
+        "usecols": [1, 3],
+        "names": {"R1": Parameter.PRCP},
+    },
+    {
+        "dir": "air_temperature",
+        "usecols": [1, 3, 4],
+        "names": {"TT_TU": Parameter.TEMP, "RF_TU": Parameter.RHUM},
+    },
+    {
+        "dir": "wind",
+        "usecols": [1, 3, 4],
+        "names": {"F": Parameter.WSPD, "D": Parameter.WDIR},
+        "convert": {"wspd": ms_to_kmh},
+    },
+    {
+        "dir": "pressure",
+        "usecols": [1, 3],
+        "names": {"P": Parameter.PRES},
+    },
+    {
+        "dir": "sun",
+        "usecols": [1, 3],
+        "names": {"SD_SO": Parameter.TSUN},
+    },
+    {
+        "dir": "cloudiness",
+        "usecols": [1, 4],
+        "names": {"V_N": Parameter.CLDC},
+    },
+    {
+        "dir": "visibility",
+        "usecols": [1, 4],
+        "names": {"V_VV": Parameter.VSBY},
+    },
+    {
+        "dir": "weather_phenomena",
+        "usecols": [1, 3],
+        "names": {"WW": Parameter.COCO},
+        "convert": {"coco": get_condicode},
+        "encoding": "latin-1",
+    },
+    # TODO: Implement solar radiation
+    # {
+    #     "dir": "solar",
+    #     "usecols": [1, 5],
+    #     "names": {"FG_LBERG": "srad"},
+    #     "convert": {"srad": jcm2_to_wm2},
+    #     "historical_only": True,
+    # },
+]
+
+
+def find_file(ftp: FTP, path: str, needle: str):
+    """
+    Find file in directory
+    """
+    match = None
+
+    try:
+        ftp.cwd(BASE_DIR + path)
+        files = ftp.nlst()
+        matching = [f for f in files if needle in f]
+        match = matching[0]
+        logger.debug(f"Found file '{match}' in '{path}' directory")
+    except IndexError:
+        logger.info(f"File '{needle}' not found in '{path}' directory")
+
+    return match
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(parameter_dir: str, mode: str, station_id: str) -> Optional[pd.DataFrame]:
+    """
+    Get a file from DWD FTP server and convert to Polars DataFrame
+    """
+    logger.debug(
+        f"Fetching {parameter_dir} data ({mode}) for DWD station '{station_id}'"
+    )
+
+    parameter = next(param for param in PARAMETERS if param["dir"] == parameter_dir)
+
+    ftp = get_ftp_connection()
+    remote_file = find_file(ftp, f"{parameter['dir']}/{mode}", station_id)
+
+    if remote_file is None:
+        return None
+
+    buffer = BytesIO()
+    ftp.retrbinary("RETR " + remote_file, buffer.write)
+
+    # Unzip file
+    with ZipFile(buffer, "r") as zipped:
+        filelist = zipped.namelist()
+        raw = None
+        for file in filelist:
+            if file[:7] == "produkt":
+                with zipped.open(file, "r") as reader:
+                    raw = BytesIO(reader.read())
+
+    # Convert raw data to DataFrame
+    df: pd.DataFrame = pd.read_csv(  # type: ignore
+        raw,
+        sep=";",
+        skipinitialspace=True,
+        date_format="%Y%m%d%H",
+        na_values=[-999, "-999"],
+        usecols=parameter["usecols"],
+        encoding=parameter["encoding"] if "encoding" in parameter else None,
+    )
+
+    df["time"] = pd.to_datetime(df.pop("MESS_DATUM"), format="%Y%m%d%H")
+
+    logger.debug(f"Found {len(df)} rows in {remote_file}")
+
+    # Rename columns
+    df = df.rename(columns=lambda x: x.strip())
+    df = df.rename(columns=parameter["names"])
+
+    # Convert column data
+    if "convert" in parameter:
+        for col, func in parameter["convert"].items():
+            df[col] = df[col].apply(func)
+
+    # Set index
+    df = df.set_index("time")
+
+    # Round decimals
+    df = df.round(1)
+
+    return df
+
+
+def get_parameter(
+    parameter_dir: str, modes: list[str], station: Station
+) -> Optional[pd.DataFrame]:
+    logger.debug(f"Fetching {parameter_dir} data ({modes}) for station '{station.id}'")
+    try:
+        data = [
+            get_df(parameter_dir, mode, station.identifiers["national"])
+            for mode in modes
+        ]
+        if all(d is None for d in data):
+            return None
+        df = pd.concat(data)
+        return df.loc[~df.index.duplicated(keep="first")]
+    except Exception as error:
+        logger.warning(error, exc_info=True)
+        return None
+
+
+def fetch(req: ProviderRequest):
+    if "national" not in req.station.identifiers:
+        return None
+
+    # Check which modes to consider for data fetching
+    #
+    # The dataset is divided into a versioned part with completed quality check ("historical"),
+    # and a part for which the quality check has not yet been completed ("recent").
+    #
+    # There is no definite answer as to when the quality check is completed. We're assuming a
+    # period of 3 years here. If the end date of the query is within this period, we will also
+    # consider the "recent" mode.
+    modes = ["historical"]
+    if abs((req.end - datetime.now()).days) < 3 * 365:
+        modes.append("recent")
+
+    columns = map(
+        lambda args: get_parameter(*args),
+        (
+            (parameter["dir"], config.dwd_hourly_modes or modes, req.station)
+            for parameter in [
+                param
+                for param in PARAMETERS
+                if not set(req.parameters).isdisjoint(param["names"].values())
+            ]
+        ),
+    )
+
+    return pd.concat(columns, axis=1)

meteostat/providers/dwd/monthly.py

@@ -0,0 +1,138 @@
+"""
+DWD national daily data import routine
+
+Get daily data for weather stations in Germany.
+
+The code is licensed under the MIT license.
+"""
+
+from datetime import datetime
+from ftplib import FTP
+from io import BytesIO
+from typing import Optional
+from zipfile import ZipFile
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.core.cache import cache_service
+from meteostat.utils.conversions import ms_to_kmh
+from meteostat.providers.dwd.shared import get_ftp_connection
+
+
+BASE_DIR = "/climate_environment/CDC/observations_germany/climate/monthly/kl/"
+USECOLS = [1, 4, 5, 6, 7, 9, 10, 11, 12, 14]  # CSV cols which should be read
+PARSE_DATES = {"time": [0]}  # Which columns should be parsed as dates?
+NAMES = {
+    "MO_N": Parameter.CLDC,
+    "MO_TT": Parameter.TEMP,
+    "MO_TX": Parameter.TMAX,
+    "MO_TN": Parameter.TMIN,
+    "MX_TX": Parameter.TXMX,
+    "MX_TN": Parameter.TXMN,
+    "MX_FX": Parameter.WPGT,
+    "MO_SD_S": Parameter.TSUN,
+    "MO_RR": Parameter.PRCP,
+}
+
+
+def find_file(ftp: FTP, mode: str, needle: str):
+    """
+    Find file in directory
+    """
+    match = None
+
+    try:
+        ftp.cwd(BASE_DIR + mode)
+        files = ftp.nlst()
+        matching = [f for f in files if needle in f]
+        match = matching[0]
+    except BaseException:
+        pass
+
+    return match
+
+
+@cache_service.cache(TTL.WEEK, "pickle")
+def get_df(station: str, mode: str) -> Optional[pd.DataFrame]:
+    """
+    Get a file from DWD FTP server and convert to Polars DataFrame
+    """
+    ftp = get_ftp_connection()
+    remote_file = find_file(ftp, mode, f"_{station}_")
+
+    if remote_file is None:
+        return None
+
+    buffer = BytesIO()
+    ftp.retrbinary("RETR " + remote_file, buffer.write)
+
+    ftp.close()
+
+    # Unzip file
+    with ZipFile(buffer, "r") as zipped:
+        filelist = zipped.namelist()
+        raw = None
+        for file in filelist:
+            if file[:7] == "produkt":
+                with zipped.open(file, "r") as reader:
+                    raw = BytesIO(reader.read())
+
+    # Convert raw data to DataFrame
+    df: pd.DataFrame = pd.read_csv(  # type: ignore
+        raw,
+        sep=r"\s*;\s*",
+        date_format="%Y%m%d",
+        na_values=["-999", -999],
+        usecols=USECOLS,
+        parse_dates=PARSE_DATES,
+        engine="python",
+    )
+
+    # Rename columns
+    df = df.rename(columns=lambda x: x.strip())
+    df = df.rename(columns=NAMES)
+
+    # Convert data
+    df["wpgt"] = df["wpgt"].apply(ms_to_kmh)
+    df["tsun"] = df["tsun"] * 60
+    df["tsun"] = df["tsun"].round()
+    df["cldc"] = df["cldc"].round()
+
+    # Set index
+    df = df.set_index("time")
+
+    # Round decimals
+    df = df.round(1)
+
+    return df
+
+
+def fetch(req: ProviderRequest):
+    if "national" not in req.station.identifiers:
+        return pd.DataFrame()
+
+    # Check which modes to consider for data fetching
+    #
+    # The dataset is divided into a versioned part with completed quality check ("historical"),
+    # and a part for which the quality check has not yet been completed ("recent").
+    #
+    # There is no definite answer as to when the quality check is completed. We're assuming a
+    # period of 3 years here. If the end date of the query is within this period, we will also
+    # consider the "recent" mode.
+    modes = ["historical"]
+    if abs((req.end - datetime.now()).days) < 3 * 365:
+        modes.append("recent")
+
+    data = [
+        get_df(
+            req.station.identifiers["national"],
+            mode,
+        )
+        for mode in modes
+    ]
+
+    df = pd.concat(data)
+
+    return df.loc[~df.index.duplicated(keep="first")]

meteostat/providers/dwd/mosmix.py

@@ -0,0 +1,351 @@
+"""
+DWD MOSMIX data provider
+
+Parameters: https://www.dwd.de/DE/leistungen/met_verfahren_mosmix/mosmix_parameteruebersicht.pdf?__blob=publicationFile&v=3
+"""
+
+import re
+from io import BytesIO
+from typing import Optional
+from datetime import datetime
+from zipfile import ZipFile
+from lxml import etree  # type: ignore
+
+import pandas as pd
+
+from meteostat.core.cache import cache_service
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import (
+    kelvin_to_celsius,
+    ms_to_kmh,
+    percentage_to_okta,
+    temp_dwpt_to_rhum,
+)
+from meteostat.core.network import network_service
+
+ENDPOINT = "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/{station}/kml/MOSMIX_L_LATEST_{station}.kmz"
+COCO_MAP = {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "45": 5,
+    "49": 5,
+    "61": 7,
+    "63": 8,
+    "65": 9,
+    "51": 7,
+    "53": 8,
+    "55": 9,
+    "68": 12,
+    "69": 13,
+    "71": 14,
+    "73": 15,
+    "75": 16,
+    "80": 17,
+    "81": 18,
+    "82": 18,
+    "83": 19,
+    "84": 20,
+    "85": 21,
+    "86": 22,
+    "66": 10,
+    "67": 11,
+    "56": 10,
+    "57": 11,
+    "95": 25,
+}
+
+
+def get_coco(code: str | int) -> Optional[int]:
+    """
+    Map DWD MOSMIX weather condition codes to Meteostat condicodes
+    """
+    return COCO_MAP.get(str(code))
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    # Fetch the KMZ file data in memory
+    response = network_service.get(ENDPOINT.format(station=station))
+    kmz_data = BytesIO(response.content)
+
+    # KMZ -> KML in memory
+    with ZipFile(kmz_data, "r") as kmz:
+        with kmz.open(kmz.infolist()[0].filename, "r") as raw:
+            kml = raw.read()
+
+    # Parse KML
+    tree = etree.fromstring(kml)
+
+    # Skip stale forecasts
+    issue_time = datetime.strptime(
+        tree.xpath(
+            "//kml:kml/kml:Document/kml:ExtendedData/"
+            + "dwd:ProductDefinition/dwd:IssueTime",
+            namespaces=tree.nsmap,
+        )[0].text,
+        "%Y-%m-%dT%H:%M:%S.%fZ",
+    )
+    if (datetime.now() - issue_time).total_seconds() > 25200:
+        return None
+
+    # Collect all time steps
+    timesteps = []
+    for step in tree.xpath(
+        "//kml:kml/kml:Document/kml:ExtendedData/dwd:ProductDefinition/"
+        + "dwd:ForecastTimeSteps/dwd:TimeStep",
+        namespaces=tree.nsmap,
+    ):
+        timesteps.append(step.text)
+
+    # COLLECT WEATHER DATA
+    # Each parameter is processed individually
+    data = {
+        "time": timesteps,
+        Parameter.TEMP: [],
+        Parameter.DWPT: [],
+        Parameter.PRCP: [],
+        Parameter.WDIR: [],
+        Parameter.WSPD: [],
+        Parameter.WPGT: [],
+        Parameter.TSUN: [],
+        Parameter.PRES: [],
+        Parameter.CLDC: [],
+        Parameter.VSBY: [],
+        Parameter.COCO: [],
+    }
+    placemark = tree.xpath(
+        "//kml:kml/kml:Document/kml:Placemark", namespaces=tree.nsmap
+    )[0]
+
+    # Pressure
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="PPPP"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.PRES].append(
+            float(value) / 100
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Air temperature
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="TTT"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.TEMP].append(
+            kelvin_to_celsius(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Dew point
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="Td"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.DWPT].append(
+            kelvin_to_celsius(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Wind direction
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="DD"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WDIR].append(
+            int(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Wind speed
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="FF"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WSPD].append(
+            ms_to_kmh(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Peak wind gust
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="FX1"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WPGT].append(
+            ms_to_kmh(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Weather condition
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="ww"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.COCO].append(
+            get_coco(int(float(value)))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Precipitation
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="RR1c"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.PRCP].append(
+            float(value) if value.lstrip("-").replace(".", "", 1).isdigit() else None
+        )
+
+    # Sunshine Duration
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="SunD1"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.TSUN].append(
+            float(value) / 60
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Cloud Cover
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="N"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.CLDC].append(
+            percentage_to_okta(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Visibility
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="VV"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.VSBY].append(
+            float(value) if value.lstrip("-").replace(".", "", 1).isdigit() else None
+        )
+
+    # Convert data dict to DataFrame
+    df = pd.DataFrame.from_dict(data)
+
+    # Convert time strings to datetime
+    df["time"] = pd.to_datetime(df["time"])
+
+    # Calculate humidity data
+    df[Parameter.RHUM] = df.apply(temp_dwpt_to_rhum, axis=1)
+
+    # Set index
+    df = df.set_index(["time"])
+
+    # Round decimals
+    df = df.round(1)
+
+    # Remove tz awareness
+    df = df.tz_convert(None, level="time")
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "mosmix" in req.station.identifiers:
+        return get_df(req.station.identifiers["mosmix"])