meteostat 1.7.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/providers/dwd/mosmix.py

@@ -0,0 +1,351 @@
+"""
+DWD MOSMIX data provider
+
+Parameters: https://www.dwd.de/DE/leistungen/met_verfahren_mosmix/mosmix_parameteruebersicht.pdf?__blob=publicationFile&v=3
+"""
+
+import re
+from io import BytesIO
+from typing import Optional
+from datetime import datetime
+from zipfile import ZipFile
+from lxml import etree  # type: ignore
+
+import pandas as pd
+
+from meteostat.core.cache import cache_service
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import (
+    kelvin_to_celsius,
+    ms_to_kmh,
+    percentage_to_okta,
+    temp_dwpt_to_rhum,
+)
+from meteostat.core.network import network_service
+
+ENDPOINT = "https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/single_stations/{station}/kml/MOSMIX_L_LATEST_{station}.kmz"
+COCO_MAP = {
+    "0": 1,
+    "1": 2,
+    "2": 3,
+    "3": 4,
+    "45": 5,
+    "49": 5,
+    "61": 7,
+    "63": 8,
+    "65": 9,
+    "51": 7,
+    "53": 8,
+    "55": 9,
+    "68": 12,
+    "69": 13,
+    "71": 14,
+    "73": 15,
+    "75": 16,
+    "80": 17,
+    "81": 18,
+    "82": 18,
+    "83": 19,
+    "84": 20,
+    "85": 21,
+    "86": 22,
+    "66": 10,
+    "67": 11,
+    "56": 10,
+    "57": 11,
+    "95": 25,
+}
+
+
+def get_coco(code: str | int) -> Optional[int]:
+    """
+    Map DWD MOSMIX weather condition codes to Meteostat condicodes
+    """
+    return COCO_MAP.get(str(code))
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    # Fetch the KMZ file data in memory
+    response = network_service.get(ENDPOINT.format(station=station))
+    kmz_data = BytesIO(response.content)
+
+    # KMZ -> KML in memory
+    with ZipFile(kmz_data, "r") as kmz:
+        with kmz.open(kmz.infolist()[0].filename, "r") as raw:
+            kml = raw.read()
+
+    # Parse KML
+    tree = etree.fromstring(kml)
+
+    # Skip stale forecasts
+    issue_time = datetime.strptime(
+        tree.xpath(
+            "//kml:kml/kml:Document/kml:ExtendedData/"
+            + "dwd:ProductDefinition/dwd:IssueTime",
+            namespaces=tree.nsmap,
+        )[0].text,
+        "%Y-%m-%dT%H:%M:%S.%fZ",
+    )
+    if (datetime.now() - issue_time).total_seconds() > 25200:
+        return None
+
+    # Collect all time steps
+    timesteps = []
+    for step in tree.xpath(
+        "//kml:kml/kml:Document/kml:ExtendedData/dwd:ProductDefinition/"
+        + "dwd:ForecastTimeSteps/dwd:TimeStep",
+        namespaces=tree.nsmap,
+    ):
+        timesteps.append(step.text)
+
+    # COLLECT WEATHER DATA
+    # Each parameter is processed individually
+    data = {
+        "time": timesteps,
+        Parameter.TEMP: [],
+        Parameter.DWPT: [],
+        Parameter.PRCP: [],
+        Parameter.WDIR: [],
+        Parameter.WSPD: [],
+        Parameter.WPGT: [],
+        Parameter.TSUN: [],
+        Parameter.PRES: [],
+        Parameter.CLDC: [],
+        Parameter.VSBY: [],
+        Parameter.COCO: [],
+    }
+    placemark = tree.xpath(
+        "//kml:kml/kml:Document/kml:Placemark", namespaces=tree.nsmap
+    )[0]
+
+    # Pressure
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="PPPP"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.PRES].append(
+            float(value) / 100
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Air temperature
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="TTT"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.TEMP].append(
+            kelvin_to_celsius(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Dew point
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="Td"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.DWPT].append(
+            kelvin_to_celsius(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Wind direction
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="DD"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WDIR].append(
+            int(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Wind speed
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="FF"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WSPD].append(
+            ms_to_kmh(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Peak wind gust
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="FX1"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.WPGT].append(
+            ms_to_kmh(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Weather condition
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="ww"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.COCO].append(
+            get_coco(int(float(value)))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Precipitation
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="RR1c"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.PRCP].append(
+            float(value) if value.lstrip("-").replace(".", "", 1).isdigit() else None
+        )
+
+    # Sunshine Duration
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="SunD1"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.TSUN].append(
+            float(value) / 60
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Cloud Cover
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="N"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.CLDC].append(
+            percentage_to_okta(float(value))
+            if value.lstrip("-").replace(".", "", 1).isdigit()
+            else None
+        )
+
+    # Visibility
+    for value in (
+        re.sub(
+            r"/\s+/",
+            " ",
+            placemark.xpath(
+                'kml:ExtendedData/dwd:Forecast[@dwd:elementName="VV"]/dwd:value',
+                namespaces=tree.nsmap,
+            )[0].text,
+        )
+        .strip()
+        .split()
+    ):
+        data[Parameter.VSBY].append(
+            float(value) if value.lstrip("-").replace(".", "", 1).isdigit() else None
+        )
+
+    # Convert data dict to DataFrame
+    df = pd.DataFrame.from_dict(data)
+
+    # Convert time strings to datetime
+    df["time"] = pd.to_datetime(df["time"])
+
+    # Calculate humidity data
+    df[Parameter.RHUM] = df.apply(temp_dwpt_to_rhum, axis=1)
+
+    # Set index
+    df = df.set_index(["time"])
+
+    # Round decimals
+    df = df.round(1)
+
+    # Remove tz awareness
+    df = df.tz_convert(None, level="time")
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "mosmix" in req.station.identifiers:
+        return get_df(req.station.identifiers["mosmix"])

meteostat/providers/dwd/poi.py

@@ -0,0 +1,117 @@
+from typing import Optional, Union
+from urllib.error import HTTPError
+
+import pandas as pd
+
+from meteostat.core.cache import cache_service
+from meteostat.core.logger import logger
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import percentage_to_okta
+
+ENDPOINT = "https://opendata.dwd.de/weather/weather_reports/poi/{station}-BEOB.csv"
+# TODO: add col 11 for solar radiation support (Globalstrahlung)
+USECOLS = [0, 1, 2, 9, 14, 21, 22, 23, 33, 35, 36, 37, 40, 41]
+NAMES = {
+    "Wolkenbedeckung": Parameter.CLDC,
+    "Temperatur (2m)": Parameter.TEMP,
+    "Sichtweite": Parameter.VSBY,
+    "Windgeschwindigkeit": Parameter.WSPD,
+    "Windboen (letzte Stunde)": Parameter.WPGT,
+    "Niederschlag (letzte Stunde)": Parameter.PRCP,
+    "Relative Feuchte": Parameter.RHUM,
+    "Windrichtung": Parameter.WDIR,
+    "Druck (auf Meereshoehe)": Parameter.PRES,
+    "Sonnenscheindauer (letzte Stunde)": Parameter.TSUN,
+    "aktuelles Wetter": Parameter.COCO,
+    "Schneehoehe": Parameter.SNWD,
+}
+COCO_MAP = {
+    "1": 1,
+    "2": 2,
+    "3": 3,
+    "4": 4,
+    "5": 5,
+    "6": 6,
+    "7": 7,
+    "8": 8,
+    "9": 9,
+    "10": 10,
+    "11": 11,
+    "12": 12,
+    "13": 13,
+    "14": 14,
+    "15": 15,
+    "16": 16,
+    "17": 24,
+    "18": 17,
+    "19": 18,
+    "20": 19,
+    "21": 20,
+    "22": 21,
+    "23": 22,
+    "24": 19,
+    "25": 20,
+    "26": 23,
+    "27": 25,
+    "28": 26,
+    "29": 25,
+    "30": 26,
+    "31": 27,
+}
+
+
+def get_coco(code: str | int) -> Union[int, None]:
+    """
+    Map DWD POI weather condition codes to Meteostat condicodes
+    """
+    return COCO_MAP.get(str(code))
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    try:
+        # Read CSV data from DWD server
+        df = pd.read_csv(  # type: ignore
+            ENDPOINT.format(station=station),
+            sep=";",
+            skiprows=2,
+            na_values="---",
+            usecols=USECOLS,
+            decimal=",",
+        )
+
+        # Rename columns
+        df = df.rename(columns=NAMES)
+
+        # Snow cm -> mm
+        df[Parameter.SNWD] = df[Parameter.SNWD].multiply(10)
+        df[Parameter.VSBY] = df[Parameter.VSBY].multiply(1000)
+
+        # Change coco
+        df[Parameter.COCO] = df[Parameter.COCO].apply(get_coco)
+        df[Parameter.CLDC] = df[Parameter.CLDC].apply(percentage_to_okta)
+
+        # Set index
+        df["time"] = pd.to_datetime(
+            df["Datum"] + " " + df["Uhrzeit (UTC)"], format="%d.%m.%y %H:%M"
+        )
+        df = df.set_index(["time"])
+        df = df.drop(["Datum", "Uhrzeit (UTC)"], axis=1)
+
+        return df
+
+    except HTTPError as error:
+        logger.info(
+            f"Couldn't load DWD POI data for weather station {station} (status: {error.status})"
+        )
+        return None
+
+    except Exception as error:
+        logger.warning(error)
+        return None
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "wmo" in req.station.identifiers:
+        return get_df(req.station.identifiers["wmo"])

meteostat/providers/dwd/shared.py

@@ -0,0 +1,155 @@
+from typing import Union
+from ftplib import FTP
+
+from meteostat.core.config import config
+
+
+DWD_FTP_SERVER = config.dwd_ftp_host
+
+
+def get_ftp_connection() -> FTP:
+    """
+    Get DWD Open Data FTP connection
+    """
+    dwd_ftp_connection = FTP(DWD_FTP_SERVER)
+    dwd_ftp_connection.login()
+    return dwd_ftp_connection
+
+
+def get_condicode(code: str) -> Union[int, None]:
+    """
+    Map DWD codes to Meteostat condicodes
+
+    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/weather_phenomena/recent/Wetter_Beschreibung.txt
+    """
+    condicodes = {
+        "0": 1,
+        "1": 2,
+        "2": 1,
+        "3": 3,
+        "11": 5,
+        "12": 5,
+        "13": 23,
+        "17": 25,
+        "21": 8,
+        "22": 15,
+        "23": 12,
+        "24": 10,
+        "25": 17,
+        "26": 21,
+        "27": 24,
+        "28": 5,
+        "29": 25,
+        "40": 5,
+        "41": 5,
+        "42": 5,
+        "43": 5,
+        "44": 5,
+        "45": 5,
+        "46": 5,
+        "47": 5,
+        "48": 5,
+        "49": 5,
+        "50": 7,
+        "51": 7,
+        "52": 8,
+        "53": 8,
+        "54": 9,
+        "55": 9,
+        "56": 10,
+        "57": 11,
+        "58": 7,
+        "59": 8,
+        "60": 7,
+        "61": 7,
+        "62": 8,
+        "63": 8,
+        "64": 9,
+        "65": 9,
+        "66": 10,
+        "67": 11,
+        "68": 12,
+        "69": 13,
+        "70": 14,
+        "71": 14,
+        "72": 15,
+        "73": 15,
+        "74": 16,
+        "75": 16,
+        "80": 17,
+        "81": 18,
+        "82": 18,
+        "83": 19,
+        "84": 20,
+        "85": 21,
+        "86": 22,
+        "87": 19,
+        "88": 20,
+        "89": 24,
+        "90": 24,
+        "91": 25,
+        "92": 26,
+        "93": 25,
+        "94": 26,
+        "95": 25,
+        "96": 25,
+        "97": 26,
+        "98": 25,
+        "99": 26,
+        "100": 3,
+        "101": 2,
+        "102": 3,
+        "103": 3,
+        "120": 5,
+        "123": 8,
+        "124": 15,
+        "125": 10,
+        "126": 25,
+        "130": 5,
+        "131": 5,
+        "132": 5,
+        "133": 5,
+        "134": 5,
+        "135": 6,
+        "150": 8,
+        "151": 7,
+        "152": 8,
+        "153": 9,
+        "154": 10,
+        "155": 10,
+        "156": 11,
+        "157": 7,
+        "158": 8,
+        "160": 8,
+        "161": 7,
+        "162": 8,
+        "163": 9,
+        "164": 10,
+        "165": 10,
+        "166": 11,
+        "167": 12,
+        "168": 13,
+        "170": 15,
+        "171": 14,
+        "172": 15,
+        "173": 16,
+        "177": 14,
+        "181": 7,
+        "182": 8,
+        "183": 9,
+        "184": 9,
+        "185": 14,
+        "186": 15,
+        "187": 16,
+        "189": 24,
+        "190": 25,
+        "191": 25,
+        "192": 25,
+        "193": 25,
+        "194": 26,
+        "195": 26,
+        "196": 26,
+        "199": 27,
+    }
+
+    return condicodes.get(str(code), None)

meteostat/providers/eccc/daily.py

@@ -0,0 +1,87 @@
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "LOCAL_DATE": "time",
+    "MAX_TEMPERATURE": Parameter.TMAX,
+    "MEAN_TEMPERATURE": Parameter.TEMP,
+    "MIN_TEMPERATURE": Parameter.TMIN,
+    "SPEED_MAX_GUST": Parameter.WPGT,
+    "TOTAL_PRECIPITATION": Parameter.PRCP,
+    "SNOW_ON_GROUND": Parameter.SNWD,
+    "TOTAL_SNOW": Parameter.SNOW,
+}
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(climate_id: str, year: int) -> Optional[pd.DataFrame]:
+    # Process start & end date
+    # ECCC uses the station's local time zone
+    start = datetime(year, 1, 1, 0, 0, 0).strftime("%Y-%m-%dT%H:%M:%S")
+    end = datetime(year, 12, 31, 23, 59, 59).strftime("%Y-%m-%dT%H:%M:%S")
+
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-daily/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "datetime": f"{start}/{end}",
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if (
+        "national" not in req.station.identifiers
+        or req.start is None
+        or req.end is None
+    ):
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+    archive_first = meta_data.get("DLY_FIRST_DATE")
+    archive_last = meta_data.get("DLY_LAST_DATE")
+
+    if not (climate_id and archive_first and archive_last):
+        return None
+
+    archive_start = datetime.strptime(archive_first, "%Y-%m-%d %H:%M:%S")
+    archive_end = datetime.strptime(archive_last, "%Y-%m-%d %H:%M:%S")
+
+    years = range(
+        max(req.start.year, archive_start.year),
+        min(req.end.year, archive_end.year) + 1,
+    )
+    data = [get_df(climate_id, year) for year in years]
+
+    return pd.concat(data) if len(data) and not all(d is None for d in data) else None