meteostat 1.7.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

meteostat/providers/dwd/poi.py

@@ -0,0 +1,117 @@
+from typing import Optional, Union
+from urllib.error import HTTPError
+
+import pandas as pd
+
+from meteostat.core.cache import cache_service
+from meteostat.core.logger import logger
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import percentage_to_okta
+
+ENDPOINT = "https://opendata.dwd.de/weather/weather_reports/poi/{station}-BEOB.csv"
+# TODO: add col 11 for solar radiation support (Globalstrahlung)
+USECOLS = [0, 1, 2, 9, 14, 21, 22, 23, 33, 35, 36, 37, 40, 41]
+NAMES = {
+    "Wolkenbedeckung": Parameter.CLDC,
+    "Temperatur (2m)": Parameter.TEMP,
+    "Sichtweite": Parameter.VSBY,
+    "Windgeschwindigkeit": Parameter.WSPD,
+    "Windboen (letzte Stunde)": Parameter.WPGT,
+    "Niederschlag (letzte Stunde)": Parameter.PRCP,
+    "Relative Feuchte": Parameter.RHUM,
+    "Windrichtung": Parameter.WDIR,
+    "Druck (auf Meereshoehe)": Parameter.PRES,
+    "Sonnenscheindauer (letzte Stunde)": Parameter.TSUN,
+    "aktuelles Wetter": Parameter.COCO,
+    "Schneehoehe": Parameter.SNWD,
+}
+COCO_MAP = {
+    "1": 1,
+    "2": 2,
+    "3": 3,
+    "4": 4,
+    "5": 5,
+    "6": 6,
+    "7": 7,
+    "8": 8,
+    "9": 9,
+    "10": 10,
+    "11": 11,
+    "12": 12,
+    "13": 13,
+    "14": 14,
+    "15": 15,
+    "16": 16,
+    "17": 24,
+    "18": 17,
+    "19": 18,
+    "20": 19,
+    "21": 20,
+    "22": 21,
+    "23": 22,
+    "24": 19,
+    "25": 20,
+    "26": 23,
+    "27": 25,
+    "28": 26,
+    "29": 25,
+    "30": 26,
+    "31": 27,
+}
+
+
+def get_coco(code: str | int) -> Union[int, None]:
+    """
+    Map DWD POI weather condition codes to Meteostat condicodes
+    """
+    return COCO_MAP.get(str(code))
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    try:
+        # Read CSV data from DWD server
+        df = pd.read_csv(  # type: ignore
+            ENDPOINT.format(station=station),
+            sep=";",
+            skiprows=2,
+            na_values="---",
+            usecols=USECOLS,
+            decimal=",",
+        )
+
+        # Rename columns
+        df = df.rename(columns=NAMES)
+
+        # Snow cm -> mm
+        df[Parameter.SNWD] = df[Parameter.SNWD].multiply(10)
+        df[Parameter.VSBY] = df[Parameter.VSBY].multiply(1000)
+
+        # Change coco
+        df[Parameter.COCO] = df[Parameter.COCO].apply(get_coco)
+        df[Parameter.CLDC] = df[Parameter.CLDC].apply(percentage_to_okta)
+
+        # Set index
+        df["time"] = pd.to_datetime(
+            df["Datum"] + " " + df["Uhrzeit (UTC)"], format="%d.%m.%y %H:%M"
+        )
+        df = df.set_index(["time"])
+        df = df.drop(["Datum", "Uhrzeit (UTC)"], axis=1)
+
+        return df
+
+    except HTTPError as error:
+        logger.info(
+            f"Couldn't load DWD POI data for weather station {station} (status: {error.status})"
+        )
+        return None
+
+    except Exception as error:
+        logger.warning(error)
+        return None
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "wmo" in req.station.identifiers:
+        return get_df(req.station.identifiers["wmo"])

meteostat/providers/dwd/shared.py

@@ -0,0 +1,155 @@
+from typing import Union
+from ftplib import FTP
+
+from meteostat.api.config import config
+
+
+DWD_FTP_SERVER = config.dwd_ftp_host
+
+
+def get_ftp_connection() -> FTP:
+    """
+    Get DWD Open Data FTP connection
+    """
+    dwd_ftp_connection = FTP(DWD_FTP_SERVER)
+    dwd_ftp_connection.login()
+    return dwd_ftp_connection
+
+
+def get_condicode(code: str) -> Union[int, None]:
+    """
+    Map DWD codes to Meteostat condicodes
+
+    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/weather_phenomena/recent/Wetter_Beschreibung.txt
+    """
+    condicodes = {
+        "0": 1,
+        "1": 2,
+        "2": 1,
+        "3": 3,
+        "11": 5,
+        "12": 5,
+        "13": 23,
+        "17": 25,
+        "21": 8,
+        "22": 15,
+        "23": 12,
+        "24": 10,
+        "25": 17,
+        "26": 21,
+        "27": 24,
+        "28": 5,
+        "29": 25,
+        "40": 5,
+        "41": 5,
+        "42": 5,
+        "43": 5,
+        "44": 5,
+        "45": 5,
+        "46": 5,
+        "47": 5,
+        "48": 5,
+        "49": 5,
+        "50": 7,
+        "51": 7,
+        "52": 8,
+        "53": 8,
+        "54": 9,
+        "55": 9,
+        "56": 10,
+        "57": 11,
+        "58": 7,
+        "59": 8,
+        "60": 7,
+        "61": 7,
+        "62": 8,
+        "63": 8,
+        "64": 9,
+        "65": 9,
+        "66": 10,
+        "67": 11,
+        "68": 12,
+        "69": 13,
+        "70": 14,
+        "71": 14,
+        "72": 15,
+        "73": 15,
+        "74": 16,
+        "75": 16,
+        "80": 17,
+        "81": 18,
+        "82": 18,
+        "83": 19,
+        "84": 20,
+        "85": 21,
+        "86": 22,
+        "87": 19,
+        "88": 20,
+        "89": 24,
+        "90": 24,
+        "91": 25,
+        "92": 26,
+        "93": 25,
+        "94": 26,
+        "95": 25,
+        "96": 25,
+        "97": 26,
+        "98": 25,
+        "99": 26,
+        "100": 3,
+        "101": 2,
+        "102": 3,
+        "103": 3,
+        "120": 5,
+        "123": 8,
+        "124": 15,
+        "125": 10,
+        "126": 25,
+        "130": 5,
+        "131": 5,
+        "132": 5,
+        "133": 5,
+        "134": 5,
+        "135": 6,
+        "150": 8,
+        "151": 7,
+        "152": 8,
+        "153": 9,
+        "154": 10,
+        "155": 10,
+        "156": 11,
+        "157": 7,
+        "158": 8,
+        "160": 8,
+        "161": 7,
+        "162": 8,
+        "163": 9,
+        "164": 10,
+        "165": 10,
+        "166": 11,
+        "167": 12,
+        "168": 13,
+        "170": 15,
+        "171": 14,
+        "172": 15,
+        "173": 16,
+        "177": 14,
+        "181": 7,
+        "182": 8,
+        "183": 9,
+        "184": 9,
+        "185": 14,
+        "186": 15,
+        "187": 16,
+        "189": 24,
+        "190": 25,
+        "191": 25,
+        "192": 25,
+        "193": 25,
+        "194": 26,
+        "195": 26,
+        "196": 26,
+        "199": 27,
+    }
+
+    return condicodes.get(str(code), None)

meteostat/providers/eccc/daily.py

@@ -0,0 +1,87 @@
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "LOCAL_DATE": "time",
+    "MAX_TEMPERATURE": Parameter.TMAX,
+    "MEAN_TEMPERATURE": Parameter.TEMP,
+    "MIN_TEMPERATURE": Parameter.TMIN,
+    "SPEED_MAX_GUST": Parameter.WPGT,
+    "TOTAL_PRECIPITATION": Parameter.PRCP,
+    "SNOW_ON_GROUND": Parameter.SNWD,
+    "TOTAL_SNOW": Parameter.SNOW,
+}
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(climate_id: str, year: int) -> Optional[pd.DataFrame]:
+    # Process start & end date
+    # ECCC uses the station's local time zone
+    start = datetime(year, 1, 1, 0, 0, 0).strftime("%Y-%m-%dT%H:%M:%S")
+    end = datetime(year, 12, 31, 23, 59, 59).strftime("%Y-%m-%dT%H:%M:%S")
+
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-daily/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "datetime": f"{start}/{end}",
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if (
+        "national" not in req.station.identifiers
+        or req.start is None
+        or req.end is None
+    ):
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+    archive_first = meta_data.get("DLY_FIRST_DATE")
+    archive_last = meta_data.get("DLY_LAST_DATE")
+
+    if not (climate_id and archive_first and archive_last):
+        return None
+
+    archive_start = datetime.strptime(archive_first, "%Y-%m-%d %H:%M:%S")
+    archive_end = datetime.strptime(archive_last, "%Y-%m-%d %H:%M:%S")
+
+    years = range(
+        max(req.start.year, archive_start.year),
+        min(req.end.year, archive_end.year) + 1,
+    )
+    data = [get_df(climate_id, year) for year in years]
+
+    return pd.concat(data) if len(data) and not all(d is None for d in data) else None

meteostat/providers/eccc/hourly.py

@@ -0,0 +1,104 @@
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+import pytz
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "UTC_DATE": "time",
+    "RELATIVE_HUMIDITY": Parameter.RHUM,
+    "WIND_DIRECTION": Parameter.WDIR,
+    "WIND_SPEED": Parameter.WSPD,
+    "VISIBILITY": Parameter.VSBY,
+    "PRECIP_AMOUNT": Parameter.PRCP,
+    "TEMP": Parameter.TEMP,
+}
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(climate_id: str, year: int, tz: str) -> Optional[pd.DataFrame]:
+    # Process start & end date
+    # ECCC uses the station's local time zone
+    from_timezone = pytz.timezone("UTC")
+    to_timezone = pytz.timezone(tz)
+    start = (
+        from_timezone.localize(datetime(year, 1, 1, 0, 0, 0))
+        .astimezone(to_timezone)
+        .strftime("%Y-%m-%dT%H:%M:%S")
+    )
+    end = (
+        from_timezone.localize(datetime(year, 12, 31, 23, 59, 59))
+        .astimezone(to_timezone)
+        .strftime("%Y-%m-%dT%H:%M:%S")
+    )
+
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-hourly/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "datetime": f"{start}/{end}",
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    # Convert data units
+    df[Parameter.WDIR] = df[Parameter.WDIR] * 10  # Wind direction is provided 10's deg
+    df[Parameter.VSBY] = (
+        df[Parameter.VSBY] * 1000
+    )  # Visibility is provided in kilometres
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if (
+        "national" not in req.station.identifiers
+        or req.start is None
+        or req.end is None
+    ):
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+    archive_first = meta_data.get("HLY_FIRST_DATE")
+    archive_last = meta_data.get("HLY_LAST_DATE")
+    timezone = meta_data.get("TIMEZONE")
+
+    if not (climate_id and archive_first and archive_last and timezone):
+        return None
+
+    archive_start = datetime.strptime(archive_first, "%Y-%m-%d %H:%M:%S")
+    archive_end = datetime.strptime(archive_last, "%Y-%m-%d %H:%M:%S")
+
+    years = range(
+        max(req.start.year, archive_start.year),
+        min(req.end.year, archive_end.year) + 1,
+    )
+    data = [get_df(climate_id, year, timezone) for year in years]
+
+    return pd.concat(data) if len(data) and not all(d is None for d in data) else None

meteostat/providers/eccc/monthly.py

@@ -0,0 +1,66 @@
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "LOCAL_DATE": "time",
+    "MAX_TEMPERATURE": Parameter.TXMX,
+    "MEAN_TEMPERATURE": Parameter.TEMP,
+    "MIN_TEMPERATURE": Parameter.TXMN,
+    "TOTAL_PRECIPITATION": Parameter.PRCP,
+    "DAYS_WITH_PRECIP_GE_1MM": Parameter.PDAY,
+    "TOTAL_SNOWFALL": Parameter.SNOW,
+}
+
+
+@cache_service.cache(TTL.WEEK, "pickle")
+def get_df(
+    climate_id: str,
+) -> Optional[pd.DataFrame]:
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-monthly/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "national" not in req.station.identifiers:
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+
+    if not climate_id:
+        return None
+
+    return get_df(climate_id)

meteostat/providers/eccc/shared.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from meteostat.enumerations import TTL
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.core.logger import logger
+
+
+ENDPOINT = "https://api.weather.gc.ca"
+
+
+@cache_service.cache(TTL.WEEK)
+def get_meta_data(station: str) -> Optional[dict]:
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-stations/items",
+        params={
+            "STN_ID": station,
+            "properties": ",".join(
+                [
+                    "CLIMATE_IDENTIFIER",
+                    "TIMEZONE",
+                    "HLY_FIRST_DATE",
+                    "HLY_LAST_DATE",
+                    "DLY_FIRST_DATE",
+                    "DLY_LAST_DATE",
+                    "MLY_FIRST_DATE",
+                    "MLY_LAST_DATE",
+                ]
+            ),
+            "f": "json",
+        },
+    )
+
+    if response.status_code == 200:
+        data = response.json()
+        try:
+            return data["features"][0]["properties"]
+        except (IndexError, KeyError):
+            logger.info(f"ECCC climate identifier for station {station} not found")
+    else:
+        logger.warning(
+            f"ECCC climate identifier for station {station} not found (status: {response.status_code})"
+        )
+
+    return None