meteostat 1.7.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/providers/eccc/hourly.py

@@ -0,0 +1,104 @@
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+import pytz
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "UTC_DATE": "time",
+    "RELATIVE_HUMIDITY": Parameter.RHUM,
+    "WIND_DIRECTION": Parameter.WDIR,
+    "WIND_SPEED": Parameter.WSPD,
+    "VISIBILITY": Parameter.VSBY,
+    "PRECIP_AMOUNT": Parameter.PRCP,
+    "TEMP": Parameter.TEMP,
+}
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(climate_id: str, year: int, tz: str) -> Optional[pd.DataFrame]:
+    # Process start & end date
+    # ECCC uses the station's local time zone
+    from_timezone = pytz.timezone("UTC")
+    to_timezone = pytz.timezone(tz)
+    start = (
+        from_timezone.localize(datetime(year, 1, 1, 0, 0, 0))
+        .astimezone(to_timezone)
+        .strftime("%Y-%m-%dT%H:%M:%S")
+    )
+    end = (
+        from_timezone.localize(datetime(year, 12, 31, 23, 59, 59))
+        .astimezone(to_timezone)
+        .strftime("%Y-%m-%dT%H:%M:%S")
+    )
+
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-hourly/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "datetime": f"{start}/{end}",
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    # Convert data units
+    df[Parameter.WDIR] = df[Parameter.WDIR] * 10  # Wind direction is provided 10's deg
+    df[Parameter.VSBY] = (
+        df[Parameter.VSBY] * 1000
+    )  # Visibility is provided in kilometres
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if (
+        "national" not in req.station.identifiers
+        or req.start is None
+        or req.end is None
+    ):
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+    archive_first = meta_data.get("HLY_FIRST_DATE")
+    archive_last = meta_data.get("HLY_LAST_DATE")
+    timezone = meta_data.get("TIMEZONE")
+
+    if not (climate_id and archive_first and archive_last and timezone):
+        return None
+
+    archive_start = datetime.strptime(archive_first, "%Y-%m-%d %H:%M:%S")
+    archive_end = datetime.strptime(archive_last, "%Y-%m-%d %H:%M:%S")
+
+    years = range(
+        max(req.start.year, archive_start.year),
+        min(req.end.year, archive_end.year) + 1,
+    )
+    data = [get_df(climate_id, year, timezone) for year in years]
+
+    return pd.concat(data) if len(data) and not all(d is None for d in data) else None

meteostat/providers/eccc/monthly.py

@@ -0,0 +1,66 @@
+from typing import Optional
+
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.providers.eccc.shared import ENDPOINT, get_meta_data
+from meteostat.typing import ProviderRequest
+
+BATCH_LIMIT = 9000
+PROPERTIES = {
+    "LOCAL_DATE": "time",
+    "MAX_TEMPERATURE": Parameter.TXMX,
+    "MEAN_TEMPERATURE": Parameter.TEMP,
+    "MIN_TEMPERATURE": Parameter.TXMN,
+    "TOTAL_PRECIPITATION": Parameter.PRCP,
+    "DAYS_WITH_PRECIP_GE_1MM": Parameter.PDAY,
+    "TOTAL_SNOWFALL": Parameter.SNOW,
+}
+
+
+@cache_service.cache(TTL.WEEK, "pickle")
+def get_df(
+    climate_id: str,
+) -> Optional[pd.DataFrame]:
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-monthly/items",
+        params={
+            "CLIMATE_IDENTIFIER": climate_id,
+            "f": "json",
+            "properties": ",".join(PROPERTIES.keys()),
+            "limit": BATCH_LIMIT,
+        },
+    )
+
+    data = response.json()
+
+    # Extract features from the response
+    features = map(
+        lambda feature: feature["properties"] if "properties" in feature else {},
+        data.get("features", []),
+    )
+
+    # Create a DataFrame from the extracted features
+    df = pd.DataFrame(features)
+    df = df.rename(columns=PROPERTIES)
+
+    # Handle time column & set index
+    df["time"] = pd.to_datetime(df["time"])
+    df = df.set_index(["time"])
+
+    return df
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "national" not in req.station.identifiers:
+        return None
+
+    meta_data = get_meta_data(req.station.identifiers["national"])
+    climate_id = meta_data.get("CLIMATE_IDENTIFIER")
+
+    if not climate_id:
+        return None
+
+    return get_df(climate_id)

meteostat/providers/eccc/shared.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from meteostat.enumerations import TTL
+from meteostat.core.cache import cache_service
+from meteostat.core.network import network_service
+from meteostat.core.logger import logger
+
+
+ENDPOINT = "https://api.weather.gc.ca"
+
+
+@cache_service.cache(TTL.WEEK)
+def get_meta_data(station: str) -> Optional[dict]:
+    response = network_service.get(
+        f"{ENDPOINT}/collections/climate-stations/items",
+        params={
+            "STN_ID": station,
+            "properties": ",".join(
+                [
+                    "CLIMATE_IDENTIFIER",
+                    "TIMEZONE",
+                    "HLY_FIRST_DATE",
+                    "HLY_LAST_DATE",
+                    "DLY_FIRST_DATE",
+                    "DLY_LAST_DATE",
+                    "MLY_FIRST_DATE",
+                    "MLY_LAST_DATE",
+                ]
+            ),
+            "f": "json",
+        },
+    )
+
+    if response.status_code == 200:
+        data = response.json()
+        try:
+            return data["features"][0]["properties"]
+        except (IndexError, KeyError):
+            logger.info(f"ECCC climate identifier for station {station} not found")
+    else:
+        logger.warning(
+            f"ECCC climate identifier for station {station} not found (status: {response.status_code})"
+        )
+
+    return None

meteostat/providers/index.py

@@ -0,0 +1,496 @@
+"""
+Provider Index
+
+This module contains a list of all available data providers.
+"""
+
+from datetime import date, timedelta
+
+from meteostat.typing import License, ProviderSpec
+from meteostat.enumerations import Grade, Granularity, Parameter, Provider, Priority
+
+PROVIDER_HOURLY = ProviderSpec(
+    id=Provider.HOURLY,
+    name="Meteostat Hourly",
+    granularity=Granularity.HOURLY,
+    priority=Priority.NONE,
+    grade=None,
+    license=None,
+    requires=["id"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.SNWD,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.CLDC,
+        Parameter.COCO,
+    ],
+    start=date(1931, 1, 1),
+    module="meteostat.providers.meteostat.hourly",
+)
+
+PROVIDER_DAILY = ProviderSpec(
+    id=Provider.DAILY,
+    name="Meteostat Daily",
+    granularity=Granularity.DAILY,
+    priority=Priority.NONE,
+    grade=None,
+    license=None,
+    requires=["id"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.SNWD,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.CLDC,
+    ],
+    start=date(1899, 1, 1),
+    module="meteostat.providers.meteostat.daily",
+)
+
+PROVIDER_DAILY_DERIVED = ProviderSpec(
+    id=Provider.DAILY_DERIVED,
+    name="Meteostat Daily Derived",
+    granularity=Granularity.DAILY,
+    priority=Priority.NONE,
+    grade=None,
+    license=None,
+    requires=["id"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.SNOW,
+        Parameter.WSPD,
+        Parameter.WDIR,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.CLDC,
+    ],
+    start=date(1931, 1, 1),
+    module="meteostat.providers.meteostat.daily_derived",
+)
+
+PROVIDER_MONTHLY = ProviderSpec(
+    id=Provider.MONTHLY,
+    name="Meteostat Monthly",
+    granularity=Granularity.MONTHLY,
+    priority=Priority.NONE,
+    grade=None,
+    license=None,
+    requires=["id"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.PRES,
+        Parameter.TSUN,
+    ],
+    start=date(1899, 1, 1),
+    module="meteostat.providers.meteostat.monthly",
+)
+
+PROVIDER_MONTHLY_DERIVED = ProviderSpec(
+    id=Provider.MONTHLY_DERIVED,
+    name="Meteostat Monthly Derived",
+    granularity=Granularity.MONTHLY,
+    priority=Priority.NONE,
+    grade=None,
+    license=None,
+    requires=["id"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.PRES,
+        Parameter.TSUN,
+    ],
+    start=date(1899, 1, 1),
+    module="meteostat.providers.meteostat.monthly_derived",
+)
+
+PROVIDER_DWD_HOURLY = ProviderSpec(
+    id=Provider.DWD_HOURLY,
+    name="DWD Hourly",
+    granularity=Granularity.HOURLY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    countries=["DE"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.VSBY,
+        Parameter.COCO,
+    ],
+    start=date(1939, 1, 1),
+    module="meteostat.providers.dwd.hourly",
+)
+
+PROVIDER_DWD_POI = ProviderSpec(
+    id=Provider.DWD_POI,
+    name="DWD POI",
+    granularity=Granularity.HOURLY,
+    priority=Priority.MEDIUM,
+    grade=Grade.OBSERVATION,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    parameters=[
+        Parameter.CLDC,
+        Parameter.TEMP,
+        Parameter.VSBY,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRCP,
+        Parameter.RHUM,
+        Parameter.WDIR,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.COCO,
+        Parameter.SNWD,
+    ],
+    start=date.today() - timedelta(days=1),
+    module="meteostat.providers.dwd.poi",
+)
+
+PROVIDER_DWD_MOSMIX = ProviderSpec(
+    id=Provider.DWD_MOSMIX,
+    name="DWD MOSMIX",
+    granularity=Granularity.HOURLY,
+    priority=Priority.LOWEST,
+    grade=Grade.FORECAST,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.CLDC,
+        Parameter.COCO,
+    ],
+    start=date.today(),
+    module="meteostat.providers.dwd.mosmix",
+)
+
+PROVIDER_DWD_DAILY = ProviderSpec(
+    id=Provider.DWD_DAILY,
+    name="DWD Daily",
+    granularity=Granularity.DAILY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    countries=["DE"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.SNWD,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.TSUN,
+        Parameter.CLDC,
+    ],
+    start=date(1781, 1, 1),
+    module="meteostat.providers.dwd.daily",
+)
+
+PROVIDER_DWD_MONTHLY = ProviderSpec(
+    id=Provider.DWD_MONTHLY,
+    name="DWD Monthly",
+    granularity=Granularity.MONTHLY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    countries=["DE"],
+    parameters=[Parameter.TEMP, Parameter.TMIN, Parameter.TMAX, Parameter.PRCP],
+    start=date(1851, 1, 1),
+    module="meteostat.providers.dwd.monthly",
+)
+
+PROVIDER_ECCC_HOURLY = ProviderSpec(
+    id=Provider.ECCC_HOURLY,
+    name="ECCC Hourly",
+    granularity=Granularity.HOURLY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Environment and Climate Change Canada",
+        name="Environment and Climate Change Canada Data Servers End-use Licence",
+        url="https://eccc-msc.github.io/open-data/licence/readme_en/",
+    ),
+    requires=["identifiers"],
+    countries=["CA"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.VSBY,
+    ],
+    start=date(1939, 1, 1),
+    module="meteostat.providers.eccc.hourly",
+)
+
+PROVIDER_ECCC_DAILY = ProviderSpec(
+    id=Provider.ECCC_DAILY,
+    name="ECCC Daily",
+    granularity=Granularity.DAILY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Environment and Climate Change Canada",
+        name="Environment and Climate Change Canada Data Servers End-use Licence",
+        url="https://eccc-msc.github.io/open-data/licence/readme_en/",
+    ),
+    requires=["identifiers"],
+    countries=["CA"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.SNOW,
+        Parameter.SNWD,
+        Parameter.WPGT,
+    ],
+    start=date(1939, 1, 1),
+    module="meteostat.providers.eccc.daily",
+)
+
+PROVIDER_ECCC_MONTHLY = ProviderSpec(
+    id=Provider.ECCC_MONTHLY,
+    name="ECCC Monthly",
+    granularity=Granularity.MONTHLY,
+    priority=Priority.HIGHEST,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Environment and Climate Change Canada",
+        name="Environment and Climate Change Canada Data Servers End-use Licence",
+        url="https://eccc-msc.github.io/open-data/licence/readme_en/",
+    ),
+    requires=["identifiers"],
+    countries=["CA"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.PDAY,
+        Parameter.SNOW,
+    ],
+    start=date(1939, 1, 1),
+    module="meteostat.providers.eccc.monthly",
+)
+
+PROVIDER_ISD_LITE = ProviderSpec(
+    id=Provider.ISD_LITE,
+    name="NOAA ISD Lite",
+    granularity=Granularity.HOURLY,
+    priority=Priority.HIGH,
+    grade=Grade.OBSERVATION,
+    license=License(
+        commercial=True,
+        name="CC0 1.0 Universal",
+        url="https://creativecommons.org/publicdomain/zero/1.0/",
+    ),  # source: https://registry.opendata.aws/noaa-isd/
+    requires=["identifiers"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.CLDC,
+    ],
+    start=date(1931, 1, 1),
+    module="meteostat.providers.noaa.isd_lite",
+)
+
+PROVIDER_GHCND = ProviderSpec(
+    id=Provider.GHCND,
+    name="NOAA GHCN-Daily",
+    granularity=Granularity.DAILY,
+    priority=Priority.HIGH,
+    grade=Grade.OBSERVATION,
+    license=License(
+        commercial=True,
+        name="CC0 1.0 Universal",
+        url="https://creativecommons.org/publicdomain/zero/1.0/",
+    ),  # source: https://registry.opendata.aws/noaa-ghcn/
+    requires=["identifiers"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.PRCP,
+        Parameter.SNOW,
+        Parameter.WSPD,
+        Parameter.WDIR,
+        Parameter.WPGT,
+        Parameter.TSUN,
+        Parameter.CLDC,
+    ],
+    start=date(1931, 1, 1),
+    module="meteostat.providers.noaa.ghcnd",
+)
+
+PROVIDER_CLIMAT = ProviderSpec(
+    id=Provider.CLIMAT,
+    name="DWD CLIMAT",
+    granularity=Granularity.MONTHLY,
+    priority=Priority.HIGH,
+    grade=Grade.RECORD,
+    license=License(
+        commercial=True,
+        attribution="Deutscher Wetterdienst",
+        name="CC BY 4.0",
+        url="https://creativecommons.org/licenses/by/4.0/",
+    ),
+    requires=["identifiers"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.TMIN,
+        Parameter.TMAX,
+        Parameter.TXMN,
+        Parameter.TXMX,
+        Parameter.PRES,
+        Parameter.PRCP,
+        Parameter.TSUN,
+    ],
+    start=date(1875, 6, 1),
+    module="meteostat.providers.dwd.climat",
+)
+
+PROVIDER_METAR = ProviderSpec(
+    id=Provider.METAR,
+    name="METAR",
+    granularity=Granularity.HOURLY,
+    priority=Priority.LOW,
+    grade=Grade.OBSERVATION,
+    license=License(
+        commercial=True,
+    ),
+    requires=["identifiers"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.PRES,
+        Parameter.COCO,
+    ],
+    start=date.today() - timedelta(days=1),
+    module="meteostat.providers.noaa.metar",
+)
+
+PROVIDER_METNO_FORECAST = ProviderSpec(
+    id=Provider.METNO_FORECAST,
+    name="MET Norway Forecast",
+    granularity=Granularity.HOURLY,
+    priority=Priority.LOWEST - 1,  # MOSMIX is more specific, therefore higher prio
+    grade=Grade.FORECAST,
+    license=License(
+        commercial=True,
+        attribution="MET Norway",
+        name="NLOD 2.0",
+        url="https://data.norge.no/nlod/en/2.0",
+    ),
+    requires=["location"],
+    parameters=[
+        Parameter.TEMP,
+        Parameter.RHUM,
+        Parameter.PRCP,
+        Parameter.WDIR,
+        Parameter.WSPD,
+        Parameter.WPGT,
+        Parameter.PRES,
+        Parameter.CLDC,
+        Parameter.COCO,
+    ],
+    start=date.today(),
+    module="meteostat.providers.metno.forecast",
+)
+
+
+DEFAULT_PROVIDERS = [
+    PROVIDER_HOURLY,
+    PROVIDER_DAILY,
+    PROVIDER_DAILY_DERIVED,
+    PROVIDER_MONTHLY,
+    PROVIDER_MONTHLY_DERIVED,
+    PROVIDER_DWD_HOURLY,
+    PROVIDER_DWD_POI,
+    PROVIDER_DWD_MOSMIX,
+    PROVIDER_DWD_DAILY,
+    PROVIDER_DWD_MONTHLY,
+    PROVIDER_ECCC_HOURLY,
+    PROVIDER_ECCC_DAILY,
+    PROVIDER_ECCC_MONTHLY,
+    PROVIDER_ISD_LITE,
+    PROVIDER_GHCND,
+    PROVIDER_CLIMAT,
+    PROVIDER_METAR,
+    PROVIDER_METNO_FORECAST,
+]