meteostat 1.7.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

meteostat/providers/noaa/ghcnd.py

@@ -0,0 +1,228 @@
+"""
+Process GHCND data
+
+This code is based on an implementation by
+Aaron Penne (https://github.com/aaronpenne).
+
+The code is licensed under the MIT license.
+"""
+
+from io import StringIO
+from ftplib import FTP
+from typing import Optional
+
+from numpy import nan
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.core.cache import cache_service
+from meteostat.utils.conversions import ms_to_kmh, percentage_to_okta
+
+FTP_SERVER = "ftp.ncdc.noaa.gov"
+COLUMN_NAMES = {
+    "MM/DD/YYYY": "time",
+    "TMAX": Parameter.TMAX,
+    "TMIN": Parameter.TMIN,
+    "TAVG": Parameter.TEMP,
+    "PRCP": Parameter.PRCP,
+    "SNWD": Parameter.SNWD,
+    "AWDR": Parameter.WDIR,
+    "AWND": Parameter.WSPD,
+    "TSUN": Parameter.TSUN,
+    "WSFG": Parameter.WPGT,
+    "ACMC": Parameter.CLDC,
+}
+
+
+def connect_to_ftp():
+    """
+    Connect to FTP server
+    """
+    ftp = FTP(FTP_SERVER)
+    ftp.login()
+
+    return ftp
+
+
+def get_flags(string):
+    """
+    Get flags, replacing empty flags with '_' for clarity (' S ' becomes '_S_')
+    """
+    m_flag = string.read(1)
+    m_flag = m_flag if m_flag.strip() else "_"
+    q_flag = string.read(1)
+    q_flag = q_flag if q_flag.strip() else "_"
+    s_flag = string.read(1)
+    s_flag = s_flag if s_flag.strip() else "_"
+
+    return [m_flag + q_flag + s_flag]
+
+
+def create_df(element, dict_element):
+    """
+    Create dataframes from dicts, turn indices into date strings (YYYY-MM-DD)
+    """
+    element = element.upper()
+    df_element = pd.DataFrame(dict_element)
+
+    # Add dates (YYYY-MM-DD) as index on df. Pad days with zeros to two places
+    df_element.index = (
+        df_element["YEAR"]
+        + "-"
+        + df_element["MONTH"]
+        + "-"
+        + df_element["DAY"].str.zfill(2)
+    )
+    df_element.index.name = "DATE"
+
+    # Arrange columns so ID, YEAR, MONTH, DAY are at front. Leaving them in
+    # for plotting later - https://stackoverflow.com/a/31396042
+    for col in ["DAY", "MONTH", "YEAR", "ID"]:
+        df_element = move_col_to_front(col, df_element)
+
+    # Convert numerical values to float
+    df_element.loc[:, element] = df_element.loc[:, element].astype(float)
+
+    return df_element
+
+
+def move_col_to_front(element, df):
+    """
+    Move DataFrame column to position 0
+    """
+    element = element.upper()
+    cols = df.columns.tolist()
+    cols.insert(0, cols.pop(cols.index(element)))
+    df = df.reindex(columns=cols)
+
+    return df
+
+
+# pylint: disable=too-many-locals
+def dly_to_df(ftp, station_id):
+    """
+    Convert .dly files to DataFrame
+    """
+    ftp_filename = station_id + ".dly"
+
+    # Write .dly file to stream using StringIO using FTP command 'RETR'
+    stream = StringIO()
+    ftp.retrlines("RETR " + "/pub/data/ghcn/daily/all/" + ftp_filename, stream.write)
+
+    # Move to first char in file
+    stream.seek(0)
+
+    # File params
+    num_chars_line = 269
+
+    # Read through entire StringIO stream (the .dly file)
+    # and collect the data
+    all_dicts = {}
+    element_flag = {}
+    index = 0
+
+    while True:
+        index += 1
+
+        # Read metadata for each line
+        # (one month of data for a particular element per line)
+        stream.read(11)  # station ID (not used in this loop)
+        year = stream.read(4)
+        month = stream.read(2)
+        day = 0
+        element = stream.read(4)
+
+        # If this is blank then we've reached EOF and should exit loop
+        if not element:
+            break
+
+        # Loop through each day in rest of row,
+        # break if current position is end of row
+        while stream.tell() % num_chars_line != 0:
+            day += 1
+            # Fill in contents of each dict depending on element type in
+            # current row
+            if day == 1:
+                try:
+                    element_flag[element]
+                except BaseException:
+                    element_flag[element] = 1
+                    all_dicts[element] = {}
+                    all_dicts[element]["ID"] = []
+                    all_dicts[element]["YEAR"] = []
+                    all_dicts[element]["MONTH"] = []
+                    all_dicts[element]["DAY"] = []
+                    all_dicts[element][element.upper()] = []
+                    all_dicts[element][element.upper() + "_FLAGS"] = []
+
+            value = stream.read(5)
+            flags = get_flags(stream)
+            if value == "-9999":
+                continue
+            all_dicts[element]["ID"] += [station_id]
+            all_dicts[element]["YEAR"] += [year]
+            all_dicts[element]["MONTH"] += [month]
+            all_dicts[element]["DAY"] += [str(day)]
+            all_dicts[element][element.upper()] += [value]
+            all_dicts[element][element.upper() + "_FLAGS"] += flags
+
+    # Create dataframes from dict
+    all_dfs = {}
+    for element in list(all_dicts.keys()):
+        all_dfs[element] = create_df(element, all_dicts[element])
+
+    # Combine all element dataframes into one dataframe,
+    # indexed on date.
+    #
+    # pd.concat automagically aligns values to matching indices, therefore the
+    # data is date aligned!
+    list_dfs = []
+    for df in list(all_dfs.keys()):
+        list_dfs += [all_dfs[df]]
+    df_all = pd.concat(list_dfs, axis=1, sort=False)
+    df_all.index.name = "MM/DD/YYYY"
+
+    # Remove duplicated/broken columns and rows
+    # https://stackoverflow.com/a/40435354
+    df_all = df_all.loc[:, ~df_all.columns.duplicated()]
+    df_all = df_all.loc[df_all["ID"].notnull(), :]
+
+    return df_all
+
+
+@cache_service.cache(TTL.DAY, "pickle")
+def get_df(station: str) -> pd.DataFrame:
+    ftp = connect_to_ftp()
+    df = dly_to_df(ftp, station)
+    # Filter relevant columns
+    df = df.drop(columns=[col for col in df if col not in COLUMN_NAMES.keys()])
+    # Add missing columns
+    for col in list(COLUMN_NAMES.keys())[1:]:
+        if col not in df.columns:
+            df[col] = nan
+
+    # Rename columns
+    df = df.reset_index().rename(columns=COLUMN_NAMES)
+
+    # Adapt columns
+    df["time"] = pd.to_datetime(df["time"])
+    df[Parameter.TEMP] = df[Parameter.TEMP].div(10)
+    df[Parameter.TMIN] = df[Parameter.TMIN].div(10)
+    df[Parameter.TMAX] = df[Parameter.TMAX].div(10)
+    df[Parameter.PRCP] = df[Parameter.PRCP].div(10)
+    df[Parameter.SNWD] = df[Parameter.SNWD].div(10)
+    df[Parameter.WSPD] = df[Parameter.WSPD].div(10).apply(ms_to_kmh)
+    df[Parameter.WPGT] = df[Parameter.WPGT].div(10).apply(ms_to_kmh)
+    df[Parameter.CLDC] = df[Parameter.CLDC].apply(percentage_to_okta)
+
+    return df.set_index("time")
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    ghcn_id = (
+        req.station.identifiers["ghcn"] if "ghcn" in req.station.identifiers else None
+    )
+    if not ghcn_id:
+        return None
+    return get_df(ghcn_id)

meteostat/providers/noaa/isd_lite.py

@@ -0,0 +1,142 @@
+from datetime import datetime
+from typing import Optional, Union
+from urllib.error import HTTPError
+
+from numpy import isnan
+import pandas as pd
+
+from meteostat.enumerations import TTL, Parameter
+from meteostat.core.logger import logger
+from meteostat.core.cache import cache_service
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import ms_to_kmh, temp_dwpt_to_rhum
+
+ISD_LITE_ENDPOINT = "https://www.ncei.noaa.gov/pub/data/noaa/isd-lite/"
+COLSPECS = [
+    (0, 4),
+    (5, 7),
+    (8, 10),
+    (11, 13),
+    (13, 19),
+    (19, 25),
+    (25, 31),
+    (31, 37),
+    (37, 43),
+    (43, 49),
+    (49, 55),
+]
+COLUMN_NAMES = [
+    "time",
+    Parameter.TEMP,
+    Parameter.DWPT,
+    Parameter.PRES,
+    Parameter.WDIR,
+    Parameter.WSPD,
+    Parameter.CLDC,
+    Parameter.PRCP,
+]
+
+
+def map_sky_code(code: Union[int, str]) -> Optional[int]:
+    """
+    Only accept okta
+    """
+    return int(code) if not isnan(code) and int(code) >= 0 and int(code) <= 8 else None
+
+
+def get_ttl(_usaf: str, _wban: str, year: int) -> int:
+    """
+    Get TTL based on year
+
+    Current + previous year = one day
+    Else = 30 days
+    """
+    current_year = datetime.now().year
+    return TTL.DAY if current_year - year < 2 else TTL.MONTH
+
+
+@cache_service.cache(get_ttl, "pickle")
+def get_df(usaf: str, wban: str, year: int) -> Optional[pd.DataFrame]:
+    if not usaf:
+        return None
+
+    filename = f"{usaf}-{wban if wban else '99999'}-{year}.gz"
+
+    try:
+        df = pd.read_fwf(
+            f"{ISD_LITE_ENDPOINT}{year}/{filename}",
+            parse_dates={"time": [0, 1, 2, 3]},
+            na_values=["-9999", -9999],
+            header=None,
+            colspecs=COLSPECS,
+            compression="gzip",
+        )
+
+        # Rename columns
+        df.columns = COLUMN_NAMES
+
+        # Adapt columns
+        df[Parameter.TEMP] = df[Parameter.TEMP].div(10)
+        df[Parameter.DWPT] = df[Parameter.DWPT].div(10)
+        df[Parameter.PRES] = df[Parameter.PRES].div(10)
+        df[Parameter.WSPD] = df[Parameter.WSPD].div(10).apply(ms_to_kmh)
+        df[Parameter.CLDC] = df[Parameter.CLDC].apply(map_sky_code)
+        df[Parameter.PRCP] = df[Parameter.PRCP].div(10)
+
+        # Calculate humidity data
+        # pylint: disable=unnecessary-lambda
+        df[Parameter.RHUM] = df.apply(lambda row: temp_dwpt_to_rhum(row), axis=1)
+
+        # Drop dew point column
+        # pylint: disable=no-member
+        df = df.drop(Parameter.DWPT, axis=1)
+
+        # Set index
+        df = df.set_index("time")
+
+        # Round decimals
+        return df.round(1)
+
+    except HTTPError as error:
+        if error.status == 404:
+            logger.info(f"ISD Lite file not found: {filename}")
+        else:
+            logger.warning(
+                f"Couldn't load ISD Lite file {filename} (status: {error.status})"
+            )
+        return None
+
+    except Exception as error:
+        logger.warning(error)
+        return None
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    """ """
+    if req.start is None or req.end is None:
+        return None
+
+    years = range(req.start.year, req.end.year + 1)
+    data = tuple(
+        map(
+            lambda i: get_df(*i),
+            (
+                (
+                    (
+                        req.station.identifiers["usaf"]
+                        if "usaf" in req.station.identifiers
+                        else None
+                    ),
+                    (
+                        req.station.identifiers["wban"]
+                        if "wban" in req.station.identifiers
+                        else None
+                    ),
+                    year,
+                )
+                for year in years
+            ),
+        )
+    )
+
+    return pd.concat(data) if len(data) and not all(d is None for d in data) else None

meteostat/providers/noaa/metar.py

@@ -0,0 +1,163 @@
+"""
+The code is licensed under the MIT license.
+"""
+
+from typing import Any, Optional
+
+import pandas as pd
+from metar import Metar
+
+from meteostat.core.logger import logger
+from meteostat.api.config import config
+from meteostat.enumerations import TTL, Frequency, Parameter
+from meteostat.typing import ProviderRequest
+from meteostat.utils.conversions import temp_dwpt_to_rhum
+from meteostat.core.cache import cache_service
+from meteostat.utils.data import enforce_freq
+from meteostat.core.network import network_service
+
+
+ENDPOINT = config.aviationweather_endpoint
+USER_AGENT = config.aviationweather_user_agent
+CLDC_MAP = {
+    "FEW": 2,  # 1-2 octas
+    "SCT": 4,  # 3-4 octas
+    "BKN": 6,  # 5-7 octas
+    "OVC": 8,  # 8 octas (fully overcast)
+}
+COCO_MAP = {
+    "RA": 8,
+    "SHRA": 17,
+    "DZ": 7,
+    "DZRA": 7,
+    "FZRA": 10,
+    "FZDZ": 10,
+    "RASN": 12,
+    "SN": 15,
+    "SHSN": 21,
+    "SG": 12,
+    "IC": 12,
+    "PL": 24,
+    "GR": 24,
+    "GS": 24,
+    "FG": 5,
+    "BR": 5,
+    "MIFG": 5,
+    "BCFG": 5,
+    "HZ": 5,
+    "TS": 25,
+    "TSRA": 25,
+    "PO": 27,
+    "SQ": 27,
+    "FC": 27,
+    "SS": 27,
+    "DS": 27,
+}
+
+
+def safe_get(obj: Optional[Any]) -> Optional[Any]:
+    try:
+        if obj is not None and hasattr(obj, "value"):
+            return obj.value()
+        return None
+    except AttributeError:
+        return None
+
+
+def get_cldc(report: Metar.Metar) -> Optional[int]:
+    """
+    Get cloud cover (octas) from METAR report
+    """
+    try:
+        cloud_cover = report.sky[0][0]
+        return CLDC_MAP.get(cloud_cover)
+    except IndexError:
+        return None
+
+
+def get_coco(report: Metar.Metar) -> Optional[int]:
+    """
+    Get weather condition code from METAR report
+    """
+    try:
+        condition_code = "".join(
+            [item for item in report.weather[0] if item is not None]
+        )
+        return COCO_MAP.get(condition_code)
+    except IndexError:
+        return None
+
+
+def map_data(record):
+    """
+    Map METAR data to Meteostat column names
+    """
+    try:
+        parsed_report = Metar.Metar(record)
+
+        return {
+            "time": parsed_report.time,
+            Parameter.TEMP: safe_get(parsed_report.temp),
+            Parameter.DWPT: safe_get(parsed_report.dewpt),
+            Parameter.PRCP: safe_get(parsed_report.precip_1hr),
+            Parameter.SNWD: safe_get(parsed_report.snowdepth),
+            Parameter.WDIR: safe_get(parsed_report.wind_dir),
+            Parameter.WSPD: safe_get(parsed_report.wind_speed),
+            Parameter.WPGT: safe_get(parsed_report.wind_gust),
+            Parameter.PRES: safe_get(parsed_report.press),
+            Parameter.VSBY: safe_get(parsed_report.vis),
+            Parameter.CLDC: get_cldc(parsed_report),
+            Parameter.COCO: get_coco(parsed_report),
+        }
+    except Metar.ParserError:
+        return None
+
+
+@cache_service.cache(TTL.HOUR, "pickle")
+def get_df(station: str) -> Optional[pd.DataFrame]:
+    """
+    Get CSV file from Meteostat and convert to DataFrame
+    """
+    url = ENDPOINT.format(station=station)
+
+    if not USER_AGENT:
+        logger.warning(
+            "Consider specifying a unique user agent when querying the Aviation Weather Center's data API."
+        )
+
+    headers = {"User-Agent": USER_AGENT}
+
+    response = network_service.get(url, headers=headers)
+
+    # Raise an exception if the request was unsuccessful
+    response.raise_for_status()
+
+    # Parse the JSON content into a DataFrame
+    data = [
+        item for item in map(map_data, response.text.splitlines()) if item is not None
+    ]
+
+    # Return None if no data is available
+    if not len(data):
+        return None
+
+    # Create DataFrame
+    df = pd.DataFrame(data)
+
+    # Return None if DataFrame is empty
+    if df.empty:
+        return None
+
+    # Add RHUM column
+    df[Parameter.RHUM] = df.apply(lambda row: temp_dwpt_to_rhum(row), axis=1)
+    df[Parameter.RHUM] = df[Parameter.RHUM].round()
+
+    # Set time index
+    df = df.set_index(["time"])
+
+    return enforce_freq(df, Frequency.HOURLY)
+
+
+def fetch(req: ProviderRequest) -> Optional[pd.DataFrame]:
+    if "icao" in req.station.identifiers:
+        return get_df(req.station.identifiers["icao"])

meteostat/typing.py

@@ -0,0 +1,113 @@
+"""
+Meteostat Typing
+"""
+
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from typing import Callable, List, Literal, Optional
+
+from meteostat.core.validator import Validator
+from meteostat.enumerations import (
+    Grade,
+    Priority,
+    Granularity,
+    Parameter,
+    Provider,
+    Unit,
+)
+
+
+@dataclass
+class Station:
+    """
+    A weather station
+
+    For virtual stations created from Point objects, some fields will be None.
+    """
+
+    id: str  # The Meteostat station ID (e.g., "10637" or "$0001" for virtual stations)
+    name: Optional[str] = None  # The (usually English) name of the station
+    country: Optional[str] = None  # ISO 3166-1 alpha-2 country code
+    region: Optional[str] = None  # ISO 3166-2 state or region code
+    identifiers: dict[str, str] = field(default_factory=dict)  # Provider identifiers
+    latitude: Optional[float] = None  # The latitude in degrees
+    longitude: Optional[float] = None  # The longitude in degrees
+    elevation: Optional[int] = None  # The elevation in meters
+    timezone: Optional[str] = None  # The IANA timezone name
+
+
+@dataclass
+class License:
+    """
+    A license
+    """
+
+    commercial: bool
+    attribution: Optional[str] = None
+    name: Optional[str] = None
+    url: Optional[str] = None
+
+
+@dataclass
+class ProviderSpec:
+    """
+    A provider's meta data
+    """
+
+    id: Provider | str  # The provider ID
+    name: str  # A descriptive provider name
+    granularity: Granularity  # The provider's time series granularity
+    priority: Priority | int  # The priority of the provider
+    grade: Optional[Grade]  # The provider's data quality grade
+    license: Optional[License]  # The provider's license
+    requires: List[
+        Literal["id", "identifiers", "location"]
+    ]  # Required station properties for the provider
+    parameters: List[Parameter]  # List of supported meteorological parameters
+    start: date  # The start date of the provider's data
+    end: Optional[datetime] = None  # The end date of the provider's data
+    countries: Optional[list[str]] = None  # List of supported countries
+    module: Optional[str] = None  # Module path to the provider's API
+
+
+@dataclass
+class ParameterSpec:
+    """
+    A parameter's meta data
+    """
+
+    id: Parameter | str  # The parameter ID
+    name: str  # A descriptive parameter name
+    granularity: Granularity  # The parameter's granularity
+    dtype: str  # The parameter's data type
+    unit: Optional[Unit] = None  # The parameter's data unit
+    validators: List[Validator | Callable] = field(
+        default_factory=list
+    )  # The parameter's validators
+
+
+@dataclass
+class Request:
+    """
+    A request to fetch meteorological time series data
+    """
+
+    granularity: Granularity  # Query's time series granularity
+    providers: List[Provider]  # Providers to query
+    parameters: List[Parameter]  # Schema of the query's data
+    station: Station | List[Station]  # Station(s) to query
+    start: Optional[datetime] = None  # Start date of the query
+    end: Optional[datetime] = None  # End date of the query
+    timezone: Optional[str] = None  # Time zone of the query's data
+
+
+@dataclass
+class ProviderRequest:
+    """
+    A query to fetch meteorological data from a provider
+    """
+
+    station: Station  # Station to query
+    parameters: List[Parameter]  # List of meteorological parameters to query
+    start: Optional[datetime]  # Start date of the query
+    end: Optional[datetime]  # End date of the query