meteostat 1.7.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/core/config.py

@@ -0,0 +1,158 @@
+"""
+Configuration Service
+
+Manages configuration settings for Meteostat, including cache, network,
+stations, interpolation, and provider-specific settings. Configuration can be
+loaded from environment variables with the MS_ prefix.
+"""
+
+import os
+import json
+from typing import Any, List, Optional
+
+from meteostat.core.logger import logger
+from meteostat.enumerations import TTL, Parameter
+from meteostat.utils.types import extract_property_type, validate_parsed_value
+
+
+class Config:
+    """
+    Configuration Base Class
+    """
+
+    prefix: str
+
+    @property
+    def _prefix(self) -> str:
+        """
+        The environment variable prefix
+        """
+        return f"{self.prefix}_" if self.prefix else ""
+
+    def _parse_env_value(self, key: str, value: str) -> Any:
+        """
+        Parse an environment variable value and validate against property type
+        """
+        # Extract the expected type for the property
+        expected_type, original_type = extract_property_type(self.__class__, key)
+
+        if expected_type is None:
+            # Fallback to JSON parsing if no type annotation is available
+            try:
+                return json.loads(value)
+            except (json.JSONDecodeError, TypeError, ValueError):
+                logger.error("Failed to parse environment variable '%s'", key)
+                return None
+
+        # Parse the value using JSON
+        try:
+            parsed_value = json.loads(value)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            logger.error("Failed to parse environment variable '%s'", key)
+            return None
+
+        # Validate and potentially convert the parsed value
+        return validate_parsed_value(parsed_value, expected_type, original_type, key)
+
+    def _set_env_value(self, key: str, value: Any) -> None:
+        """
+        Set a configuration using a key-value pair
+        """
+        setattr(self, key, value)
+
+    def __init__(self, prefix: str = "MS") -> None:
+        """
+        Initialize configuration service
+        """
+        self.prefix = prefix
+        self.load_env()
+
+    def get_env_name(self, key: str) -> str:
+        """
+        Get the environment variable name for a given key
+        """
+        if not hasattr(self, key):
+            raise KeyError(f"Configuration has no key '{key}'")
+
+        key = f"{self._prefix}{key}"
+        return key.upper()
+
+    def load_env(self) -> None:
+        """
+        Update configuration from environment variables with a given prefix.
+        """
+        for key, value in os.environ.items():
+            if not key.startswith(self._prefix):
+                continue
+
+            key = key.replace(self._prefix, "").lower()
+            value = self._parse_env_value(key, value)
+
+            if value is not None:
+                self._set_env_value(key, value)
+
+
+class ConfigService(Config):
+    """
+    Configuration Service for Meteostat
+
+    Manages all configuration settings including cache, network, stations,
+    interpolation, and provider-specific settings. Supports loading configuration
+    from environment variables.
+    """
+
+    # General settings
+    block_large_requests: bool = True  # Block requests that include too many stations
+
+    # Cache settings
+    cache_enable: bool = True
+    cache_directory: str = (
+        os.path.expanduser("~") + os.sep + ".meteostat" + os.sep + "cache"
+    )
+    cache_ttl: int = TTL.MONTH
+    cache_autoclean: bool = True
+
+    # Network settings
+    network_proxies: Optional[dict] = None
+
+    # Station meta data settings
+    stations_db_ttl: int = TTL.WEEK
+    stations_db_endpoints: List[str] = [
+        "https://data.meteostat.net/stations.db",
+        "https://raw.githubusercontent.com/meteostat/weather-stations/master/stations.db",
+    ]
+    stations_db_file: str = (
+        os.path.expanduser("~") + os.sep + ".meteostat" + os.sep + "stations.db"
+    )
+
+    # Interpolation settings
+    lapse_rate_parameters = [Parameter.TEMP, Parameter.TMIN, Parameter.TMAX]
+
+    # [Provider] Meteostat settings
+    include_model_data: bool = True
+    hourly_endpoint: str = "https://data.meteostat.net/hourly/{year}/{station}.csv.gz"
+    daily_endpoint: str = "https://data.meteostat.net/daily/{year}/{station}.csv.gz"
+    monthly_endpoint: str = "https://data.meteostat.net/monthly/{station}.csv.gz"
+
+    # [Provider] DWD settings
+    dwd_ftp_host: str = "opendata.dwd.de"
+    dwd_hourly_modes: Optional[List[str]] = None
+    dwd_daily_modes: Optional[List[str]] = None
+    dwd_climat_modes: Optional[List[str]] = None
+
+    # [Provider] NOAA settings
+    aviationweather_endpoint: str = (
+        "https://aviationweather.gov/api/data/metar?"
+        "ids={station}&format=raw&taf=false&hours=24"
+    )
+    aviationweather_user_agent: Optional[str] = None
+
+    # [Provider] Met.no settings
+    metno_forecast_endpoint: str = (
+        "https://api.met.no/weatherapi/locationforecast/2.0/compact?"
+        "lat={latitude}&lon={longitude}&altitude={elevation}"
+    )
+    metno_user_agent: Optional[str] = None
+
+
+config = ConfigService("MS")

meteostat/core/data.py

@@ -0,0 +1,199 @@
+"""
+Data Service
+
+The Data Service is responsible for fetching meteorological data from
+different providers and merging it into a single time series.
+"""
+
+from datetime import datetime
+from typing import List, Optional, Union, cast
+
+import pandas as pd
+
+from meteostat.api.timeseries import TimeSeries
+from meteostat.core.logger import logger
+from meteostat.core.parameters import parameter_service
+from meteostat.core.providers import provider_service
+from meteostat.core.schema import schema_service
+from meteostat.enumerations import Parameter, Provider
+from meteostat.typing import Station, Request
+from meteostat.utils.data import stations_to_df
+
+
+class DataService:
+    """
+    Data Service
+    """
+
+    @staticmethod
+    def _add_source(df: pd.DataFrame, provider_id: str) -> pd.DataFrame:
+        """
+        Add source column to DataFrame
+        """
+        if "source" not in df.index.names:
+            df["source"] = provider_id
+            df = df.set_index(["source"], append=True)
+
+        return df
+
+    @staticmethod
+    def filter_time(
+        df: pd.DataFrame,
+        start: Union[datetime, None] = None,
+        end: Union[datetime, None] = None,
+    ) -> pd.DataFrame:
+        """
+        Filter time series data based on start and end date
+        """
+
+        # Return empty DataFrame if input is empty
+        if df.empty:
+            return df
+
+        # Get time index
+        time = df.index.get_level_values("time")
+
+        # Filter & return
+        try:
+            return df.loc[(time >= start) & (time <= end)] if start and end else df
+        except TypeError:
+            return (
+                df.loc[(time >= start.date()) & (time <= end.date())]
+                if start and end
+                else df
+            )
+
+    @staticmethod
+    def concat_fragments(
+        fragments: List[pd.DataFrame],
+        parameters: List[Parameter],
+    ) -> pd.DataFrame:
+        """
+        Concatenate multiple fragments into a single DataFrame
+        """
+        try:
+            cleaned = [
+                df.dropna(how="all", axis=1) if not df.empty else None
+                for df in fragments
+            ]
+            filtered = [df for df in cleaned if df is not None]
+            if not filtered:
+                return pd.DataFrame()
+            df = pd.concat(filtered)
+            df = schema_service.fill(df, parameters)
+            df = schema_service.purge(df, parameters)
+            return df
+        except ValueError:
+            return pd.DataFrame()
+
+    def _fetch_provider_data(
+        self, req: Request, station: Station, provider: Provider
+    ) -> Optional[pd.DataFrame]:
+        """
+        Fetch data for a single weather station and provider
+        """
+        try:
+            # Fetch DataFrame for current provider
+            df = provider_service.fetch_data(provider, req, station)
+
+            # Continue if no data was returned
+            if df is None:
+                return None
+
+            # Add current station ID to DataFrame
+            df = pd.concat([df], keys=[station.id], names=["station"])
+
+            # Add source index column to DataFrame
+            df = self._add_source(df, provider)
+
+            # Filter DataFrame for requested parameters and time range
+            df = self.filter_time(df, req.start, req.end)
+
+            # Drop empty rows
+            df = df.dropna(how="all")
+
+            return df
+
+        except Exception:
+            logger.error(
+                'Could not fetch data for provider "%s"',
+                provider,
+                exc_info=True,
+            )
+
+    def _fetch_station_data(self, req: Request, station: Station) -> List[pd.DataFrame]:
+        """
+        Fetch data for a single weather station
+        """
+        fragments = []
+
+        filtered_providers = provider_service.filter_providers(req, station)
+
+        for provider in filtered_providers:
+            df = self._fetch_provider_data(req, station, provider)
+
+            # Continue if no data was returned
+            if df is None:
+                continue
+
+            fragments.append(df)
+
+        return fragments
+
+    def fetch(
+        self,
+        req: Request,
+    ) -> TimeSeries:
+        """
+        Load meteorological time series data from different providers
+        """
+        # Convert stations to list if single Station
+        stations: List[Station] = (
+            cast(List[Station], req.station)
+            if isinstance(req.station, list)
+            else [req.station]
+        )
+
+        logger.debug(
+            "%s time series requested for %s station(s)", req.granularity, len(stations)
+        )
+
+        # Filter parameters
+        req.parameters = parameter_service.filter_parameters(
+            req.granularity, req.parameters
+        )
+
+        fragments = []
+
+        # Go through all weather stations
+        for station in stations:
+            station_fragments = self._fetch_station_data(req, station)
+
+            if station_fragments:
+                fragments.extend(station_fragments)
+
+        # Merge data in a single DataFrame
+        if fragments:
+            df = self.concat_fragments(fragments, req.parameters)
+        else:
+            df = pd.DataFrame()
+
+        # Set data types
+        df = schema_service.format(df, req.granularity)
+
+        # Create time series
+        ts = TimeSeries(
+            req.granularity,
+            stations_to_df(stations),
+            df,
+            req.start,
+            req.end,
+            req.timezone,
+            multi_station=isinstance(req.station, list),
+        )
+
+        # Return time series
+        return ts
+
+
+data_service = DataService()

meteostat/core/logger.py

@@ -0,0 +1,9 @@
+"""
+Logger Module
+
+Provides a centralized logger instance for the Meteostat package.
+"""
+
+import logging
+
+logger = logging.getLogger("meteostat")

meteostat/core/network.py

@@ -0,0 +1,82 @@
+"""
+Network Service
+
+The Network Service provides methods to send HTTP requests
+considering the Meteostat configuration.
+"""
+
+from typing import Optional
+
+import requests
+
+from meteostat import __version__
+from meteostat.core.logger import logger
+from meteostat.core.config import config
+
+
+class NetworkService:
+    """
+    Network Service
+    """
+
+    @staticmethod
+    def _process_headers(headers: dict) -> dict:
+        """
+        Process headers
+        """
+
+        headers["X-Meteostat-Version"] = __version__
+
+        return headers
+
+    def get(
+        self,
+        url: str,
+        params=None,
+        headers: Optional[dict] = None,
+        stream: Optional[bool] = None,
+    ) -> requests.Response:
+        """
+        Send a GET request using the Meteostat configuration
+        """
+        if headers is None:
+            headers = {}
+
+        headers = self._process_headers(headers)
+
+        return requests.get(
+            url,
+            params,
+            headers=headers,
+            stream=stream,
+            proxies=config.network_proxies,
+            timeout=30,
+        )
+
+    def get_from_mirrors(
+        self,
+        mirrors: list[str],
+        params=None,
+        headers: Optional[dict] = None,
+        stream: Optional[bool] = None,
+    ) -> Optional[requests.Response]:
+        """
+        Send a GET request to multiple mirrors using the Meteostat configuration
+        """
+        for mirror in mirrors:
+            try:
+                response = self.get(
+                    mirror,
+                    params=params,
+                    headers=headers,
+                    stream=stream,
+                )
+                if response.status_code == 200:
+                    return response
+            except requests.RequestException:
+                logger.warning("Could not fetch data from '%s'", mirror)
+                continue
+        return None
+
+
+network_service = NetworkService()

meteostat/core/parameters.py

@@ -0,0 +1,112 @@
+"""
+Parameter Service
+
+The Parameter Service provides methods to manage and access
+supported parameters for data requests.
+"""
+
+from typing import List, Optional
+
+from meteostat.core.logger import logger
+from meteostat.enumerations import Granularity, Parameter
+from meteostat.parameters import DEFAULT_PARAMETERS
+from meteostat.typing import ParameterSpec
+
+
+class ParameterService:
+    """
+    Parameter Service
+    """
+
+    _parameters: List[ParameterSpec]
+
+    @staticmethod
+    def _has_duplicates(parameter_specs: List[ParameterSpec]) -> bool:
+        """
+        Check if parameter list contains duplicates
+        """
+        seen = set()
+        for spec in parameter_specs:
+            key = (spec.id, spec.granularity)
+            if key in seen:
+                return True  # Duplicate found
+            seen.add(key)
+        return False  # No duplicates found
+
+    def _parameter_exists(self, parameter: ParameterSpec) -> bool:
+        """
+        Check if a parameter already exists
+        """
+        key = (parameter.id, parameter.granularity)
+        return any((spec.id, spec.granularity) == key for spec in self.parameters)
+
+    def __init__(self, parameters: List[ParameterSpec]) -> None:
+        if self._has_duplicates(parameters):
+            raise ValueError("List of parameters contains duplicates")
+
+        self._parameters = parameters
+
+    @property
+    def parameters(self) -> List[ParameterSpec]:
+        """
+        Get supported parameters
+        """
+        return self._parameters
+
+    def register(self, parameter: ParameterSpec) -> None:
+        """
+        Register a parameter
+        """
+        if self._parameter_exists(parameter):
+            raise ValueError("The parameter already exists")
+
+        self._parameters.append(parameter)
+
+    def get_parameter(
+        self, parameter_id: Parameter, granularity: Granularity
+    ) -> Optional[ParameterSpec]:
+        """
+        Get parameter by ID and granularity
+        """
+        return next(
+            (
+                parameter
+                for parameter in self.parameters
+                if parameter.id == parameter_id and parameter.granularity == granularity
+            ),
+            None,
+        )
+
+    def filter_parameters(
+        self, granularity: Granularity, parameters: List[Parameter]
+    ) -> List[Parameter]:
+        """
+        Raise exception if a requested parameter is not part of the schema
+        """
+        supported_parameters = list(
+            map(
+                lambda parameter: parameter.id,
+                filter(
+                    lambda parameter: parameter.granularity == granularity,
+                    self.parameters,
+                ),
+            )
+        )
+        # Get difference between requested parameters and root schema
+        diff = set(parameters).difference(supported_parameters)
+        # Log warning
+        if diff:
+            logger.error(
+                "Tried to request data for unsupported parameter(s): %s",
+                ", ".join(diff),
+            )
+        # Return intersection
+        return list(
+            filter(
+                lambda parameter: parameter in parameters,
+                supported_parameters,
+            )
+        )
+
+
+parameter_service = ParameterService(DEFAULT_PARAMETERS)