meteostat 1.7.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

meteostat/core/data.py

@@ -0,0 +1,203 @@
+"""
+Data Service
+
+The Data Service is responsible for fetching meteorological data from
+different providers and merging it into a single time series.
+"""
+
+from datetime import datetime
+from typing import List, Optional, Union, cast
+
+import pandas as pd
+
+from meteostat.api.timeseries import TimeSeries
+from meteostat.core.logger import logger
+from meteostat.core.parameters import parameter_service
+from meteostat.core.providers import provider_service
+from meteostat.core.schema import schema_service
+from meteostat.enumerations import Parameter, Provider
+from meteostat.typing import Station, Request
+from meteostat.utils.data import stations_to_df
+from meteostat.utils.guards import request_size_guard
+
+
+class DataService:
+    """
+    Data Service
+    """
+
+    @staticmethod
+    def _add_source(df: pd.DataFrame, provider_id: str) -> pd.DataFrame:
+        """
+        Add source column to DataFrame
+        """
+        if "source" not in df.index.names:
+            df["source"] = provider_id
+            df = df.set_index(["source"], append=True)
+
+        return df
+
+    @staticmethod
+    def filter_time(
+        df: pd.DataFrame,
+        start: Union[datetime, None] = None,
+        end: Union[datetime, None] = None,
+    ) -> pd.DataFrame:
+        """
+        Filter time series data based on start and end date
+        """
+
+        # Return empty DataFrame if input is empty
+        if df.empty:
+            return df
+
+        # Get time index
+        time = df.index.get_level_values("time")
+
+        # Filter & return
+        try:
+            return df.loc[(time >= start) & (time <= end)] if start and end else df
+        except TypeError:
+            return (
+                df.loc[(time >= start.date()) & (time <= end.date())]
+                if start and end
+                else df
+            )
+
+    @staticmethod
+    def concat_fragments(
+        fragments: List[pd.DataFrame],
+        parameters: List[Parameter],
+    ) -> pd.DataFrame:
+        """
+        Concatenate multiple fragments into a single DataFrame
+        """
+        try:
+            cleaned = [
+                df.dropna(how="all", axis=1) if not df.empty else None
+                for df in fragments
+            ]
+            filtered = [df for df in cleaned if df is not None]
+            if not filtered:
+                return pd.DataFrame()
+            df = pd.concat(filtered)
+            df = schema_service.fill(df, parameters)
+            df = schema_service.purge(df, parameters)
+            return df
+        except ValueError:
+            return pd.DataFrame()
+
+    def _fetch_provider_data(
+        self, req: Request, station: Station, provider: Provider
+    ) -> Optional[pd.DataFrame]:
+        """
+        Fetch data for a single weather station and provider
+        """
+        try:
+            # Fetch DataFrame for current provider
+            df = provider_service.fetch_data(provider, req, station)
+
+            # Continue if no data was returned
+            if df is None:
+                return None
+
+            # Add current station ID to DataFrame
+            df = pd.concat([df], keys=[station.id], names=["station"])
+
+            # Add source index column to DataFrame
+            df = self._add_source(df, provider)
+
+            # Filter DataFrame for requested parameters and time range
+            df = self.filter_time(df, req.start, req.end)
+
+            # Drop empty rows
+            df = df.dropna(how="all")
+
+            return df
+
+        except Exception:
+            logger.error(
+                'Could not fetch data for provider "%s"',
+                provider,
+                exc_info=True,
+            )
+
+    def _fetch_station_data(self, req: Request, station: Station) -> List[pd.DataFrame]:
+        """
+        Fetch data for a single weather station
+        """
+        fragments = []
+
+        filtered_providers = provider_service.filter_providers(req, station)
+
+        for provider in filtered_providers:
+            df = self._fetch_provider_data(req, station, provider)
+
+            # Continue if no data was returned
+            if df is None:
+                continue
+
+            fragments.append(df)
+
+        return fragments
+
+    def fetch(
+        self,
+        req: Request,
+    ) -> TimeSeries:
+        """
+        Load meteorological time series data from different providers
+        """
+        # Guard request
+        request_size_guard(req)
+
+        # Convert stations to list if single Station
+        stations: List[Station] = (
+            cast(List[Station], req.station)
+            if isinstance(req.station, list)
+            else [req.station]
+        )
+
+        logger.debug(
+            "%s time series requested for %s station(s)", req.granularity, len(stations)
+        )
+
+        # Filter parameters
+        req.parameters = parameter_service.filter_parameters(
+            req.granularity, req.parameters
+        )
+
+        fragments = []
+
+        # Go through all weather stations
+        for station in stations:
+            station_fragments = self._fetch_station_data(req, station)
+
+            if station_fragments:
+                fragments.extend(station_fragments)
+
+        # Merge data in a single DataFrame
+        if fragments:
+            df = self.concat_fragments(fragments, req.parameters)
+        else:
+            df = pd.DataFrame()
+
+        # Set data types
+        df = schema_service.format(df, req.granularity)
+
+        # Create time series
+        ts = TimeSeries(
+            req.granularity,
+            stations_to_df(stations),
+            df,
+            req.start,
+            req.end,
+            req.timezone,
+            multi_station=isinstance(req.station, list),
+        )
+
+        # Return time series
+        return ts
+
+
+data_service = DataService()

meteostat/core/logger.py

@@ -0,0 +1,9 @@
+"""
+Logger Module
+
+Provides a centralized logger instance for the Meteostat package.
+"""
+
+import logging
+
+logger = logging.getLogger("meteostat")

meteostat/core/network.py

@@ -0,0 +1,82 @@
+"""
+Network Service
+
+The Network Service provides methods to send HTTP requests
+considering the Meteostat configuration.
+"""
+
+from typing import Optional
+
+import requests
+
+from meteostat import __version__
+from meteostat.core.logger import logger
+from meteostat.api.config import config
+
+
+class NetworkService:
+    """
+    Network Service
+    """
+
+    @staticmethod
+    def _process_headers(headers: dict) -> dict:
+        """
+        Process headers
+        """
+
+        headers["X-Meteostat-Version"] = __version__
+
+        return headers
+
+    def get(
+        self,
+        url: str,
+        params=None,
+        headers: Optional[dict] = None,
+        stream: Optional[bool] = None,
+    ) -> requests.Response:
+        """
+        Send a GET request using the Meteostat configuration
+        """
+        if headers is None:
+            headers = {}
+
+        headers = self._process_headers(headers)
+
+        return requests.get(
+            url,
+            params,
+            headers=headers,
+            stream=stream,
+            proxies=config.network_proxies,
+            timeout=30,
+        )
+
+    def get_from_mirrors(
+        self,
+        mirrors: list[str],
+        params=None,
+        headers: Optional[dict] = None,
+        stream: Optional[bool] = None,
+    ) -> Optional[requests.Response]:
+        """
+        Send a GET request to multiple mirrors using the Meteostat configuration
+        """
+        for mirror in mirrors:
+            try:
+                response = self.get(
+                    mirror,
+                    params=params,
+                    headers=headers,
+                    stream=stream,
+                )
+                if response.status_code == 200:
+                    return response
+            except requests.RequestException:
+                logger.warning("Could not fetch data from '%s'", mirror)
+                continue
+        return None
+
+
+network_service = NetworkService()

meteostat/core/parameters.py

@@ -0,0 +1,112 @@
+"""
+Parameter Service
+
+The Parameter Service provides methods to manage and access
+supported parameters for data requests.
+"""
+
+from typing import List, Optional
+
+from meteostat.core.logger import logger
+from meteostat.enumerations import Granularity, Parameter
+from meteostat.parameters import DEFAULT_PARAMETERS
+from meteostat.typing import ParameterSpec
+
+
+class ParameterService:
+    """
+    Parameter Service
+    """
+
+    _parameters: List[ParameterSpec]
+
+    @staticmethod
+    def _has_duplicates(parameter_specs: List[ParameterSpec]) -> bool:
+        """
+        Check if parameter list contains duplicates
+        """
+        seen = set()
+        for spec in parameter_specs:
+            key = (spec.id, spec.granularity)
+            if key in seen:
+                return True  # Duplicate found
+            seen.add(key)
+        return False  # No duplicates found
+
+    def _parameter_exists(self, parameter: ParameterSpec) -> bool:
+        """
+        Check if a parameter already exists
+        """
+        key = (parameter.id, parameter.granularity)
+        return any((spec.id, spec.granularity) == key for spec in self.parameters)
+
+    def __init__(self, parameters: List[ParameterSpec]) -> None:
+        if self._has_duplicates(parameters):
+            raise ValueError("List of parameters contains duplicates")
+
+        self._parameters = parameters
+
+    @property
+    def parameters(self) -> List[ParameterSpec]:
+        """
+        Get supported parameters
+        """
+        return self._parameters
+
+    def register(self, parameter: ParameterSpec) -> None:
+        """
+        Register a parameter
+        """
+        if self._parameter_exists(parameter):
+            raise ValueError("The parameter already exists")
+
+        self._parameters.append(parameter)
+
+    def get_parameter(
+        self, parameter_id: Parameter, granularity: Granularity
+    ) -> Optional[ParameterSpec]:
+        """
+        Get parameter by ID and granularity
+        """
+        return next(
+            (
+                parameter
+                for parameter in self.parameters
+                if parameter.id == parameter_id and parameter.granularity == granularity
+            ),
+            None,
+        )
+
+    def filter_parameters(
+        self, granularity: Granularity, parameters: List[Parameter]
+    ) -> List[Parameter]:
+        """
+        Raise exception if a requested parameter is not part of the schema
+        """
+        supported_parameters = list(
+            map(
+                lambda parameter: parameter.id,
+                filter(
+                    lambda parameter: parameter.granularity == granularity,
+                    self.parameters,
+                ),
+            )
+        )
+        # Get difference between requested parameters and root schema
+        diff = set(parameters).difference(supported_parameters)
+        # Log warning
+        if diff:
+            logger.error(
+                "Tried to request data for unsupported parameter(s): %s",
+                ", ".join(diff),
+            )
+        # Return intersection
+        return list(
+            filter(
+                lambda parameter: parameter in parameters,
+                supported_parameters,
+            )
+        )
+
+
+parameter_service = ParameterService(DEFAULT_PARAMETERS)

meteostat/core/providers.py

@@ -0,0 +1,184 @@
+"""
+Provider Service
+
+The Provider Service provides methods to interact with data providers.
+"""
+
+from datetime import datetime
+from importlib import import_module
+from statistics import fmean
+from typing import List, Optional, TypeGuard, cast
+
+import pandas as pd
+
+from meteostat.core.logger import logger
+from meteostat.enumerations import Granularity, Priority, Provider
+from meteostat.providers.index import DEFAULT_PROVIDERS
+from meteostat.typing import (
+    ProviderRequest,
+    ProviderSpec,
+    Station,
+    Request,
+)
+
+
+class ProviderService:
+    """
+    Provider Service
+    """
+
+    _providers: List[ProviderSpec]
+
+    def __init__(self, providers: List[ProviderSpec]) -> None:
+        self._providers = providers
+
+    @property
+    def providers(self) -> List[ProviderSpec]:
+        """
+        Get supported providers
+        """
+        return self._providers
+
+    def register(self, provider: ProviderSpec) -> None:
+        """
+        Register a provider
+        """
+        self._providers.append(provider)
+
+    def get_provider(self, provider_id: Provider | str) -> Optional[ProviderSpec]:
+        """
+        Get provider by ID
+        """
+        return next(
+            (provider for provider in self._providers if provider.id == provider_id),
+            None,
+        )
+
+    def _get_provider_priority(self, provider_id: Provider | str) -> int:
+        """
+        Get priority of a provider by its ID
+        """
+        baselines = {
+            Granularity.HOURLY: 0,
+            Granularity.DAILY: 100,
+            Granularity.MONTHLY: 200,
+            Granularity.NORMALS: 300,
+        }
+
+        provider = self.get_provider(provider_id)
+
+        if not provider:
+            return Priority.NONE
+
+        baseline = baselines[provider.granularity]
+
+        return int(provider.priority + baseline)
+
+    def get_source_priority(self, source: str) -> float:
+        """
+        Get priority of a source string
+        """
+        provider_ids = source.split(" ")
+
+        if len(provider_ids) == 1:
+            return self._get_provider_priority(provider_ids[0])
+
+        priorities = [
+            self._get_provider_priority(provider) for provider in provider_ids
+        ]
+
+        return fmean(priorities)
+
+    def filter_providers(self, query: Request, station: Station) -> List[Provider]:
+        """
+        Get a filtered list of providers
+        """
+
+        def _filter(provider_id: Provider) -> TypeGuard[Provider]:
+            provider = self.get_provider(provider_id)
+
+            if provider is None:
+                return False
+
+            # Filter out providers with diverging granularities
+            if provider.granularity is not query.granularity:
+                logger.error(
+                    "Provider '%s' does not support granularity '%s'",
+                    provider_id,
+                    query.granularity,
+                )
+                return False
+
+            # Filter out providers with no overlap in parameters
+            if set(provider.parameters).isdisjoint(query.parameters):
+                logger.info(
+                    "Provider '%s' does not support any requested parameter",
+                    provider_id,
+                )
+                return False
+
+            # Filter out providers which do not serve the station's country
+            if provider.countries and station.country not in provider.countries:
+                logger.info(
+                    "Skipping provider '%s' as it does not serve the station's country ('%s')",
+                    provider_id,
+                    station.country,
+                )
+                return False
+
+            # Filter out providers which stopped providing data before the request's start date
+            if query.end and query.end < datetime.combine(
+                provider.start, datetime.min.time()
+            ):
+                logger.info(
+                    "Skipping provider '%s' as it stopped providing data before request start",
+                    provider_id,
+                )
+                return False
+
+            # Filter out providers which only started providing data after the request's end date
+            if (
+                provider.end is not None
+                and query.start is not None
+                and query.start > datetime.combine(provider.end, datetime.max.time())
+            ):
+                logger.info(
+                    "Skipping provider '%s' as it only started providing data after request end",
+                    provider_id,
+                )
+                return False
+
+            return True
+
+        return list(filter(_filter, query.providers))
+
+    def fetch_data(
+        self, provider_id: Provider, req: Request, station: Station
+    ) -> Optional[pd.DataFrame]:
+        """
+        Fetch data from a given provider
+        """
+        provider = self.get_provider(provider_id)
+
+        if not provider:
+            return None
+
+        query = ProviderRequest(
+            station=station,
+            start=req.start
+            or (
+                datetime.combine(provider.start, datetime.min.time())
+                if provider.start
+                else None
+            ),
+            end=req.end or (provider.end or datetime.now()),
+            parameters=req.parameters,
+        )
+
+        module = import_module(cast(str, provider.module))
+        df = module.fetch(query)
+
+        return df
+
+
+provider_service = ProviderService(providers=DEFAULT_PROVIDERS)