meteostat 1.7.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

meteostat/api/timeseries.py

@@ -0,0 +1,334 @@
+"""
+TimeSeries Class
+
+A class to handle meteorological time series data.
+"""
+
+from copy import copy
+from datetime import datetime
+from itertools import chain
+from math import floor
+from statistics import mean
+from typing import List, Optional
+
+import pandas as pd
+
+from meteostat.core.parameters import parameter_service
+from meteostat.core.validator import Validator
+from meteostat.core.providers import provider_service
+from meteostat.core.schema import schema_service
+from meteostat.enumerations import Parameter, Granularity, Provider, UnitSystem
+from meteostat.typing import License
+from meteostat.utils.data import fill_df, localize, squash_df
+
+
+class TimeSeries:
+    """
+    TimeSeries class which provides features which are
+    used across all granularities
+    """
+
+    granularity: Granularity
+    stations: pd.DataFrame
+    start: Optional[datetime] = None
+    end: Optional[datetime] = None
+    timezone: Optional[str] = None
+
+    _df: Optional[pd.DataFrame] = None
+    _multi_station: bool = False
+
+    def __init__(
+        self,
+        granularity: Granularity,
+        stations: pd.DataFrame,
+        df: Optional[pd.DataFrame],
+        start: Optional[datetime] = None,
+        end: Optional[datetime] = None,
+        timezone: Optional[str] = None,
+        multi_station: bool = False,
+    ) -> None:
+        self.granularity = granularity
+        self.stations = stations
+        self.timezone = timezone
+        self._multi_station = multi_station
+        if df is not None and not df.empty:
+            self._df = df
+            self.start = start if start else df.index.get_level_values("time").min()
+            self.end = end if end else df.index.get_level_values("time").max()
+
+    def __len__(self) -> int:
+        """
+        Return number of rows in DataFrame
+        """
+        return len(self._df) if self._df is not None else 0
+
+    def __str__(self) -> str:
+        """
+        Return a stringified version of the DataFrame
+        """
+        return self._df.__str__() if self._df is not None else "Empty time series"
+
+    @property
+    def _target_length(self) -> int:
+        """
+        Expected number of non-NaN values
+        """
+        if not self.start or not self.end:
+            return 0
+
+        diff = self.end - self.start
+
+        return (
+            diff.days + 1
+            if self.granularity is Granularity.DAILY
+            else floor(diff.total_seconds() / 3600) + 1
+        ) * len(self.stations)
+
+    @property
+    def parameters(self) -> List[Parameter]:
+        """
+        Get parameters
+        """
+        return self._df.columns.to_list() if self._df is not None else []
+
+    @property
+    def freq(self) -> Optional[str]:
+        """
+        The time series frequency.
+
+        Returns a Pandas offset alias string (e.g. "1h", "1D", "1M") or
+        `None` for granularities where a regular frequency does not apply
+        (e.g. normals).
+        """
+        if self.granularity is Granularity.HOURLY:
+            return "1h"
+
+        if self.granularity is Granularity.DAILY:
+            return "1D"
+
+        if self.granularity is Granularity.MONTHLY:
+            return "1MS"
+
+        # Normals (climatological normals) do not have a regular time
+        # frequency in the same sense as timeseries data.
+        if self.granularity is Granularity.NORMALS:
+            return None
+
+        return None
+
+    @property
+    def empty(self) -> bool:
+        """
+        Is the time series empty?
+        """
+        return True if self._df is None else self._df.empty
+
+    @property
+    def providers(self) -> List[Provider]:
+        """
+        Get included providers
+        """
+        if self._df is None:
+            return []
+        providers: List[str] = (
+            self._df.index.get_level_values("source").unique().to_list()
+        )
+        return list(
+            set(chain.from_iterable([provider.split(" ") for provider in providers]))
+        )
+
+    @property
+    def licenses(self) -> List[License]:
+        """
+        Get licenses
+        """
+        providers = [
+            provider
+            for provider_id in self.providers
+            if (provider := provider_service.get_provider(provider_id)) is not None
+        ]
+
+        return [
+            provider.license for provider in providers if provider.license is not None
+        ]
+
+    @property
+    def attribution(self) -> str:
+        """
+        Attribution string
+        """
+        attributions = [
+            "Meteostat",
+            *set(
+                [
+                    license.attribution
+                    for license in self.licenses
+                    if license.attribution
+                ]
+            ),
+        ]
+
+        return ", ".join(attributions)
+
+    @property
+    def commercial(self) -> bool:
+        """
+        Is commercial use allowed?
+        """
+        return all(license.commercial for license in self.licenses)
+
+    def fetch(
+        self,
+        squash=True,
+        fill=False,
+        sources=False,
+        location=False,
+        clean=True,
+        humanize=False,
+        units: UnitSystem = UnitSystem.METRIC,
+    ) -> Optional[pd.DataFrame]:
+        """
+        Fetch the time series data as a DataFrame.
+
+        Parameters
+        ----------
+        squash : bool, optional
+            Whether to squash the DataFrame by source. Defaults to True.
+        fill : bool, optional
+            Whether to fill missing rows in the DataFrame. Defaults to False.
+        sources : bool, optional
+            Whether to include source information in the DataFrame. Defaults to False.
+        location : bool, optional
+            Whether to include location information (latitude, longitude, elevation)
+            in the DataFrame. Defaults to False.
+        clean : bool, optional
+            Whether to clean the DataFrame according to the schema. Defaults to True.
+        humanize : bool, optional
+            Whether to convert wind direction and condition codes to human-readable values. Defaults to False.
+        units : UnitSystem, optional
+            The unit system to use for the DataFrame. Defaults to metric units.
+
+        Returns
+        -------
+        pd.DataFrame or None
+            The time series data as a DataFrame, or None if no data is available.
+        """
+        df = copy(self._df)
+
+        if df is None:
+            return None
+
+        if squash:
+            df = squash_df(df, sources=sources)
+
+        if clean:
+            df = schema_service.clean(df, self.granularity)
+
+        if (
+            fill
+            and self.start is not None
+            and self.end is not None
+            and self.freq is not None
+        ):
+            df = fill_df(df, self.start, self.end, self.freq)
+
+        if self.timezone:
+            df = localize(df, self.timezone)
+
+        if location:
+            df = df.join(
+                self.stations[["latitude", "longitude", "elevation"]], on="station"
+            )
+
+        if humanize:
+            df = schema_service.humanize(df)
+
+        if units != UnitSystem.METRIC:
+            df = schema_service.convert(df, self.granularity, units)
+
+        # Remove station index level if not a multi-station query
+        if not self._multi_station and "station" in df.index.names:
+            df = df.droplevel("station")
+
+        return df.sort_index()
+
+    def count(self, parameter: Optional[Parameter | str] = None) -> int:
+        """
+        Get number of non-NaN values for a specific parameter.
+        If no parameter is specified, it returns the count for the entire DataFrame.
+
+        Parameters
+        ----------
+        parameter : Parameter or str
+            The parameter to count non-NaN values for. If None, counts for the entire DataFrame.
+
+        Returns
+        -------
+        int
+            The count of non-NaN values for the specified parameter or the entire DataFrame.
+        """
+        if self._df is None:
+            return 0
+
+        if parameter is None:
+            return self._df.count().max()
+
+        return self._df[
+            parameter if isinstance(parameter, Parameter) else parameter
+        ].count()
+
+    def completeness(self, parameter: Optional[Parameter | str] = None) -> float:
+        """
+        Get completeness for a specific parameter or the entire DataFrame.
+
+        Parameters
+        ----------
+        parameter : Parameter or str, optional
+            The parameter to calculate completeness for.
+            If None, calculates for the entire DataFrame.
+
+        Returns
+        -------
+        float
+            The completeness ratio for the specified parameter or the entire DataFrame.
+            Returns 0 if no data is available, 1 if complete, or a value between 0 and 1 otherwise.
+        """
+        df = self.fetch()
+
+        if df is None:
+            return 0
+
+        if parameter:
+            return round(
+                self.count(parameter) / self._target_length,
+                2,
+            )
+
+        return round(mean([self.completeness(p) for p in df.columns]), 2)
+
+    def validate(self) -> bool:
+        """
+        Does the time series pass all validations?
+        """
+        df = self.fetch(fill=True, clean=False)
+
+        if df is None:
+            return True
+
+        for col in df:
+            parameter = parameter_service.get_parameter(col, self.granularity)
+            if parameter is None or not hasattr(parameter, "validators"):
+                continue
+            for validator in parameter.validators:
+                if isinstance(validator, Validator):
+                    test = validator.test(df[col], df, col)
+                else:
+                    test = validator(df[col], df, col)
+
+                if isinstance(test, bool):
+                    if not test:
+                        return False
+                elif not test.all():
+                    return False
+
+        return True

meteostat/core/cache.py

@@ -1,71 +1,224 @@
 """
-Core Class - Cache
+Cache Service
 
-Meteorological data provided by Meteostat (https://dev.meteostat.net)
-under the terms of the Creative Commons Attribution-NonCommercial
-4.0 International Public License.
-
-The code is licensed under the MIT license.
+The Cache Service provides utilities for caching data on the local file system.
 """
 
+from functools import wraps
+from hashlib import md5
+import json
 import os
-import time
-import hashlib
-
-
-def get_local_file_path(cache_dir: str, cache_subdir: str, path: str) -> str:
-    """
-    Get the local file path
-    """
-
-    # Get file ID
-    file = hashlib.md5(path.encode("utf-8")).hexdigest()
-
-    return f"{cache_dir}/{cache_subdir}/{file}"
-
+from os.path import exists
+from time import time
+from typing import Any, Callable, Optional
 
-def file_in_cache(path: str, max_age: int = 0) -> bool:
-    """
-    Check if a file exists in the local cache
-    """
-
-    # Get directory
-    directory = os.path.dirname(path)
-
-    # Make sure the cache directory exists
-    if not os.path.exists(directory):
-        try:
-            os.makedirs(directory)
-        except FileExistsError:
-            pass
+import pandas as pd
 
-    # Return the file path if it exists
-    if os.path.isfile(path) and time.time() - os.path.getmtime(path) <= max_age:
-        return True
+from meteostat.core.config import config
+from meteostat.core.logger import logger
 
-    return False
 
-
-@classmethod
-def clear_cache(cls, max_age: int = None) -> None:
+class CacheService:
     """
-    Clear the cache
+    Cache Service
     """
 
-    if os.path.exists(cls.cache_dir + os.sep + cls.cache_subdir):
-        # Set max_age
-        if max_age is None:
-            max_age = cls.max_age
-
-        # Get current time
-        now = time.time()
-
-        # Go through all files
-        for file in os.listdir(cls.cache_dir + os.sep + cls.cache_subdir):
-            # Get full path
-            path = os.path.join(cls.cache_dir + os.sep + cls.cache_subdir, file)
-
-            # Check if file is older than max_age
-            if now - os.path.getmtime(path) > max_age and os.path.isfile(path):
-                # Delete file
-                os.remove(path)
+    _purged = False  # Flag to indicate if cache has been purged automatically
+
+    @staticmethod
+    def _write_pickle(path: str, df: Optional[pd.DataFrame]) -> None:
+        """
+        Persist a DataFrame in Pickle format
+        """
+        if df is None:
+            pd.DataFrame().to_pickle(path)
+        else:
+            df.to_pickle(path)
+
+    @staticmethod
+    def _read_pickle(path) -> Optional[pd.DataFrame]:
+        """
+        Read a pickle file into a DataFrame
+        """
+        df: pd.DataFrame = pd.read_pickle(path)
+        return None if df.empty else df
+
+    @staticmethod
+    def _write_json(path: str, data: dict | list) -> None:
+        """
+        Persist data in JSON format
+        """
+        with open(path, "w", encoding="utf-8") as file:
+            json.dump(data, file)
+
+    @staticmethod
+    def _read_json(path) -> dict | list:
+        """
+        Read JSON data into memory
+        """
+        with open(path, "r", encoding="utf-8") as file:
+            raw = file.read()
+        return json.loads(raw)
+
+    @staticmethod
+    def _func_to_uid(func, args: tuple, kwargs: dict[str, Any]) -> str:
+        """
+        Get a unique ID from a function call based on its module, name and arguments
+        """
+        return md5(
+            ";".join(
+                (
+                    func.__module__,
+                    func.__name__,
+                    *map(str, args),
+                    *[f"{key}:{str(value)}" for key, value in kwargs.items()],
+                )
+            ).encode("utf-8")
+        ).hexdigest()
+
+    @staticmethod
+    def create_cache_dir() -> None:
+        """
+        Create the cache directory if it doesn't exist
+        """
+        cache_dir = config.cache_directory
+
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+
+    @staticmethod
+    def get_cache_path(uid: str, filetype: str):
+        """
+        Get path of a cached file based on its uid and file type
+        """
+        return config.cache_directory + os.sep + f"{uid}.{filetype}"
+
+    @staticmethod
+    def is_stale(path: str, ttl: int) -> bool:
+        """
+        Check if a cached file is stale based on its age and TTL
+        """
+        return time() - os.path.getmtime(path) > max([ttl, config.cache_ttl])
+
+    @staticmethod
+    def purge(ttl: Optional[int] = None) -> None:
+        """
+        Remove stale files from disk cache
+        """
+        if ttl is None:
+            ttl = config.cache_ttl
+
+        logger.debug("Removing cached files older than %s seconds", ttl)
+
+        cache_dir = config.cache_directory
+
+        if os.path.exists(cache_dir):
+            # Get current time
+            now = time()
+            # Go through all files
+            for file in os.listdir(cache_dir):
+                # Get full path
+                path = os.path.join(cache_dir, file)
+                # Check if file is older than TTL
+                if now - os.path.getmtime(path) > ttl and os.path.isfile(path):
+                    # Delete file
+                    os.remove(path)
+
+    def persist(
+        self, path: str, data: pd.DataFrame | dict | list, data_type: str
+    ) -> None:
+        """
+        Persist any given data under a specific path
+        """
+        # Create cache directory if it doesn't exist
+        self.create_cache_dir()
+        # Save data locally
+        if data_type == "json" and isinstance(data, (dict, list)):
+            self._write_json(path, data)
+        elif data_type == "pickle" and (isinstance(data, pd.DataFrame) or data is None):
+            self._write_pickle(path, data)
+
+    def fetch(self, path, data_type: str) -> pd.DataFrame | dict | list | None:
+        """
+        Fetch data from a given path
+        """
+        if data_type == "json":
+            return self._read_json(path)
+        return self._read_pickle(path)
+
+    def from_func(
+        self, func, args, kwargs, ttl: int, data_format: str
+    ) -> pd.DataFrame | dict | list:
+        """
+        Cache a function's return value
+        """
+        uid = self._func_to_uid(func, args, kwargs)  # Get UID for function call
+        path = self.get_cache_path(uid, data_format)  # Get the local cache path
+        result = (
+            self.fetch(path, data_format)
+            if ttl > 0 and exists(path) and not self.is_stale(path, ttl)
+            else False
+        )
+
+        cache_status = "is" if isinstance(result, pd.DataFrame) or result else "is not"
+        logger.debug(
+            "%s from module %s with args=%s and kwargs=%s returns %s and %s served from cache",
+            func.__name__,
+            func.__module__,
+            args,
+            kwargs,
+            data_format,
+            cache_status,
+        )
+
+        if isinstance(result, pd.DataFrame) or result:
+            return result
+
+        result = func(*args, **kwargs)
+        if ttl > 0:
+            self.persist(path, result, data_format)
+
+        return result
+
+    def cache(
+        self,
+        ttl: int | Callable[..., int] = 60 * 60 * 24,
+        data_format: str = "json",
+    ):
+        """
+        A simple decorator which caches a function's return value
+        based on its payload.
+
+        All data is persisted in either JSON or Pickle format.
+        """
+
+        def decorator(func):
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                if not config.cache_enable:
+                    logger.debug(
+                        "Omitting cache for %s from module %s with args=%s and kwargs=%s",
+                        func.__name__,
+                        func.__module__,
+                        args,
+                        kwargs,
+                    )
+                    return func(*args, **kwargs)
+                if config.cache_autoclean and not self._purged:
+                    self.purge()
+                    self._purged = True
+                return self.from_func(
+                    func,
+                    args,
+                    kwargs,
+                    ttl if isinstance(ttl, int) else ttl(*args, **kwargs),
+                    data_format,
+                )
+
+            return wrapper
+
+        return decorator
+
+
+cache_service = CacheService()
+purge = cache_service.purge