freva-client 2410.0.0b1__tar.gz → 2411.0.0__tar.gz

{freva_client-2410.0.0b1 → freva_client-2411.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: freva-client
-Version: 2410.0.0b1
+Version: 2411.0.0
 Summary: Search for climate data based on key-value pairs
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.8
@@ -22,6 +22,7 @@ Requires-Dist: authlib
 Requires-Dist: requests
 Requires-Dist: intake_esm
 Requires-Dist: rich
+Requires-Dist: setuptools
 Requires-Dist: tomli
 Requires-Dist: typer
 Requires-Dist: tox ; extra == "dev"

{freva_client-2410.0.0b1 → freva_client-2411.0.0}/pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
 "requests",
 "intake_esm",
 "rich",
+"setuptools",
 "tomli",
 "typer",
 ]

{freva_client-2410.0.0b1 → freva_client-2411.0.0}/src/freva_client/__init__.py

@@ -17,5 +17,5 @@ need to apply data analysis plugins, please visit the
 from .auth import authenticate
 from .query import databrowser
 
-__version__ = "2410.0.0-beta1"
+__version__ = "2411.0.0"
 __all__ = ["authenticate", "databrowser", "__version__"]

{freva_client-2410.0.0b1 → freva_client-2411.0.0}/src/freva_client/cli/databrowser_cli.py

@@ -10,6 +10,7 @@ from tempfile import NamedTemporaryFile
 from typing import Dict, List, Literal, Optional, Union, cast
 
 import typer
+import xarray as xr
 from freva_client import databrowser
 from freva_client.auth import Auth
 from freva_client.utils import exception_handler, logger
@@ -42,6 +43,7 @@ class Flavours(str, Enum):
     cmip5: str = "cmip5"
     cordex: str = "cordex"
     nextgems: str = "nextgems"
+    user: str = "user"
 
 
 class TimeSelect(str, Enum):
@@ -187,7 +189,7 @@ def metadata_search(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select.value),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         host=host,
@@ -310,7 +312,7 @@ def data_search(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -441,7 +443,7 @@ def intake_catalogue(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -564,7 +566,7 @@ def count_values(
             time=time or "",
             time_select=cast(Literal["file", "flexible", "strict"], time_select),
             flavour=cast(
-                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
                 flavour.value,
             ),
             host=host,
@@ -580,7 +582,7 @@ def count_values(
                 time=time or "",
                 time_select=cast(Literal["file", "flexible", "strict"], time_select),
                 flavour=cast(
-                    Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                    Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
                     flavour.value,
                 ),
                 host=host,
@@ -611,7 +613,6 @@ databrowser_app.add_typer(user_data_app, name="user-data")
 @user_data_app.command(name="add", help="Add user data into the databrowser.")
 @exception_handler
 def user_data_add(
-    username: str = typer.Argument(..., help="Username of the data owner"),
     paths: List[str] = typer.Option(
         ...,
         "--path",
@@ -621,8 +622,8 @@ def user_data_add(
     facets: Optional[List[str]] = typer.Option(
         None,
         "--facet",
-        "-f",
-        help="Facet key-value pairs for metadata in the format key=value.",
+        help="Key-value metadata pairs to categorize the user"
+        "input data in the format key=value.",
     ),
     host: Optional[str] = typer.Option(
         None,
@@ -656,22 +657,24 @@ def user_data_add(
             key, value = facet.split("=", 1)
             facet_dict[key] = value
 
-    logger.debug(
-        f"Adding user data for {username} with paths {paths} and facets {facet_dict}"
+    logger.debug(f"Adding user data with paths {paths} and facets {facet_dict}")
+    databrowser.userdata(
+        action="add",
+        userdata_items=cast(List[Union[str, xr.Dataset]], paths),
+        metadata=facet_dict,
+        host=host
     )
-    result.add_user_data(username=username, paths=paths, facets=facet_dict)
-    logger.info("User data started crawling. Check the Databrowser to see the updates.")
 
 
 @user_data_app.command(name="delete", help="Delete user data from the databrowser.")
 @exception_handler
-def user_data_remove(
-    username: str = typer.Argument(..., help="Username of the data owner"),
+def user_data_delete(
     search_keys: List[str] = typer.Option(
         None,
         "--search-key",
         "-s",
-        help="Search keys for the data to be deleted in the format key=value.",
+        help="Key-value metadata pairs to search and identify user data "
+        "for deletion in the format key=value.",
     ),
     host: Optional[str] = typer.Option(
         None,
@@ -705,5 +708,4 @@ def user_data_remove(
                 raise typer.Exit(code=1)
             key, value = search_key.split("=", 1)
             search_key_dict[key] = value
-    result.delete_user_data(username=username, search_keys=search_key_dict)
-    logger.info("User data deleted successfully.")
+    databrowser.userdata(action="delete", metadata=search_key_dict, host=host)

{freva_client-2410.0.0b1 → freva_client-2411.0.0}/src/freva_client/query.py

@@ -1,5 +1,6 @@
 """Query climate data sets by using-key value pair search queries."""
 
+
 import sys
 from collections import defaultdict
 from fnmatch import fnmatch
@@ -8,6 +9,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import (
     Any,
+    Collection,
     Dict,
     Iterator,
     List,
@@ -21,12 +23,13 @@ from typing import (
 import intake
 import intake_esm
 import requests
+import xarray as xr
 import yaml
 from rich import print as pprint
 
 from .auth import Auth
 from .utils import logger
-from .utils.databrowser_utils import Config
+from .utils.databrowser_utils import Config, UserDataHandler
 
 __all__ = ["databrowser"]
 
@@ -206,7 +209,9 @@ class databrowser:
         self,
         *facets: str,
         uniq_key: Literal["file", "uri"] = "file",
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -266,7 +271,7 @@ class databrowser:
             query_url = self._cfg.zarr_loader_url
             token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
             headers = {"Authorization": f"Bearer {token['access_token']}"}
-        result = self._get(query_url, headers=headers, stream=True)
+        result = self._request("GET", query_url, headers=headers, stream=True)
         if result is not None:
             try:
                 for res in result.iter_lines():
@@ -328,7 +333,7 @@ class databrowser:
 
 
         """
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result:
             return cast(int, result.json().get("total_count", 0))
         return 0
@@ -342,7 +347,7 @@ class databrowser:
             url = self._cfg.zarr_loader_url
             kwargs["headers"] = {"Authorization": f"Bearer {token['access_token']}"}
             kwargs["params"] = {"catalogue-type": "intake"}
-        result = self._get(url, **kwargs)
+        result = self._request("GET", url, **kwargs)
         if result is None:
             raise ValueError("No results found")
 
@@ -377,7 +382,7 @@ class databrowser:
         .. execute_code::
 
             from freva_client import databrowser
-            db = databrowser(dataset="cmip6-fs", stream_zarr=True)
+            db = databrowser(dataset="cmip6-hsm", stream_zarr=True)
             cat = db.intake_catalogue()
             print(cat.df)
         """
@@ -389,7 +394,9 @@ class databrowser:
     def count_values(
         cls,
         *facets: str,
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -438,6 +445,7 @@ class databrowser:
             Select all versions and not just the latest version (default).
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str
             The search constraints to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -519,7 +527,9 @@ class databrowser:
     def metadata_search(
         cls,
         *facets: str,
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -571,6 +581,7 @@ class databrowser:
             name will be taken from the freva config file.
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str, list[str]
             The facets to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -699,7 +710,7 @@ class databrowser:
         self,
         extended_search: bool = False,
     ) -> Dict[str, List[str]]:
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result is None:
             return {}
         data = result.json()
@@ -709,172 +720,172 @@ class databrowser:
             constraints = data["primary_facets"]
         return {f: v for f, v in data["facets"].items() if f in constraints}
 
-    def add_user_data(
-        self, username: str, paths: List[str], facets: Dict[str, str]
+    @classmethod
+    def userdata(
+        cls,
+        action: Literal["add", "delete"],
+        userdata_items: Optional[List[Union[str, xr.Dataset]]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        host: Optional[str] = None,
+        fail_on_error: bool = False,
     ) -> None:
-        """Add user data to the databrowser.
+        """Add or delete user data in the databrowser system.
 
-        Via this functionality, user would be able to add data to the databrowser.
-        It accepts file paths and metadata facets to categorize and store the user's
-        data.
+        Manage user data in the databrowser system by adding new data or
+        deleting existing data.
+
+        For the "``add``" action, the user can provide data items (file paths
+        or xarray datasets) along with metadata (key-value pairs) to
+        categorize and organize the data.
+
+        For the "``delete``" action, the user provides metadata as search
+        criteria to identify and remove the existing data from the
+        system.
 
         Parameters
         ~~~~~~~~~~
-        username: str
-            The username of user.
-        paths: list[str]
-            A list of paths to the data files that should be uploaded or cataloged.
-        facets: dict[str, str]
-            A dictionary containing metadata facets (key-value pairs) to describe the
-            data.
-
-        Returns
-        ~~~~~~~~
-        None
-            If the operation is successful, no return value is provided.
+        action : Literal["add", "delete"]
+            The action to perform: "add" to add new data, or "delete"
+            to remove existing data.
+        userdata_items : List[Union[str, xr.Dataset]], optional
+            A list of user file paths or xarray datasets to add to the
+            databrowser (required for "add").
+        metadata : Dict[str, str], optional
+            Key-value metadata pairs to categorize the data (for "add")
+            or search and identify data for
+            deletion (for "delete").
+        host : str, optional
+            Override the host name of the databrowser server. This is usually
+            the url where the freva web site can be found. Such as
+            www.freva.dkrz.de. By default no host name is given and the host
+            name will be taken from the freva config file.
+        fail_on_error : bool, optional
+            Make the call fail if the connection to the databrowser could not
+            be established.
 
         Raises
-        ~~~~~~~
+        ~~~~~~
         ValueError
-            If the operation fails to add the user data.
+            If the operation fails or required parameters are missing
+            for the specified action.
+        FileNotFoundError
+            If no user data is provided for the "add" action.
 
         Example
         ~~~~~~~
+
+        Adding user data:
+
         .. execute_code::
 
             from freva_client import authenticate, databrowser
+            import xarray as xr
             token_info = authenticate(username="janedoe")
-            db = databrowser()
-            db.add_user_data(
-                "janedoe",
-                ["."],
-                {"project": "cmip5", "experiment": "something"}
+            filenames = (
+                "../freva-rest/src/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "GERICS/NCC-NorESM1-M/rcp85/r1i1p1/GERICS-REMO2015/v1/3hr/pr/v20181212/*.nc"
+            )
+            filename1 = (
+                "../freva-rest/src/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "CLMcom/MPI-M-MPI-ESM-LR/historical/r0i0p0/CLMcom-CCLM4-8-17/v1/fx/orog/v20140515/"
+                "orog_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_fx.nc"
+            )
+            xarray_data = xr.open_dataset(filename1)
+            databrowser.userdata(
+                action="add",
+                userdata_items=[xarray_data, filenames],
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
             )
-        """
-        url = f"{self._cfg.userdata_url}/{username}"
-        token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
-        headers = {"Authorization": f"Bearer {token['access_token']}"}
-        params = {"paths": paths}
-        if "username" in facets:
-            del facets["username"]
-        data = facets
-        result = self._put(url, data=data, headers=headers, params=params)
-
-        if result is None:
-            raise ValueError("Failed to add user data")
-
-    def delete_user_data(self, username: str, search_keys: Dict[str, str]) -> None:
-        """
-        Delete user data from the databrowser.
-
-        Uing this, user would be able to delete the user's data from the databrowser
-        based on the provided search keys.
-
-        Parameters
-        ~~~~~~~~~~
-        username: str
-            The username associated with the data to be deleted.
-        search_keys: dict[str, str]
-            A dictionary containing the search keys to identify the data to be deleted.
 
-        Returns
-        ~~~~~~~~
-        None
-            If the operation is successful, no return value is provided.
+        Deleting user data:
 
-        Raises
-        ~~~~~~~
-        ValueError
-            If the operation fails to delete the user data.
-
-        Example
-        ~~~~~~~
         .. execute_code::
 
-            from freva_client import databrowser, authenticate
+            from freva_client import authenticate, databrowser
             token_info = authenticate(username="janedoe")
-            db = databrowser()
-            db.delete_user_data(
-                "janedoe",
-                {"project": "cmip5", "experiment": "something"}
+            databrowser.userdata(
+                action="delete",
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
             )
         """
-        url = f"{self._cfg.userdata_url}/{username}"
-        token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
+        this = cls(
+            host=host,
+            fail_on_error=fail_on_error,
+        )
+        userdata_items = userdata_items or []
+        metadata = metadata or {}
+        url = f"{this._cfg.userdata_url}"
+        token = this._auth.check_authentication(auth_url=this._cfg.auth_url)
         headers = {"Authorization": f"Bearer {token['access_token']}"}
-        data = search_keys
-        result = self._delete(url, headers=headers, json=data)
-        if result is None:
-            raise ValueError("Failed to delete user data")
+        payload_metadata: dict[str, Collection[Collection[str]]] = {}
+
+        if action == "add":
+            user_data_handler = UserDataHandler(userdata_items)
+            if user_data_handler.user_metadata:
+                payload_metadata = {
+                    "user_metadata": user_data_handler.user_metadata,
+                    "facets": metadata,
+                }
+                result = this._request(
+                    "POST", url, data=payload_metadata, headers=headers
+                )
+                if result is not None:
+                    response_data = result.json()
+                    status_message = response_data.get("status")
+                else:
+                    raise ValueError("Failed to add user data")
+                pprint(f"[b][green]{status_message}[green][b]")
+            else:
+                raise ValueError("No metadata generated from the input data.")
 
-    def _get(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
-        """Apply the get method to the databrowser."""
-        logger.debug("Searching %s with parameters: %s", url, self._params)
-        params = kwargs.pop("params", {})
-        kwargs.setdefault("timeout", 30)
-        try:
-            res = requests.get(url, params={**self._params, **params}, **kwargs)
-            res.raise_for_status()
-            return res
-        except KeyboardInterrupt:
-            pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"Search request failed with {error}"
-            if self._fail_on_error:
-                raise ValueError(msg) from None
-            logger.warning(msg)
-        return None
+        if action == "delete":
+            if userdata_items:
+                logger.info(
+                    "'userdata_items' are not needed for the 'delete'"
+                    "action and will be ignored."
+                )
+
+            result = this._request("DELETE", url, data=metadata, headers=headers)
 
-    def _put(
-        self, url: str, data: Dict[str, Any], **kwargs: Any
+            if result is None:
+                raise ValueError("Failed to delete user data")
+            pprint("[b][green]User data deleted successfully[green][b]")
+
+    def _request(
+        self,
+        method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"],
+        url: str,
+        data: Optional[Dict[str, Any]] = None,
+        **kwargs: Any
     ) -> Optional[requests.models.Response]:
-        """Apply the PUT method to the databrowser."""
-        logger.debug(
-            "PUT request to %s with data: %s and parameters: %s",
-            url,
-            data,
-            self._params,
-        )
-        kwargs.setdefault("timeout", 30)
+        """Request method to handle CRUD operations (GET, POST, PUT, PATCH, DELETE)."""
+        method_upper = method.upper()
+        timeout = kwargs.pop("timeout", 30)
         params = kwargs.pop("params", {})
-        try:
-            res = requests.put(
-                url, json=data, params={**self._params, **params}, **kwargs
-            )
-            res.raise_for_status()
-            return res
-        except KeyboardInterrupt:
-            pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
+        stream = kwargs.pop("stream", False)
 
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"adding user data request failed with {error}"
-            if self._fail_on_error:
-                raise ValueError(msg) from None
-            logger.warning(msg)
-        return None
+        logger.debug("%s request to %s with data: %s and parameters: %s",
+                     method_upper, url, data, {**self._params, **params})
 
-    def _delete(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
-        """Apply the DELETE method to the databrowser."""
-        logger.debug("DELETE request to %s with parameters: %s", url, self._params)
-        params = kwargs.pop("params", {})
-        kwargs.setdefault("timeout", 30)
         try:
-            res = requests.delete(url, params={**self._params, **params}, **kwargs)
-            res.raise_for_status()
-            return res
+            req = requests.Request(
+                method=method_upper,
+                url=url,
+                params={**self._params, **params},
+                json=None if method_upper in "GET" else data,
+                **kwargs
+            )
+            with requests.Session() as session:
+                prepared = session.prepare_request(req)
+                res = session.send(prepared, timeout=timeout, stream=stream)
+                res.raise_for_status()
+                return res
+
         except KeyboardInterrupt:
             pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"DELETE request failed with {error}"
+        except (requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError) as error:
+            msg = f"{method_upper} request failed with {error}"
             if self._fail_on_error:
                 raise ValueError(msg) from None
             logger.warning(msg)

freva_client-2411.0.0/src/freva_client/utils/databrowser_utils.py

@@ -0,0 +1,368 @@
+"""Various utilities for getting the databrowser working."""
+
+import concurrent.futures
+import os
+import sys
+import sysconfig
+from configparser import ConfigParser, ExtendedInterpolation
+from functools import cached_property
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
+
+import appdirs
+import numpy as np
+import requests
+import tomli
+import xarray as xr
+
+from . import logger
+
+
+class Config:
+    """Client config class.
+
+    This class is used for basic configuration of the databrowser
+    client.
+    """
+
+    def __init__(
+        self,
+        host: Optional[str] = None,
+        uniq_key: Literal["file", "uri"] = "file",
+        flavour: str = "freva",
+    ) -> None:
+        self.databrowser_url = f"{self.get_api_url(host)}/databrowser"
+        self.auth_url = f"{self.get_api_url(host)}/auth/v2"
+        self.uniq_key = uniq_key
+        self._flavour = flavour
+
+    def _read_ini(self, path: Path) -> str:
+        """Read an ini file."""
+        ini_parser = ConfigParser(interpolation=ExtendedInterpolation())
+        ini_parser.read_string(path.read_text())
+        config = ini_parser["evaluation_system"]
+        scheme, host = self._split_url(
+            config.get("databrowser.host") or config.get("solr.host")
+        )
+        host, _, port = (host or "").partition(":")
+        port = port or config.get("databrowser.port", "")
+        if port:
+            host = f"{host}:{port}"
+        return f"{scheme}://{host}"
+
+    def _read_toml(self, path: Path) -> str:
+        """Read a new style toml config file."""
+        try:
+            config = tomli.loads(path.read_text()).get("freva", {})
+            scheme, host = self._split_url(cast(str, config["host"]))
+        except (tomli.TOMLDecodeError, KeyError):
+            return ""
+        host, _, port = host.partition(":")
+        if port:
+            host = f"{host}:{port}"
+        return f"{scheme}://{host}"
+
+    def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
+        """Read the configuration."""
+        data_types = {"toml": self._read_toml, "ini": self._read_ini}
+        try:
+            return data_types[file_type](path)
+        except KeyError:
+            pass
+        return ""
+
+    @cached_property
+    def overview(self) -> Dict[str, Any]:
+        """Get an overview of the all databrowser flavours and search keys."""
+        try:
+            res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
+        except requests.exceptions.ConnectionError:
+            raise ValueError(f"Could not connect to {self.databrowser_url}") from None
+        return cast(Dict[str, Any], res.json())
+
+    def _get_databrowser_host_from_config(self) -> str:
+        """Get the config file order."""
+
+        eval_conf = self.get_dirs(user=False) / "evaluation_system.conf"
+        freva_config = Path(
+            os.environ.get("FREVA_CONFIG")
+            or Path(self.get_dirs(user=False)) / "freva.toml"
+        )
+        paths: Dict[Path, Literal["toml", "ini"]] = {
+            Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
+            Path(self.get_dirs(user=True)) / "freva.toml": "toml",
+            freva_config: "toml",
+            Path(os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf): "ini",
+        }
+        for config_path, config_type in paths.items():
+            if config_path.is_file():
+                host = self._read_config(config_path, config_type)
+                if host:
+                    return host
+        raise ValueError(
+            "No databrowser host configured, please use a"
+            " configuration defining a databrowser host or"
+            " set a host name using the `host` key"
+        )
+
+    @cached_property
+    def flavour(self) -> str:
+        """Get the flavour."""
+        flavours = self.overview.get("flavours", [])
+        if self._flavour not in flavours:
+            raise ValueError(
+                f"Search {self._flavour} not available, select from"
+                f" {','.join(flavours)}"
+            )
+        return self._flavour
+
+    @property
+    def search_url(self) -> str:
+        """Define the data search endpoint."""
+        return f"{self.databrowser_url}/data_search/{self.flavour}/{self.uniq_key}"
+
+    @property
+    def zarr_loader_url(self) -> str:
+        """Define the url for getting zarr files."""
+        return f"{self.databrowser_url}/load/{self.flavour}/"
+
+    @property
+    def intake_url(self) -> str:
+        """Define the url for creating intake catalogues."""
+        return f"{self.databrowser_url}/intake_catalogue/{self.flavour}/{self.uniq_key}"
+
+    @property
+    def metadata_url(self) -> str:
+        """Define the endpoint for the metadata search."""
+        return (
+            f"{self.databrowser_url}/metadata_search/" f"{self.flavour}/{self.uniq_key}"
+        )
+
+    @staticmethod
+    def _split_url(url: str) -> Tuple[str, str]:
+        scheme, _, hostname = url.partition("://")
+        if not hostname:
+            hostname = scheme
+            scheme = ""
+        scheme = scheme or "http"
+        return scheme, hostname
+
+    def get_api_url(self, url: Optional[str]) -> str:
+        """Construct the databrowser url from a given hostname."""
+        url = url or self._get_databrowser_host_from_config()
+        scheme, hostname = self._split_url(url)
+        hostname, _, port = hostname.partition(":")
+        if port:
+            hostname = f"{hostname}:{port}"
+        hostname = hostname.partition("/")[0]
+        return f"{scheme}://{hostname}/api"
+
+    @staticmethod
+    def get_dirs(user: bool = True) -> Path:
+        """Get the 'scripts' and 'purelib' directories we'll install into.
+
+        This is now a thin wrapper around sysconfig.get_paths(). It's not inlined,
+        because some tests mock it out to install to a different location.
+        """
+        if user:
+            if (sys.platform == "darwin") and sysconfig.get_config_var(
+                "PYTHONFRAMEWORK"
+            ):
+                scheme = "osx_framework_user"
+            else:
+                scheme = f"{os.name}_user"
+            return Path(sysconfig.get_path("data", scheme)) / "share" / "freva"
+        # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
+        # installing into a virtualenv
+        return Path(sysconfig.get_path("data")) / "share" / "freva"
+
+    @property
+    def userdata_url(self) -> str:
+        """Define the url for adding and deleting user-data."""
+        return f"{self.databrowser_url}/userdata"
+
+
+class UserDataHandler:
+    """Class for processing user data.
+
+    This class is used for processing user data and extracting metadata
+    from the data files.
+    """
+    def __init__(self, userdata_items: List[Union[str, xr.Dataset]]) -> None:
+        self._suffixes = [".nc", ".nc4", ".grb", ".grib", ".zarr", "zar"]
+        self.user_metadata: List[Dict[str, Union[str, List[str], Dict[str, str]]]] = []
+        self._metadata_collection: List[Dict[str, Union[str, List[str]]]] = []
+        try:
+            self._executor = concurrent.futures.ThreadPoolExecutor(
+                max_workers=min(int(os.cpu_count() or 4), 15)
+            )
+            self._process_user_data(userdata_items)
+        finally:
+            self._executor.shutdown(wait=True)
+
+    def _gather_files(self, path: Path, pattern: str = "*") -> Iterator[Path]:
+        """Gather all valid files from directory and wildcard pattern."""
+        for item in path.rglob(pattern):
+            if item.is_file() and item.suffix in self._suffixes:
+                yield item
+
+    def _validate_user_data(
+        self,
+        user_data: Sequence[Union[str, xr.Dataset]],
+    ) -> Dict[str, Union[List[Path], List[xr.Dataset]]]:
+        validated_paths: List[Path] = []
+        validated_xarray_datasets: List[xr.Dataset] = []
+        for data in user_data:
+            if isinstance(data, (str, Path)):
+                path = Path(data).expanduser().absolute()
+                if path.is_dir():
+                    validated_paths.extend(self._gather_files(path))
+                elif path.is_file() and path.suffix in self._suffixes:
+                    validated_paths.append(path)
+                else:
+                    validated_paths.extend(
+                        self._gather_files(path.parent, pattern=path.name)
+                    )
+            elif isinstance(data, xr.Dataset):
+                validated_xarray_datasets.append(data)
+
+        if not validated_paths and not validated_xarray_datasets:
+            raise FileNotFoundError("No valid file paths or xarray datasets found.")
+        return {
+            "validated_user_paths": validated_paths,
+            "validated_user_xrdatasets": validated_xarray_datasets,
+        }
+
+    def _process_user_data(self, userdata_items: List[Union[str, xr.Dataset]],
+                           ) -> None:
+        """Process xarray datasets and file paths using thread pool."""
+        futures = []
+        validated_userdata: Dict[str, Union[List[Path], List[xr.Dataset]]] = \
+            self._validate_user_data(userdata_items)
+        if validated_userdata["validated_user_xrdatasets"]:
+            futures.append(
+                self._executor.submit(self._process_userdata_in_executor,
+                                      validated_userdata["validated_user_xrdatasets"])
+            )
+
+        if validated_userdata["validated_user_paths"]:
+            futures.append(
+                self._executor.submit(self._process_userdata_in_executor,
+                                      validated_userdata["validated_user_paths"])
+            )
+        for future in futures:
+            try:
+                future.result()
+            except Exception as e:  # pragma: no cover
+                logger.error(f"Error processing batch: {e}")
+
+    def _process_userdata_in_executor(
+        self, validated_userdata: Union[List[Path], List[xr.Dataset]]
+    ) -> None:
+        for data in validated_userdata:
+            metadata = self._get_metadata(data)
+            if isinstance(metadata, Exception) or metadata == {}:
+                logger.warning("Error getting metadata: %s", metadata)
+            else:
+                self.user_metadata.append(metadata)
+
+    def _timedelta_to_cmor_frequency(self, dt: float) -> str:
+        for total_seconds, frequency in self._time_table.items():
+            if dt >= total_seconds:
+                return frequency
+        return "fx"  # pragma: no cover
+
+    @property
+    def _time_table(self) -> dict[int, str]:
+        return {
+            315360000: "dec",  # Decade
+            31104000: "yr",  # Year
+            2538000: "mon",  # Month
+            1296000: "sem",  # Seasonal (half-year)
+            84600: "day",  # Day
+            21600: "6h",  # Six-hourly
+            10800: "3h",  # Three-hourly
+            3600: "hr",  # Hourly
+            1: "subhr",  # Sub-hourly
+        }
+
+    def _get_time_frequency(self, time_delta: int, freq_attr: str = "") -> str:
+        if freq_attr in self._time_table.values():
+            return freq_attr
+        return self._timedelta_to_cmor_frequency(time_delta)
+
+    def _get_metadata(
+        self, path: Union[os.PathLike[str], xr.Dataset]
+    ) -> Dict[str, Union[str, List[str], Dict[str, str]]]:
+        """Get metadata from a path or xarray dataset."""
+
+        try:
+            dset = (
+                path if isinstance(path, xr.Dataset)
+                else xr.open_mfdataset(str(path),
+                                       parallel=False,
+                                       use_cftime=True,
+                                       lock=False)
+            )
+            time_freq = dset.attrs.get("frequency", "")
+            data_vars = list(map(str, dset.data_vars))
+            coords = list(map(str, dset.coords))
+            try:
+                times = dset["time"].values[:]
+            except (KeyError, IndexError, TypeError):
+                times = np.array([])
+
+        except Exception as error:
+            logger.error("Failed to open data file %s: %s", str(path), error)
+            return {}
+        if len(times) > 0:
+            try:
+                time_str = f"[{times[0].isoformat()}Z TO {times[-1].isoformat()}Z]"
+                dt = abs((times[1] - times[0]).total_seconds()) if len(times) > 1 else 0
+            except Exception as non_cftime:
+                logger.info("The time var is not based on the cftime: %s", non_cftime)
+                time_str = (
+                    f"[{np.datetime_as_string(times[0], unit='s')}Z TO "
+                    f"{np.datetime_as_string(times[-1], unit='s')}Z]"
+                )
+                dt = (
+                    abs((times[1] - times[0]).astype("timedelta64[s]").astype(int))
+                    if len(times) > 1
+                    else 0
+                )
+        else:
+            time_str = "fx"
+            dt = 0
+
+        variables = [
+            var
+            for var in data_vars
+            if var not in coords
+            and not any(
+                term in var.lower() for term in ["lon", "lat", "bnds", "x", "y"]
+            )
+            and var.lower() not in ["rotated_pole", "rot_pole"]
+        ]
+
+        _data: Dict[str, Union[str, List[str], Dict[str, str]]] = {}
+        _data.setdefault("variable", variables[0])
+        _data.setdefault("time_frequency", self._get_time_frequency(dt, time_freq))
+        _data["time"] = time_str
+        _data.setdefault("cmor_table", _data["time_frequency"])
+        _data.setdefault("version", "")
+        if isinstance(path, Path):
+            _data["file"] = str(path)
+        if isinstance(path, xr.Dataset):
+            _data["file"] = str(dset.encoding["source"])
+        return _data

freva_client-2410.0.0b1/src/freva_client/utils/databrowser_utils.py

@@ -1,177 +0,0 @@
-"""Various utilities for getting the databrowser working."""
-
-import os
-import sys
-import sysconfig
-from configparser import ConfigParser, ExtendedInterpolation
-from functools import cached_property
-from pathlib import Path
-from typing import Any, Dict, Literal, Optional, Tuple, cast
-
-import appdirs
-import requests
-import tomli
-
-
-class Config:
-    """Client config class.
-
-    This class is used for basic configuration of the databrowser
-    client.
-    """
-
-    def __init__(
-        self,
-        host: Optional[str] = None,
-        uniq_key: Literal["file", "uri"] = "file",
-        flavour: str = "freva",
-    ) -> None:
-        self.databrowser_url = f"{self.get_api_url(host)}/databrowser"
-        self.auth_url = f"{self.get_api_url(host)}/auth/v2"
-        self.uniq_key = uniq_key
-        self._flavour = flavour
-
-    def _read_ini(self, path: Path) -> str:
-        """Read an ini file."""
-        ini_parser = ConfigParser(interpolation=ExtendedInterpolation())
-        ini_parser.read_string(path.read_text())
-        config = ini_parser["evaluation_system"]
-        scheme, host = self._split_url(
-            config.get("databrowser.host") or config.get("solr.host")
-        )
-        host, _, port = (host or "").partition(":")
-        port = port or config.get("databrowser.port", "")
-        if port:
-            host = f"{host}:{port}"
-        return f"{scheme}://{host}"
-
-    def _read_toml(self, path: Path) -> str:
-        """Read a new style toml config file."""
-        try:
-            config = tomli.loads(path.read_text()).get("freva", {})
-            scheme, host = self._split_url(cast(str, config["host"]))
-        except (tomli.TOMLDecodeError, KeyError):
-            return ""
-        host, _, port = host.partition(":")
-        if port:
-            host = f"{host}:{port}"
-        return f"{scheme}://{host}"
-
-    def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
-        """Read the configuration."""
-        data_types = {"toml": self._read_toml, "ini": self._read_ini}
-        try:
-            return data_types[file_type](path)
-        except KeyError:
-            pass
-        return ""
-
-    @cached_property
-    def overview(self) -> Dict[str, Any]:
-        """Get an overview of the all databrowser flavours and search keys."""
-        try:
-            res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
-        except requests.exceptions.ConnectionError:
-            raise ValueError(f"Could not connect to {self.databrowser_url}") from None
-        return cast(Dict[str, Any], res.json())
-
-    def _get_databrowser_host_from_config(self) -> str:
-        """Get the config file order."""
-
-        eval_conf = self.get_dirs(user=False) / "evaluation_system.conf"
-        freva_config = Path(
-            os.environ.get("FREVA_CONFIG")
-            or Path(self.get_dirs(user=False)) / "freva.toml"
-        )
-        paths: Dict[Path, Literal["toml", "ini"]] = {
-            Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
-            Path(self.get_dirs(user=True)) / "freva.toml": "toml",
-            freva_config: "toml",
-            Path(os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf): "ini",
-        }
-        for config_path, config_type in paths.items():
-            if config_path.is_file():
-                host = self._read_config(config_path, config_type)
-                if host:
-                    return host
-        raise ValueError(
-            "No databrowser host configured, please use a"
-            " configuration defining a databrowser host or"
-            " set a host name using the `host` key"
-        )
-
-    @cached_property
-    def flavour(self) -> str:
-        """Get the flavour."""
-        flavours = self.overview.get("flavours", [])
-        if self._flavour not in flavours:
-            raise ValueError(
-                f"Search {self._flavour} not available, select from"
-                f" {','.join(flavours)}"
-            )
-        return self._flavour
-
-    @property
-    def search_url(self) -> str:
-        """Define the data search endpoint."""
-        return f"{self.databrowser_url}/data_search/{self.flavour}/{self.uniq_key}"
-
-    @property
-    def zarr_loader_url(self) -> str:
-        """Define the url for getting zarr files."""
-        return f"{self.databrowser_url}/load/{self.flavour}/"
-
-    @property
-    def intake_url(self) -> str:
-        """Define the url for creating intake catalogues."""
-        return f"{self.databrowser_url}/intake_catalogue/{self.flavour}/{self.uniq_key}"
-
-    @property
-    def metadata_url(self) -> str:
-        """Define the endpoint for the metadata search."""
-        return (
-            f"{self.databrowser_url}/metadata_search/" f"{self.flavour}/{self.uniq_key}"
-        )
-
-    @staticmethod
-    def _split_url(url: str) -> Tuple[str, str]:
-        scheme, _, hostname = url.partition("://")
-        if not hostname:
-            hostname = scheme
-            scheme = ""
-        scheme = scheme or "http"
-        return scheme, hostname
-
-    def get_api_url(self, url: Optional[str]) -> str:
-        """Construct the databrowser url from a given hostname."""
-        url = url or self._get_databrowser_host_from_config()
-        scheme, hostname = self._split_url(url)
-        hostname, _, port = hostname.partition(":")
-        if port:
-            hostname = f"{hostname}:{port}"
-        hostname = hostname.partition("/")[0]
-        return f"{scheme}://{hostname}/api"
-
-    @staticmethod
-    def get_dirs(user: bool = True) -> Path:
-        """Get the 'scripts' and 'purelib' directories we'll install into.
-
-        This is now a thin wrapper around sysconfig.get_paths(). It's not inlined,
-        because some tests mock it out to install to a different location.
-        """
-        if user:
-            if (sys.platform == "darwin") and sysconfig.get_config_var(
-                "PYTHONFRAMEWORK"
-            ):
-                scheme = "osx_framework_user"
-            else:
-                scheme = f"{os.name}_user"
-            return Path(sysconfig.get_path("data", scheme)) / "share" / "freva"
-        # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
-        # installing into a virtualenv
-        return Path(sysconfig.get_path("data")) / "share" / "freva"
-
-    @property
-    def userdata_url(self) -> str:
-        """Define the url for adding and deleting user-data."""
-        return f"{self.databrowser_url}/userdata"