freva-client 2410.0.1__tar.gz → 2411.0.0__tar.gz

{freva_client-2410.0.1 → freva_client-2411.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: freva-client
-Version: 2410.0.1
+Version: 2411.0.0
 Summary: Search for climate data based on key-value pairs
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.8

{freva_client-2410.0.1 → freva_client-2411.0.0}/src/freva_client/__init__.py

@@ -17,5 +17,5 @@ need to apply data analysis plugins, please visit the
 from .auth import authenticate
 from .query import databrowser
 
-__version__ = "2410.0.1"
+__version__ = "2411.0.0"
 __all__ = ["authenticate", "databrowser", "__version__"]

{freva_client-2410.0.1 → freva_client-2411.0.0}/src/freva_client/cli/auth_cli.py

@@ -49,9 +49,7 @@ def authenticate_cli(
         "-f",
         help="Force token recreation, even if current token is still valid.",
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",

{freva_client-2410.0.1 → freva_client-2411.0.0}/src/freva_client/cli/databrowser_cli.py

@@ -10,6 +10,7 @@ from tempfile import NamedTemporaryFile
 from typing import Dict, List, Literal, Optional, Union, cast
 
 import typer
+import xarray as xr
 from freva_client import databrowser
 from freva_client.auth import Auth
 from freva_client.utils import exception_handler, logger
@@ -42,6 +43,7 @@ class Flavours(str, Enum):
     cmip5: str = "cmip5"
     cordex: str = "cordex"
     nextgems: str = "nextgems"
+    user: str = "user"
 
 
 class TimeSelect(str, Enum):
@@ -163,9 +165,7 @@ def metadata_search(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -187,11 +187,9 @@ def metadata_search(
     result = databrowser.metadata_search(
         *(facets or []),
         time=time or "",
-        time_select=cast(
-            Literal["file", "flexible", "strict"], time_select.value
-        ),
+        time_select=cast(Literal["file", "flexible", "strict"], time_select.value),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         host=host,
@@ -253,9 +251,7 @@ def data_search(
         "--time-select",
         help=TimeSelect.get_help(),
     ),
-    zarr: bool = typer.Option(
-        False, "--zarr", help="Create zarr stream files."
-    ),
+    zarr: bool = typer.Option(False, "--zarr", help="Create zarr stream files."),
     access_token: Optional[str] = typer.Option(
         None,
         "--access-token",
@@ -289,9 +285,7 @@ def data_search(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -318,7 +312,7 @@ def data_search(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -425,9 +419,7 @@ def intake_catalogue(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -451,7 +443,7 @@ def intake_catalogue(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -469,9 +461,7 @@ def intake_catalogue(
             print(Path(temp_f.name).read_text())
 
 
-@databrowser_app.command(
-    name="data-count", help="Count the databrowser search results"
-)
+@databrowser_app.command(name="data-count", help="Count the databrowser search results")
 @exception_handler
 def count_values(
     search_keys: Optional[List[str]] = typer.Argument(
@@ -547,9 +537,7 @@ def count_values(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -576,11 +564,9 @@ def count_values(
         result = databrowser.count_values(
             *facets,
             time=time or "",
-            time_select=cast(
-                Literal["file", "flexible", "strict"], time_select
-            ),
+            time_select=cast(Literal["file", "flexible", "strict"], time_select),
             flavour=cast(
-                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
                 flavour.value,
             ),
             host=host,
@@ -594,11 +580,9 @@ def count_values(
             databrowser(
                 *facets,
                 time=time or "",
-                time_select=cast(
-                    Literal["file", "flexible", "strict"], time_select
-                ),
+                time_select=cast(Literal["file", "flexible", "strict"], time_select),
                 flavour=cast(
-                    Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                    Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
                     flavour.value,
                 ),
                 host=host,
@@ -620,3 +604,108 @@ def count_values(
             print(f"{key}: {', '.join(counts)}")
     else:
         print(result)
+
+
+user_data_app = typer.Typer(help="Add or delete user data.")
+databrowser_app.add_typer(user_data_app, name="user-data")
+
+
+@user_data_app.command(name="add", help="Add user data into the databrowser.")
+@exception_handler
+def user_data_add(
+    paths: List[str] = typer.Option(
+        ...,
+        "--path",
+        "-p",
+        help="Paths to the user's data to be added.",
+    ),
+    facets: Optional[List[str]] = typer.Option(
+        None,
+        "--facet",
+        help="Key-value metadata pairs to categorize the user"
+        "input data in the format key=value.",
+    ),
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser. If not set (default), "
+            "the hostname is read from a config file."
+        ),
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help="Access token for authentication when adding user data.",
+    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+) -> None:
+    """Add user data into the databrowser."""
+    logger.set_verbosity(verbose)
+    logger.debug("Checking if the user has the right to add data")
+    result = databrowser(host=host)
+    _auth(result._cfg.auth_url, access_token)
+
+    facet_dict = {}
+    if facets:
+        for facet in facets:
+            if "=" not in facet:
+                logger.error(
+                    f"Invalid facet format: {facet}. Expected format: key=value."
+                )
+                raise typer.Exit(code=1)
+            key, value = facet.split("=", 1)
+            facet_dict[key] = value
+
+    logger.debug(f"Adding user data with paths {paths} and facets {facet_dict}")
+    databrowser.userdata(
+        action="add",
+        userdata_items=cast(List[Union[str, xr.Dataset]], paths),
+        metadata=facet_dict,
+        host=host
+    )
+
+
+@user_data_app.command(name="delete", help="Delete user data from the databrowser.")
+@exception_handler
+def user_data_delete(
+    search_keys: List[str] = typer.Option(
+        None,
+        "--search-key",
+        "-s",
+        help="Key-value metadata pairs to search and identify user data "
+        "for deletion in the format key=value.",
+    ),
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser. If not set (default), "
+            "the hostname is read from a config file."
+        ),
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help="Access token for authentication when deleting user data.",
+    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+) -> None:
+    """Delete user data from the databrowser."""
+    logger.set_verbosity(verbose)
+    logger.debug("Checking if the user has the right to delete data")
+    result = databrowser(host=host)
+    _auth(result._cfg.auth_url, access_token)
+
+    search_key_dict = {}
+    if search_keys:
+        for search_key in search_keys:
+            if "=" not in search_key:
+                logger.error(
+                    f"Invalid search key format: {search_key}. "
+                    "Expected format: key=value."
+                )
+                raise typer.Exit(code=1)
+            key, value = search_key.split("=", 1)
+            search_key_dict[key] = value
+    databrowser.userdata(action="delete", metadata=search_key_dict, host=host)

{freva_client-2410.0.1 → freva_client-2411.0.0}/src/freva_client/query.py

@@ -1,5 +1,6 @@
 """Query climate data sets by using-key value pair search queries."""
 
+
 import sys
 from collections import defaultdict
 from fnmatch import fnmatch
@@ -8,6 +9,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import (
     Any,
+    Collection,
     Dict,
     Iterator,
     List,
@@ -21,12 +23,13 @@ from typing import (
 import intake
 import intake_esm
 import requests
+import xarray as xr
 import yaml
 from rich import print as pprint
 
 from .auth import Auth
 from .utils import logger
-from .utils.databrowser_utils import Config
+from .utils.databrowser_utils import Config, UserDataHandler
 
 __all__ = ["databrowser"]
 
@@ -207,7 +210,7 @@ class databrowser:
         *facets: str,
         uniq_key: Literal["file", "uri"] = "file",
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -243,8 +246,7 @@ class databrowser:
         self, facets: Tuple[str, ...], search_kw: Dict[str, List[str]]
     ) -> None:
         metadata = {
-            k: v[::2]
-            for (k, v) in self._facet_search(extended_search=True).items()
+            k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
         }
         primary_key = list(metadata.keys() or ["project"])[0]
         num_facets = 0
@@ -267,19 +269,15 @@ class databrowser:
         headers = {}
         if self._stream_zarr:
             query_url = self._cfg.zarr_loader_url
-            token = self._auth.check_authentication(
-                auth_url=self._cfg.auth_url
-            )
+            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
             headers = {"Authorization": f"Bearer {token['access_token']}"}
-        result = self._get(query_url, headers=headers, stream=True)
+        result = self._request("GET", query_url, headers=headers, stream=True)
         if result is not None:
             try:
                 for res in result.iter_lines():
                     yield res.decode("utf-8")
             except KeyboardInterrupt:
-                pprint(
-                    "[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr
-                )
+                pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
 
     def __repr__(self) -> str:
         params = ", ".join(
@@ -306,9 +304,7 @@ class databrowser:
 
         # Create a table-like structure for available flavors and search facets
         style = 'style="text-align: left"'
-        facet_heading = (
-            f"Available search facets for <em>{self._flavour}</em> flavour"
-        )
+        facet_heading = f"Available search facets for <em>{self._flavour}</em> flavour"
         html_repr = (
             "<table>"
             f"<tr><th colspan='2' {style}>{self.__class__.__name__}"
@@ -337,7 +333,7 @@ class databrowser:
 
 
         """
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result:
             return cast(int, result.json().get("total_count", 0))
         return 0
@@ -347,15 +343,11 @@ class databrowser:
         kwargs: Dict[str, Any] = {"stream": True}
         url = self._cfg.intake_url
         if self._stream_zarr:
-            token = self._auth.check_authentication(
-                auth_url=self._cfg.auth_url
-            )
+            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
             url = self._cfg.zarr_loader_url
-            kwargs["headers"] = {
-                "Authorization": f"Bearer {token['access_token']}"
-            }
+            kwargs["headers"] = {"Authorization": f"Bearer {token['access_token']}"}
             kwargs["params"] = {"catalogue-type": "intake"}
-        result = self._get(url, **kwargs)
+        result = self._request("GET", url, **kwargs)
         if result is None:
             raise ValueError("No results found")
 
@@ -365,9 +357,7 @@ class databrowser:
                 for content in result.iter_content(decode_unicode=False):
                     stream.write(content)
         except Exception as error:
-            raise ValueError(
-                f"Couldn't write catalogue content: {error}"
-            ) from None
+            raise ValueError(f"Couldn't write catalogue content: {error}") from None
 
     def intake_catalogue(self) -> intake_esm.core.esm_datastore:
         """Create an intake esm catalogue object from the search.
@@ -392,7 +382,7 @@ class databrowser:
         .. execute_code::
 
             from freva_client import databrowser
-            db = databrowser(dataset="cmip6-fs", stream_zarr=True)
+            db = databrowser(dataset="cmip6-hsm", stream_zarr=True)
             cat = db.intake_catalogue()
             print(cat.df)
         """
@@ -405,7 +395,7 @@ class databrowser:
         cls,
         *facets: str,
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -455,6 +445,7 @@ class databrowser:
             Select all versions and not just the latest version (default).
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str
             The search constraints to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -504,9 +495,7 @@ class databrowser:
         result = this._facet_search(extended_search=extended_search)
         counts = {}
         for facet, value_counts in result.items():
-            counts[facet] = dict(
-                zip(value_counts[::2], map(int, value_counts[1::2]))
-            )
+            counts[facet] = dict(zip(value_counts[::2], map(int, value_counts[1::2])))
         return counts
 
     @cached_property
@@ -531,8 +520,7 @@ class databrowser:
 
         """
         return {
-            k: v[::2]
-            for (k, v) in self._facet_search(extended_search=True).items()
+            k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
         }
 
     @classmethod
@@ -540,7 +528,7 @@ class databrowser:
         cls,
         *facets: str,
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -593,6 +581,7 @@ class databrowser:
             name will be taken from the freva config file.
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str, list[str]
             The facets to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -664,9 +653,7 @@ class databrowser:
         )
         return {
             k: v[::2]
-            for (k, v) in this._facet_search(
-                extended_search=extended_search
-            ).items()
+            for (k, v) in this._facet_search(extended_search=extended_search).items()
         }
 
     @classmethod
@@ -723,7 +710,7 @@ class databrowser:
         self,
         extended_search: bool = False,
     ) -> Dict[str, List[str]]:
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result is None:
             return {}
         data = result.json()
@@ -733,26 +720,172 @@ class databrowser:
             constraints = data["primary_facets"]
         return {f: v for f, v in data["facets"].items() if f in constraints}
 
-    def _get(
-        self, url: str, **kwargs: Any
+    @classmethod
+    def userdata(
+        cls,
+        action: Literal["add", "delete"],
+        userdata_items: Optional[List[Union[str, xr.Dataset]]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        host: Optional[str] = None,
+        fail_on_error: bool = False,
+    ) -> None:
+        """Add or delete user data in the databrowser system.
+
+        Manage user data in the databrowser system by adding new data or
+        deleting existing data.
+
+        For the "``add``" action, the user can provide data items (file paths
+        or xarray datasets) along with metadata (key-value pairs) to
+        categorize and organize the data.
+
+        For the "``delete``" action, the user provides metadata as search
+        criteria to identify and remove the existing data from the
+        system.
+
+        Parameters
+        ~~~~~~~~~~
+        action : Literal["add", "delete"]
+            The action to perform: "add" to add new data, or "delete"
+            to remove existing data.
+        userdata_items : List[Union[str, xr.Dataset]], optional
+            A list of user file paths or xarray datasets to add to the
+            databrowser (required for "add").
+        metadata : Dict[str, str], optional
+            Key-value metadata pairs to categorize the data (for "add")
+            or search and identify data for
+            deletion (for "delete").
+        host : str, optional
+            Override the host name of the databrowser server. This is usually
+            the url where the freva web site can be found. Such as
+            www.freva.dkrz.de. By default no host name is given and the host
+            name will be taken from the freva config file.
+        fail_on_error : bool, optional
+            Make the call fail if the connection to the databrowser could not
+            be established.
+
+        Raises
+        ~~~~~~
+        ValueError
+            If the operation fails or required parameters are missing
+            for the specified action.
+        FileNotFoundError
+            If no user data is provided for the "add" action.
+
+        Example
+        ~~~~~~~
+
+        Adding user data:
+
+        .. execute_code::
+
+            from freva_client import authenticate, databrowser
+            import xarray as xr
+            token_info = authenticate(username="janedoe")
+            filenames = (
+                "../freva-rest/src/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "GERICS/NCC-NorESM1-M/rcp85/r1i1p1/GERICS-REMO2015/v1/3hr/pr/v20181212/*.nc"
+            )
+            filename1 = (
+                "../freva-rest/src/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "CLMcom/MPI-M-MPI-ESM-LR/historical/r0i0p0/CLMcom-CCLM4-8-17/v1/fx/orog/v20140515/"
+                "orog_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_fx.nc"
+            )
+            xarray_data = xr.open_dataset(filename1)
+            databrowser.userdata(
+                action="add",
+                userdata_items=[xarray_data, filenames],
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
+            )
+
+        Deleting user data:
+
+        .. execute_code::
+
+            from freva_client import authenticate, databrowser
+            token_info = authenticate(username="janedoe")
+            databrowser.userdata(
+                action="delete",
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
+            )
+        """
+        this = cls(
+            host=host,
+            fail_on_error=fail_on_error,
+        )
+        userdata_items = userdata_items or []
+        metadata = metadata or {}
+        url = f"{this._cfg.userdata_url}"
+        token = this._auth.check_authentication(auth_url=this._cfg.auth_url)
+        headers = {"Authorization": f"Bearer {token['access_token']}"}
+        payload_metadata: dict[str, Collection[Collection[str]]] = {}
+
+        if action == "add":
+            user_data_handler = UserDataHandler(userdata_items)
+            if user_data_handler.user_metadata:
+                payload_metadata = {
+                    "user_metadata": user_data_handler.user_metadata,
+                    "facets": metadata,
+                }
+                result = this._request(
+                    "POST", url, data=payload_metadata, headers=headers
+                )
+                if result is not None:
+                    response_data = result.json()
+                    status_message = response_data.get("status")
+                else:
+                    raise ValueError("Failed to add user data")
+                pprint(f"[b][green]{status_message}[green][b]")
+            else:
+                raise ValueError("No metadata generated from the input data.")
+
+        if action == "delete":
+            if userdata_items:
+                logger.info(
+                    "'userdata_items' are not needed for the 'delete'"
+                    "action and will be ignored."
+                )
+
+            result = this._request("DELETE", url, data=metadata, headers=headers)
+
+            if result is None:
+                raise ValueError("Failed to delete user data")
+            pprint("[b][green]User data deleted successfully[green][b]")
+
+    def _request(
+        self,
+        method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"],
+        url: str,
+        data: Optional[Dict[str, Any]] = None,
+        **kwargs: Any
     ) -> Optional[requests.models.Response]:
-        """Apply the get method to the databrowser."""
-        logger.debug("Searching %s with parameters: %s", url, self._params)
+        """Request method to handle CRUD operations (GET, POST, PUT, PATCH, DELETE)."""
+        method_upper = method.upper()
+        timeout = kwargs.pop("timeout", 30)
         params = kwargs.pop("params", {})
-        kwargs.setdefault("timeout", 30)
+        stream = kwargs.pop("stream", False)
+
+        logger.debug("%s request to %s with data: %s and parameters: %s",
+                     method_upper, url, data, {**self._params, **params})
+
         try:
-            res = requests.get(
-                url, params={**self._params, **params}, **kwargs
+            req = requests.Request(
+                method=method_upper,
+                url=url,
+                params={**self._params, **params},
+                json=None if method_upper in "GET" else data,
+                **kwargs
             )
-            res.raise_for_status()
-            return res
+            with requests.Session() as session:
+                prepared = session.prepare_request(req)
+                res = session.send(prepared, timeout=timeout, stream=stream)
+                res.raise_for_status()
+                return res
+
         except KeyboardInterrupt:
             pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"Search request failed with {error}"
+        except (requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError) as error:
+            msg = f"{method_upper} request failed with {error}"
             if self._fail_on_error:
                 raise ValueError(msg) from None
             logger.warning(msg)

freva_client-2411.0.0/src/freva_client/utils/databrowser_utils.py

@@ -0,0 +1,368 @@
+"""Various utilities for getting the databrowser working."""
+
+import concurrent.futures
+import os
+import sys
+import sysconfig
+from configparser import ConfigParser, ExtendedInterpolation
+from functools import cached_property
+from pathlib import Path
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
+
+import appdirs
+import numpy as np
+import requests
+import tomli
+import xarray as xr
+
+from . import logger
+
+
+class Config:
+    """Client config class.
+
+    This class is used for basic configuration of the databrowser
+    client.
+    """
+
+    def __init__(
+        self,
+        host: Optional[str] = None,
+        uniq_key: Literal["file", "uri"] = "file",
+        flavour: str = "freva",
+    ) -> None:
+        self.databrowser_url = f"{self.get_api_url(host)}/databrowser"
+        self.auth_url = f"{self.get_api_url(host)}/auth/v2"
+        self.uniq_key = uniq_key
+        self._flavour = flavour
+
+    def _read_ini(self, path: Path) -> str:
+        """Read an ini file."""
+        ini_parser = ConfigParser(interpolation=ExtendedInterpolation())
+        ini_parser.read_string(path.read_text())
+        config = ini_parser["evaluation_system"]
+        scheme, host = self._split_url(
+            config.get("databrowser.host") or config.get("solr.host")
+        )
+        host, _, port = (host or "").partition(":")
+        port = port or config.get("databrowser.port", "")
+        if port:
+            host = f"{host}:{port}"
+        return f"{scheme}://{host}"
+
+    def _read_toml(self, path: Path) -> str:
+        """Read a new style toml config file."""
+        try:
+            config = tomli.loads(path.read_text()).get("freva", {})
+            scheme, host = self._split_url(cast(str, config["host"]))
+        except (tomli.TOMLDecodeError, KeyError):
+            return ""
+        host, _, port = host.partition(":")
+        if port:
+            host = f"{host}:{port}"
+        return f"{scheme}://{host}"
+
+    def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
+        """Read the configuration."""
+        data_types = {"toml": self._read_toml, "ini": self._read_ini}
+        try:
+            return data_types[file_type](path)
+        except KeyError:
+            pass
+        return ""
+
+    @cached_property
+    def overview(self) -> Dict[str, Any]:
+        """Get an overview of the all databrowser flavours and search keys."""
+        try:
+            res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
+        except requests.exceptions.ConnectionError:
+            raise ValueError(f"Could not connect to {self.databrowser_url}") from None
+        return cast(Dict[str, Any], res.json())
+
+    def _get_databrowser_host_from_config(self) -> str:
+        """Get the config file order."""
+
+        eval_conf = self.get_dirs(user=False) / "evaluation_system.conf"
+        freva_config = Path(
+            os.environ.get("FREVA_CONFIG")
+            or Path(self.get_dirs(user=False)) / "freva.toml"
+        )
+        paths: Dict[Path, Literal["toml", "ini"]] = {
+            Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
+            Path(self.get_dirs(user=True)) / "freva.toml": "toml",
+            freva_config: "toml",
+            Path(os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf): "ini",
+        }
+        for config_path, config_type in paths.items():
+            if config_path.is_file():
+                host = self._read_config(config_path, config_type)
+                if host:
+                    return host
+        raise ValueError(
+            "No databrowser host configured, please use a"
+            " configuration defining a databrowser host or"
+            " set a host name using the `host` key"
+        )
+
+    @cached_property
+    def flavour(self) -> str:
+        """Get the flavour."""
+        flavours = self.overview.get("flavours", [])
+        if self._flavour not in flavours:
+            raise ValueError(
+                f"Search {self._flavour} not available, select from"
+                f" {','.join(flavours)}"
+            )
+        return self._flavour
+
+    @property
+    def search_url(self) -> str:
+        """Define the data search endpoint."""
+        return f"{self.databrowser_url}/data_search/{self.flavour}/{self.uniq_key}"
+
+    @property
+    def zarr_loader_url(self) -> str:
+        """Define the url for getting zarr files."""
+        return f"{self.databrowser_url}/load/{self.flavour}/"
+
+    @property
+    def intake_url(self) -> str:
+        """Define the url for creating intake catalogues."""
+        return f"{self.databrowser_url}/intake_catalogue/{self.flavour}/{self.uniq_key}"
+
+    @property
+    def metadata_url(self) -> str:
+        """Define the endpoint for the metadata search."""
+        return (
+            f"{self.databrowser_url}/metadata_search/" f"{self.flavour}/{self.uniq_key}"
+        )
+
+    @staticmethod
+    def _split_url(url: str) -> Tuple[str, str]:
+        scheme, _, hostname = url.partition("://")
+        if not hostname:
+            hostname = scheme
+            scheme = ""
+        scheme = scheme or "http"
+        return scheme, hostname
+
+    def get_api_url(self, url: Optional[str]) -> str:
+        """Construct the databrowser url from a given hostname."""
+        url = url or self._get_databrowser_host_from_config()
+        scheme, hostname = self._split_url(url)
+        hostname, _, port = hostname.partition(":")
+        if port:
+            hostname = f"{hostname}:{port}"
+        hostname = hostname.partition("/")[0]
+        return f"{scheme}://{hostname}/api"
+
+    @staticmethod
+    def get_dirs(user: bool = True) -> Path:
+        """Get the 'scripts' and 'purelib' directories we'll install into.
+
+        This is now a thin wrapper around sysconfig.get_paths(). It's not inlined,
+        because some tests mock it out to install to a different location.
+        """
+        if user:
+            if (sys.platform == "darwin") and sysconfig.get_config_var(
+                "PYTHONFRAMEWORK"
+            ):
+                scheme = "osx_framework_user"
+            else:
+                scheme = f"{os.name}_user"
+            return Path(sysconfig.get_path("data", scheme)) / "share" / "freva"
+        # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
+        # installing into a virtualenv
+        return Path(sysconfig.get_path("data")) / "share" / "freva"
+
+    @property
+    def userdata_url(self) -> str:
+        """Define the url for adding and deleting user-data."""
+        return f"{self.databrowser_url}/userdata"
+
+
+class UserDataHandler:
+    """Class for processing user data.
+
+    This class is used for processing user data and extracting metadata
+    from the data files.
+    """
+    def __init__(self, userdata_items: List[Union[str, xr.Dataset]]) -> None:
+        self._suffixes = [".nc", ".nc4", ".grb", ".grib", ".zarr", "zar"]
+        self.user_metadata: List[Dict[str, Union[str, List[str], Dict[str, str]]]] = []
+        self._metadata_collection: List[Dict[str, Union[str, List[str]]]] = []
+        try:
+            self._executor = concurrent.futures.ThreadPoolExecutor(
+                max_workers=min(int(os.cpu_count() or 4), 15)
+            )
+            self._process_user_data(userdata_items)
+        finally:
+            self._executor.shutdown(wait=True)
+
+    def _gather_files(self, path: Path, pattern: str = "*") -> Iterator[Path]:
+        """Gather all valid files from directory and wildcard pattern."""
+        for item in path.rglob(pattern):
+            if item.is_file() and item.suffix in self._suffixes:
+                yield item
+
+    def _validate_user_data(
+        self,
+        user_data: Sequence[Union[str, xr.Dataset]],
+    ) -> Dict[str, Union[List[Path], List[xr.Dataset]]]:
+        validated_paths: List[Path] = []
+        validated_xarray_datasets: List[xr.Dataset] = []
+        for data in user_data:
+            if isinstance(data, (str, Path)):
+                path = Path(data).expanduser().absolute()
+                if path.is_dir():
+                    validated_paths.extend(self._gather_files(path))
+                elif path.is_file() and path.suffix in self._suffixes:
+                    validated_paths.append(path)
+                else:
+                    validated_paths.extend(
+                        self._gather_files(path.parent, pattern=path.name)
+                    )
+            elif isinstance(data, xr.Dataset):
+                validated_xarray_datasets.append(data)
+
+        if not validated_paths and not validated_xarray_datasets:
+            raise FileNotFoundError("No valid file paths or xarray datasets found.")
+        return {
+            "validated_user_paths": validated_paths,
+            "validated_user_xrdatasets": validated_xarray_datasets,
+        }
+
+    def _process_user_data(self, userdata_items: List[Union[str, xr.Dataset]],
+                           ) -> None:
+        """Process xarray datasets and file paths using thread pool."""
+        futures = []
+        validated_userdata: Dict[str, Union[List[Path], List[xr.Dataset]]] = \
+            self._validate_user_data(userdata_items)
+        if validated_userdata["validated_user_xrdatasets"]:
+            futures.append(
+                self._executor.submit(self._process_userdata_in_executor,
+                                      validated_userdata["validated_user_xrdatasets"])
+            )
+
+        if validated_userdata["validated_user_paths"]:
+            futures.append(
+                self._executor.submit(self._process_userdata_in_executor,
+                                      validated_userdata["validated_user_paths"])
+            )
+        for future in futures:
+            try:
+                future.result()
+            except Exception as e:  # pragma: no cover
+                logger.error(f"Error processing batch: {e}")
+
+    def _process_userdata_in_executor(
+        self, validated_userdata: Union[List[Path], List[xr.Dataset]]
+    ) -> None:
+        for data in validated_userdata:
+            metadata = self._get_metadata(data)
+            if isinstance(metadata, Exception) or metadata == {}:
+                logger.warning("Error getting metadata: %s", metadata)
+            else:
+                self.user_metadata.append(metadata)
+
+    def _timedelta_to_cmor_frequency(self, dt: float) -> str:
+        for total_seconds, frequency in self._time_table.items():
+            if dt >= total_seconds:
+                return frequency
+        return "fx"  # pragma: no cover
+
+    @property
+    def _time_table(self) -> dict[int, str]:
+        return {
+            315360000: "dec",  # Decade
+            31104000: "yr",  # Year
+            2538000: "mon",  # Month
+            1296000: "sem",  # Seasonal (half-year)
+            84600: "day",  # Day
+            21600: "6h",  # Six-hourly
+            10800: "3h",  # Three-hourly
+            3600: "hr",  # Hourly
+            1: "subhr",  # Sub-hourly
+        }
+
+    def _get_time_frequency(self, time_delta: int, freq_attr: str = "") -> str:
+        if freq_attr in self._time_table.values():
+            return freq_attr
+        return self._timedelta_to_cmor_frequency(time_delta)
+
+    def _get_metadata(
+        self, path: Union[os.PathLike[str], xr.Dataset]
+    ) -> Dict[str, Union[str, List[str], Dict[str, str]]]:
+        """Get metadata from a path or xarray dataset."""
+
+        try:
+            dset = (
+                path if isinstance(path, xr.Dataset)
+                else xr.open_mfdataset(str(path),
+                                       parallel=False,
+                                       use_cftime=True,
+                                       lock=False)
+            )
+            time_freq = dset.attrs.get("frequency", "")
+            data_vars = list(map(str, dset.data_vars))
+            coords = list(map(str, dset.coords))
+            try:
+                times = dset["time"].values[:]
+            except (KeyError, IndexError, TypeError):
+                times = np.array([])
+
+        except Exception as error:
+            logger.error("Failed to open data file %s: %s", str(path), error)
+            return {}
+        if len(times) > 0:
+            try:
+                time_str = f"[{times[0].isoformat()}Z TO {times[-1].isoformat()}Z]"
+                dt = abs((times[1] - times[0]).total_seconds()) if len(times) > 1 else 0
+            except Exception as non_cftime:
+                logger.info("The time var is not based on the cftime: %s", non_cftime)
+                time_str = (
+                    f"[{np.datetime_as_string(times[0], unit='s')}Z TO "
+                    f"{np.datetime_as_string(times[-1], unit='s')}Z]"
+                )
+                dt = (
+                    abs((times[1] - times[0]).astype("timedelta64[s]").astype(int))
+                    if len(times) > 1
+                    else 0
+                )
+        else:
+            time_str = "fx"
+            dt = 0
+
+        variables = [
+            var
+            for var in data_vars
+            if var not in coords
+            and not any(
+                term in var.lower() for term in ["lon", "lat", "bnds", "x", "y"]
+            )
+            and var.lower() not in ["rotated_pole", "rot_pole"]
+        ]
+
+        _data: Dict[str, Union[str, List[str], Dict[str, str]]] = {}
+        _data.setdefault("variable", variables[0])
+        _data.setdefault("time_frequency", self._get_time_frequency(dt, time_freq))
+        _data["time"] = time_str
+        _data.setdefault("cmor_table", _data["time_frequency"])
+        _data.setdefault("version", "")
+        if isinstance(path, Path):
+            _data["file"] = str(path)
+        if isinstance(path, xr.Dataset):
+            _data["file"] = str(dset.encoding["source"])
+        return _data

freva_client-2410.0.1/src/freva_client/utils/databrowser_utils.py

@@ -1,179 +0,0 @@
-"""Various utilities for getting the databrowser working."""
-
-import os
-import sys
-import sysconfig
-from configparser import ConfigParser, ExtendedInterpolation
-from functools import cached_property
-from pathlib import Path
-from typing import Any, Dict, Literal, Optional, Tuple, cast
-
-import appdirs
-import requests
-import tomli
-
-
-class Config:
-    """Client config class.
-
-    This class is used for basic configuration of the databrowser
-    client.
-    """
-
-    def __init__(
-        self,
-        host: Optional[str] = None,
-        uniq_key: Literal["file", "uri"] = "file",
-        flavour: str = "freva",
-    ) -> None:
-        self.databrowser_url = f"{self.get_api_url(host)}/databrowser"
-        self.auth_url = f"{self.get_api_url(host)}/auth/v2"
-        self.uniq_key = uniq_key
-        self._flavour = flavour
-
-    def _read_ini(self, path: Path) -> str:
-        """Read an ini file."""
-        ini_parser = ConfigParser(interpolation=ExtendedInterpolation())
-        ini_parser.read_string(path.read_text())
-        config = ini_parser["evaluation_system"]
-        scheme, host = self._split_url(
-            config.get("databrowser.host") or config.get("solr.host")
-        )
-        host, _, port = (host or "").partition(":")
-        port = port or config.get("databrowser.port", "")
-        if port:
-            host = f"{host}:{port}"
-        return f"{scheme}://{host}"
-
-    def _read_toml(self, path: Path) -> str:
-        """Read a new style toml config file."""
-        try:
-            config = tomli.loads(path.read_text()).get("freva", {})
-            scheme, host = self._split_url(cast(str, config["host"]))
-        except (tomli.TOMLDecodeError, KeyError):
-            return ""
-        host, _, port = host.partition(":")
-        if port:
-            host = f"{host}:{port}"
-        return f"{scheme}://{host}"
-
-    def _read_config(
-        self, path: Path, file_type: Literal["toml", "ini"]
-    ) -> str:
-        """Read the configuration."""
-        data_types = {"toml": self._read_toml, "ini": self._read_ini}
-        try:
-            return data_types[file_type](path)
-        except KeyError:
-            pass
-        return ""
-
-    @cached_property
-    def overview(self) -> Dict[str, Any]:
-        """Get an overview of the all databrowser flavours and search keys."""
-        try:
-            res = requests.get(f"{self.databrowser_url}/overview", timeout=3)
-        except requests.exceptions.ConnectionError:
-            raise ValueError(
-                f"Could not connect to {self.databrowser_url}"
-            ) from None
-        return cast(Dict[str, Any], res.json())
-
-    def _get_databrowser_host_from_config(self) -> str:
-        """Get the config file order."""
-
-        eval_conf = self.get_dirs(user=False) / "evaluation_system.conf"
-        freva_config = Path(
-            os.environ.get("FREVA_CONFIG")
-            or Path(self.get_dirs(user=False)) / "freva.toml"
-        )
-        paths: Dict[Path, Literal["toml", "ini"]] = {
-            Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
-            Path(self.get_dirs(user=True)) / "freva.toml": "toml",
-            freva_config: "toml",
-            Path(
-                os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf
-            ): "ini",
-        }
-        for config_path, config_type in paths.items():
-            if config_path.is_file():
-                host = self._read_config(config_path, config_type)
-                if host:
-                    return host
-        raise ValueError(
-            "No databrowser host configured, please use a"
-            " configuration defining a databrowser host or"
-            " set a host name using the `host` key"
-        )
-
-    @cached_property
-    def flavour(self) -> str:
-        """Get the flavour."""
-        flavours = self.overview.get("flavours", [])
-        if self._flavour not in flavours:
-            raise ValueError(
-                f"Search {self._flavour} not available, select from"
-                f" {','.join(flavours)}"
-            )
-        return self._flavour
-
-    @property
-    def search_url(self) -> str:
-        """Define the data search endpoint."""
-        return f"{self.databrowser_url}/data_search/{self.flavour}/{self.uniq_key}"
-
-    @property
-    def zarr_loader_url(self) -> str:
-        """Define the url for getting zarr files."""
-        return f"{self.databrowser_url}/load/{self.flavour}/"
-
-    @property
-    def intake_url(self) -> str:
-        """Define the url for creating intake catalogues."""
-        return f"{self.databrowser_url}/intake_catalogue/{self.flavour}/{self.uniq_key}"
-
-    @property
-    def metadata_url(self) -> str:
-        """Define the endpoint for the metadata search."""
-        return (
-            f"{self.databrowser_url}/metadata_search/"
-            f"{self.flavour}/{self.uniq_key}"
-        )
-
-    @staticmethod
-    def _split_url(url: str) -> Tuple[str, str]:
-        scheme, _, hostname = url.partition("://")
-        if not hostname:
-            hostname = scheme
-            scheme = ""
-        scheme = scheme or "http"
-        return scheme, hostname
-
-    def get_api_url(self, url: Optional[str]) -> str:
-        """Construct the databrowser url from a given hostname."""
-        url = url or self._get_databrowser_host_from_config()
-        scheme, hostname = self._split_url(url)
-        hostname, _, port = hostname.partition(":")
-        if port:
-            hostname = f"{hostname}:{port}"
-        hostname = hostname.partition("/")[0]
-        return f"{scheme}://{hostname}/api"
-
-    @staticmethod
-    def get_dirs(user: bool = True) -> Path:
-        """Get the 'scripts' and 'purelib' directories we'll install into.
-
-        This is now a thin wrapper around sysconfig.get_paths(). It's not inlined,
-        because some tests mock it out to install to a different location.
-        """
-        if user:
-            if (sys.platform == "darwin") and sysconfig.get_config_var(
-                "PYTHONFRAMEWORK"
-            ):
-                scheme = "osx_framework_user"
-            else:
-                scheme = f"{os.name}_user"
-            return Path(sysconfig.get_path("data", scheme)) / "share" / "freva"
-        # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
-        # installing into a virtualenv
-        return Path(sysconfig.get_path("data")) / "share" / "freva"