freva-client 2410.0.1__py3-none-any.whl → 2502.0.0__py3-none-any.whl

freva_client/__init__.py

@@ -17,5 +17,5 @@ need to apply data analysis plugins, please visit the
 from .auth import authenticate
 from .query import databrowser
 
-__version__ = "2410.0.1"
+__version__ = "2502.0.0"
 __all__ = ["authenticate", "databrowser", "__version__"]

freva_client/cli/auth_cli.py

@@ -5,6 +5,7 @@ from getpass import getuser
 from typing import Optional
 
 import typer
+
 from freva_client import authenticate
 from freva_client.utils import exception_handler, logger
 
@@ -49,9 +50,7 @@ def authenticate_cli(
         "-f",
         help="Force token recreation, even if current token is still valid.",
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",

freva_client/cli/cli_app.py

@@ -4,6 +4,7 @@ import os
 from typing import Optional
 
 import typer
+
 from freva_client.utils import logger
 
 from .auth_cli import authenticate_cli

freva_client/cli/cli_utils.py

@@ -3,9 +3,10 @@
 from typing import Dict, List
 
 import typer
+from rich import print as pprint
+
 from freva_client import __version__
 from freva_client.utils import logger
-from rich import print as pprint
 
 APP_NAME: str = "freva-client"
 

freva_client/cli/databrowser_cli.py

@@ -10,6 +10,8 @@ from tempfile import NamedTemporaryFile
 from typing import Dict, List, Literal, Optional, Union, cast
 
 import typer
+import xarray as xr
+
 from freva_client import databrowser
 from freva_client.auth import Auth
 from freva_client.utils import exception_handler, logger
@@ -30,26 +32,27 @@ def _auth(url: str, token: Optional[str]) -> None:
 class UniqKeys(str, Enum):
     """Literal implementation for the cli."""
 
-    file: str = "file"
-    uri: str = "uri"
+    file = "file"
+    uri = "uri"
 
 
 class Flavours(str, Enum):
     """Literal implementation for the cli."""
 
-    freva: str = "freva"
-    cmip6: str = "cmip6"
-    cmip5: str = "cmip5"
-    cordex: str = "cordex"
-    nextgems: str = "nextgems"
+    freva = "freva"
+    cmip6 = "cmip6"
+    cmip5 = "cmip5"
+    cordex = "cordex"
+    nextgems = "nextgems"
+    user = "user"
 
 
 class TimeSelect(str, Enum):
     """Literal implementation for the cli."""
 
-    strict: str = "strict"
-    flexible: str = "flexible"
-    file: str = "file"
+    strict = "strict"
+    flexible = "flexible"
+    file = "file"
 
     @staticmethod
     def get_help() -> str:
@@ -163,9 +166,7 @@ def metadata_search(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -191,7 +192,7 @@ def metadata_search(
             Literal["file", "flexible", "strict"], time_select.value
         ),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         host=host,
@@ -253,9 +254,7 @@ def data_search(
         "--time-select",
         help=TimeSelect.get_help(),
     ),
-    zarr: bool = typer.Option(
-        False, "--zarr", help="Create zarr stream files."
-    ),
+    zarr: bool = typer.Option(False, "--zarr", help="Create zarr stream files."),
     access_token: Optional[str] = typer.Option(
         None,
         "--access-token",
@@ -289,9 +288,7 @@ def data_search(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -318,7 +315,7 @@ def data_search(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -425,9 +422,7 @@ def intake_catalogue(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -451,7 +446,7 @@ def intake_catalogue(
         time=time or "",
         time_select=cast(Literal["file", "flexible", "strict"], time_select),
         flavour=cast(
-            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
             flavour.value,
         ),
         uniq_key=cast(Literal["uri", "file"], uniq_key.value),
@@ -547,9 +542,7 @@ def count_values(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(
-        0, "-v", help="Increase verbosity", count=True
-    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -576,11 +569,9 @@ def count_values(
         result = databrowser.count_values(
             *facets,
             time=time or "",
-            time_select=cast(
-                Literal["file", "flexible", "strict"], time_select
-            ),
+            time_select=cast(Literal["file", "flexible", "strict"], time_select),
             flavour=cast(
-                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                Literal["freva", "cmip6", "cmip5", "cordex", "nextgems", "user"],
                 flavour.value,
             ),
             host=host,
@@ -598,7 +589,9 @@ def count_values(
                     Literal["file", "flexible", "strict"], time_select
                 ),
                 flavour=cast(
-                    Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+                    Literal[
+                        "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
+                    ],
                     flavour.value,
                 ),
                 host=host,
@@ -620,3 +613,110 @@ def count_values(
             print(f"{key}: {', '.join(counts)}")
     else:
         print(result)
+
+
+user_data_app = typer.Typer(help="Add or delete user data.")
+databrowser_app.add_typer(user_data_app, name="user-data")
+
+
+@user_data_app.command(name="add", help="Add user data into the databrowser.")
+@exception_handler
+def user_data_add(
+    paths: List[str] = typer.Option(
+        ...,
+        "--path",
+        "-p",
+        help="Paths to the user's data to be added.",
+    ),
+    facets: Optional[List[str]] = typer.Option(
+        None,
+        "--facet",
+        help="Key-value metadata pairs to categorize the user"
+        "input data in the format key=value.",
+    ),
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser. If not set (default), "
+            "the hostname is read from a config file."
+        ),
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help="Access token for authentication when adding user data.",
+    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+) -> None:
+    """Add user data into the databrowser."""
+    logger.set_verbosity(verbose)
+    logger.debug("Checking if the user has the right to add data")
+    result = databrowser(host=host)
+    _auth(result._cfg.auth_url, access_token)
+
+    facet_dict = {}
+    if facets:
+        for facet in facets:
+            if "=" not in facet:
+                logger.error(
+                    f"Invalid facet format: {facet}. Expected format: key=value."
+                )
+                raise typer.Exit(code=1)
+            key, value = facet.split("=", 1)
+            facet_dict[key] = value
+
+    logger.debug(f"Adding user data with paths {paths} and facets {facet_dict}")
+    databrowser.userdata(
+        action="add",
+        userdata_items=cast(List[Union[str, xr.Dataset]], paths),
+        metadata=facet_dict,
+        host=host,
+    )
+
+
+@user_data_app.command(
+    name="delete", help="Delete user data from the databrowser."
+)
+@exception_handler
+def user_data_delete(
+    search_keys: List[str] = typer.Option(
+        None,
+        "--search-key",
+        "-s",
+        help="Key-value metadata pairs to search and identify user data "
+        "for deletion in the format key=value.",
+    ),
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser. If not set (default), "
+            "the hostname is read from a config file."
+        ),
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help="Access token for authentication when deleting user data.",
+    ),
+    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+) -> None:
+    """Delete user data from the databrowser."""
+    logger.set_verbosity(verbose)
+    logger.debug("Checking if the user has the right to delete data")
+    result = databrowser(host=host)
+    _auth(result._cfg.auth_url, access_token)
+
+    search_key_dict = {}
+    if search_keys:
+        for search_key in search_keys:
+            if "=" not in search_key:
+                logger.error(
+                    f"Invalid search key format: {search_key}. "
+                    "Expected format: key=value."
+                )
+                raise typer.Exit(code=1)
+            key, value = search_key.split("=", 1)
+            search_key_dict[key] = value
+    databrowser.userdata(action="delete", metadata=search_key_dict, host=host)

freva_client/query.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import (
     Any,
+    Collection,
     Dict,
     Iterator,
     List,
@@ -21,12 +22,13 @@ from typing import (
 import intake
 import intake_esm
 import requests
+import xarray as xr
 import yaml
 from rich import print as pprint
 
 from .auth import Auth
 from .utils import logger
-from .utils.databrowser_utils import Config
+from .utils.databrowser_utils import Config, UserDataHandler
 
 __all__ = ["databrowser"]
 
@@ -207,7 +209,7 @@ class databrowser:
         *facets: str,
         uniq_key: Literal["file", "uri"] = "file",
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -267,19 +269,15 @@ class databrowser:
         headers = {}
         if self._stream_zarr:
             query_url = self._cfg.zarr_loader_url
-            token = self._auth.check_authentication(
-                auth_url=self._cfg.auth_url
-            )
+            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
             headers = {"Authorization": f"Bearer {token['access_token']}"}
-        result = self._get(query_url, headers=headers, stream=True)
+        result = self._request("GET", query_url, headers=headers, stream=True)
         if result is not None:
             try:
                 for res in result.iter_lines():
                     yield res.decode("utf-8")
             except KeyboardInterrupt:
-                pprint(
-                    "[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr
-                )
+                pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
 
     def __repr__(self) -> str:
         params = ", ".join(
@@ -337,7 +335,7 @@ class databrowser:
 
 
         """
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result:
             return cast(int, result.json().get("total_count", 0))
         return 0
@@ -347,15 +345,13 @@ class databrowser:
         kwargs: Dict[str, Any] = {"stream": True}
         url = self._cfg.intake_url
         if self._stream_zarr:
-            token = self._auth.check_authentication(
-                auth_url=self._cfg.auth_url
-            )
+            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
             url = self._cfg.zarr_loader_url
             kwargs["headers"] = {
                 "Authorization": f"Bearer {token['access_token']}"
             }
             kwargs["params"] = {"catalogue-type": "intake"}
-        result = self._get(url, **kwargs)
+        result = self._request("GET", url, **kwargs)
         if result is None:
             raise ValueError("No results found")
 
@@ -392,20 +388,23 @@ class databrowser:
         .. execute_code::
 
             from freva_client import databrowser
-            db = databrowser(dataset="cmip6-fs", stream_zarr=True)
+            db = databrowser(dataset="cmip6-hsm", stream_zarr=True)
             cat = db.intake_catalogue()
             print(cat.df)
         """
         with NamedTemporaryFile(suffix=".json") as temp_f:
             self._create_intake_catalogue_file(temp_f.name)
-            return intake.open_esm_datastore(temp_f.name)
+            return cast(
+                intake_esm.core.esm_datastore,
+                intake.open_esm_datastore(temp_f.name),
+            )
 
     @classmethod
     def count_values(
         cls,
         *facets: str,
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -455,6 +454,7 @@ class databrowser:
             Select all versions and not just the latest version (default).
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str
             The search constraints to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -540,7 +540,7 @@ class databrowser:
         cls,
         *facets: str,
         flavour: Literal[
-            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+            "freva", "cmip6", "cmip5", "cordex", "nextgems", "user"
         ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
@@ -593,6 +593,7 @@ class databrowser:
             name will be taken from the freva config file.
         fail_on_error: bool, default: False
             Make the call fail if the connection to the databrowser could not
+            be established.
         **search_keys: str, list[str]
             The facets to be applied in the data search. If not given
             the whole dataset will be queried.
@@ -723,7 +724,7 @@ class databrowser:
         self,
         extended_search: bool = False,
     ) -> Dict[str, List[str]]:
-        result = self._get(self._cfg.metadata_url)
+        result = self._request("GET", self._cfg.metadata_url)
         if result is None:
             return {}
         data = result.json()
@@ -733,26 +734,179 @@ class databrowser:
             constraints = data["primary_facets"]
         return {f: v for f, v in data["facets"].items() if f in constraints}
 
-    def _get(
-        self, url: str, **kwargs: Any
+    @classmethod
+    def userdata(
+        cls,
+        action: Literal["add", "delete"],
+        userdata_items: Optional[List[Union[str, xr.Dataset]]] = None,
+        metadata: Optional[Dict[str, str]] = None,
+        host: Optional[str] = None,
+        fail_on_error: bool = False,
+    ) -> None:
+        """Add or delete user data in the databrowser system.
+
+        Manage user data in the databrowser system by adding new data or
+        deleting existing data.
+
+        For the "``add``" action, the user can provide data items (file paths
+        or xarray datasets) along with metadata (key-value pairs) to
+        categorize and organize the data.
+
+        For the "``delete``" action, the user provides metadata as search
+        criteria to identify and remove the existing data from the
+        system.
+
+        Parameters
+        ~~~~~~~~~~
+        action : Literal["add", "delete"]
+            The action to perform: "add" to add new data, or "delete"
+            to remove existing data.
+        userdata_items : List[Union[str, xr.Dataset]], optional
+            A list of user file paths or xarray datasets to add to the
+            databrowser (required for "add").
+        metadata : Dict[str, str], optional
+            Key-value metadata pairs to categorize the data (for "add")
+            or search and identify data for
+            deletion (for "delete").
+        host : str, optional
+            Override the host name of the databrowser server. This is usually
+            the url where the freva web site can be found. Such as
+            www.freva.dkrz.de. By default no host name is given and the host
+            name will be taken from the freva config file.
+        fail_on_error : bool, optional
+            Make the call fail if the connection to the databrowser could not
+            be established.
+
+        Raises
+        ~~~~~~
+        ValueError
+            If the operation fails or required parameters are missing
+            for the specified action.
+        FileNotFoundError
+            If no user data is provided for the "add" action.
+
+        Example
+        ~~~~~~~
+
+        Adding user data:
+
+        .. execute_code::
+
+            from freva_client import authenticate, databrowser
+            import xarray as xr
+            token_info = authenticate(username="janedoe")
+            filenames = (
+                "../freva-rest/src/freva_rest/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "GERICS/NCC-NorESM1-M/rcp85/r1i1p1/GERICS-REMO2015/v1/3hr/pr/v20181212/*.nc"
+            )
+            filename1 = (
+                "../freva-rest/src/freva_rest/databrowser_api/mock/data/model/regional/cordex/output/EUR-11/"
+                "CLMcom/MPI-M-MPI-ESM-LR/historical/r0i0p0/CLMcom-CCLM4-8-17/v1/fx/orog/v20140515/"
+                "orog_EUR-11_MPI-M-MPI-ESM-LR_historical_r1i1p1_CLMcom-CCLM4-8-17_v1_fx.nc"
+            )
+            xarray_data = xr.open_dataset(filename1)
+            databrowser.userdata(
+                action="add",
+                userdata_items=[xarray_data, filenames],
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
+            )
+
+        Deleting user data:
+
+        .. execute_code::
+
+            from freva_client import authenticate, databrowser
+            token_info = authenticate(username="janedoe")
+            databrowser.userdata(
+                action="delete",
+                metadata={"project": "cmip5", "experiment": "myFavExp"}
+            )
+        """
+        this = cls(
+            host=host,
+            fail_on_error=fail_on_error,
+        )
+        userdata_items = userdata_items or []
+        metadata = metadata or {}
+        url = f"{this._cfg.userdata_url}"
+        token = this._auth.check_authentication(auth_url=this._cfg.auth_url)
+        headers = {"Authorization": f"Bearer {token['access_token']}"}
+        payload_metadata: dict[str, Collection[Collection[str]]] = {}
+
+        if action == "add":
+            user_data_handler = UserDataHandler(userdata_items)
+            if user_data_handler.user_metadata:
+                payload_metadata = {
+                    "user_metadata": user_data_handler.user_metadata,
+                    "facets": metadata,
+                }
+                result = this._request(
+                    "POST", url, data=payload_metadata, headers=headers
+                )
+                if result is not None:
+                    response_data = result.json()
+                    status_message = response_data.get("status")
+                else:
+                    raise ValueError("Failed to add user data")
+                pprint(f"[b][green]{status_message}[green][b]")
+            else:
+                raise ValueError("No metadata generated from the input data.")
+
+        if action == "delete":
+            if userdata_items:
+                logger.info(
+                    "'userdata_items' are not needed for the 'delete'"
+                    "action and will be ignored."
+                )
+
+            result = this._request("DELETE", url, data=metadata, headers=headers)
+
+            if result is None:
+                raise ValueError("Failed to delete user data")
+            pprint("[b][green]User data deleted successfully[green][b]")
+
+    def _request(
+        self,
+        method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"],
+        url: str,
+        data: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
     ) -> Optional[requests.models.Response]:
-        """Apply the get method to the databrowser."""
-        logger.debug("Searching %s with parameters: %s", url, self._params)
+        """Request method to handle CRUD operations (GET, POST, PUT, PATCH, DELETE)."""
+        method_upper = method.upper()
+        timeout = kwargs.pop("timeout", 30)
         params = kwargs.pop("params", {})
-        kwargs.setdefault("timeout", 30)
+        stream = kwargs.pop("stream", False)
+
+        logger.debug(
+            "%s request to %s with data: %s and parameters: %s",
+            method_upper,
+            url,
+            data,
+            {**self._params, **params},
+        )
+
         try:
-            res = requests.get(
-                url, params={**self._params, **params}, **kwargs
+            req = requests.Request(
+                method=method_upper,
+                url=url,
+                params={**self._params, **params},
+                json=None if method_upper in "GET" else data,
+                **kwargs,
             )
-            res.raise_for_status()
-            return res
+            with requests.Session() as session:
+                prepared = session.prepare_request(req)
+                res = session.send(prepared, timeout=timeout, stream=stream)
+                res.raise_for_status()
+                return res
+
         except KeyboardInterrupt:
             pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
         except (
             requests.exceptions.ConnectionError,
             requests.exceptions.HTTPError,
         ) as error:
-            msg = f"Search request failed with {error}"
+            msg = f"{method_upper} request failed with {error}"
             if self._fail_on_error:
                 raise ValueError(msg) from None
             logger.warning(msg)

freva_client/utils/databrowser_utils.py

@@ -1,16 +1,32 @@
 """Various utilities for getting the databrowser working."""
 
+import concurrent.futures
 import os
 import sys
 import sysconfig
 from configparser import ConfigParser, ExtendedInterpolation
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Dict, Literal, Optional, Tuple, cast
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
 
 import appdirs
+import numpy as np
 import requests
 import tomli
+import xarray as xr
+
+from . import logger
 
 
 class Config:
@@ -37,7 +53,7 @@ class Config:
         ini_parser.read_string(path.read_text())
         config = ini_parser["evaluation_system"]
         scheme, host = self._split_url(
-            config.get("databrowser.host") or config.get("solr.host")
+            config.get("databrowser.host", config.get("solr.host", ""))
         )
         host, _, port = (host or "").partition(":")
         port = port or config.get("databrowser.port", "")
@@ -57,9 +73,7 @@ class Config:
             host = f"{host}:{port}"
         return f"{scheme}://{host}"
 
-    def _read_config(
-        self, path: Path, file_type: Literal["toml", "ini"]
-    ) -> str:
+    def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
         """Read the configuration."""
         data_types = {"toml": self._read_toml, "ini": self._read_ini}
         try:
@@ -72,7 +86,7 @@ class Config:
     def overview(self) -> Dict[str, Any]:
         """Get an overview of the all databrowser flavours and search keys."""
         try:
-            res = requests.get(f"{self.databrowser_url}/overview", timeout=3)
+            res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
         except requests.exceptions.ConnectionError:
             raise ValueError(
                 f"Could not connect to {self.databrowser_url}"
@@ -120,7 +134,9 @@ class Config:
     @property
     def search_url(self) -> str:
         """Define the data search endpoint."""
-        return f"{self.databrowser_url}/data_search/{self.flavour}/{self.uniq_key}"
+        return (
+            f"{self.databrowser_url}/data-search/{self.flavour}/{self.uniq_key}"
+        )
 
     @property
     def zarr_loader_url(self) -> str:
@@ -130,13 +146,13 @@ class Config:
     @property
     def intake_url(self) -> str:
         """Define the url for creating intake catalogues."""
-        return f"{self.databrowser_url}/intake_catalogue/{self.flavour}/{self.uniq_key}"
+        return f"{self.databrowser_url}/intake-catalogue/{self.flavour}/{self.uniq_key}"
 
     @property
     def metadata_url(self) -> str:
         """Define the endpoint for the metadata search."""
         return (
-            f"{self.databrowser_url}/metadata_search/"
+            f"{self.databrowser_url}/metadata-search/"
             f"{self.flavour}/{self.uniq_key}"
         )
 
@@ -157,7 +173,7 @@ class Config:
         if port:
             hostname = f"{hostname}:{port}"
         hostname = hostname.partition("/")[0]
-        return f"{scheme}://{hostname}/api"
+        return f"{scheme}://{hostname}/api/freva-nextgen"
 
     @staticmethod
     def get_dirs(user: bool = True) -> Path:
@@ -177,3 +193,207 @@ class Config:
         # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
         # installing into a virtualenv
         return Path(sysconfig.get_path("data")) / "share" / "freva"
+
+    @property
+    def userdata_url(self) -> str:
+        """Define the url for adding and deleting user-data."""
+        return f"{self.databrowser_url}/userdata"
+
+
+class UserDataHandler:
+    """Class for processing user data.
+
+    This class is used for processing user data and extracting metadata
+    from the data files.
+    """
+
+    def __init__(self, userdata_items: List[Union[str, xr.Dataset]]) -> None:
+        self._suffixes = [".nc", ".nc4", ".grb", ".grib", ".zarr", "zar"]
+        self.user_metadata: List[
+            Dict[str, Union[str, List[str], Dict[str, str]]]
+        ] = []
+        self._metadata_collection: List[Dict[str, Union[str, List[str]]]] = []
+        try:
+            self._executor = concurrent.futures.ThreadPoolExecutor(
+                max_workers=min(int(os.cpu_count() or 4), 15)
+            )
+            self._process_user_data(userdata_items)
+        finally:
+            self._executor.shutdown(wait=True)
+
+    def _gather_files(self, path: Path, pattern: str = "*") -> Iterator[Path]:
+        """Gather all valid files from directory and wildcard pattern."""
+        for item in path.rglob(pattern):
+            if item.is_file() and item.suffix in self._suffixes:
+                yield item
+
+    def _validate_user_data(
+        self,
+        user_data: Sequence[Union[str, xr.Dataset]],
+    ) -> Dict[str, Union[List[Path], List[xr.Dataset]]]:
+        validated_paths: List[Path] = []
+        validated_xarray_datasets: List[xr.Dataset] = []
+        for data in user_data:
+            if isinstance(data, (str, Path)):
+                path = Path(data).expanduser().absolute()
+                if path.is_dir():
+                    validated_paths.extend(self._gather_files(path))
+                elif path.is_file() and path.suffix in self._suffixes:
+                    validated_paths.append(path)
+                else:
+                    validated_paths.extend(
+                        self._gather_files(path.parent, pattern=path.name)
+                    )
+            elif isinstance(data, xr.Dataset):
+                validated_xarray_datasets.append(data)
+
+        if not validated_paths and not validated_xarray_datasets:
+            raise FileNotFoundError(
+                "No valid file paths or xarray datasets found."
+            )
+        return {
+            "validated_user_paths": validated_paths,
+            "validated_user_xrdatasets": validated_xarray_datasets,
+        }
+
+    def _process_user_data(
+        self,
+        userdata_items: List[Union[str, xr.Dataset]],
+    ) -> None:
+        """Process xarray datasets and file paths using thread pool."""
+        futures = []
+        validated_userdata: Dict[str, Union[List[Path], List[xr.Dataset]]] = (
+            self._validate_user_data(userdata_items)
+        )
+        if validated_userdata["validated_user_xrdatasets"]:
+            futures.append(
+                self._executor.submit(
+                    self._process_userdata_in_executor,
+                    validated_userdata["validated_user_xrdatasets"],
+                )
+            )
+
+        if validated_userdata["validated_user_paths"]:
+            futures.append(
+                self._executor.submit(
+                    self._process_userdata_in_executor,
+                    validated_userdata["validated_user_paths"],
+                )
+            )
+        for future in futures:
+            try:
+                future.result()
+            except Exception as e:  # pragma: no cover
+                logger.error(f"Error processing batch: {e}")
+
+    def _process_userdata_in_executor(
+        self, validated_userdata: Union[List[Path], List[xr.Dataset]]
+    ) -> None:
+        for data in validated_userdata:
+            metadata = self._get_metadata(data)
+            if isinstance(metadata, Exception) or metadata == {}:
+                logger.warning("Error getting metadata: %s", metadata)
+            else:
+                self.user_metadata.append(metadata)
+
+    def _timedelta_to_cmor_frequency(self, dt: float) -> str:
+        for total_seconds, frequency in self._time_table.items():
+            if dt >= total_seconds:
+                return frequency
+        return "fx"  # pragma: no cover
+
+    @property
+    def _time_table(self) -> dict[int, str]:
+        return {
+            315360000: "dec",  # Decade
+            31104000: "yr",  # Year
+            2538000: "mon",  # Month
+            1296000: "sem",  # Seasonal (half-year)
+            84600: "day",  # Day
+            21600: "6h",  # Six-hourly
+            10800: "3h",  # Three-hourly
+            3600: "hr",  # Hourly
+            1: "subhr",  # Sub-hourly
+        }
+
+    def _get_time_frequency(self, time_delta: int, freq_attr: str = "") -> str:
+        if freq_attr in self._time_table.values():
+            return freq_attr
+        return self._timedelta_to_cmor_frequency(time_delta)
+
+    def _get_metadata(
+        self, path: Union[os.PathLike[str], xr.Dataset]
+    ) -> Dict[str, Union[str, List[str], Dict[str, str]]]:
+        """Get metadata from a path or xarray dataset."""
+
+        try:
+            dset = (
+                path
+                if isinstance(path, xr.Dataset)
+                else xr.open_mfdataset(
+                    str(path), parallel=False, use_cftime=True, lock=False
+                )
+            )
+            time_freq = dset.attrs.get("frequency", "")
+            data_vars = list(map(str, dset.data_vars))
+            coords = list(map(str, dset.coords))
+            try:
+                times = dset["time"].values[:]
+            except (KeyError, IndexError, TypeError):
+                times = np.array([])
+
+        except Exception as error:
+            logger.error("Failed to open data file %s: %s", str(path), error)
+            return {}
+        if len(times) > 0:
+            try:
+                time_str = (
+                    f"[{times[0].isoformat()}Z TO {times[-1].isoformat()}Z]"
+                )
+                dt = (
+                    abs((times[1] - times[0]).total_seconds())
+                    if len(times) > 1
+                    else 0
+                )
+            except Exception as non_cftime:
+                logger.info(
+                    "The time var is not based on the cftime: %s", non_cftime
+                )
+                time_str = (
+                    f"[{np.datetime_as_string(times[0], unit='s')}Z TO "
+                    f"{np.datetime_as_string(times[-1], unit='s')}Z]"
+                )
+                dt = (
+                    abs(
+                        (times[1] - times[0]).astype("timedelta64[s]").astype(int)
+                    )
+                    if len(times) > 1
+                    else 0
+                )
+        else:
+            time_str = "fx"
+            dt = 0
+
+        variables = [
+            var
+            for var in data_vars
+            if var not in coords
+            and not any(
+                term in var.lower() for term in ["lon", "lat", "bnds", "x", "y"]
+            )
+            and var.lower() not in ["rotated_pole", "rot_pole"]
+        ]
+
+        _data: Dict[str, Union[str, List[str], Dict[str, str]]] = {}
+        _data.setdefault("variable", variables[0])
+        _data.setdefault(
+            "time_frequency", self._get_time_frequency(dt, time_freq)
+        )
+        _data["time"] = time_str
+        _data.setdefault("cmor_table", _data["time_frequency"])
+        _data.setdefault("version", "")
+        if isinstance(path, Path):
+            _data["file"] = str(path)
+        if isinstance(path, xr.Dataset):
+            _data["file"] = str(dset.encoding["source"])
+        return _data

{freva_client-2410.0.1.dist-info → freva_client-2502.0.0.dist-info}/METADATA

@@ -1,11 +1,11 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: freva-client
-Version: 2410.0.1
+Version: 2502.0.0
 Summary: Search for climate data based on key-value pairs
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Classifier: Development Status :: 3 - Alpha
+Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: appdirs
 Requires-Dist: pyyaml
 Requires-Dist: authlib

freva_client-2502.0.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+freva_client/__init__.py,sha256=T8gGhWl4PHUFdDJ0kLyxUZhTVwxzwMB6IUY0f6OOfcc,851
+freva_client/__main__.py,sha256=JVj12puT4o8JfhKLAggR2-NKCZa3wKwsYGi4HQ61DOQ,149
+freva_client/auth.py,sha256=o33EKI5CxMOBY0nWFLmZT-V6v2U9L1qGjLcE7y4PIoE,6814
+freva_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+freva_client/query.py,sha256=ZxGPkU9B5JROCMIQ_AmhTKLlVPOCE2ENs4p7o9Jxge0,34095
+freva_client/cli/__init__.py,sha256=NgTqBZGdozmTZtJduJUMrZj-opGw2KoT20tg6sc_xqo,149
+freva_client/cli/auth_cli.py,sha256=3sThrF0YUgSzdwzzc5ww_GJlzUDK3jx-gQG9Z0hneRk,1843
+freva_client/cli/cli_app.py,sha256=QH6cb9XZTx8S9L0chPWOVd9Jn4GD1hd3sENAhdzmLZU,887
+freva_client/cli/cli_parser.py,sha256=c-_WG56g6arLvnayb9SO1OeHt7AjBV4egI_vTEYee4I,5042
+freva_client/cli/cli_utils.py,sha256=Ev9UxM4S1ZDbZSAGHFe5dMjVGot75w3muNKH3P80bHY,842
+freva_client/cli/databrowser_cli.py,sha256=r_Tl1yxLBZPMqY8vmJiOAvF_envTtf6c6KwQKjwdAq8,24387
+freva_client/utils/__init__.py,sha256=ySHn-3CZBwfZW2s0EpZ3INxUWOw1V4LOlKIxSLYr52U,1000
+freva_client/utils/databrowser_utils.py,sha256=kGTYjeHWBOIEcLkFuXhZ7ai6_j_DUzVq7hq7HEOYaho,14179
+freva_client/utils/logger.py,sha256=xd_3jjbsD1UBWlZZe8OUtKLpG7lbLcH46yiJ_bftyKg,2464
+freva_client-2502.0.0.data/data/share/freva/freva.toml,sha256=64Rh4qvWc9TaGJMXMi8tZW14FnESt5Z24y17BfD2VyM,736
+freva_client-2502.0.0.dist-info/entry_points.txt,sha256=zGyEwHrH_kAGLsCXv00y7Qnp-WjXkUuIomHkfGMCxtA,53
+freva_client-2502.0.0.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+freva_client-2502.0.0.dist-info/METADATA,sha256=G2VWBzP6OFpIOpCY-tPE2mipQVoYvaBF9gO-nrG-yEs,2526
+freva_client-2502.0.0.dist-info/RECORD,,

{freva_client-2410.0.1.dist-info → freva_client-2502.0.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.9.0
+Generator: flit 3.11.0
 Root-Is-Purelib: true
 Tag: py3-none-any

freva_client-2410.0.1.dist-info/RECORD

@@ -1,19 +0,0 @@
-freva_client/__init__.py,sha256=v82gUih2G4ZlHhKautrsVEn3KkWnvI2Hwg5LLYk4AC4,851
-freva_client/__main__.py,sha256=JVj12puT4o8JfhKLAggR2-NKCZa3wKwsYGi4HQ61DOQ,149
-freva_client/auth.py,sha256=o33EKI5CxMOBY0nWFLmZT-V6v2U9L1qGjLcE7y4PIoE,6814
-freva_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-freva_client/query.py,sha256=VJHUxuESORsXgyqFs7xbCpoGjCmQikM3453yMtOEmFw,27950
-freva_client/cli/__init__.py,sha256=NgTqBZGdozmTZtJduJUMrZj-opGw2KoT20tg6sc_xqo,149
-freva_client/cli/auth_cli.py,sha256=_WaEL6iSeE2agv-G-qka-cC6UoedtfoCAC_Vqfk0jKw,1856
-freva_client/cli/cli_app.py,sha256=AE7nSnm5okwrp6y4a8vas8QzPZGyGhhCUl8bc8jKBUg,886
-freva_client/cli/cli_parser.py,sha256=c-_WG56g6arLvnayb9SO1OeHt7AjBV4egI_vTEYee4I,5042
-freva_client/cli/cli_utils.py,sha256=ygwkNYqnjYJFRQu1M0r1VFQLAAl5lT05bM4tVsR0Xpc,841
-freva_client/cli/databrowser_cli.py,sha256=kFu44cAzwQ-HLd--d649ER6BCCxjRt9LeJA7xgIBlJ4,20924
-freva_client/utils/__init__.py,sha256=ySHn-3CZBwfZW2s0EpZ3INxUWOw1V4LOlKIxSLYr52U,1000
-freva_client/utils/databrowser_utils.py,sha256=sPnPk8KgQSztfr7sDrBYillFiSzYIB5if66pv-VFR40,6380
-freva_client/utils/logger.py,sha256=xd_3jjbsD1UBWlZZe8OUtKLpG7lbLcH46yiJ_bftyKg,2464
-freva_client-2410.0.1.data/data/share/freva/freva.toml,sha256=64Rh4qvWc9TaGJMXMi8tZW14FnESt5Z24y17BfD2VyM,736
-freva_client-2410.0.1.dist-info/entry_points.txt,sha256=zGyEwHrH_kAGLsCXv00y7Qnp-WjXkUuIomHkfGMCxtA,53
-freva_client-2410.0.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-freva_client-2410.0.1.dist-info/METADATA,sha256=naCBSzYCuR8mFwvbYwSonFdnet0sjKv7ev47wLsgQUk,2476
-freva_client-2410.0.1.dist-info/RECORD,,