freva-client 2410.0.0b1__tar.gz → 2410.0.1__tar.gz

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freva-client
-Version: 2410.0.0b1
+Version: 2410.0.1
 Summary: Search for climate data based on key-value pairs
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.8
@@ -22,6 +22,7 @@ Requires-Dist: authlib
 Requires-Dist: requests
 Requires-Dist: intake_esm
 Requires-Dist: rich
+Requires-Dist: setuptools
 Requires-Dist: tomli
 Requires-Dist: typer
 Requires-Dist: tox ; extra == "dev"

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
 "requests",
 "intake_esm",
 "rich",
+"setuptools",
 "tomli",
 "typer",
 ]

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/src/freva_client/__init__.py

@@ -17,5 +17,5 @@ need to apply data analysis plugins, please visit the
 from .auth import authenticate
 from .query import databrowser
 
-__version__ = "2410.0.0-beta1"
+__version__ = "2410.0.1"
 __all__ = ["authenticate", "databrowser", "__version__"]

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/src/freva_client/cli/auth_cli.py

@@ -49,7 +49,9 @@ def authenticate_cli(
         "-f",
         help="Force token recreation, even if current token is still valid.",
     ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
     version: Optional[bool] = typer.Option(
         False,
         "-V",

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/src/freva_client/cli/databrowser_cli.py

@@ -163,7 +163,9 @@ def metadata_search(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -185,7 +187,9 @@ def metadata_search(
     result = databrowser.metadata_search(
         *(facets or []),
         time=time or "",
-        time_select=cast(Literal["file", "flexible", "strict"], time_select.value),
+        time_select=cast(
+            Literal["file", "flexible", "strict"], time_select.value
+        ),
         flavour=cast(
             Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
             flavour.value,
@@ -249,7 +253,9 @@ def data_search(
         "--time-select",
         help=TimeSelect.get_help(),
     ),
-    zarr: bool = typer.Option(False, "--zarr", help="Create zarr stream files."),
+    zarr: bool = typer.Option(
+        False, "--zarr", help="Create zarr stream files."
+    ),
     access_token: Optional[str] = typer.Option(
         None,
         "--access-token",
@@ -283,7 +289,9 @@ def data_search(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -417,7 +425,9 @@ def intake_catalogue(
             "the hostname is read from a config file"
         ),
     ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
     multiversion: bool = typer.Option(
         False,
         "--multi-version",
@@ -459,7 +469,9 @@ def intake_catalogue(
             print(Path(temp_f.name).read_text())
 
 
-@databrowser_app.command(name="data-count", help="Count the databrowser search results")
+@databrowser_app.command(
+    name="data-count", help="Count the databrowser search results"
+)
 @exception_handler
 def count_values(
     search_keys: Optional[List[str]] = typer.Argument(
@@ -535,7 +547,9 @@ def count_values(
     parse_json: bool = typer.Option(
         False, "-j", "--json", help="Parse output in json format."
     ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
@@ -562,7 +576,9 @@ def count_values(
         result = databrowser.count_values(
             *facets,
             time=time or "",
-            time_select=cast(Literal["file", "flexible", "strict"], time_select),
+            time_select=cast(
+                Literal["file", "flexible", "strict"], time_select
+            ),
             flavour=cast(
                 Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
                 flavour.value,
@@ -578,7 +594,9 @@ def count_values(
             databrowser(
                 *facets,
                 time=time or "",
-                time_select=cast(Literal["file", "flexible", "strict"], time_select),
+                time_select=cast(
+                    Literal["file", "flexible", "strict"], time_select
+                ),
                 flavour=cast(
                     Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
                     flavour.value,
@@ -602,108 +620,3 @@ def count_values(
             print(f"{key}: {', '.join(counts)}")
     else:
         print(result)
-
-
-user_data_app = typer.Typer(help="Add or delete user data.")
-databrowser_app.add_typer(user_data_app, name="user-data")
-
-
-@user_data_app.command(name="add", help="Add user data into the databrowser.")
-@exception_handler
-def user_data_add(
-    username: str = typer.Argument(..., help="Username of the data owner"),
-    paths: List[str] = typer.Option(
-        ...,
-        "--path",
-        "-p",
-        help="Paths to the user's data to be added.",
-    ),
-    facets: Optional[List[str]] = typer.Option(
-        None,
-        "--facet",
-        "-f",
-        help="Facet key-value pairs for metadata in the format key=value.",
-    ),
-    host: Optional[str] = typer.Option(
-        None,
-        "--host",
-        help=(
-            "Set the hostname of the databrowser. If not set (default), "
-            "the hostname is read from a config file."
-        ),
-    ),
-    access_token: Optional[str] = typer.Option(
-        None,
-        "--access-token",
-        help="Access token for authentication when adding user data.",
-    ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
-) -> None:
-    """Add user data into the databrowser."""
-    logger.set_verbosity(verbose)
-    logger.debug("Checking if the user has the right to add data")
-    result = databrowser(host=host)
-    _auth(result._cfg.auth_url, access_token)
-
-    facet_dict = {}
-    if facets:
-        for facet in facets:
-            if "=" not in facet:
-                logger.error(
-                    f"Invalid facet format: {facet}. Expected format: key=value."
-                )
-                raise typer.Exit(code=1)
-            key, value = facet.split("=", 1)
-            facet_dict[key] = value
-
-    logger.debug(
-        f"Adding user data for {username} with paths {paths} and facets {facet_dict}"
-    )
-    result.add_user_data(username=username, paths=paths, facets=facet_dict)
-    logger.info("User data started crawling. Check the Databrowser to see the updates.")
-
-
-@user_data_app.command(name="delete", help="Delete user data from the databrowser.")
-@exception_handler
-def user_data_remove(
-    username: str = typer.Argument(..., help="Username of the data owner"),
-    search_keys: List[str] = typer.Option(
-        None,
-        "--search-key",
-        "-s",
-        help="Search keys for the data to be deleted in the format key=value.",
-    ),
-    host: Optional[str] = typer.Option(
-        None,
-        "--host",
-        help=(
-            "Set the hostname of the databrowser. If not set (default), "
-            "the hostname is read from a config file."
-        ),
-    ),
-    access_token: Optional[str] = typer.Option(
-        None,
-        "--access-token",
-        help="Access token for authentication when deleting user data.",
-    ),
-    verbose: int = typer.Option(0, "-v", help="Increase verbosity", count=True),
-) -> None:
-    """Delete user data from the databrowser."""
-    logger.set_verbosity(verbose)
-    logger.debug("Checking if the user has the right to delete data")
-    result = databrowser(host=host)
-    _auth(result._cfg.auth_url, access_token)
-
-    search_key_dict = {}
-    if search_keys:
-        for search_key in search_keys:
-            if "=" not in search_key:
-                logger.error(
-                    f"Invalid search key format: {search_key}. "
-                    "Expected format: key=value."
-                )
-                raise typer.Exit(code=1)
-            key, value = search_key.split("=", 1)
-            search_key_dict[key] = value
-    result.delete_user_data(username=username, search_keys=search_key_dict)
-    logger.info("User data deleted successfully.")

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/src/freva_client/query.py

@@ -206,7 +206,9 @@ class databrowser:
         self,
         *facets: str,
         uniq_key: Literal["file", "uri"] = "file",
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -241,7 +243,8 @@ class databrowser:
         self, facets: Tuple[str, ...], search_kw: Dict[str, List[str]]
     ) -> None:
         metadata = {
-            k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
+            k: v[::2]
+            for (k, v) in self._facet_search(extended_search=True).items()
         }
         primary_key = list(metadata.keys() or ["project"])[0]
         num_facets = 0
@@ -264,7 +267,9 @@ class databrowser:
         headers = {}
         if self._stream_zarr:
             query_url = self._cfg.zarr_loader_url
-            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
+            token = self._auth.check_authentication(
+                auth_url=self._cfg.auth_url
+            )
             headers = {"Authorization": f"Bearer {token['access_token']}"}
         result = self._get(query_url, headers=headers, stream=True)
         if result is not None:
@@ -272,7 +277,9 @@ class databrowser:
                 for res in result.iter_lines():
                     yield res.decode("utf-8")
             except KeyboardInterrupt:
-                pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
+                pprint(
+                    "[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr
+                )
 
     def __repr__(self) -> str:
         params = ", ".join(
@@ -299,7 +306,9 @@ class databrowser:
 
         # Create a table-like structure for available flavors and search facets
         style = 'style="text-align: left"'
-        facet_heading = f"Available search facets for <em>{self._flavour}</em> flavour"
+        facet_heading = (
+            f"Available search facets for <em>{self._flavour}</em> flavour"
+        )
         html_repr = (
             "<table>"
             f"<tr><th colspan='2' {style}>{self.__class__.__name__}"
@@ -338,9 +347,13 @@ class databrowser:
         kwargs: Dict[str, Any] = {"stream": True}
         url = self._cfg.intake_url
         if self._stream_zarr:
-            token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
+            token = self._auth.check_authentication(
+                auth_url=self._cfg.auth_url
+            )
             url = self._cfg.zarr_loader_url
-            kwargs["headers"] = {"Authorization": f"Bearer {token['access_token']}"}
+            kwargs["headers"] = {
+                "Authorization": f"Bearer {token['access_token']}"
+            }
             kwargs["params"] = {"catalogue-type": "intake"}
         result = self._get(url, **kwargs)
         if result is None:
@@ -352,7 +365,9 @@ class databrowser:
                 for content in result.iter_content(decode_unicode=False):
                     stream.write(content)
         except Exception as error:
-            raise ValueError(f"Couldn't write catalogue content: {error}") from None
+            raise ValueError(
+                f"Couldn't write catalogue content: {error}"
+            ) from None
 
     def intake_catalogue(self) -> intake_esm.core.esm_datastore:
         """Create an intake esm catalogue object from the search.
@@ -389,7 +404,9 @@ class databrowser:
     def count_values(
         cls,
         *facets: str,
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -487,7 +504,9 @@ class databrowser:
         result = this._facet_search(extended_search=extended_search)
         counts = {}
         for facet, value_counts in result.items():
-            counts[facet] = dict(zip(value_counts[::2], map(int, value_counts[1::2])))
+            counts[facet] = dict(
+                zip(value_counts[::2], map(int, value_counts[1::2]))
+            )
         return counts
 
     @cached_property
@@ -512,14 +531,17 @@ class databrowser:
 
         """
         return {
-            k: v[::2] for (k, v) in self._facet_search(extended_search=True).items()
+            k: v[::2]
+            for (k, v) in self._facet_search(extended_search=True).items()
         }
 
     @classmethod
     def metadata_search(
         cls,
         *facets: str,
-        flavour: Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"] = "freva",
+        flavour: Literal[
+            "freva", "cmip6", "cmip5", "cordex", "nextgems"
+        ] = "freva",
         time: Optional[str] = None,
         host: Optional[str] = None,
         time_select: Literal["flexible", "strict", "file"] = "flexible",
@@ -642,7 +664,9 @@ class databrowser:
         )
         return {
             k: v[::2]
-            for (k, v) in this._facet_search(extended_search=extended_search).items()
+            for (k, v) in this._facet_search(
+                extended_search=extended_search
+            ).items()
         }
 
     @classmethod
@@ -709,172 +733,26 @@ class databrowser:
             constraints = data["primary_facets"]
         return {f: v for f, v in data["facets"].items() if f in constraints}
 
-    def add_user_data(
-        self, username: str, paths: List[str], facets: Dict[str, str]
-    ) -> None:
-        """Add user data to the databrowser.
-
-        Via this functionality, user would be able to add data to the databrowser.
-        It accepts file paths and metadata facets to categorize and store the user's
-        data.
-
-        Parameters
-        ~~~~~~~~~~
-        username: str
-            The username of user.
-        paths: list[str]
-            A list of paths to the data files that should be uploaded or cataloged.
-        facets: dict[str, str]
-            A dictionary containing metadata facets (key-value pairs) to describe the
-            data.
-
-        Returns
-        ~~~~~~~~
-        None
-            If the operation is successful, no return value is provided.
-
-        Raises
-        ~~~~~~~
-        ValueError
-            If the operation fails to add the user data.
-
-        Example
-        ~~~~~~~
-        .. execute_code::
-
-            from freva_client import authenticate, databrowser
-            token_info = authenticate(username="janedoe")
-            db = databrowser()
-            db.add_user_data(
-                "janedoe",
-                ["."],
-                {"project": "cmip5", "experiment": "something"}
-            )
-        """
-        url = f"{self._cfg.userdata_url}/{username}"
-        token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
-        headers = {"Authorization": f"Bearer {token['access_token']}"}
-        params = {"paths": paths}
-        if "username" in facets:
-            del facets["username"]
-        data = facets
-        result = self._put(url, data=data, headers=headers, params=params)
-
-        if result is None:
-            raise ValueError("Failed to add user data")
-
-    def delete_user_data(self, username: str, search_keys: Dict[str, str]) -> None:
-        """
-        Delete user data from the databrowser.
-
-        Uing this, user would be able to delete the user's data from the databrowser
-        based on the provided search keys.
-
-        Parameters
-        ~~~~~~~~~~
-        username: str
-            The username associated with the data to be deleted.
-        search_keys: dict[str, str]
-            A dictionary containing the search keys to identify the data to be deleted.
-
-        Returns
-        ~~~~~~~~
-        None
-            If the operation is successful, no return value is provided.
-
-        Raises
-        ~~~~~~~
-        ValueError
-            If the operation fails to delete the user data.
-
-        Example
-        ~~~~~~~
-        .. execute_code::
-
-            from freva_client import databrowser, authenticate
-            token_info = authenticate(username="janedoe")
-            db = databrowser()
-            db.delete_user_data(
-                "janedoe",
-                {"project": "cmip5", "experiment": "something"}
-            )
-        """
-        url = f"{self._cfg.userdata_url}/{username}"
-        token = self._auth.check_authentication(auth_url=self._cfg.auth_url)
-        headers = {"Authorization": f"Bearer {token['access_token']}"}
-        data = search_keys
-        result = self._delete(url, headers=headers, json=data)
-        if result is None:
-            raise ValueError("Failed to delete user data")
-
-    def _get(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
+    def _get(
+        self, url: str, **kwargs: Any
+    ) -> Optional[requests.models.Response]:
         """Apply the get method to the databrowser."""
         logger.debug("Searching %s with parameters: %s", url, self._params)
         params = kwargs.pop("params", {})
         kwargs.setdefault("timeout", 30)
         try:
-            res = requests.get(url, params={**self._params, **params}, **kwargs)
-            res.raise_for_status()
-            return res
-        except KeyboardInterrupt:
-            pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"Search request failed with {error}"
-            if self._fail_on_error:
-                raise ValueError(msg) from None
-            logger.warning(msg)
-        return None
-
-    def _put(
-        self, url: str, data: Dict[str, Any], **kwargs: Any
-    ) -> Optional[requests.models.Response]:
-        """Apply the PUT method to the databrowser."""
-        logger.debug(
-            "PUT request to %s with data: %s and parameters: %s",
-            url,
-            data,
-            self._params,
-        )
-        kwargs.setdefault("timeout", 30)
-        params = kwargs.pop("params", {})
-        try:
-            res = requests.put(
-                url, json=data, params={**self._params, **params}, **kwargs
+            res = requests.get(
+                url, params={**self._params, **params}, **kwargs
             )
             res.raise_for_status()
             return res
         except KeyboardInterrupt:
             pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-
         except (
             requests.exceptions.ConnectionError,
             requests.exceptions.HTTPError,
         ) as error:
-            msg = f"adding user data request failed with {error}"
-            if self._fail_on_error:
-                raise ValueError(msg) from None
-            logger.warning(msg)
-        return None
-
-    def _delete(self, url: str, **kwargs: Any) -> Optional[requests.models.Response]:
-        """Apply the DELETE method to the databrowser."""
-        logger.debug("DELETE request to %s with parameters: %s", url, self._params)
-        params = kwargs.pop("params", {})
-        kwargs.setdefault("timeout", 30)
-        try:
-            res = requests.delete(url, params={**self._params, **params}, **kwargs)
-            res.raise_for_status()
-            return res
-        except KeyboardInterrupt:
-            pprint("[red][b]User interrupt: Exit[/red][/b]", file=sys.stderr)
-        except (
-            requests.exceptions.ConnectionError,
-            requests.exceptions.HTTPError,
-        ) as error:
-            msg = f"DELETE request failed with {error}"
+            msg = f"Search request failed with {error}"
             if self._fail_on_error:
                 raise ValueError(msg) from None
             logger.warning(msg)

{freva_client-2410.0.0b1 → freva_client-2410.0.1}/src/freva_client/utils/databrowser_utils.py

@@ -57,7 +57,9 @@ class Config:
             host = f"{host}:{port}"
         return f"{scheme}://{host}"
 
-    def _read_config(self, path: Path, file_type: Literal["toml", "ini"]) -> str:
+    def _read_config(
+        self, path: Path, file_type: Literal["toml", "ini"]
+    ) -> str:
         """Read the configuration."""
         data_types = {"toml": self._read_toml, "ini": self._read_ini}
         try:
@@ -70,9 +72,11 @@ class Config:
     def overview(self) -> Dict[str, Any]:
         """Get an overview of the all databrowser flavours and search keys."""
         try:
-            res = requests.get(f"{self.databrowser_url}/overview", timeout=15)
+            res = requests.get(f"{self.databrowser_url}/overview", timeout=3)
         except requests.exceptions.ConnectionError:
-            raise ValueError(f"Could not connect to {self.databrowser_url}") from None
+            raise ValueError(
+                f"Could not connect to {self.databrowser_url}"
+            ) from None
         return cast(Dict[str, Any], res.json())
 
     def _get_databrowser_host_from_config(self) -> str:
@@ -87,7 +91,9 @@ class Config:
             Path(appdirs.user_config_dir("freva")) / "freva.toml": "toml",
             Path(self.get_dirs(user=True)) / "freva.toml": "toml",
             freva_config: "toml",
-            Path(os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf): "ini",
+            Path(
+                os.environ.get("EVALUATION_SYSTEM_CONFIG_FILE") or eval_conf
+            ): "ini",
         }
         for config_path, config_type in paths.items():
             if config_path.is_file():
@@ -130,7 +136,8 @@ class Config:
     def metadata_url(self) -> str:
         """Define the endpoint for the metadata search."""
         return (
-            f"{self.databrowser_url}/metadata_search/" f"{self.flavour}/{self.uniq_key}"
+            f"{self.databrowser_url}/metadata_search/"
+            f"{self.flavour}/{self.uniq_key}"
         )
 
     @staticmethod
@@ -170,8 +177,3 @@ class Config:
         # The default scheme is 'posix_prefix' or 'nt', and should work for e.g.
         # installing into a virtualenv
         return Path(sysconfig.get_path("data")) / "share" / "freva"
-
-    @property
-    def userdata_url(self) -> str:
-        """Define the url for adding and deleting user-data."""
-        return f"{self.databrowser_url}/userdata"