freva-client 2404.0.1__tar.gz → 2408.0.0.dev1__tar.gz

{freva_client-2404.0.1 → freva_client-2408.0.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freva-client
-Version: 2404.0.1
+Version: 2408.0.0.dev1
 Summary: Search for climate data based on key-value pairs
 Author-email: "DKRZ, Clint" <freva@dkrz.de>
 Requires-Python: >=3.8
@@ -18,7 +18,9 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: appdirs
 Requires-Dist: pyyaml
+Requires-Dist: authlib
 Requires-Dist: requests
+Requires-Dist: intake_esm
 Requires-Dist: rich
 Requires-Dist: tomli
 Requires-Dist: typer

{freva_client-2404.0.1 → freva_client-2408.0.0.dev1}/pyproject.toml

@@ -24,7 +24,9 @@ requires-python = ">=3.8"
 dependencies = [
 "appdirs",
 "pyyaml",
+"authlib",
 "requests",
+"intake_esm",
 "rich",
 "tomli",
 "typer",

{freva_client-2404.0.1 → freva_client-2408.0.0.dev1}/src/freva_client/__init__.py

@@ -14,7 +14,8 @@ library described in the documentation only support searching for data. If you
 need to apply data analysis plugins, please visit the
 """
 
+from .auth import authenticate
 from .query import databrowser
 
-__version__ = "2404.0.1"
-__all__ = ["databrowser", "__version__"]
+__version__ = "2408.0.0.dev1"
+__all__ = ["authenticate", "databrowser", "__version__"]

freva_client-2408.0.0.dev1/src/freva_client/auth.py

@@ -0,0 +1,197 @@
+"""Module that handles the authentication at the rest service."""
+
+from datetime import datetime
+from getpass import getpass, getuser
+from typing import Optional, TypedDict
+
+from authlib.integrations.requests_client import OAuth2Session
+
+from .utils import logger
+from .utils.databrowser_utils import Config
+
+Token = TypedDict(
+    "Token",
+    {
+        "access_token": str,
+        "token_type": str,
+        "expires": float,
+        "refresh_token": str,
+        "refresh_expires": float,
+    },
+)
+
+
+class Auth:
+    """Helper class for authentication."""
+
+    _instance: Optional["Auth"] = None
+    _auth_token: Optional[Token] = None
+
+    def __new__(cls) -> "Auth":
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self) -> None:
+        self._auth_cls = OAuth2Session()
+
+    @property
+    def token_expiration_time(self) -> datetime:
+        """Get the expiration time of an access token."""
+        if self._auth_token is None:
+            exp = 0.0
+        else:
+            exp = self._auth_token["expires"]
+        return datetime.fromtimestamp(exp)
+
+    def set_token(
+        self,
+        access_token: str,
+        refresh_token: Optional[str] = None,
+        expires_in: int = 10,
+        refresh_expires_in: int = 10,
+        expires: Optional[float] = None,
+        refresh_expires: Optional[float] = None,
+        token_type: str = "Bearer",
+    ) -> Token:
+        """Override the existing auth token."""
+        now = datetime.now().timestamp()
+
+        self._auth_token = Token(
+            access_token=access_token or "",
+            refresh_token=refresh_token or "",
+            token_type=token_type,
+            expires=expires or now + expires_in,
+            refresh_expires=refresh_expires or now + refresh_expires_in,
+        )
+        return self._auth_token
+
+    def _refresh(
+        self, url: str, refresh_token: str, username: Optional[str] = None
+    ) -> Token:
+        """Refresh the access_token with a refresh token."""
+        auth = self._auth_cls.refresh_token(f"{url}/token", refresh_token or " ")
+        try:
+            return self.set_token(
+                access_token=auth["access_token"],
+                token_type=auth["token_type"],
+                expires_in=auth["expires_in"],
+                refresh_token=auth["refresh_token"],
+                refresh_expires_in=auth["refresh_expires_in"],
+            )
+        except KeyError:
+            logger.warning("Failed to refresh token: %s", auth.get("detail", ""))
+            if username:
+                return self._login_with_password(url, username)
+            raise ValueError("Could not use refresh token") from None
+
+    def check_authentication(self, auth_url: Optional[str] = None) -> Token:
+        """Check the status of the authentication.
+
+        Raises
+        ------
+        ValueError: If user isn't or is no longer authenticated.
+        """
+        if not self._auth_token:
+            raise ValueError("You must authenticate first.")
+        now = datetime.now().timestamp()
+        if now > self._auth_token["refresh_expires"]:
+            raise ValueError("Refresh token has expired.")
+        if now > self._auth_token["expires"] and auth_url:
+            self._refresh(auth_url, self._auth_token["refresh_token"])
+        return self._auth_token
+
+    def _login_with_password(self, auth_url: str, username: str) -> Token:
+        """Create a new token."""
+        pw_msg = "Give password for server authentication: "
+        auth = self._auth_cls.fetch_token(
+            f"{auth_url}/token", username=username, password=getpass(pw_msg)
+        )
+        try:
+            return self.set_token(
+                access_token=auth["access_token"],
+                token_type=auth["token_type"],
+                expires_in=auth["expires_in"],
+                refresh_token=auth["refresh_token"],
+                refresh_expires_in=auth["refresh_expires_in"],
+            )
+        except KeyError:
+            logger.error("Failed to authenticate: %s", auth.get("detail", ""))
+            raise ValueError("Token creation failed") from None
+
+    def authenticate(
+        self,
+        host: Optional[str] = None,
+        refresh_token: Optional[str] = None,
+        username: Optional[str] = None,
+        force: bool = False,
+    ) -> Token:
+        """Authenticate the user to the host."""
+        cfg = Config(host)
+        if refresh_token:
+            try:
+                return self._refresh(cfg.auth_url, refresh_token)
+            except ValueError:
+                logger.warning(
+                    (
+                        "Could not use refresh token, falling back "
+                        "to username/password"
+                    )
+                )
+        username = username or getuser()
+        if self._auth_token is None or force:
+            return self._login_with_password(cfg.auth_url, username)
+        if self.token_expiration_time < datetime.now():
+            self._refresh(cfg.auth_url, self._auth_token["refresh_token"], username)
+        return self._auth_token
+
+
+def authenticate(
+    *,
+    refresh_token: Optional[str] = None,
+    username: Optional[str] = None,
+    host: Optional[str] = None,
+    force: bool = False,
+) -> Token:
+    """Authenticate to the host.
+
+    This method generates a new access token that should be used for restricted methods.
+
+    Parameters
+    ----------
+    refresh_token: str, optional
+        Instead of setting a password, you can set a refresh token to refresh
+        the access token. This is recommended for non-interactive environments.
+    username: str, optional
+        The username used for authentication. By default, the current
+        system username is used.
+    host: str, optional
+        The hostname of the REST server.
+    force: bool, default: False
+        Force token recreation, even if current token is still valid.
+
+    Returns
+    -------
+    Token: The authentication token.
+
+    Examples
+    --------
+    Interactive authentication:
+
+    .. code-block:: python
+
+        from freva_client import authenticate
+        token = authenticate(username="janedoe")
+        print(token)
+
+    Batch mode authentication with a refresh token:
+
+    .. code-block:: python
+
+        from freva_client import authenticate
+        token = authenticate(refresh_token="MYTOKEN")
+    """
+    auth = Auth()
+    return auth.authenticate(
+        host=host, username=username, refresh_token=refresh_token, force=force
+    )

freva_client-2408.0.0.dev1/src/freva_client/cli/auth_cli.py

@@ -0,0 +1,71 @@
+"""Command line interface for authentication."""
+
+import json
+from getpass import getuser
+from typing import Optional
+
+import typer
+from freva_client import authenticate
+from freva_client.utils import exception_handler, logger
+
+from .cli_utils import version_callback
+
+auth_app = typer.Typer(
+    name="auth",
+    help="Create OAuth2 access and refresh token.",
+    pretty_exceptions_short=False,
+)
+
+
+@exception_handler
+def authenticate_cli(
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser, if not set (default) "
+            "the hostname is read from a config file"
+        ),
+    ),
+    username: str = typer.Option(
+        getuser(),
+        "--username",
+        "-u",
+        help="The username used for authentication.",
+    ),
+    refresh_token: Optional[str] = typer.Option(
+        None,
+        "--refresh-token",
+        "-r",
+        help=(
+            "Instead of using a password, you can use a refresh token. "
+            "refresh the access token. This is recommended for non-interactive"
+            " environments."
+        ),
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="Force token recreation, even if current token is still valid.",
+    ),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
+    version: Optional[bool] = typer.Option(
+        False,
+        "-V",
+        "--version",
+        help="Show version an exit",
+        callback=version_callback,
+    ),
+) -> None:
+    """Create OAuth2 access and refresh token."""
+    logger.set_verbosity(verbose)
+    token_data = authenticate(
+        host=host,
+        username=username,
+        refresh_token=refresh_token,
+        force=force,
+    )
+    print(json.dumps(token_data, indent=3))

freva_client-2408.0.0.dev1/src/freva_client/cli/cli_app.py

@@ -0,0 +1,35 @@
+"""Freva the Free Evaluation System command line interface."""
+
+from typing import Optional
+
+import typer
+from freva_client.utils import logger
+
+from .auth_cli import authenticate_cli
+from .cli_utils import APP_NAME, version_callback
+from .databrowser_cli import databrowser_app
+
+app = typer.Typer(
+    name=APP_NAME,
+    help=__doc__,
+    add_completion=True,
+    callback=logger.set_cli,
+)
+
+
+@app.callback()
+def main(
+    version: Optional[bool] = typer.Option(
+        None,
+        "--version",
+        "-V",
+        help="Show version and exit",
+        callback=version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """The main cli app."""
+
+
+app.add_typer(databrowser_app, name="databrowser")
+app.command(name="auth", help=authenticate_cli.__doc__)(authenticate_cli)

freva_client-2404.0.1/src/freva_client/cli/cli_utils.py → freva_client-2408.0.0.dev1/src/freva_client/cli/cli_parser.py

@@ -1,4 +1,4 @@
-"""Utilities for the command line interface."""
+"""Command line argument completion definitions."""
 
 import argparse
 import inspect
@@ -6,21 +6,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
 from freva_client import databrowser
 from freva_client.cli.cli_app import app
-from freva_client.utils import logger
 
-
-def parse_cli_args(cli_args: List[str]) -> Dict[str, List[str]]:
-    """Convert the cli arguments to a dictionary."""
-    logger.debug("parsing command line arguments.")
-    kwargs = {}
-    for entry in cli_args:
-        key, _, value = entry.partition("=")
-        if value and key not in kwargs:
-            kwargs[key] = [value]
-        elif value:
-            kwargs[key].append(value)
-    logger.debug(kwargs)
-    return kwargs
+from .cli_utils import parse_cli_args
 
 
 class Completer:
@@ -103,7 +90,7 @@ class Completer:
         choices = self._get_choices()
         return {**self.choices, **choices}
 
-    def formated_print(self) -> None:
+    def formatted_print(self) -> None:
         """Print all choices to be processed by the shell completion function."""
 
         out = self.get_print(self.command_choices)

freva_client-2408.0.0.dev1/src/freva_client/cli/cli_utils.py

@@ -0,0 +1,31 @@
+"""Utilities for the command line interface."""
+
+from typing import Dict, List
+
+import typer
+from freva_client import __version__
+from freva_client.utils import logger
+from rich import print as pprint
+
+APP_NAME: str = "freva-client"
+
+
+def version_callback(version: bool) -> None:
+    """Print the version and exit."""
+    if version:
+        pprint(f"{APP_NAME}: {__version__}")
+        raise typer.Exit()
+
+
+def parse_cli_args(cli_args: List[str]) -> Dict[str, List[str]]:
+    """Convert the cli arguments to a dictionary."""
+    logger.debug("parsing command line arguments.")
+    kwargs = {}
+    for entry in cli_args:
+        key, _, value = entry.partition("=")
+        if value and key not in kwargs:
+            kwargs[key] = [value]
+        elif value:
+            kwargs[key].append(value)
+    logger.debug(kwargs)
+    return kwargs

{freva_client-2404.0.1 → freva_client-2408.0.0.dev1}/src/freva_client/cli/databrowser_cli.py

@@ -5,14 +5,26 @@ Search quickly and intuitively for many different climate datasets.
 
 import json
 from enum import Enum
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Dict, List, Literal, Optional, Union, cast
 
 import typer
 from freva_client import databrowser
+from freva_client.auth import Auth
 from freva_client.utils import exception_handler, logger
 
-from .cli_app import app, version_callback
-from .cli_utils import parse_cli_args
+from .cli_utils import parse_cli_args, version_callback
+
+
+def _auth(url: str, token: Optional[str]) -> None:
+    if token:
+        auth = Auth()
+        auth.set_token(
+            access_token=token, expires=auth.token_expiration_time.timestamp()
+        )
+    else:
+        raise ValueError("`--access-token` is required for authentication.")
 
 
 class UniqKeys(str, Enum):
@@ -55,7 +67,12 @@ class TimeSelect(str, Enum):
         )
 
 
-@app.command(
+databrowser_app = typer.Typer(
+    help="Data search related commands", callback=logger.set_cli
+)
+
+
+@databrowser_app.command(
     name="data-overview",
     help="Get an overview over what is available in the databrowser.",
 )
@@ -74,7 +91,7 @@ def overview(
     print(databrowser.overview(host=host))
 
 
-@app.command(
+@databrowser_app.command(
     name="metadata-search", help="Search databrowser for metadata (facets)."
 )
 @exception_handler
@@ -190,7 +207,9 @@ def metadata_search(
         print(f"{key}: {', '.join(values)}")
 
 
-@app.command(name="data-search", help="Search the databrowser for datasets.")
+@databrowser_app.command(
+    name="data-search", help="Search the databrowser for datasets."
+)
 @exception_handler
 def data_search(
     search_keys: Optional[List[str]] = typer.Argument(
@@ -234,6 +253,17 @@ def data_search(
         "--time-select",
         help=TimeSelect.get_help(),
     ),
+    zarr: bool = typer.Option(
+        False, "--zarr", help="Create zarr stream files."
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help=(
+            "Use this access token for authentication"
+            " when creating a zarr stream files."
+        ),
+    ),
     time: Optional[str] = typer.Option(
         None,
         "-t",
@@ -264,14 +294,14 @@ def data_search(
     ),
     multiversion: bool = typer.Option(
         False,
-        "--mulit-version",
+        "--multi-version",
         help="Select all versions and not just the latest version (default).",
     ),
     version: Optional[bool] = typer.Option(
         False,
         "-V",
         "--version",
-        help="Show verion an exit",
+        help="Show version an exit",
         callback=version_callback,
     ),
 ) -> None:
@@ -295,8 +325,11 @@ def data_search(
         host=host,
         fail_on_error=False,
         multiversion=multiversion,
+        stream_zarr=zarr,
         **(parse_cli_args(search_keys or [])),
     )
+    if zarr:
+        _auth(result._cfg.auth_url, access_token)
     if parse_json:
         print(json.dumps(sorted(result)))
     else:
@@ -304,7 +337,141 @@ def data_search(
             print(res)
 
 
-@app.command(name="data-count", help="Count the databrowser search results")
+@databrowser_app.command(
+    name="intake-catalogue", help="Create an intake catalogue from the search."
+)
+@exception_handler
+def intake_catalogue(
+    search_keys: Optional[List[str]] = typer.Argument(
+        default=None,
+        help="Refine your data search with this `key=value` pair search "
+        "parameters. The parameters could be, depending on the DRS standard, "
+        "flavour product, project model etc.",
+    ),
+    facets: Optional[List[str]] = typer.Option(
+        None,
+        "--facet",
+        help=(
+            "If you are not sure about the correct search key's you can use"
+            " the ``--facet`` flag to search of any matching entries. For "
+            "example --facet 'era5' would allow you to search for any entries"
+            " containing era5, regardless of project, product etc."
+        ),
+    ),
+    uniq_key: UniqKeys = typer.Option(
+        "file",
+        "--uniq-key",
+        "-u",
+        help=(
+            "The type of search result, which can be either “file” "
+            "or “uri”. This parameter determines whether the search will be "
+            "based on file paths or Uniform Resource Identifiers"
+        ),
+    ),
+    flavour: Flavours = typer.Option(
+        "freva",
+        "--flavour",
+        "-f",
+        help=(
+            "The Data Reference Syntax (DRS) standard specifying the type "
+            "of climate datasets to query."
+        ),
+    ),
+    time_select: TimeSelect = typer.Option(
+        "flexible",
+        "-ts",
+        "--time-select",
+        help=TimeSelect.get_help(),
+    ),
+    time: Optional[str] = typer.Option(
+        None,
+        "-t",
+        "--time",
+        help=(
+            "Special search facet to refine/subset search results by time. "
+            "This can be a string representation of a time range or a single "
+            "time step. The time steps have to follow ISO-8601. Valid strings "
+            "are ``%Y-%m-%dT%H:%M`` to ``%Y-%m-%dT%H:%M`` for time ranges and "
+            "``%Y-%m-%dT%H:%M``. **Note**: You don't have to give the full "
+            "string format to subset time steps ``%Y``, ``%Y-%m`` etc are also"
+            " valid."
+        ),
+    ),
+    zarr: bool = typer.Option(
+        False, "--zarr", help="Create zarr stream files, as catalogue targets."
+    ),
+    access_token: Optional[str] = typer.Option(
+        None,
+        "--access-token",
+        help=(
+            "Use this access token for authentication"
+            " when creating a zarr based intake catalogue."
+        ),
+    ),
+    filename: Optional[Path] = typer.Option(
+        None,
+        "-f",
+        "--filename",
+        help=(
+            "Path to the file where the catalogue, should be written to. "
+            "if None given (default) the catalogue is parsed to stdout."
+        ),
+    ),
+    host: Optional[str] = typer.Option(
+        None,
+        "--host",
+        help=(
+            "Set the hostname of the databrowser, if not set (default) "
+            "the hostname is read from a config file"
+        ),
+    ),
+    verbose: int = typer.Option(
+        0, "-v", help="Increase verbosity", count=True
+    ),
+    multiversion: bool = typer.Option(
+        False,
+        "--multi-version",
+        help="Select all versions and not just the latest version (default).",
+    ),
+    version: Optional[bool] = typer.Option(
+        False,
+        "-V",
+        "--version",
+        help="Show version an exit",
+        callback=version_callback,
+    ),
+) -> None:
+    """Create an intake catalogue for climate datasets based on the specified "
+    "Data Reference Syntax (DRS) standard (flavour) and the type of search "
+    result (uniq_key), which can be either “file” or “uri”."""
+    logger.set_verbosity(verbose)
+    logger.debug("Search the databrowser")
+    result = databrowser(
+        *(facets or []),
+        time=time or "",
+        time_select=cast(Literal["file", "flexible", "strict"], time_select),
+        flavour=cast(
+            Literal["freva", "cmip6", "cmip5", "cordex", "nextgems"],
+            flavour.value,
+        ),
+        uniq_key=cast(Literal["uri", "file"], uniq_key.value),
+        host=host,
+        fail_on_error=False,
+        multiversion=multiversion,
+        stream_zarr=zarr,
+        **(parse_cli_args(search_keys or [])),
+    )
+    if zarr:
+        _auth(result._cfg.auth_url, access_token)
+    with NamedTemporaryFile(suffix=".json") as temp_f:
+        result._create_intake_catalogue_file(str(filename or temp_f.name))
+        if not filename:
+            print(Path(temp_f.name).read_text())
+
+
+@databrowser_app.command(
+    name="data-count", help="Count the databrowser search results"
+)
 @exception_handler
 def count_values(
     search_keys: Optional[List[str]] = typer.Argument(
@@ -387,7 +554,7 @@ def count_values(
         False,
         "-V",
         "--version",
-        help="Show verion an exit",
+        help="Show version an exit",
         callback=version_callback,
     ),
 ) -> None:
@@ -438,6 +605,7 @@ def count_values(
                 multiversion=multiversion,
                 fail_on_error=False,
                 uniq_key="file",
+                stream_zarr=False,
                 **search_kws,
             )
         )