esgpull 0.9.2__tar.gz → 0.9.3__tar.gz

{esgpull-0.9.2 → esgpull-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.2
+Version: 0.9.3
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -21,10 +21,11 @@ Requires-Dist: cattrs>=22.2.0
 Requires-Dist: click-params>=0.4.0
 Requires-Dist: click>=8.1.3
 Requires-Dist: httpx>=0.23.0
-Requires-Dist: myproxyclient>=2.1.0
 Requires-Dist: nest-asyncio>=1.5.6
 Requires-Dist: packaging>=25.0
 Requires-Dist: platformdirs>=2.6.2
+Requires-Dist: pydantic-settings>=2.10.1
+Requires-Dist: pydantic>=2.11.7
 Requires-Dist: pyopenssl>=22.1.0
 Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0

{esgpull-0.9.2 → esgpull-0.9.3}/docs/docs/configuration.md

@@ -9,7 +9,6 @@ $ esgpull config
 
 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────── /home/me/.esgpull/config.toml ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
 [paths]
-auth = "/home/me/.esgpull/auth"
 data = "/home/me/.esgpull/data"
 db = "/home/me/.esgpull/db"
 log = "/home/me/.esgpull/log"
@@ -98,35 +97,49 @@ $ esgpull config --generate
     ```
 
 
-## Login
+## Index Node Configuration
 
-Although most data on ESGF can be downloaded without authentication, some datasets require a valid OpenID login and password.
+The `api.index_node` setting determines which ESGF index node `esgpull` will query for dataset metadata and search results.
 
-The ESGF OpenID authentication system is described on the [ESGF user documentation](http://www.esgf.io/esgf-user-support/user_guide.html).
+### `esg-search` Index Nodes
 
-
-This can be provided from the command line by running the following:
+`esgpull` uses a single regional ESGF index node, backed by the `esg-search` API, the endpoint can be changed through configuration:
 
 ```shell
-$ esgpull login
+$ esgpull config api.index_node esgf-node.ipsl.upmc.fr
 ```
+
+Other available index nodes include:
+
+- `esgf.ceda.ac.uk` (UK)
+- `esgf-data.dkrz.de` (Germany)
+- `esgf-node.ornl.gov/esgf-1-5-bridge` (USA, Bridge API)
+- find more at https://esgf.github.io/nodes.html
+
+### Bridge API
+
+The ORNL Bridge API node provides a temporary wrapper mimicking the API surface of the legacy esg-search code. This Bridge API has replaced the regular ORNL endpoint.
+
+To use the ORNL node, you must specify the Bridge API path:
+
 ```shell
-No credentials found.
-  [0] esg-dn1.nsc.liu.se
-  [1] esgf-data.dkrz.de
-  [2] ceda.ac.uk
-  [3] esgf-node.ipsl.upmc.fr
-  [4] esgf-node.llnl.gov
-  [5] esgf.nci.org.au
-Select a provider: 0
-User: MyESGFusername
-Password: <hidden>
-Certificates are missing.
-👍 Renewed successfully
+$ esgpull config api.index_node esgf-node.ornl.gov/esgf-1-5-bridge
 ```
 
-The credentials will then be saved under the ``~/.esgpull/auth`` directory, within
-``credentials.toml``, which can then be used for future sessions.
+!!! note "Bridge API Compatibility"
+
+    The Bridge API has some limitations compared to standard ESGF index nodes:
+
+    - The `retracted` option is not supported and will be automatically removed from queries
+    - The `fields` parameter is not supported
+    - Facet queries may behave differently than on standard nodes
+
+    `esgpull` automatically detects when you're using a Bridge API endpoint (by checking for `esgf-1-5-bridge` in the URL) and adjusts query parameters accordingly to ensure compatibility.
+
+## Login (deprecated)
+
+Login is not longer required and has been removed from `esgpull`.
+Existing `auth` folder in esgpull installation folder can be removed safely.
 
 ## Plugins
 

{esgpull-0.9.2 → esgpull-0.9.3}/esgpull/cli/__init__.py

@@ -8,7 +8,7 @@ from esgpull.cli.add import add
 from esgpull.cli.config import config
 from esgpull.cli.convert import convert
 from esgpull.cli.download import download
-from esgpull.cli.login import login
+from esgpull.cli.index_nodes import index_nodes
 from esgpull.cli.plugins import plugins
 from esgpull.cli.remove import remove
 from esgpull.cli.retry import retry
@@ -40,7 +40,6 @@ SUBCOMMANDS: list[click.Command] = [
     # get,
     self,
     # install,
-    login,
     plugins,
     remove,
     retry,
@@ -51,6 +50,7 @@ SUBCOMMANDS: list[click.Command] = [
     status,
     # # stats,
     update,
+    index_nodes,
 ]
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])

esgpull-0.9.3/esgpull/cli/index_nodes.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+
+import click
+from click.exceptions import Abort, Exit
+
+from esgpull import Context
+from esgpull.cli.decorators import groups, opts
+from esgpull.cli.utils import init_esgpull, totable
+from esgpull.models import Query
+from esgpull.tui import Verbosity, logger
+
+
+def check_node(c: Context, node: str) -> bool:
+    try:
+        c.probe(index_node=node)
+        return True
+    except:
+        return False
+
+
+def find_nodes(c: Context, node: str) -> list[str]:
+    try:
+        hints = c.hints(
+            Query(), file=False, index_node=node, facets=["index_node"]
+        )
+        return list(hints[0]["index_node"])
+    except:
+        return []
+
+
+@click.command()
+@groups.json_yaml
+@opts.record
+@opts.verbosity
+def index_nodes(
+    ## json_yaml
+    json: bool,
+    yaml: bool,
+    record: bool,
+    verbosity: Verbosity,
+) -> None:
+    """
+    Test index nodes for their current status
+    """
+    esg = init_esgpull(
+        verbosity,
+        safe=False,
+        record=record,
+    )
+    with esg.ui.logging("scan", onraise=Abort):
+        node_status: dict[str, bool] = {}
+        nodes = [
+            "esgf-node.ipsl.upmc.fr",
+            "esgf-data.dkrz.de",
+            "esgf.ceda.ac.uk",
+            "esgf-node.ornl.gov/esgf-1-5-bridge",
+        ]
+
+        esg.config.api.http_timeout = 3
+        with esg.ui.spinner("Fetching index nodes status"):
+            while True:
+                if not nodes:
+                    break
+                node = nodes.pop()
+                if node in node_status:
+                    continue
+                logger.info(f"check: {node}")
+                node_status[node] = check_node(esg.context, node)
+                logger.info(f"{node}\tok: {node_status[node]}")
+                for node in find_nodes(esg.context, node):
+                    if node not in node_status:
+                        nodes.append(node)
+                        logger.info(f"found index_node: {node}")
+        if json:
+            esg.ui.print(node_status, json=True)
+        elif yaml:
+            esg.ui.print(node_status, yaml=True)
+        else:
+            table = [
+                OrderedDict(
+                    [
+                        ("node", node),
+                        (
+                            "status",
+                            "[green]OK" if status else "[red]no response",
+                        ),
+                    ]
+                )
+                for node, status in node_status.items()
+            ]
+            esg.ui.print(totable(table))
+        esg.ui.raise_maybe_record(Exit(0))

{esgpull-0.9.2 → esgpull-0.9.3}/esgpull/cli/search.py

@@ -99,6 +99,7 @@ def search(
             esg.ui.raise_maybe_record(Exit(0))
         esg.graph.add(query, force=True)
         query = esg.graph.expand(query.sha)
+        esg.context.probe()
         hits = esg.context.hits(
             query,
             file=file,

{esgpull-0.9.2 → esgpull-0.9.3}/esgpull/cli/update.py

@@ -78,6 +78,7 @@ def update(
         if not qfs:
             esg.ui.print(":stop_sign: Trying to update untracked queries.")
             esg.ui.raise_maybe_record(Exit(0))
+        esg.context.probe()
         hints = esg.context.hints(
             *[qf.expanded for qf in qfs],
             file=True,

esgpull-0.9.3/esgpull/config.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator, Mapping
+from enum import Enum, auto
+from pathlib import Path
+from typing import Any, TypeVar
+
+import tomlkit
+from pydantic import BaseModel, Field, PrivateAttr, field_validator
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    TomlConfigSettingsSource,
+)
+
+from esgpull.constants import CONFIG_FILENAME
+from esgpull.exceptions import BadConfigError, VirtualConfigError
+from esgpull.install_config import InstallConfig
+from esgpull.models.options import Option, Options
+
+logger = logging.getLogger("esgpull")
+T = TypeVar("T")
+
+
+def _get_root() -> Path:
+    if InstallConfig.current is not None:
+        return InstallConfig.current.path
+    else:
+        return InstallConfig.default
+
+
+class Paths(BaseModel, validate_assignment=True, validate_default=True):
+    data: Path = Path("data")
+    db: Path = Path("db")
+    log: Path = Path("log")
+    tmp: Path = Path("tmp")
+    plugins: Path = Path("plugins")
+
+    @field_validator(
+        "data",
+        "db",
+        "log",
+        "tmp",
+        "plugins",
+        mode="after",
+    )
+    @classmethod
+    def _set_path_from_root(cls, value: Path) -> Path:
+        root = _get_root()
+        if not value.is_absolute():
+            value = root / value
+        return value
+
+    def values(self) -> Iterator[Path]:
+        yield self.data
+        yield self.db
+        yield self.log
+        yield self.tmp
+        yield self.plugins
+
+
+class Credentials(BaseModel, validate_assignment=True):
+    filename: str = "credentials.toml"
+
+
+class Cli(BaseModel, validate_assignment=True):
+    page_size: int = 20
+
+
+class Db(BaseModel, validate_assignment=True):
+    filename: str = "esgpull.db"
+
+
+class Download(BaseModel, validate_assignment=True):
+    chunk_size: int = 1 << 26
+    http_timeout: int = 20
+    max_concurrent: int = 5
+    disable_ssl: bool = False
+    disable_checksum: bool = False
+    show_filename: bool = False
+
+
+class DefaultOptions(BaseModel, validate_assignment=True):
+    distrib: str = Options._distrib_.name
+    latest: str = Options._latest_.name
+    replica: str = Options._replica_.name
+    retracted: str = Options._retracted_.name
+
+    @field_validator(
+        "distrib", "latest", "replica", "retracted", mode="before"
+    )
+    @classmethod
+    def _is_valid_option(cls, value: str | Option) -> str:
+        if isinstance(value, str):
+            return Option(value.lower()).name
+        else:
+            return value.name
+
+    def asdict(self) -> dict[str, str]:
+        return dict(
+            distrib=self.distrib,
+            latest=self.latest,
+            replica=self.replica,
+            retracted=self.retracted,
+        )
+
+
+class API(BaseModel, validate_assignment=True):
+    index_node: str = "esgf-node.ipsl.upmc.fr"
+    http_timeout: int = 20
+    max_concurrent: int = 5
+    page_limit: int = 50
+    default_options: DefaultOptions = Field(default_factory=DefaultOptions)
+    default_query_id: str = ""
+    use_custom_distribution_algorithm: bool = False
+
+
+class Plugins(BaseModel, validate_assignment=True):
+    enabled: bool = False
+
+
+class ConfigKind(Enum):
+    Virtual = auto()
+    NoFile = auto()
+    Partial = auto()
+    Complete = auto()
+
+
+class ConfigKey:
+    path: tuple[str, ...]
+
+    def __init__(
+        self,
+        first: str | tuple[str, ...] | list[str],
+        *rest: str,
+    ) -> None:
+        if isinstance(first, (tuple, list)):
+            self.path = tuple(first) + rest
+        elif "." in first:
+            self.path = tuple(first.split(".")) + rest
+        else:
+            self.path = (first,) + rest
+
+    def __iter__(self) -> Iterator[str]:
+        yield from self.path
+
+    def __hash__(self) -> int:
+        return hash(str(self.path))
+
+    def __eq__(self, other: object) -> bool:
+        match other:
+            case ConfigKey():
+                return self.path == other.path
+            case _:
+                raise TypeError(type(other))
+
+    def __repr__(self) -> str:
+        return ".".join(self)
+
+    def __add__(self, path: str) -> ConfigKey:
+        return ConfigKey(self.path, path)
+
+    def __len__(self) -> int:
+        return len(self.path)
+
+    def exists_in(self, source: Mapping | None) -> bool:
+        if source is None:
+            return False
+        doc = source
+        for key in self:
+            if key in doc:
+                doc = doc[key]
+            else:
+                return False
+        return True
+
+    def value_of(self, source: Any) -> Any:
+        doc = source
+        for key in self:
+            try:
+                doc = doc[key]
+            except TypeError:
+                doc = getattr(doc, key)
+        return doc
+
+
+def fix_rename_search_api(doc: dict) -> dict:
+    if "api" in doc and "search" in doc:
+        raise KeyError(
+            "Both 'api' and 'search' (deprecated) are used in your "
+            "config, please use 'api' only."
+        )
+    elif "search" in doc:
+        logger.warn(
+            "Deprecated key 'search' is used in your config, "
+            "please use 'api' instead."
+        )
+        doc["api"] = doc.pop("search")
+    return doc
+
+
+def fix_remove_auth(doc: dict) -> dict:
+    if "paths" in doc and "auth" in doc["paths"]:
+        logger.warn(
+            "Deprecated 'paths.auth' is present in your config, "
+            "you can remove it safely."
+        )
+        doc["paths"].pop("auth")
+    return doc
+
+
+def iter_keys(
+    source: Mapping,
+    path: ConfigKey | None = None,
+) -> Iterator[ConfigKey]:
+    for key in source.keys():
+        if path is None:
+            local_path = ConfigKey(key)
+        else:
+            local_path = path + key
+        if isinstance(source[key], Mapping):
+            yield from iter_keys(source[key], local_path)
+        else:
+            yield local_path
+
+
+def pop_and_clear_empty_parents(source: Mapping, ckey: ConfigKey):
+    *parent_path, last_key = ckey.path
+    parent_ckey = ConfigKey(parent_path)
+    parent_ckey.value_of(source).pop(last_key)
+
+    for i in range(len(parent_path), 0, -1):
+        parent_ckey = ConfigKey(parent_path[: i - 1])
+        container_ckey = ConfigKey(parent_path[:i])
+        parent = parent_ckey.value_of(source)
+        container = container_ckey.value_of(source)
+        if isinstance(container, dict) and len(container) == 0:
+            parent.pop(container_ckey.path[-1])
+        else:
+            break  # Stop if we hit a non-empty container
+
+
+config_fixers = [fix_rename_search_api, fix_remove_auth]
+
+
+class TomlKitConfigSettingsSource(TomlConfigSettingsSource):
+    def _read_file(self, file_path: Path) -> dict[str, Any]:
+        with open(file_path, mode="rb") as toml_file:
+            doc = tomlkit.load(toml_file)
+            for fixer in config_fixers:
+                try:
+                    doc = fixer(doc)
+                except Exception:
+                    raise BadConfigError(file_path)
+            return dict(doc)
+
+
+class Config(BaseSettings):
+    # TODO: set in a load method instead
+    # model_config = SettingsConfigDict(toml_file=_get_root() / "config.toml")
+    model_config = SettingsConfigDict(toml_file=None)
+
+    paths: Paths = Field(default_factory=Paths)
+    credentials: Credentials = Field(default_factory=Credentials)
+    cli: Cli = Field(default_factory=Cli)
+    db: Db = Field(default_factory=Db)
+    download: Download = Field(default_factory=Download)
+    api: API = Field(default_factory=API)
+    plugins: Plugins = Field(default_factory=Plugins)
+    _raw: dict[str, Any] | None = PrivateAttr(default=None)
+    _config_file: Path | None = PrivateAttr(default=None)
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            TomlKitConfigSettingsSource(settings_cls),
+        )
+
+    @classmethod
+    def load(cls, path: Path) -> Config:
+        try:
+            file_path = path / CONFIG_FILENAME
+            cls.model_config["toml_file"] = file_path
+            instance = cls()
+        finally:
+            cls.model_config["toml_file"] = None
+        instance._config_file = file_path
+        if file_path.is_file():
+            with file_path.open("rb") as toml_file:
+                instance._raw = tomlkit.load(toml_file)
+        else:
+            instance._raw = None
+        return instance
+
+    @classmethod
+    def default(cls) -> Config:
+        ## TODO: rename+deprecate
+        ## very bad name since this is loading from the **default** config
+        ## file path, as set in InstallConfig
+        return cls.load(path=_get_root())
+
+    @property
+    def kind(self) -> ConfigKind:
+        if self._config_file is None:
+            return ConfigKind.Virtual
+        elif not self._config_file.is_file():
+            return ConfigKind.NoFile
+        elif self.unset_options():
+            return ConfigKind.Partial
+        else:
+            return ConfigKind.Complete
+
+    def dump(self, with_defaults: bool = True) -> dict:
+        result = self.model_dump(mode="json")
+        if not with_defaults:
+            unset = set(self.unset_options())
+            for ckey in iter_keys(self.model_dump()):
+                if ckey in unset:
+                    pop_and_clear_empty_parents(result, ckey)
+        return result
+
+    def unset_options(self) -> list[ConfigKey]:
+        result: list[ConfigKey] = []
+        dump = self.model_dump()
+        for ckey in iter_keys(dump):
+            if not ckey.exists_in(self._raw):
+                result.append(ckey)
+        return result
+
+    def update_item(
+        self,
+        key: str,
+        value: T,
+        empty_ok: bool = False,
+    ) -> T:
+        if self._raw is None and empty_ok:
+            self._raw = {}
+        elif self._raw is None:
+            raise VirtualConfigError
+        doc = self._raw
+        obj = self
+        ckey = ConfigKey(key)
+        *parts, last = ckey
+        for part in parts:
+            doc.setdefault(part, {})
+            doc = doc[part]
+            obj = getattr(obj, part)
+        old_value = getattr(obj, last)
+        setattr(obj, last, value)
+        doc[last] = value
+        return old_value
+
+    def set_default(self, key: str) -> Any:
+        ckey = ConfigKey(key)
+        if self._raw is None:
+            raise VirtualConfigError()
+        elif not ckey.exists_in(self._raw):
+            return None
+        default_config = Config()
+        default_value: Any = ckey.value_of(default_config)
+        old_value: Any = ckey.value_of(self)
+
+        *parent_path, last_key = ckey.path
+        parent_ckey = ConfigKey(parent_path)
+        obj = parent_ckey.value_of(self)
+        setattr(obj, last_key, default_value)
+        pop_and_clear_empty_parents(self._raw, ckey)
+        return old_value
+
+    def generate(self, overwrite: bool = False) -> None:
+        match (self.kind, overwrite):
+            case (ConfigKind.Virtual, _):
+                raise VirtualConfigError
+            case (ConfigKind.Partial, overwrite):
+                defaults = self.model_dump()
+                for ckey in self.unset_options():
+                    self.update_item(str(ckey), ckey.value_of(defaults))
+            case (ConfigKind.Partial | ConfigKind.Complete, _):
+                raise FileExistsError(self._config_file)
+            case (ConfigKind.NoFile, _):
+                self._raw = self.model_dump(mode="json")
+            case _:
+                raise ValueError(self.kind)
+        self.write()
+
+    def write(self) -> None:
+        if self._config_file is None or self._raw is None:
+            raise VirtualConfigError
+        self._config_file.parent.mkdir(parents=True, exist_ok=True)
+        with self._config_file.open("w") as f:
+            tomlkit.dump(self._raw, f)

{esgpull-0.9.2 → esgpull-0.9.3}/esgpull/constants.py

@@ -4,6 +4,7 @@ CONFIG_FILENAME = "config.toml"
 INSTALLS_PATH_ENV = "ESGPULL_INSTALLS_PATH"
 ROOT_ENV = "ESGPULL_CURRENT"
 ESGPULL_DEBUG = os.environ.get("ESGPULL_DEBUG", "0") == "1"
+ESGPULL_DEBUG_LOCALS = os.environ.get("ESGPULL_DEBUG_LOCALS", "0") == "1"
 
 IDP = "/esgf-idp/openid/"
 CEDA_IDP = "/OpenID/Provider/server/"