esgpull 0.9.1__tar.gz → 0.9.2__tar.gz

{esgpull-0.9.1 → esgpull-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.1
+Version: 0.9.2
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/

{esgpull-0.9.1 → esgpull-0.9.2}/esgpull/cli/plugins.py

@@ -337,7 +337,7 @@ from datetime import datetime
 from pathlib import Path
 from logging import Logger
 
-from esgpull.models import File, Query
+from esgpull.models import Dataset, File, Query
 from esgpull.plugin import Event, on
 
 # Specify version compatibility (optional)

{esgpull-0.9.1 → esgpull-0.9.2}/esgpull/cli/search.py

@@ -7,6 +7,7 @@ from click.exceptions import Abort, Exit
 
 from esgpull.cli.decorators import args, groups, opts
 from esgpull.cli.utils import filter_keys, init_esgpull, parse_query, totable
+from esgpull.context import IndexNode
 from esgpull.exceptions import PageIndexError
 from esgpull.graph import Graph
 from esgpull.models import Query
@@ -135,14 +136,26 @@ def search(
             esg.ui.raise_maybe_record(Exit(0))
         if facets_hints:
             not_distrib_query = query << Query(options=dict(distrib=False))
-            facet_counts = esg.context.hints(
-                not_distrib_query,
-                file=file,
-                facets=["*"],
-                date_from=date_from,
-                date_to=date_to,
-            )
-            esg.ui.print(list(facet_counts[0]), json=True)
+            index = IndexNode(esg.config.api.index_node)
+            if index.is_bridge():
+                first_file_result = esg.context.search_as_queries(
+                    not_distrib_query,
+                    file=True,
+                    max_hits=1,
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                first_file = first_file_result[0].selection.asdict()
+                esg.ui.print(list(first_file), json=True)
+            else:
+                facet_counts = esg.context.hints(
+                    not_distrib_query,
+                    file=file,
+                    facets=["*"],
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                esg.ui.print(list(facet_counts[0]), json=True)
             esg.ui.raise_maybe_record(Exit(0))
         if hints is not None:
             facet_counts = esg.context.hints(

{esgpull-0.9.1 → esgpull-0.9.2}/esgpull/context.py

@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Any, TypeAlias, TypeVar
+from urllib.parse import urlparse
 
 if sys.version_info < (3, 11):
     from exceptiongroup import BaseExceptionGroup
@@ -18,7 +19,7 @@ from esgpull.config import Config
 from esgpull.exceptions import SolrUnstableQueryError
 from esgpull.models import DatasetRecord, File, Query
 from esgpull.tui import logger
-from esgpull.utils import format_date_iso, index2url, sync
+from esgpull.utils import format_date_iso, sync
 
 # workaround for notebooks with running event loop
 if asyncio.get_event_loop().is_running():
@@ -39,6 +40,29 @@ DangerousFacets = {
 }
 
 
+@dataclass
+class IndexNode:
+    value: str
+
+    def is_bridge(self) -> bool:
+        return "esgf-1-5-bridge" in self.value
+
+    @property
+    def url(self) -> str:
+        parsed = urlparse(self.value)
+        result: str
+        match (parsed.scheme, parsed.netloc, parsed.path, self.is_bridge()):
+            case ("", "", path, True):
+                result = "https://" + parsed.path
+            case ("", "", path, False):
+                result = "https://" + parsed.path + "/esg-search/search"
+            case _:
+                result = self.value
+        if "." not in result:
+            raise ValueError(self.value)
+        return result
+
+
 @dataclass
 class Result:
     query: Query
@@ -70,12 +94,12 @@ class Result:
             "format": "application/solr+json",
             # "from": self.since,
         }
-        if index_url is None:
-            index_url = index2url(index_node)
-        if fields_param is not None:
-            params["fields"] = ",".join(fields_param)
-        else:
-            params["fields"] = "instance_id"
+        index = IndexNode(value=index_url or index_node)
+        if not index.is_bridge():
+            if fields_param is not None:
+                params["fields"] = ",".join(fields_param)
+            else:
+                params["fields"] = "instance_id"
         if date_from is not None:
             params["from"] = format_date_iso(date_from)
         if date_to is not None:
@@ -101,15 +125,20 @@ class Result:
             else:
                 if len(values) > 1:
                     value_term = f"({value_term})"
-                solr_terms.append(f"{name}:{value_term}")
+                if name.startswith("!"):
+                    solr_terms.append(f"(NOT {name[1:]}:{value_term})")
+                else:
+                    solr_terms.append(f"{name}:{value_term}")
         if solr_terms:
             params["query"] = " AND ".join(solr_terms)
         for name, option in self.query.options.items(use_default=True):
             if option.is_bool():
                 params[name] = option.name
+        if index.is_bridge():
+            _ = params.pop("retracted", None)  # not supported in bridge API
         if params.get("distrib") == "true" and facets_star:
             raise SolrUnstableQueryError(pretty_repr(self.query))
-        self.request = Request("GET", index_url, params=params)
+        self.request = Request("GET", index.url, params=params)
 
     def to(self, subtype: type[RT]) -> RT:
         result: RT = subtype(self.query, self.file)

esgpull-0.9.2/esgpull/migrations/versions/0.9.2_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.2
+Revises: 0.9.1
+Create Date: 2025-09-04 16:56:29.263007
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.2'
+down_revision = '0.9.1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.9.1 → esgpull-0.9.2}/esgpull/plugin.py

@@ -151,7 +151,11 @@ class PluginConfig:
     enabled: set[str] = field(default_factory=set)
     disabled: set[str] = field(default_factory=set)
     plugins: dict[str, dict[str, Any]] = field(default_factory=dict)
-    _raw: dict[str, dict[str, Any]] = field(default_factory=dict)
+    _raw: tomlkit.TOMLDocument = field(default_factory=tomlkit.TOMLDocument)
+
+    def __post_init__(self):
+        if "plugins" not in self._raw:
+            self._raw["plugins"] = {}
 
 
 class PluginManager:
@@ -199,12 +203,13 @@ class PluginManager:
 
         try:
             with open(self.config_path, "r") as f:
-                raw = tomlkit.parse(f.read())
-                self.config.enabled = set(raw.get("enabled", []))
-                self.config.disabled = set(raw.get("disabled", []))
-                self.config.plugins = raw.get("plugins", {})
-                # Store the raw plugin configuration to preserve what's on disk
-                self.config._raw = dict(raw.get("plugins", {}))
+                raw = tomlkit.load(f)
+                unwrap = raw.unwrap()
+            self.config.enabled = set(unwrap.get("enabled", []))
+            self.config.disabled = set(unwrap.get("disabled", []))
+            self.config.plugins = unwrap.get("plugins", {})
+            # Store the raw plugin configuration to preserve what's on disk
+            self.config._raw = raw
         except Exception as e:
             logger.error(f"Failed to load plugin config: {e}")
 
@@ -219,18 +224,16 @@ class PluginManager:
             return
 
         try:
-            doc = tomlkit.document()
+            doc = self.config._raw
             doc["enabled"] = list(self.config.enabled)
             doc["disabled"] = list(self.config.disabled)
 
             # For plugins section, handle differently based on generate_full_config flag
             if generate_full_config:
                 doc["plugins"] = self.config.plugins
-            else:
-                doc["plugins"] = self.config._raw
 
             with open(self.config_path, "w") as f:
-                f.write(tomlkit.dumps(doc))
+                tomlkit.dump(doc, f)
         except Exception as e:
             logger.error(f"Failed to save plugin config: {e}")
 
@@ -464,8 +467,8 @@ class PluginManager:
         # Make sure plugin exists in both plugins and _raw dicts
         if plugin_name not in self.config.plugins:
             self.config.plugins[plugin_name] = {}
-        if plugin_name not in self.config._raw:
-            self.config._raw[plugin_name] = {}
+        if plugin_name not in self.config._raw["plugins"]:
+            self.config._raw["plugins"][plugin_name] = {}
 
         # Update the value in both places
         if key in self.config.plugins[plugin_name]:
@@ -473,7 +476,7 @@ class PluginManager:
             new_value = cast_value(old_value, value, key)
             self.config.plugins[plugin_name][key] = new_value
             # Also update in _raw to keep in sync
-            self.config._raw[plugin_name][key] = new_value
+            self.config._raw["plugins"][plugin_name][key] = new_value
         else:
             raise KeyError(key, self.config.plugins[plugin_name])
 
@@ -497,8 +500,8 @@ class PluginManager:
         # Make sure the plugin exists in both configs
         if plugin_name not in self.config.plugins:
             self.config.plugins[plugin_name] = {}
-        if plugin_name not in self.config._raw:
-            self.config._raw[plugin_name] = {}
+        if plugin_name not in self.config._raw["plugins"]:
+            self.config._raw["plugins"][plugin_name] = {}
 
         # Remove the key from both configs
         if key in self.config.plugins[plugin_name]:
@@ -508,10 +511,10 @@ class PluginManager:
 
         # Also remove from _raw if it exists
         if (
-            plugin_name in self.config._raw
-            and key in self.config._raw[plugin_name]
+            plugin_name in self.config._raw["plugins"]
+            and key in self.config._raw["plugins"][plugin_name]
         ):
-            self.config._raw[plugin_name].pop(key)
+            self.config._raw["plugins"][plugin_name].pop(key)
 
         self.write_config()
 

{esgpull-0.9.1 → esgpull-0.9.2}/esgpull/utils.py

@@ -1,7 +1,6 @@
 import asyncio
 import datetime
 from typing import Callable, Coroutine, TypeVar
-from urllib.parse import urlparse
 
 from rich.filesize import _to_str
 
@@ -52,19 +51,3 @@ def format_date_iso(
     date: str | datetime.datetime, fmt: str = "%Y-%m-%d"
 ) -> str:
     return parse_date(date, fmt).replace(microsecond=0).isoformat() + "Z"
-
-
-def url2index(url: str) -> str:
-    parsed = urlparse(url)
-    if parsed.netloc == "":
-        return parsed.path
-    else:
-        return parsed.netloc
-
-
-def index2url(index: str) -> str:
-    url = "https://" + url2index(index)
-    if "esgf-1-5-bridge" in index:
-        return url
-    else:
-        return url + "/esg-search/search"

{esgpull-0.9.1 → esgpull-0.9.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.9.1"
+version = "0.9.2"
 classifiers = [
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python :: 3",

{esgpull-0.9.1 → esgpull-0.9.2}/requirements-dev.lock

@@ -14,7 +14,7 @@ aiofiles==24.1.0
     # via esgpull
 aiostream==0.7.0
     # via esgpull
-alembic==1.16.4
+alembic==1.16.5
     # via esgpull
 anyio==4.10.0
     # via httpx
@@ -27,7 +27,7 @@ babel==2.17.0
     # via mkdocs-material
 backrefs==5.9
     # via mkdocs-material
-cattrs==25.1.1
+cattrs==25.2.0
     # via esgpull
 certifi==2025.8.3
     # via httpcore
@@ -35,21 +35,22 @@ certifi==2025.8.3
     # via requests
 cffi==1.17.1
     # via cryptography
-charset-normalizer==3.4.2
+charset-normalizer==3.4.3
     # via requests
 click==8.2.1
     # via click-params
     # via esgpull
     # via mkdocs
+    # via mkdocs-material
 click-params==0.5.0
     # via esgpull
 colorama==0.4.6
     # via mkdocs-material
 comm==0.2.3
     # via ipykernel
-coverage==7.10.2
+coverage==7.10.6
     # via pytest-cov
-cryptography==45.0.6
+cryptography==45.0.7
     # via pyopenssl
 debugpy==1.8.16
     # via ipykernel
@@ -65,9 +66,9 @@ exceptiongroup==1.3.0
     # via pytest
 execnet==2.1.1
     # via pytest-xdist
-executing==2.2.0
+executing==2.2.1
     # via stack-data
-filelock==3.18.0
+filelock==3.19.1
     # via pytest-mypy
 ghp-import==2.1.0
     # via mkdocs
@@ -111,7 +112,7 @@ markdown==3.8.2
     # via mkdocs
     # via mkdocs-material
     # via pymdown-extensions
-markdown-it-py==3.0.0
+markdown-it-py==4.0.0
     # via rich
 markupsafe==3.0.2
     # via jinja2
@@ -129,7 +130,7 @@ mkdocs==1.6.1
     # via mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.16
+mkdocs-material==9.6.18
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
 myproxyclient==2.1.1
@@ -141,7 +142,7 @@ mypy-extensions==1.1.0
 nest-asyncio==1.6.0
     # via esgpull
     # via ipykernel
-orjson==3.11.1
+orjson==3.11.3
 packaging==25.0
     # via esgpull
     # via ipykernel
@@ -149,21 +150,21 @@ packaging==25.0
     # via pytest
 paginate==0.5.7
     # via mkdocs-material
-parso==0.8.4
+parso==0.8.5
     # via jedi
 pathspec==0.12.1
     # via mkdocs
     # via mypy
 pexpect==4.9.0
     # via ipython
-platformdirs==4.3.8
+platformdirs==4.4.0
     # via esgpull
     # via jupyter-core
     # via mkdocs-get-deps
 pluggy==1.6.0
     # via pytest
     # via pytest-cov
-prompt-toolkit==3.0.51
+prompt-toolkit==3.0.52
     # via ipython
     # via jupyter-console
 psutil==7.0.0
@@ -187,7 +188,7 @@ pyopenssl==25.1.0
     # via myproxyclient
 pyparsing==3.2.3
     # via esgpull
-pytest==8.4.1
+pytest==8.4.2
     # via pytest-cov
     # via pytest-mypy
     # via pytest-xdist
@@ -205,11 +206,11 @@ pyyaml==6.0.2
     # via pyyaml-env-tag
 pyyaml-env-tag==1.1
     # via mkdocs
-pyzmq==27.0.1
+pyzmq==27.0.2
     # via ipykernel
     # via jupyter-client
     # via jupyter-console
-requests==2.32.4
+requests==2.32.5
     # via mkdocs-material
 rich==14.1.0
     # via esgpull
@@ -220,7 +221,7 @@ six==1.17.0
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
-sqlalchemy==2.0.42
+sqlalchemy==2.0.43
     # via alembic
     # via esgpull
 stack-data==0.6.3
@@ -233,7 +234,7 @@ tomli==2.2.1
     # via pytest
 tomlkit==0.13.3
     # via esgpull
-tornado==6.5.1
+tornado==6.5.2
     # via ipykernel
     # via jupyter-client
 traitlets==5.14.3
@@ -243,9 +244,9 @@ traitlets==5.14.3
     # via jupyter-console
     # via jupyter-core
     # via matplotlib-inline
-types-aiofiles==24.1.0.20250801
-types-pyyaml==6.0.12.20250516
-typing-extensions==4.14.1
+types-aiofiles==24.1.0.20250822
+types-pyyaml==6.0.12.20250822
+typing-extensions==4.15.0
     # via aiostream
     # via alembic
     # via anyio
@@ -263,5 +264,5 @@ watchdog==6.0.0
     # via mkdocs
 wcwidth==0.2.13
     # via prompt-toolkit
-wrapt==1.17.2
+wrapt==1.17.3
     # via deprecated

{esgpull-0.9.1 → esgpull-0.9.2}/requirements.lock

@@ -14,14 +14,14 @@ aiofiles==24.1.0
     # via esgpull
 aiostream==0.7.0
     # via esgpull
-alembic==1.16.4
+alembic==1.16.5
     # via esgpull
 anyio==4.10.0
     # via httpx
 attrs==25.3.0
     # via cattrs
     # via esgpull
-cattrs==25.1.1
+cattrs==25.2.0
     # via esgpull
 certifi==2025.8.3
     # via httpcore
@@ -33,7 +33,7 @@ click==8.2.1
     # via esgpull
 click-params==0.5.0
     # via esgpull
-cryptography==45.0.6
+cryptography==45.0.7
     # via pyopenssl
 deprecated==1.2.18
     # via click-params
@@ -53,7 +53,7 @@ idna==3.10
     # via httpx
 mako==1.3.10
     # via alembic
-markdown-it-py==3.0.0
+markdown-it-py==4.0.0
     # via rich
 markupsafe==3.0.2
     # via mako
@@ -65,7 +65,7 @@ nest-asyncio==1.6.0
     # via esgpull
 packaging==25.0
     # via esgpull
-platformdirs==4.3.8
+platformdirs==4.4.0
     # via esgpull
 pycparser==2.22
     # via cffi
@@ -86,14 +86,14 @@ six==1.17.0
     # via myproxyclient
 sniffio==1.3.1
     # via anyio
-sqlalchemy==2.0.42
+sqlalchemy==2.0.43
     # via alembic
     # via esgpull
 tomli==2.2.1
     # via alembic
 tomlkit==0.13.3
     # via esgpull
-typing-extensions==4.14.1
+typing-extensions==4.15.0
     # via aiostream
     # via alembic
     # via anyio
@@ -103,5 +103,5 @@ typing-extensions==4.14.1
     # via sqlalchemy
 validators==0.22.0
     # via click-params
-wrapt==1.17.2
+wrapt==1.17.3
     # via deprecated

{esgpull-0.9.1 → esgpull-0.9.2}/tests/test_context.py

@@ -3,7 +3,7 @@ from time import perf_counter
 
 import pytest
 
-from esgpull.context import Context
+from esgpull.context import Context, IndexNode
 from esgpull.models import Query
 
 
@@ -104,6 +104,10 @@ class Timer:
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(
+    raises=ValueError,
+    reason="ESGF bridge API gives index_node values that are not valid URLs",
+)
 def test_search_distributed(ctx):
     query = Query()
     # ctx.config.api.http_timeout = 60
@@ -138,7 +142,12 @@ def test_search_distributed(ctx):
 
 
 def test_ipsl_hits_between_1_and_2_million(ctx, cmip6_ipsl):
-    assert 1_000_000 < ctx.hits(cmip6_ipsl, file=False)[0] < 2_000_000
+    hits = ctx.hits(
+        cmip6_ipsl,
+        file=False,
+        index_node="esgf-node.ipsl.upmc.fr",
+    )
+    assert 1_000_000 < hits[0] < 2_000_000
 
 
 def test_more_files_than_datasets(ctx, query):
@@ -168,3 +177,35 @@ def test_ignore_facet_hits(ctx):
     hits_not_ipsl = ctx.hits(query_not_ipsl, file=False)[0]
     assert all(hits > 0 for hits in [hits_all, hits_ipsl, hits_not_ipsl])
     assert hits_all == hits_ipsl + hits_not_ipsl
+
+
+@pytest.mark.parametrize(
+    "index,url,is_bridge",
+    [
+        (
+            "esgf-node.ipsl.upmc.fr",
+            "https://esgf-node.ipsl.upmc.fr/esg-search/search",
+            False,
+        ),
+        (
+            "https://esgf-node.ipsl.upmc.fr/esg-search/search",
+            "https://esgf-node.ipsl.upmc.fr/esg-search/search",
+            False,
+        ),
+        (
+            "esgf-node.ornl.gov/esgf-1-5-bridge",
+            "https://esgf-node.ornl.gov/esgf-1-5-bridge",
+            True,
+        ),
+        (
+            "https://esgf-node.ornl.gov/esgf-1-5-bridge",
+            "https://esgf-node.ornl.gov/esgf-1-5-bridge",
+            True,
+        ),
+    ],
+)
+def test_index2url(index: str, url: str, is_bridge: bool):
+    for value in (index, url):
+        index_node = IndexNode(value=value)
+        assert index_node.url == url
+        assert index_node.is_bridge() == is_bridge

{esgpull-0.9.1 → esgpull-0.9.2}/tests/test_utils.py

@@ -2,10 +2,7 @@ from datetime import datetime
 
 import pytest
 
-from esgpull.utils import format_date, format_date_iso, index2url, parse_date
-
-ESGF_INDEX = "esgf-node.ipsl.upmc.fr"
-ESGF_URL = "https://esgf-node.ipsl.upmc.fr/esg-search/search"
+from esgpull.utils import format_date, format_date_iso, parse_date
 
 
 def test_parse_date():
@@ -40,8 +37,3 @@ def test_format_date_iso():
         format_date_iso("20220101")
     with pytest.raises(ValueError):
         format_date_iso(20220101)
-
-
-def test_index2url():
-    assert index2url(ESGF_INDEX) == ESGF_URL
-    assert index2url(ESGF_URL) == ESGF_URL

{esgpull-0.9.1 → esgpull-0.9.2}/uv.lock

@@ -370,7 +370,7 @@ wheels = [
 
 [[package]]
 name = "esgpull"
-version = "0.9.1"
+version = "0.9.2"
 source = { editable = "." }
 dependencies = [
     { name = "aiofiles" },