esgpull 0.9.1__py3-none-any.whl → 0.9.2__py3-none-any.whl

esgpull/cli/plugins.py

@@ -337,7 +337,7 @@ from datetime import datetime
 from pathlib import Path
 from logging import Logger
 
-from esgpull.models import File, Query
+from esgpull.models import Dataset, File, Query
 from esgpull.plugin import Event, on
 
 # Specify version compatibility (optional)

esgpull/cli/search.py

@@ -7,6 +7,7 @@ from click.exceptions import Abort, Exit
 
 from esgpull.cli.decorators import args, groups, opts
 from esgpull.cli.utils import filter_keys, init_esgpull, parse_query, totable
+from esgpull.context import IndexNode
 from esgpull.exceptions import PageIndexError
 from esgpull.graph import Graph
 from esgpull.models import Query
@@ -135,14 +136,26 @@ def search(
             esg.ui.raise_maybe_record(Exit(0))
         if facets_hints:
             not_distrib_query = query << Query(options=dict(distrib=False))
-            facet_counts = esg.context.hints(
-                not_distrib_query,
-                file=file,
-                facets=["*"],
-                date_from=date_from,
-                date_to=date_to,
-            )
-            esg.ui.print(list(facet_counts[0]), json=True)
+            index = IndexNode(esg.config.api.index_node)
+            if index.is_bridge():
+                first_file_result = esg.context.search_as_queries(
+                    not_distrib_query,
+                    file=True,
+                    max_hits=1,
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                first_file = first_file_result[0].selection.asdict()
+                esg.ui.print(list(first_file), json=True)
+            else:
+                facet_counts = esg.context.hints(
+                    not_distrib_query,
+                    file=file,
+                    facets=["*"],
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                esg.ui.print(list(facet_counts[0]), json=True)
             esg.ui.raise_maybe_record(Exit(0))
         if hints is not None:
             facet_counts = esg.context.hints(

esgpull/context.py

@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Any, TypeAlias, TypeVar
+from urllib.parse import urlparse
 
 if sys.version_info < (3, 11):
     from exceptiongroup import BaseExceptionGroup
@@ -18,7 +19,7 @@ from esgpull.config import Config
 from esgpull.exceptions import SolrUnstableQueryError
 from esgpull.models import DatasetRecord, File, Query
 from esgpull.tui import logger
-from esgpull.utils import format_date_iso, index2url, sync
+from esgpull.utils import format_date_iso, sync
 
 # workaround for notebooks with running event loop
 if asyncio.get_event_loop().is_running():
@@ -39,6 +40,29 @@ DangerousFacets = {
 }
 
 
+@dataclass
+class IndexNode:
+    value: str
+
+    def is_bridge(self) -> bool:
+        return "esgf-1-5-bridge" in self.value
+
+    @property
+    def url(self) -> str:
+        parsed = urlparse(self.value)
+        result: str
+        match (parsed.scheme, parsed.netloc, parsed.path, self.is_bridge()):
+            case ("", "", path, True):
+                result = "https://" + parsed.path
+            case ("", "", path, False):
+                result = "https://" + parsed.path + "/esg-search/search"
+            case _:
+                result = self.value
+        if "." not in result:
+            raise ValueError(self.value)
+        return result
+
+
 @dataclass
 class Result:
     query: Query
@@ -70,12 +94,12 @@ class Result:
             "format": "application/solr+json",
             # "from": self.since,
         }
-        if index_url is None:
-            index_url = index2url(index_node)
-        if fields_param is not None:
-            params["fields"] = ",".join(fields_param)
-        else:
-            params["fields"] = "instance_id"
+        index = IndexNode(value=index_url or index_node)
+        if not index.is_bridge():
+            if fields_param is not None:
+                params["fields"] = ",".join(fields_param)
+            else:
+                params["fields"] = "instance_id"
         if date_from is not None:
             params["from"] = format_date_iso(date_from)
         if date_to is not None:
@@ -101,15 +125,20 @@ class Result:
             else:
                 if len(values) > 1:
                     value_term = f"({value_term})"
-                solr_terms.append(f"{name}:{value_term}")
+                if name.startswith("!"):
+                    solr_terms.append(f"(NOT {name[1:]}:{value_term})")
+                else:
+                    solr_terms.append(f"{name}:{value_term}")
         if solr_terms:
             params["query"] = " AND ".join(solr_terms)
         for name, option in self.query.options.items(use_default=True):
             if option.is_bool():
                 params[name] = option.name
+        if index.is_bridge():
+            _ = params.pop("retracted", None)  # not supported in bridge API
         if params.get("distrib") == "true" and facets_star:
             raise SolrUnstableQueryError(pretty_repr(self.query))
-        self.request = Request("GET", index_url, params=params)
+        self.request = Request("GET", index.url, params=params)
 
     def to(self, subtype: type[RT]) -> RT:
         result: RT = subtype(self.query, self.file)

esgpull/migrations/versions/0.9.2_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.2
+Revises: 0.9.1
+Create Date: 2025-09-04 16:56:29.263007
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.2'
+down_revision = '0.9.1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/plugin.py

@@ -151,7 +151,11 @@ class PluginConfig:
     enabled: set[str] = field(default_factory=set)
     disabled: set[str] = field(default_factory=set)
     plugins: dict[str, dict[str, Any]] = field(default_factory=dict)
-    _raw: dict[str, dict[str, Any]] = field(default_factory=dict)
+    _raw: tomlkit.TOMLDocument = field(default_factory=tomlkit.TOMLDocument)
+
+    def __post_init__(self):
+        if "plugins" not in self._raw:
+            self._raw["plugins"] = {}
 
 
 class PluginManager:
@@ -199,12 +203,13 @@ class PluginManager:
 
         try:
             with open(self.config_path, "r") as f:
-                raw = tomlkit.parse(f.read())
-                self.config.enabled = set(raw.get("enabled", []))
-                self.config.disabled = set(raw.get("disabled", []))
-                self.config.plugins = raw.get("plugins", {})
-                # Store the raw plugin configuration to preserve what's on disk
-                self.config._raw = dict(raw.get("plugins", {}))
+                raw = tomlkit.load(f)
+                unwrap = raw.unwrap()
+            self.config.enabled = set(unwrap.get("enabled", []))
+            self.config.disabled = set(unwrap.get("disabled", []))
+            self.config.plugins = unwrap.get("plugins", {})
+            # Store the raw plugin configuration to preserve what's on disk
+            self.config._raw = raw
         except Exception as e:
             logger.error(f"Failed to load plugin config: {e}")
 
@@ -219,18 +224,16 @@ class PluginManager:
             return
 
         try:
-            doc = tomlkit.document()
+            doc = self.config._raw
             doc["enabled"] = list(self.config.enabled)
             doc["disabled"] = list(self.config.disabled)
 
             # For plugins section, handle differently based on generate_full_config flag
             if generate_full_config:
                 doc["plugins"] = self.config.plugins
-            else:
-                doc["plugins"] = self.config._raw
 
             with open(self.config_path, "w") as f:
-                f.write(tomlkit.dumps(doc))
+                tomlkit.dump(doc, f)
         except Exception as e:
             logger.error(f"Failed to save plugin config: {e}")
 
@@ -464,8 +467,8 @@ class PluginManager:
         # Make sure plugin exists in both plugins and _raw dicts
         if plugin_name not in self.config.plugins:
             self.config.plugins[plugin_name] = {}
-        if plugin_name not in self.config._raw:
-            self.config._raw[plugin_name] = {}
+        if plugin_name not in self.config._raw["plugins"]:
+            self.config._raw["plugins"][plugin_name] = {}
 
         # Update the value in both places
         if key in self.config.plugins[plugin_name]:
@@ -473,7 +476,7 @@ class PluginManager:
             new_value = cast_value(old_value, value, key)
             self.config.plugins[plugin_name][key] = new_value
             # Also update in _raw to keep in sync
-            self.config._raw[plugin_name][key] = new_value
+            self.config._raw["plugins"][plugin_name][key] = new_value
         else:
             raise KeyError(key, self.config.plugins[plugin_name])
 
@@ -497,8 +500,8 @@ class PluginManager:
         # Make sure the plugin exists in both configs
         if plugin_name not in self.config.plugins:
             self.config.plugins[plugin_name] = {}
-        if plugin_name not in self.config._raw:
-            self.config._raw[plugin_name] = {}
+        if plugin_name not in self.config._raw["plugins"]:
+            self.config._raw["plugins"][plugin_name] = {}
 
         # Remove the key from both configs
         if key in self.config.plugins[plugin_name]:
@@ -508,10 +511,10 @@ class PluginManager:
 
         # Also remove from _raw if it exists
         if (
-            plugin_name in self.config._raw
-            and key in self.config._raw[plugin_name]
+            plugin_name in self.config._raw["plugins"]
+            and key in self.config._raw["plugins"][plugin_name]
         ):
-            self.config._raw[plugin_name].pop(key)
+            self.config._raw["plugins"][plugin_name].pop(key)
 
         self.write_config()
 

esgpull/utils.py

@@ -1,7 +1,6 @@
 import asyncio
 import datetime
 from typing import Callable, Coroutine, TypeVar
-from urllib.parse import urlparse
 
 from rich.filesize import _to_str
 
@@ -52,19 +51,3 @@ def format_date_iso(
     date: str | datetime.datetime, fmt: str = "%Y-%m-%d"
 ) -> str:
     return parse_date(date, fmt).replace(microsecond=0).isoformat() + "Z"
-
-
-def url2index(url: str) -> str:
-    parsed = urlparse(url)
-    if parsed.netloc == "":
-        return parsed.path
-    else:
-        return parsed.netloc
-
-
-def index2url(index: str) -> str:
-    url = "https://" + url2index(index)
-    if "esgf-1-5-bridge" in index:
-        return url
-    else:
-        return url + "/esg-search/search"

{esgpull-0.9.1.dist-info → esgpull-0.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.1
+Version: 0.9.2
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/

{esgpull-0.9.1.dist-info → esgpull-0.9.2.dist-info}/RECORD

@@ -2,7 +2,7 @@ esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
 esgpull/auth.py,sha256=QZ-l1ySLMP0fvuwYHRLv9FZYp1gqfju_eGaTMDByUxw,5205
 esgpull/config.py,sha256=9atYqxc3PiJKY1hfoEWZxf5Ba6U9SzMUs4QvcWWGFy0,13423
 esgpull/constants.py,sha256=WjG7xzMZNckOv5GhRehBtI7hoSwwLZvwkvEq5RG-dv4,1189
-esgpull/context.py,sha256=_XBAA9XM-fcJXLsqFlg1ziwQdndDoFxketdcFhi4oSM,23430
+esgpull/context.py,sha256=iyHAJ9OXU5_0SYcXNlmJI_M6x3vphJjnnAcsfB4MxQ8,24383
 esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
 esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
 esgpull/esgpull.py,sha256=-OwaWvwmkur_mdVNKKxdxlASeK93tmOceV__jGRMwqo,19666
@@ -10,12 +10,12 @@ esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
 esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
 esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
 esgpull/install_config.py,sha256=hzYpcHMtPMOK9fYcvVH-Hn_8zYsbs3yXlYgMumXo1zE,5598
-esgpull/plugin.py,sha256=wW19V55E1Fn2_ooZ5aTc58uJVSQ0WT43P29G3TlMD6A,18748
+esgpull/plugin.py,sha256=do6b4duPyAhNnYXWmFiVAlgNKj2FJcEjAMPqT8JqxtA,18871
 esgpull/processor.py,sha256=WLf4NFO_dp27E0GhZAiekiRD-gT6uyFJJnFZKbFV5vU,5484
 esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
 esgpull/tui.py,sha256=jpfDK_g_taNPEu0FeJ4MM1vtNJfOJY5x3dkUxRkoPEA,11210
-esgpull/utils.py,sha256=Rw_pU_rDvD6b2_h4Hzi8b5Hmb0ufuhrdL0Xwr8kLFcg,1584
+esgpull/utils.py,sha256=pLMQfY0p2oNIFyCZhHBD74BOAJtmt9QV6dTJ79VsfF8,1213
 esgpull/version.py,sha256=IHT4mKrIr8eV-C3HtmIVD85iGVH25n2ohoff31kaJ1A,93
 esgpull/cli/__init__.py,sha256=Q-U7opBTHxF5yUyyv15Cj-4Y9MaDMJ4i5FxNL-EmHfs,1683
 esgpull/cli/add.py,sha256=h8xFy5ORzY9O4SPPo4VCtTybeJJ5GcA18w4TZeL0DqU,3507
@@ -28,10 +28,10 @@ esgpull/cli/facet.py,sha256=V1u-DxNkhswwSt0qpXvuHrCI_tE8jAJGEe6_fMhYbaM,584
 esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
 esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
 esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
-esgpull/cli/plugins.py,sha256=eTAwWqvrzDKs122_Wa3sOkuriSoInymxWySQQahrlHg,13357
+esgpull/cli/plugins.py,sha256=lM8eQIH4H_bIy49rC2flmTstpr9ILDBYXp1YYYMv7qw,13366
 esgpull/cli/remove.py,sha256=9fqE8NJdr1mHypu_N-TBJy_yl3elUBSfzEARxxTkqKg,2662
 esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
-esgpull/cli/search.py,sha256=c2OI0Xvw6h2zBnC8msjxidKJZ0KKUuWJK9FuAmQ15ZU,6369
+esgpull/cli/search.py,sha256=J41dRi0mS-XsJyWsDzXsv04yWHPwhVG2VOEcdPnaak8,6940
 esgpull/cli/self.py,sha256=psgFcgkDyemquZEpoWp2cyjgampCgDzRc1QBvzqGs24,7941
 esgpull/cli/show.py,sha256=B-h7bKMrwgjnTHio2du8IPOLlKCaan56RQKAtzlQzcw,2822
 esgpull/cli/status.py,sha256=HEyj6QFABblADtYf1PWmSzghKX3fs33x9p5vpSqA514,2521
@@ -70,6 +70,7 @@ esgpull/migrations/versions/0.7.3_update_tables.py,sha256=4pNkM7VaGQqkLuIoLKuM0f
 esgpull/migrations/versions/0.8.0_update_tables.py,sha256=5rr3guWipnnuciFuviUxZUasdPw6x06WZCmY5nNCn5k,555
 esgpull/migrations/versions/0.9.0_update_tables.py,sha256=nXfPiyuseD5BXvu59zkeSTzb6EA7txpBudclU-MIyU0,555
 esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7ENt_6KecG3z5xfRr_LwrtY,541
+esgpull/migrations/versions/0.9.2_update_tables.py,sha256=aux--HPA_jUtvk7Zb2ZRXRhnRLT-whqZ0lLWt8Apyac,541
 esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
 esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
 esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
@@ -86,8 +87,8 @@ esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
 esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
 esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
 esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
-esgpull-0.9.1.dist-info/METADATA,sha256=Iy0nz7xstS0pywi5LNuLt-giW-5ng-X_lYb7WTrv8eE,3781
-esgpull-0.9.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-esgpull-0.9.1.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
-esgpull-0.9.1.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
-esgpull-0.9.1.dist-info/RECORD,,
+esgpull-0.9.2.dist-info/METADATA,sha256=58JNkrLxhc7gH87SOzgLz6VTIZX5ylTAGfbYFMcv0Jg,3781
+esgpull-0.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+esgpull-0.9.2.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
+esgpull-0.9.2.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
+esgpull-0.9.2.dist-info/RECORD,,