esgpull 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl

esgpull/cli/__init__.py

@@ -8,7 +8,7 @@ from esgpull.cli.add import add
 from esgpull.cli.config import config
 from esgpull.cli.convert import convert
 from esgpull.cli.download import download
-from esgpull.cli.login import login
+from esgpull.cli.index_nodes import index_nodes
 from esgpull.cli.plugins import plugins
 from esgpull.cli.remove import remove
 from esgpull.cli.retry import retry
@@ -40,7 +40,6 @@ SUBCOMMANDS: list[click.Command] = [
     # get,
     self,
     # install,
-    login,
     plugins,
     remove,
     retry,
@@ -51,6 +50,7 @@ SUBCOMMANDS: list[click.Command] = [
     status,
     # # stats,
     update,
+    index_nodes,
 ]
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])

esgpull/cli/index_nodes.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+
+import click
+from click.exceptions import Abort, Exit
+
+from esgpull import Context
+from esgpull.cli.decorators import groups, opts
+from esgpull.cli.utils import init_esgpull, totable
+from esgpull.models import Query
+from esgpull.tui import Verbosity, logger
+
+
+def check_node(c: Context, node: str) -> bool:
+    try:
+        c.probe(index_node=node)
+        return True
+    except:
+        return False
+
+
+def find_nodes(c: Context, node: str) -> list[str]:
+    try:
+        hints = c.hints(
+            Query(), file=False, index_node=node, facets=["index_node"]
+        )
+        return list(hints[0]["index_node"])
+    except:
+        return []
+
+
+@click.command()
+@groups.json_yaml
+@opts.record
+@opts.verbosity
+def index_nodes(
+    ## json_yaml
+    json: bool,
+    yaml: bool,
+    record: bool,
+    verbosity: Verbosity,
+) -> None:
+    """
+    Test index nodes for their current status
+    """
+    esg = init_esgpull(
+        verbosity,
+        safe=False,
+        record=record,
+    )
+    with esg.ui.logging("scan", onraise=Abort):
+        node_status: dict[str, bool] = {}
+        nodes = [
+            "esgf-node.ipsl.upmc.fr",
+            "esgf-data.dkrz.de",
+            "esgf.ceda.ac.uk",
+            "esgf-node.ornl.gov/esgf-1-5-bridge",
+        ]
+
+        esg.config.api.http_timeout = 3
+        with esg.ui.spinner("Fetching index nodes status"):
+            while True:
+                if not nodes:
+                    break
+                node = nodes.pop()
+                if node in node_status:
+                    continue
+                logger.info(f"check: {node}")
+                node_status[node] = check_node(esg.context, node)
+                logger.info(f"{node}\tok: {node_status[node]}")
+                for node in find_nodes(esg.context, node):
+                    if node not in node_status:
+                        nodes.append(node)
+                        logger.info(f"found index_node: {node}")
+        if json:
+            esg.ui.print(node_status, json=True)
+        elif yaml:
+            esg.ui.print(node_status, yaml=True)
+        else:
+            table = [
+                OrderedDict(
+                    [
+                        ("node", node),
+                        (
+                            "status",
+                            "[green]OK" if status else "[red]no response",
+                        ),
+                    ]
+                )
+                for node, status in node_status.items()
+            ]
+            esg.ui.print(totable(table))
+        esg.ui.raise_maybe_record(Exit(0))

esgpull/cli/plugins.py

@@ -337,7 +337,7 @@ from datetime import datetime
 from pathlib import Path
 from logging import Logger
 
-from esgpull.models import File, Query
+from esgpull.models import Dataset, File, Query
 from esgpull.plugin import Event, on
 
 # Specify version compatibility (optional)

esgpull/cli/search.py

@@ -7,6 +7,7 @@ from click.exceptions import Abort, Exit
 
 from esgpull.cli.decorators import args, groups, opts
 from esgpull.cli.utils import filter_keys, init_esgpull, parse_query, totable
+from esgpull.context import IndexNode
 from esgpull.exceptions import PageIndexError
 from esgpull.graph import Graph
 from esgpull.models import Query
@@ -98,6 +99,7 @@ def search(
             esg.ui.raise_maybe_record(Exit(0))
         esg.graph.add(query, force=True)
         query = esg.graph.expand(query.sha)
+        esg.context.probe()
         hits = esg.context.hits(
             query,
             file=file,
@@ -135,14 +137,26 @@ def search(
             esg.ui.raise_maybe_record(Exit(0))
         if facets_hints:
             not_distrib_query = query << Query(options=dict(distrib=False))
-            facet_counts = esg.context.hints(
-                not_distrib_query,
-                file=file,
-                facets=["*"],
-                date_from=date_from,
-                date_to=date_to,
-            )
-            esg.ui.print(list(facet_counts[0]), json=True)
+            index = IndexNode(esg.config.api.index_node)
+            if index.is_bridge():
+                first_file_result = esg.context.search_as_queries(
+                    not_distrib_query,
+                    file=True,
+                    max_hits=1,
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                first_file = first_file_result[0].selection.asdict()
+                esg.ui.print(list(first_file), json=True)
+            else:
+                facet_counts = esg.context.hints(
+                    not_distrib_query,
+                    file=file,
+                    facets=["*"],
+                    date_from=date_from,
+                    date_to=date_to,
+                )
+                esg.ui.print(list(facet_counts[0]), json=True)
             esg.ui.raise_maybe_record(Exit(0))
         if hints is not None:
             facet_counts = esg.context.hints(

esgpull/cli/update.py

@@ -78,6 +78,7 @@ def update(
         if not qfs:
             esg.ui.print(":stop_sign: Trying to update untracked queries.")
             esg.ui.raise_maybe_record(Exit(0))
+        esg.context.probe()
         hints = esg.context.hints(
             *[qf.expanded for qf in qfs],
             file=True,