esgpull 0.6.3__py3-none-any.whl

esgpull/__init__.py

@@ -0,0 +1,12 @@
+from esgpull.context import Context
+from esgpull.esgpull import Esgpull
+from esgpull.models import File, Query
+from esgpull.version import __version__
+
+__all__ = [
+    "Context",
+    "Esgpull",
+    "File",
+    "Query",
+    "__version__",
+]

esgpull/auth.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from enum import Enum, unique
+from pathlib import Path
+from shutil import rmtree
+from typing import Any
+from urllib.parse import (
+    ParseResult,
+    ParseResultBytes,
+    urljoin,
+    urlparse,
+    urlunparse,
+)
+from xml.etree import ElementTree
+
+import httpx
+import tomlkit
+from attrs import Factory, define, field
+from myproxy.client import MyProxyClient
+from OpenSSL import crypto
+
+from esgpull.config import Config
+from esgpull.constants import PROVIDERS
+
+
+class Secret:
+    def __init__(self, value: str | None = None) -> None:
+        self._value = value
+
+    def get_value(self) -> str | None:
+        return self._value
+
+    def __str__(self) -> str:
+        if self.get_value() is None:
+            return str(None)
+        else:
+            return "*" * 10
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
+@define
+class Credentials:
+    provider: str | None = None
+    user: str | None = None
+    password: Secret = field(default=None, converter=Secret)
+
+    @staticmethod
+    def from_config(config: Config) -> Credentials:
+        path = config.paths.auth / config.credentials.filename
+        return Credentials.from_path(path)
+
+    @staticmethod
+    def from_path(path: Path) -> Credentials:
+        if path.is_file():
+            with path.open() as fh:
+                doc = tomlkit.load(fh)
+            return Credentials(**doc)
+        else:
+            return Credentials()
+
+    def write(self, path: Path) -> None:
+        if path.is_file():
+            raise FileExistsError(path)
+        with path.open("w") as f:
+            cred_dict = dict(
+                provider=self.provider,
+                user=self.user,
+                password=self.password.get_value(),
+            )
+            tomlkit.dump(cred_dict, f)
+
+    def parse_openid(self) -> ParseResult | ParseResultBytes | Any:
+        if self.provider not in PROVIDERS:
+            raise ValueError(f"unknown provider: {self.provider}")
+        ns = {"x": "xri://$xrd*($v*2.0)"}
+        provider = urlunparse(
+            [
+                "https",
+                self.provider,
+                urljoin(PROVIDERS[self.provider], self.user),
+                "",
+                "",
+                "",
+            ]
+        )
+        resp = httpx.get(str(provider))
+        resp.raise_for_status()
+        root = ElementTree.fromstring(resp.text)
+        services = root.findall(".//x:Service", namespaces=ns)
+        for service in services:
+            t = service.find("x:Type", namespaces=ns)
+            if t is None:
+                continue
+            elif t.text == "urn:esg:security:myproxy-service":
+                url = service.find("x:URI", namespaces=ns)
+                if url is not None:
+                    return urlparse(url.text)
+        raise ValueError("did not found host/port")
+
+
+@unique
+class AuthStatus(Enum):
+    Valid = ("valid", "green")
+    Expired = ("expired", "orange")
+    Missing = ("missing", "red")
+
+
+@define
+class Auth:
+    cert_dir: Path
+    cert_file: Path
+    credentials: Credentials = Factory(Credentials)
+    __status: AuthStatus | None = field(init=False, default=None, repr=False)
+
+    Valid = AuthStatus.Valid
+    Expired = AuthStatus.Expired
+    Missing = AuthStatus.Missing
+
+    @staticmethod
+    def from_config(
+        config: Config, credentials: Credentials = Credentials()
+    ) -> Auth:
+        return Auth.from_path(config.paths.auth, credentials)
+
+    @staticmethod
+    def from_path(
+        path: Path, credentials: Credentials = Credentials()
+    ) -> Auth:
+        cert_dir = path / "certificates"
+        cert_file = path / "credentials.pem"
+        return Auth(cert_dir, cert_file, credentials)
+
+    @property
+    def cert(self) -> str | None:
+        if self.status == AuthStatus.Valid:
+            return str(self.cert_file)
+        else:
+            return None
+
+    @property
+    def status(self) -> AuthStatus:
+        if self.__status is None:
+            self.__status = self._get_status()
+        return self.__status
+
+    def _get_status(self) -> AuthStatus:
+        if not self.cert_file.exists():
+            return AuthStatus.Missing
+        with self.cert_file.open("rb") as f:
+            content = f.read()
+        filetype = crypto.FILETYPE_PEM
+        pem = crypto.load_certificate(filetype, content)
+        if pem.has_expired():
+            return AuthStatus.Expired
+        return AuthStatus.Valid
+
+    # TODO: review this
+    def renew(self) -> None:
+        if self.cert_dir.is_dir():
+            rmtree(self.cert_dir)
+        self.cert_file.unlink(missing_ok=True)
+        openid = self.credentials.parse_openid()
+        client = MyProxyClient(
+            hostname=openid.hostname,
+            port=openid.port,
+            caCertDir=str(self.cert_dir),
+            proxyCertLifetime=12 * 60 * 60,
+        )
+        creds = client.logon(
+            self.credentials.user,
+            self.credentials.password.get_value(),
+            bootstrap=True,
+            updateTrustRoots=True,
+            authnGetTrustRootsCall=False,
+        )
+        with self.cert_file.open("wb") as file:
+            for cred in creds:
+                file.write(cred)
+        self.__status = None

esgpull/cli/__init__.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+# https://click.palletsprojects.com/en/latest/
+import click
+
+from esgpull import __version__
+from esgpull.cli.add import add
+from esgpull.cli.config import config
+from esgpull.cli.convert import convert
+from esgpull.cli.download import download
+from esgpull.cli.login import login
+from esgpull.cli.remove import remove
+from esgpull.cli.retry import retry
+from esgpull.cli.search import search
+from esgpull.cli.self import self
+from esgpull.cli.show import show
+from esgpull.cli.status import status
+from esgpull.cli.track import track, untrack
+from esgpull.cli.update import update
+from esgpull.tui import UI
+
+# from esgpull.cli.autoremove import autoremove
+# from esgpull.cli.facet import facet
+# from esgpull.cli.get import get
+# from esgpull.cli.install import install
+
+# [-]TODO: stats
+#   - speed per index/data node
+#   - total disk usage
+#   - log config for later optimisation ?
+
+SUBCOMMANDS: list[click.Command] = [
+    add,
+    # autoremove,
+    config,
+    convert,
+    download,
+    # facet,
+    # get,
+    self,
+    # install,
+    login,
+    remove,
+    retry,
+    search,
+    show,
+    track,
+    untrack,
+    status,
+    # # stats,
+    update,
+]
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+_ui = UI("/tmp")
+version_msg = _ui.render(f"esgpull, version [green]{__version__}[/]")
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option(None, "-V", "--version", message=version_msg)
+def cli():
+    """
+    esgpull is a management utility for files and datasets from ESGF.
+    """
+
+
+for subcmd in SUBCOMMANDS:
+    cli.add_command(subcmd)
+
+
+def main():
+    cli()

esgpull/cli/add.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from click.exceptions import Abort, Exit
+
+from esgpull.cli.decorators import args, groups, opts
+from esgpull.cli.utils import (
+    init_esgpull,
+    parse_query,
+    serialize_queries_from_file,
+)
+from esgpull.graph import Graph
+from esgpull.models import Query
+from esgpull.tui import Verbosity
+
+
+@click.command()
+@args.facets
+@groups.query_def
+@opts.query_file
+@opts.track
+@opts.record
+@opts.verbosity
+def add(
+    facets: list[str],
+    # query options
+    tags: list[str],
+    require: str | None,
+    distrib: str | None,
+    latest: str | None,
+    replica: str | None,
+    retracted: str | None,
+    # since: str | None,
+    query_file: Path | None,
+    track: bool,
+    record: bool,
+    verbosity: Verbosity,
+) -> None:
+    """
+    Add queries to the database
+
+    OPTIONS / FACETS examples:
+
+        esgpull add --distrib true variable_id:co2,co3 mip_era:CMIP6
+
+            Syntax reference: http://www.esgf.io/esgf-download/search/
+
+        esgpull add --query-file path/to/query.yaml
+
+            Valid query files are usually created with either `show --json/--yaml` or `convert` commands.
+
+    Queries are `untracked` by default.
+
+    To fetch files from ESGF and link them to a query, see the `track` and `update` commands.
+    """
+    esg = init_esgpull(verbosity, record=record)
+    with esg.ui.logging("add", onraise=Abort):
+        if query_file is not None:
+            queries = serialize_queries_from_file(query_file)
+        else:
+            query = parse_query(
+                facets=facets,
+                tags=tags,
+                require=require,
+                distrib=distrib,
+                latest=latest,
+                replica=replica,
+                retracted=retracted,
+            )
+            esg.graph.resolve_require(query)
+            if track:
+                if query.require is not None:
+                    expanded = esg.graph.expand(query.require)
+                else:
+                    expanded = query
+                query.track(expanded.options)
+            queries = [query]
+        subgraph = Graph(None)
+        subgraph.add(*queries)
+        esg.ui.print(subgraph)
+        empty = Query()
+        empty.compute_sha()
+        for query in queries:
+            query.compute_sha()
+            esg.graph.resolve_require(query)
+            if query.sha == empty.sha:
+                esg.ui.print(":stop_sign: Trying to add empty query.")
+                esg.ui.raise_maybe_record(Exit(1))
+            if query.sha in esg.graph:  # esg.graph.has(sha=query.sha):
+                esg.ui.print(f"Skipping existing query: {query.rich_name}")
+            else:
+                esg.graph.add(query)
+                esg.ui.print(f"New query added: {query.rich_name}")
+        new_queries = esg.graph.merge()
+        nb = len(new_queries)
+        ies = "ies" if nb > 1 else "y"
+        if new_queries:
+            esg.ui.print(f":+1: {nb} new quer{ies} added.")
+        else:
+            esg.ui.print(":stop_sign: No new query was added.")
+        esg.ui.raise_maybe_record(Exit(0))

esgpull/cli/autoremove.py

@@ -0,0 +1,38 @@
+# import click
+# from click.exceptions import Abort, Exit
+
+# from esgpull import Esgpull
+# from esgpull.cli.decorators import opts
+# from esgpull.cli.utils import filter_docs, totable
+# from esgpull.tui import Verbosity
+
+
+# @click.command()
+# @opts.force
+# @opts.verbosity
+# def autoremove(
+#     force: bool,
+#     verbosity: Verbosity,
+# ):
+#     esg = Esgpull(verbosity=verbosity)
+#     with esg.ui.logging("autoremove", onraise=Abort):
+#         deprecated = esg.db.get_deprecated_files()
+#         nb = len(deprecated)
+#         if not nb:
+#             esg.ui.print("All files are up to date.")
+#             raise Exit(0)
+#         if not force:
+#             docs = filter_docs([file.raw for file in deprecated])
+#             esg.ui.print(totable(docs))
+#             s = "s" if nb > 1 else ""
+#             esg.ui.print(f"Found {nb} file{s} to remove.")
+#             if not esg.ui.ask("Continue?", default=True):
+#                 raise Abort
+#         removed = esg.remove(*deprecated)
+#         esg.ui.print(f"Removed {len(removed)} files with newer version.")
+#         nb_remain = len(removed) - nb
+#         if nb_remain:
+#             esg.ui.print(f"{nb_remain} files could not be removed.")
+#         if force:
+#             docs = filter_docs([file.raw for file in removed])
+#             esg.ui.print(totable(docs))

esgpull/cli/config.py

@@ -0,0 +1,116 @@
+from textwrap import dedent
+
+import click
+from click.exceptions import Abort, BadOptionUsage, Exit
+
+from esgpull.cli.decorators import args, opts
+from esgpull.cli.utils import init_esgpull
+from esgpull.config import ConfigKind
+from esgpull.tui import Verbosity
+
+
+def extract_command(doc: dict, key: str | None) -> dict:
+    if key is None:
+        return doc
+    for part in key.split("."):
+        if not part:
+            raise KeyError(key)
+        elif part in doc:
+            doc = doc[part]
+        else:
+            raise KeyError(part)
+    for part in key.split(".")[::-1]:
+        doc = {part: doc}
+    return doc
+
+
+@click.command()
+@args.key
+@args.value
+@opts.default
+@opts.generate
+@opts.record
+@opts.verbosity
+def config(
+    key: str | None,
+    value: str | None,
+    default: bool,
+    generate: bool,
+    record: bool,
+    verbosity: Verbosity,
+):
+    """
+    View/modify config
+
+    The full config is shown when no arguments are provided. It includes all items overwritten
+    in the `config.toml` file, and default values otherwise.
+    Note that the `config.toml` file does not exist by default, an empty file will be created
+    on the first modification of any item. Otherwise one can generate a config file containing
+    every default value using the `--generate` flag.
+
+    To view a specific config section/item, the dot-separated path to that section/item must
+    be provided as the only argument.
+
+    To modify a config item, the dot-separated path to that item must be provided as the first
+    argument, along with the new value that item should get as the second argument.
+
+    Only config items can be modified.
+    """
+    esg = init_esgpull(verbosity=verbosity, load_db=False, record=record)
+    with esg.ui.logging("config", onraise=Abort):
+        if key is not None and value is not None:
+            if default:
+                raise BadOptionUsage(
+                    "default",
+                    dedent(
+                        f"""
+                        --default/-d is invalid with a value.
+                        Instead use:
+
+                        $ esgpull config {key} -d
+                        """
+                    ),
+                )
+            kind = esg.config.kind
+            old_value = esg.config.update_item(key, value, empty_ok=True)
+            info = extract_command(esg.config.dump(), key)
+            esg.config.write()
+            esg.ui.print(info, toml=True)
+            if kind == ConfigKind.NoFile:
+                esg.ui.print(
+                    ":+1: New config file created at "
+                    f"{esg.config._config_file}."
+                )
+            else:
+                esg.ui.print(f"Previous value: {old_value}")
+        elif key is not None:
+            if default:
+                old_value = esg.config.set_default(key)
+                info = extract_command(esg.config.dump(), key)
+                esg.config.write()
+                esg.ui.print(info, toml=True)
+                esg.ui.print(f"Previous value: {old_value}")
+            else:
+                info = extract_command(esg.config.dump(), key)
+                esg.ui.print(info, toml=True)
+        elif generate:
+            overwrite = False
+            if esg.config.kind == ConfigKind.Complete:
+                esg.ui.print(
+                    f"{esg.config._config_file}\n"
+                    ":+1: Your config file is already complete."
+                )
+                esg.ui.raise_maybe_record(Exit(0))
+            elif esg.config.kind == ConfigKind.Partial and esg.ui.ask(
+                "A config file already exists,"
+                " fill it with missing defaults?",
+                default=False,
+            ):
+                overwrite = True
+            esg.config.generate(overwrite=overwrite)
+            msg = f":+1: Config generated at {esg.config._config_file}"
+            esg.ui.print(msg)
+        else:
+            esg.ui.rule(str(esg.config._config_file))
+            esg.ui.print(esg.config.dump(), toml=True)
+        esg.ui.raise_maybe_record(Exit(0))