esgpull 0.6.3__tar.gz

esgpull-0.6.3/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

esgpull-0.6.3/PKG-INFO

@@ -0,0 +1,110 @@
+Metadata-Version: 2.1
+Name: esgpull
+Version: 0.6.3
+Summary: ESGF data discovery, download, replication tool
+Author-Email: Sven Rodriguez <srodriguez@ipsl.fr>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        3. Neither the name of the copyright holder nor the names of its
+           contributors may be used to endorse or promote products derived from
+           this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+Project-URL: Repository, https://github.com/ESGF/esgf-download
+Project-URL: Documentation, https://esgf.github.io/esgf-download/
+Requires-Python: >=3.10
+Requires-Dist: MyProxyClient>=2.1.0
+Requires-Dist: aiofiles>=22.1.0
+Requires-Dist: alembic>=1.8.1
+Requires-Dist: click>=8.1.3
+Requires-Dist: click-params>=0.4.0
+Requires-Dist: httpx>=0.23.0
+Requires-Dist: nest-asyncio>=1.5.6
+Requires-Dist: pyOpenSSL>=22.1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: tomlkit>=0.11.5
+Requires-Dist: rich>=12.6.0
+Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: setuptools>=65.4.1
+Requires-Dist: aiostream>=0.4.5
+Requires-Dist: attrs>=22.1.0
+Requires-Dist: cattrs>=22.2.0
+Requires-Dist: platformdirs>=2.6.2
+Requires-Dist: pyparsing>=3.0.9
+Description-Content-Type: text/markdown
+
+[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
+
+# esgpull - ESGF data management utility
+
+`esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
+
+```py
+from esgpull import Esgpull, Query
+
+query = Query()
+query.selection.project = "CMIP6"
+query.options.distrib = True  # default=False
+esg = Esgpull()
+nb_datasets = esg.context.hits(query, file=False)[0]
+nb_files = esg.context.hits(query, file=True)[0]
+datasets = esg.context.datasets(query, max_hits=5)
+print(f"Number of CMIP6 datasets: {nb_datasets}")
+print(f"Number of CMIP6 files: {nb_files}")
+for dataset in datasets:
+    print(dataset)
+```
+
+## Features
+
+- Command-line interface
+- HTTP download (async multi-file)
+
+## Usage
+
+```console
+Usage: esgpull [OPTIONS] COMMAND [ARGS]...
+
+  esgpull is a management utility for files and datasets from ESGF.
+
+Options:
+  -V, --version  Show the version and exit.
+  -h, --help     Show this message and exit.
+
+Commands:
+  add       Add queries to the database
+  config    View/modify config
+  convert   Convert synda selection files to esgpull queries
+  download  Asynchronously download files linked to queries
+  login     OpenID authentication and certificates renewal
+  remove    Remove queries from the database
+  retry     Re-queue failed and cancelled downloads
+  search    Search datasets and files on ESGF
+  self      Manage esgpull installations / import synda database
+  show      View query tree
+  status    View file queue status
+  track     Track queries
+  untrack   Untrack queries
+  update    Fetch files, link files <-> queries, send files to download...
+```

esgpull-0.6.3/README.md

@@ -0,0 +1,54 @@
+[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
+
+# esgpull - ESGF data management utility
+
+`esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
+
+```py
+from esgpull import Esgpull, Query
+
+query = Query()
+query.selection.project = "CMIP6"
+query.options.distrib = True  # default=False
+esg = Esgpull()
+nb_datasets = esg.context.hits(query, file=False)[0]
+nb_files = esg.context.hits(query, file=True)[0]
+datasets = esg.context.datasets(query, max_hits=5)
+print(f"Number of CMIP6 datasets: {nb_datasets}")
+print(f"Number of CMIP6 files: {nb_files}")
+for dataset in datasets:
+    print(dataset)
+```
+
+## Features
+
+- Command-line interface
+- HTTP download (async multi-file)
+
+## Usage
+
+```console
+Usage: esgpull [OPTIONS] COMMAND [ARGS]...
+
+  esgpull is a management utility for files and datasets from ESGF.
+
+Options:
+  -V, --version  Show the version and exit.
+  -h, --help     Show this message and exit.
+
+Commands:
+  add       Add queries to the database
+  config    View/modify config
+  convert   Convert synda selection files to esgpull queries
+  download  Asynchronously download files linked to queries
+  login     OpenID authentication and certificates renewal
+  remove    Remove queries from the database
+  retry     Re-queue failed and cancelled downloads
+  search    Search datasets and files on ESGF
+  self      Manage esgpull installations / import synda database
+  show      View query tree
+  status    View file queue status
+  track     Track queries
+  untrack   Untrack queries
+  update    Fetch files, link files <-> queries, send files to download...
+```

esgpull-0.6.3/esgpull/__init__.py

@@ -0,0 +1,12 @@
+from esgpull.context import Context
+from esgpull.esgpull import Esgpull
+from esgpull.models import File, Query
+from esgpull.version import __version__
+
+__all__ = [
+    "Context",
+    "Esgpull",
+    "File",
+    "Query",
+    "__version__",
+]

esgpull-0.6.3/esgpull/auth.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from enum import Enum, unique
+from pathlib import Path
+from shutil import rmtree
+from typing import Any
+from urllib.parse import (
+    ParseResult,
+    ParseResultBytes,
+    urljoin,
+    urlparse,
+    urlunparse,
+)
+from xml.etree import ElementTree
+
+import httpx
+import tomlkit
+from attrs import Factory, define, field
+from myproxy.client import MyProxyClient
+from OpenSSL import crypto
+
+from esgpull.config import Config
+from esgpull.constants import PROVIDERS
+
+
+class Secret:
+    def __init__(self, value: str | None = None) -> None:
+        self._value = value
+
+    def get_value(self) -> str | None:
+        return self._value
+
+    def __str__(self) -> str:
+        if self.get_value() is None:
+            return str(None)
+        else:
+            return "*" * 10
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
+@define
+class Credentials:
+    provider: str | None = None
+    user: str | None = None
+    password: Secret = field(default=None, converter=Secret)
+
+    @staticmethod
+    def from_config(config: Config) -> Credentials:
+        path = config.paths.auth / config.credentials.filename
+        return Credentials.from_path(path)
+
+    @staticmethod
+    def from_path(path: Path) -> Credentials:
+        if path.is_file():
+            with path.open() as fh:
+                doc = tomlkit.load(fh)
+            return Credentials(**doc)
+        else:
+            return Credentials()
+
+    def write(self, path: Path) -> None:
+        if path.is_file():
+            raise FileExistsError(path)
+        with path.open("w") as f:
+            cred_dict = dict(
+                provider=self.provider,
+                user=self.user,
+                password=self.password.get_value(),
+            )
+            tomlkit.dump(cred_dict, f)
+
+    def parse_openid(self) -> ParseResult | ParseResultBytes | Any:
+        if self.provider not in PROVIDERS:
+            raise ValueError(f"unknown provider: {self.provider}")
+        ns = {"x": "xri://$xrd*($v*2.0)"}
+        provider = urlunparse(
+            [
+                "https",
+                self.provider,
+                urljoin(PROVIDERS[self.provider], self.user),
+                "",
+                "",
+                "",
+            ]
+        )
+        resp = httpx.get(str(provider))
+        resp.raise_for_status()
+        root = ElementTree.fromstring(resp.text)
+        services = root.findall(".//x:Service", namespaces=ns)
+        for service in services:
+            t = service.find("x:Type", namespaces=ns)
+            if t is None:
+                continue
+            elif t.text == "urn:esg:security:myproxy-service":
+                url = service.find("x:URI", namespaces=ns)
+                if url is not None:
+                    return urlparse(url.text)
+        raise ValueError("did not found host/port")
+
+
+@unique
+class AuthStatus(Enum):
+    Valid = ("valid", "green")
+    Expired = ("expired", "orange")
+    Missing = ("missing", "red")
+
+
+@define
+class Auth:
+    cert_dir: Path
+    cert_file: Path
+    credentials: Credentials = Factory(Credentials)
+    __status: AuthStatus | None = field(init=False, default=None, repr=False)
+
+    Valid = AuthStatus.Valid
+    Expired = AuthStatus.Expired
+    Missing = AuthStatus.Missing
+
+    @staticmethod
+    def from_config(
+        config: Config, credentials: Credentials = Credentials()
+    ) -> Auth:
+        return Auth.from_path(config.paths.auth, credentials)
+
+    @staticmethod
+    def from_path(
+        path: Path, credentials: Credentials = Credentials()
+    ) -> Auth:
+        cert_dir = path / "certificates"
+        cert_file = path / "credentials.pem"
+        return Auth(cert_dir, cert_file, credentials)
+
+    @property
+    def cert(self) -> str | None:
+        if self.status == AuthStatus.Valid:
+            return str(self.cert_file)
+        else:
+            return None
+
+    @property
+    def status(self) -> AuthStatus:
+        if self.__status is None:
+            self.__status = self._get_status()
+        return self.__status
+
+    def _get_status(self) -> AuthStatus:
+        if not self.cert_file.exists():
+            return AuthStatus.Missing
+        with self.cert_file.open("rb") as f:
+            content = f.read()
+        filetype = crypto.FILETYPE_PEM
+        pem = crypto.load_certificate(filetype, content)
+        if pem.has_expired():
+            return AuthStatus.Expired
+        return AuthStatus.Valid
+
+    # TODO: review this
+    def renew(self) -> None:
+        if self.cert_dir.is_dir():
+            rmtree(self.cert_dir)
+        self.cert_file.unlink(missing_ok=True)
+        openid = self.credentials.parse_openid()
+        client = MyProxyClient(
+            hostname=openid.hostname,
+            port=openid.port,
+            caCertDir=str(self.cert_dir),
+            proxyCertLifetime=12 * 60 * 60,
+        )
+        creds = client.logon(
+            self.credentials.user,
+            self.credentials.password.get_value(),
+            bootstrap=True,
+            updateTrustRoots=True,
+            authnGetTrustRootsCall=False,
+        )
+        with self.cert_file.open("wb") as file:
+            for cred in creds:
+                file.write(cred)
+        self.__status = None

esgpull-0.6.3/esgpull/cli/__init__.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+# https://click.palletsprojects.com/en/latest/
+import click
+
+from esgpull import __version__
+from esgpull.cli.add import add
+from esgpull.cli.config import config
+from esgpull.cli.convert import convert
+from esgpull.cli.download import download
+from esgpull.cli.login import login
+from esgpull.cli.remove import remove
+from esgpull.cli.retry import retry
+from esgpull.cli.search import search
+from esgpull.cli.self import self
+from esgpull.cli.show import show
+from esgpull.cli.status import status
+from esgpull.cli.track import track, untrack
+from esgpull.cli.update import update
+from esgpull.tui import UI
+
+# from esgpull.cli.autoremove import autoremove
+# from esgpull.cli.facet import facet
+# from esgpull.cli.get import get
+# from esgpull.cli.install import install
+
+# [-]TODO: stats
+#   - speed per index/data node
+#   - total disk usage
+#   - log config for later optimisation ?
+
+SUBCOMMANDS: list[click.Command] = [
+    add,
+    # autoremove,
+    config,
+    convert,
+    download,
+    # facet,
+    # get,
+    self,
+    # install,
+    login,
+    remove,
+    retry,
+    search,
+    show,
+    track,
+    untrack,
+    status,
+    # # stats,
+    update,
+]
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+_ui = UI("/tmp")
+version_msg = _ui.render(f"esgpull, version [green]{__version__}[/]")
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option(None, "-V", "--version", message=version_msg)
+def cli():
+    """
+    esgpull is a management utility for files and datasets from ESGF.
+    """
+
+
+for subcmd in SUBCOMMANDS:
+    cli.add_command(subcmd)
+
+
+def main():
+    cli()

esgpull-0.6.3/esgpull/cli/add.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+from click.exceptions import Abort, Exit
+
+from esgpull.cli.decorators import args, groups, opts
+from esgpull.cli.utils import (
+    init_esgpull,
+    parse_query,
+    serialize_queries_from_file,
+)
+from esgpull.graph import Graph
+from esgpull.models import Query
+from esgpull.tui import Verbosity
+
+
+@click.command()
+@args.facets
+@groups.query_def
+@opts.query_file
+@opts.track
+@opts.record
+@opts.verbosity
+def add(
+    facets: list[str],
+    # query options
+    tags: list[str],
+    require: str | None,
+    distrib: str | None,
+    latest: str | None,
+    replica: str | None,
+    retracted: str | None,
+    # since: str | None,
+    query_file: Path | None,
+    track: bool,
+    record: bool,
+    verbosity: Verbosity,
+) -> None:
+    """
+    Add queries to the database
+
+    OPTIONS / FACETS examples:
+
+        esgpull add --distrib true variable_id:co2,co3 mip_era:CMIP6
+
+            Syntax reference: http://www.esgf.io/esgf-download/search/
+
+        esgpull add --query-file path/to/query.yaml
+
+            Valid query files are usually created with either `show --json/--yaml` or `convert` commands.
+
+    Queries are `untracked` by default.
+
+    To fetch files from ESGF and link them to a query, see the `track` and `update` commands.
+    """
+    esg = init_esgpull(verbosity, record=record)
+    with esg.ui.logging("add", onraise=Abort):
+        if query_file is not None:
+            queries = serialize_queries_from_file(query_file)
+        else:
+            query = parse_query(
+                facets=facets,
+                tags=tags,
+                require=require,
+                distrib=distrib,
+                latest=latest,
+                replica=replica,
+                retracted=retracted,
+            )
+            esg.graph.resolve_require(query)
+            if track:
+                if query.require is not None:
+                    expanded = esg.graph.expand(query.require)
+                else:
+                    expanded = query
+                query.track(expanded.options)
+            queries = [query]
+        subgraph = Graph(None)
+        subgraph.add(*queries)
+        esg.ui.print(subgraph)
+        empty = Query()
+        empty.compute_sha()
+        for query in queries:
+            query.compute_sha()
+            esg.graph.resolve_require(query)
+            if query.sha == empty.sha:
+                esg.ui.print(":stop_sign: Trying to add empty query.")
+                esg.ui.raise_maybe_record(Exit(1))
+            if query.sha in esg.graph:  # esg.graph.has(sha=query.sha):
+                esg.ui.print(f"Skipping existing query: {query.rich_name}")
+            else:
+                esg.graph.add(query)
+                esg.ui.print(f"New query added: {query.rich_name}")
+        new_queries = esg.graph.merge()
+        nb = len(new_queries)
+        ies = "ies" if nb > 1 else "y"
+        if new_queries:
+            esg.ui.print(f":+1: {nb} new quer{ies} added.")
+        else:
+            esg.ui.print(":stop_sign: No new query was added.")
+        esg.ui.raise_maybe_record(Exit(0))

esgpull-0.6.3/esgpull/cli/autoremove.py

@@ -0,0 +1,38 @@
+# import click
+# from click.exceptions import Abort, Exit
+
+# from esgpull import Esgpull
+# from esgpull.cli.decorators import opts
+# from esgpull.cli.utils import filter_docs, totable
+# from esgpull.tui import Verbosity
+
+
+# @click.command()
+# @opts.force
+# @opts.verbosity
+# def autoremove(
+#     force: bool,
+#     verbosity: Verbosity,
+# ):
+#     esg = Esgpull(verbosity=verbosity)
+#     with esg.ui.logging("autoremove", onraise=Abort):
+#         deprecated = esg.db.get_deprecated_files()
+#         nb = len(deprecated)
+#         if not nb:
+#             esg.ui.print("All files are up to date.")
+#             raise Exit(0)
+#         if not force:
+#             docs = filter_docs([file.raw for file in deprecated])
+#             esg.ui.print(totable(docs))
+#             s = "s" if nb > 1 else ""
+#             esg.ui.print(f"Found {nb} file{s} to remove.")
+#             if not esg.ui.ask("Continue?", default=True):
+#                 raise Abort
+#         removed = esg.remove(*deprecated)
+#         esg.ui.print(f"Removed {len(removed)} files with newer version.")
+#         nb_remain = len(removed) - nb
+#         if nb_remain:
+#             esg.ui.print(f"{nb_remain} files could not be removed.")
+#         if force:
+#             docs = filter_docs([file.raw for file in removed])
+#             esg.ui.print(totable(docs))