esgpull 0.6.4__tar.gz → 0.7.1__tar.gz

esgpull-0.7.1/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "*.py"
+      - esgpull/**
+      - tests/**
+      - .github/workflows/ci.yml
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "*.py"
+      - esgpull/**
+      - tests/**
+      - .github/workflows/ci.yml
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+          cache-prefix: ${{ matrix.python-version }}
+
+      - name: Pin python-version ${{ matrix.python-version }}
+        run: rye pin ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: rye sync
+
+      - name: Run tests
+        run: rye test

esgpull-0.7.1/.github/workflows/doc.yml

@@ -0,0 +1,34 @@
+name: documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "*.md"
+      - docs/**
+      - .github/workflows/doc.yml
+
+jobs:
+  documentation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+
+      - name: Sync dependencies
+        run: rye sync
+
+      - name: build
+        run: rye run mkdocs build -f docs/mkdocs.yml
+
+      - name: deploy
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/site

esgpull-0.7.1/.github/workflows/pypi-publish.yml

@@ -0,0 +1,32 @@
+name: pypi-publish
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  pypi-publish:
+    name: upload release to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      # This permission is needed for private repositories.
+      contents: read
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+
+      - name: Sync dependencies
+        run: rye sync
+
+      - name: Build package
+        run: rye build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

esgpull-0.7.1/.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+.notebooks
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm-python
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+/docs/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/

esgpull-0.7.1/.pre-commit-config.yaml

@@ -0,0 +1,57 @@
+default_language_version:
+    python: python3
+
+repos:
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.16.0
+    hooks:
+      - id: pyupgrade
+        args: [ '--py310-plus' ]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: '(setup.cfg|.svg)'
+      - id: end-of-file-fixer
+      - id: check-toml
+      - id: debug-statements
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.2
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.10.1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - attrs==22.1.0
+          - sqlalchemy==2.0.0b2
+          - types-pyopenssl==22.1.0.1
+          - types-aiofiles==22.1.0
+          - types-python-dateutil==2.8.19.2
+          - types-pymysql==1.0.19.1
+          - types-pyyaml==6.0.12
+          - types-cryptography==3.3.23.1
+          - types-setuptools==65.5.0.1
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.29.0
+    hooks:
+      - id: check-github-workflows
+
+ci:
+  autofix_commit_msg: |
+    [pre-commit.ci] auto fixes from pre-commit.com hooks
+
+    for more information, see https://pre-commit.ci
+  autofix_prs: true
+  autoupdate_branch: ''
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+  autoupdate_schedule: monthly
+  skip: [ ]
+  submodules: false

{esgpull-0.6.4 → esgpull-0.7.1}/PKG-INFO

@@ -1,63 +1,42 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: esgpull
-Version: 0.6.4
+Version: 0.7.1
 Summary: ESGF data discovery, download, replication tool
-Author-Email: Sven Rodriguez <srodriguez@ipsl.fr>
-License: BSD 3-Clause License
-        
-        Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        
-        1. Redistributions of source code must retain the above copyright notice, this
-           list of conditions and the following disclaimer.
-        
-        2. Redistributions in binary form must reproduce the above copyright notice,
-           this list of conditions and the following disclaimer in the documentation
-           and/or other materials provided with the distribution.
-        
-        3. Neither the name of the copyright holder nor the names of its
-           contributors may be used to endorse or promote products derived from
-           this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
+Author-email: Sven Rodriguez <srodriguez@ipsl.fr>
+License: BSD-3-Clause
+License-File: LICENSE
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
-Requires-Dist: MyProxyClient>=2.1.0
 Requires-Dist: aiofiles>=22.1.0
+Requires-Dist: aiostream>=0.4.5
 Requires-Dist: alembic>=1.8.1
-Requires-Dist: click>=8.1.3
+Requires-Dist: attrs>=22.1.0
+Requires-Dist: cattrs>=22.2.0
 Requires-Dist: click-params>=0.4.0
+Requires-Dist: click>=8.1.3
 Requires-Dist: httpx>=0.23.0
+Requires-Dist: myproxyclient>=2.1.0
 Requires-Dist: nest-asyncio>=1.5.6
-Requires-Dist: pyOpenSSL>=22.1.0
+Requires-Dist: platformdirs>=2.6.2
+Requires-Dist: pyopenssl>=22.1.0
+Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0
-Requires-Dist: tomlkit>=0.11.5
 Requires-Dist: rich>=12.6.0
-Requires-Dist: sqlalchemy>=2.0.0b2
 Requires-Dist: setuptools>=65.4.1
-Requires-Dist: aiostream>=0.4.5
-Requires-Dist: attrs>=22.1.0
-Requires-Dist: cattrs>=22.2.0
-Requires-Dist: platformdirs>=2.6.2
-Requires-Dist: pyparsing>=3.0.9
+Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: tomlkit>=0.11.5
 Description-Content-Type: text/markdown
 
-[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
-
 # esgpull - ESGF data management utility
 
+[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 
 ```py

{esgpull-0.6.4 → esgpull-0.7.1}/README.md

@@ -1,7 +1,7 @@
-[![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
-
 # esgpull - ESGF data management utility
 
+[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 
 ```py

esgpull-0.7.1/alembic.ini

@@ -0,0 +1,45 @@
+[alembic]
+script_location = esgpull/migrations
+prepend_sys_path = .
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+[post_write_hooks]
+hooks = black
+black.type = console_scripts
+black.entrypoint = black
+black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

esgpull-0.7.1/docs/docs/configuration.md

@@ -0,0 +1,126 @@
+This page details some of the options available for configuring your `esgpull` installation.
+
+## Configuration
+
+On invocation, `$ esgpull config` will show the base configurations in the terminal.
+
+```shell
+$ esgpull config
+
+──────────────────────────────────────────────────────────────────────────────────────────────────────────────────── /home/me/.esgpull/config.toml ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+[paths]
+auth = "/home/me/.esgpull/auth"
+data = "/home/me/.esgpull/data"
+db = "/home/me/.esgpull/db"
+log = "/home/me/.esgpull/log"
+tmp = "/home/me/.esgpull/tmp"
+
+[credentials]
+filename = "credentials.toml"
+
+[cli]
+page_size = 20
+
+[db]
+filename = "esgpull.db"
+
+[download]
+chunk_size = 67108864
+http_timeout = 20
+max_concurrent = 5
+disable_ssl = false
+
+[api]
+index_node = "esgf-node.ipsl.upmc.fr"
+http_timeout = 20
+max_concurrent = 5
+page_limit = 50
+
+[api.default_options]
+distrib = "false"
+latest = "true"
+replica = "none"
+retracted = "false"
+```
+
+To modify a config item from the command line, the dot-separated path to that item must
+be provided as the first argument along with the new value that item should get as the second argument:
+
+```shell
+$ esgpull config api.index_node esgf-data.dkrz.de
+```
+```shell
+[api]
+index_node = "esgf-data.dkrz.de"
+
+Previous value: esgf-node.ipsl.upmc.fr
+```
+
+On first call, this will generate a ``config.toml`` file in the ``~/.esgpull``
+directory with only the modified values:
+
+```shell
+$ esgpull config api.index_node esgf-data.dkrz.de
+```
+```shell
+[api]
+index_node = "esgf-data.dkrz.de"
+
+👍 New config file created at /home/srodriguez/.esgpull_test_config_generate/config.toml.
+```
+
+If a user wishes to simply generate the ``config.toml`` file without modifying any values, they
+simply must run the following:
+
+```shell
+$ esgpull config --generate
+```
+```shell
+👍 Config generated at /home/me/.esgpull/config.toml
+```
+
+!!! note "Complete existing config with defaults"
+
+    The `--generate` flag also works when a configuration file already exists.
+
+    In this case, a prompt will ask for permission to fill the existing file
+    with default values for all missing options:
+
+    ```shell
+    $ esgpull config --generate
+    ```
+    ```shell
+    A config file already exists, fill it with missing defaults? [y/n] (n): y
+    👍 Config generated at /home/me/.esgpull/config.toml
+    ```
+
+
+## Login
+
+Although most data on ESGF can be downloaded without authentication, some datasets require a valid OpenID login and password.
+
+The ESGF OpenID authentication system is described on the [ESGF user documentation](http://www.esgf.io/esgf-user-support/user_guide.html).
+
+
+This can be provided from the command line by running the following:
+
+```shell
+$ esgpull login
+```
+```shell
+No credentials found.
+  [0] esg-dn1.nsc.liu.se
+  [1] esgf-data.dkrz.de
+  [2] ceda.ac.uk
+  [3] esgf-node.ipsl.upmc.fr
+  [4] esgf-node.llnl.gov
+  [5] esgf.nci.org.au
+Select a provider: 0
+User: MyESGFusername
+Password: <hidden>
+Certificates are missing.
+👍 Renewed successfully
+```
+
+The credentials will then be saved under the ``~/.esgpull/auth`` directory, within
+``credentials.toml``, which can then be used for future sessions.