anemoi-utils 0.3.0__tar.gz → 0.3.2__tar.gz

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/.github/workflows/python-publish.yml

@@ -41,9 +41,7 @@ jobs:
 
     - name: Install
       run: |
-        pip install pytest
-        pip install -e .[all]
-        pip install -r tests/requirements.txt
+        pip install -e .[all,tests]
         pip freeze
 
     - name: Tests

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/.pre-commit-config.yaml

@@ -1,17 +1,14 @@
 repos:
-
 # Empty notebookds
 - repo: local
   hooks:
-    - id: clear-notebooks-output
-      name: clear-notebooks-output
-      files: tools/.*\.ipynb$
-      stages: [commit]
-      language: python
-      entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
-      additional_dependencies: [jupyter]
-
-
+  - id: clear-notebooks-output
+    name: clear-notebooks-output
+    files: tools/.*\.ipynb$
+    stages: [commit]
+    language: python
+    entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+    additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
   hooks:
@@ -23,13 +20,11 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
-
 - repo: https://github.com/psf/black-pre-commit-mirror
   rev: 24.4.2
   hooks:
-    - id: black
-      args: [--line-length=120]
-
+  - id: black
+    args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
   rev: 5.13.2
   hooks:
@@ -38,44 +33,38 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
-
-
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.4.6
   hooks:
   - id: ruff
-    exclude: '(dev/.*|.*_)\.py$'
+    # Next line if for documenation cod snippets
+    exclude: '^(dev/.*|[A-Za-z].*_)\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
-
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v0.9.1
   hooks:
-    - id: sphinx-lint
-
+  - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
 - repo: https://github.com/dzhu/rstfmt
   rev: v0.0.14
   hooks:
-    - id: rstfmt
-      exclude: 'cli/.*' # Because we use argparse
-
+  - id: rstfmt
+    exclude: 'cli/.*' # Because we use argparse
 - repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.4"
+  rev: "0.1.5"
   hooks:
   - id: docconvert
     args: ["numpy"]
-
 - repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.2"
+  rev: "0.0.6"
   hooks:
   - id: optional-dependencies-all
-    args: ["--inplace", "--all-key", "all", "--exclude-keys", "dev,docs"]
-
+    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "2.1.3"
   hooks:
-    - id: pyproject-fmt
+  - id: pyproject-fmt

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.0
+Version: 0.3.2
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -229,15 +229,18 @@ Requires-Dist: tqdm
 Provides-Extra: all
 Requires-Dist: gitpython; extra == "all"
 Requires-Dist: nvsmi; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: termcolor; extra == "all"
-Requires-Dist: tomli; extra == "all"
-Requires-Dist: tqdm; extra == "all"
 Provides-Extra: dev
 Requires-Dist: gitpython; extra == "dev"
+Requires-Dist: nbsphinx; extra == "dev"
 Requires-Dist: nvsmi; extra == "dev"
+Requires-Dist: pandoc; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx-argparse; extra == "dev"
+Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Requires-Dist: termcolor; extra == "dev"
 Requires-Dist: tomli; extra == "dev"
 Provides-Extra: docs
@@ -254,5 +257,7 @@ Requires-Dist: requests; extra == "grib"
 Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
 Requires-Dist: termcolor; extra == "text"

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/pyproject.toml

@@ -57,18 +57,21 @@ dependencies = [
 optional-dependencies.all = [
   "gitpython",
   "nvsmi",
-  "pyyaml",
   "requests",
   "termcolor",
-  "tomli",
-  "tqdm",
 ]
 optional-dependencies.dev = [
   "gitpython",
+  "nbsphinx",
   "nvsmi",
+  "pandoc",
+  "pytest",
   "requests",
+  "sphinx",
+  "sphinx-argparse",
+  "sphinx-rtd-theme",
   "termcolor",
-  "tomli",     # Only needed before 3.11
+  "tomli",
 ]
 # Loaded by read-the-docs
 # `pip install .[docs]`
@@ -80,7 +83,7 @@ optional-dependencies.docs = [
   "sphinx-argparse",
   "sphinx-rtd-theme",
   "termcolor",
-  "tomli",            # Only needed before 3.11
+  "tomli",
 ]
 optional-dependencies.grib = [
   "requests",
@@ -90,6 +93,10 @@ optional-dependencies.provenance = [
   "gitpython",
   "nvsmi",
 ]
+optional-dependencies.tests = [
+  "pytest",
+]
+
 optional-dependencies.text = [
   "termcolor",
 ]

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.0'
-__version_tuple__ = version_tuple = (0, 3, 0)
+__version__ = version = '0.3.2'
+__version_tuple__ = version_tuple = (0, 3, 2)

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/src/anemoi/utils/cli.py

@@ -41,7 +41,7 @@ def make_parser(description, commands):
 
     subparsers = parser.add_subparsers(help="commands:", dest="command")
     for name, command in commands.items():
-        command_parser = subparsers.add_parser(name, help=command.__doc__)
+        command_parser = subparsers.add_parser(name, description=command.__doc__, help=command.__doc__)
         command.add_arguments(command_parser)
 
     return parser

anemoi_utils-0.3.2/src/anemoi/utils/s3.py

@@ -0,0 +1,57 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+import logging
+import os
+from contextlib import closing
+
+import boto3
+import tqdm
+
+LOG = logging.getLogger(__name__)
+
+
+def upload(source, target, overwrite=False, ignore_existing=False):
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
+    assert target.startswith("s3://")
+
+    _, _, bucket, key = target.split("/", 3)
+
+    LOG.info(f"Uploading {source} to {target}")
+    s3_client = boto3.client("s3")
+
+    if not overwrite:
+        results = s3_client.list_objects(Bucket=bucket, Prefix=key)
+        if results.get("Contents"):
+            if ignore_existing:
+                LOG.info(f"{target} already exists, skipping")
+                return
+            else:
+                raise ValueError(f"{target} already exists, use --overwrite to replace")
+
+    size = os.path.getsize(source)
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3_client.upload_file(source, bucket, key, Callback=lambda x: t.update(x))
+
+    LOG.info(f"{target} is ready")
+
+
+def download(source, target, overwrite=False):
+    assert source.startswith("s3://")
+
+    _, _, bucket, key = source.split("/", 3)
+
+    s3 = boto3.client("s3")
+    response = s3.head_object(Bucket=bucket, Key=key)
+    size = response["ContentLength"]
+
+    if not overwrite:
+        if os.path.exists(source) and os.path.getsize(source) == size:
+            LOG.info(f"{source} already exists, skipping")
+            return
+
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3.download_file(bucket, key, target, Callback=lambda x: t.update(x))

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.0
+Version: 0.3.2
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -229,15 +229,18 @@ Requires-Dist: tqdm
 Provides-Extra: all
 Requires-Dist: gitpython; extra == "all"
 Requires-Dist: nvsmi; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: requests; extra == "all"
 Requires-Dist: termcolor; extra == "all"
-Requires-Dist: tomli; extra == "all"
-Requires-Dist: tqdm; extra == "all"
 Provides-Extra: dev
 Requires-Dist: gitpython; extra == "dev"
+Requires-Dist: nbsphinx; extra == "dev"
 Requires-Dist: nvsmi; extra == "dev"
+Requires-Dist: pandoc; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
+Requires-Dist: sphinx; extra == "dev"
+Requires-Dist: sphinx-argparse; extra == "dev"
+Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Requires-Dist: termcolor; extra == "dev"
 Requires-Dist: tomli; extra == "dev"
 Provides-Extra: docs
@@ -254,5 +257,7 @@ Requires-Dist: requests; extra == "grib"
 Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
 Requires-Dist: termcolor; extra == "text"

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ src/anemoi/utils/dates.py
 src/anemoi/utils/grib.py
 src/anemoi/utils/humanize.py
 src/anemoi/utils/provenance.py
+src/anemoi/utils/s3.py
 src/anemoi/utils/text.py
 src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py
@@ -43,6 +44,5 @@ src/anemoi_utils.egg-info/dependency_links.txt
 src/anemoi_utils.egg-info/entry_points.txt
 src/anemoi_utils.egg-info/requires.txt
 src/anemoi_utils.egg-info/top_level.txt
-tests/requirements.txt
 tests/test_dates.py
 tests/test_utils.py

{anemoi_utils-0.3.0 → anemoi_utils-0.3.2}/src/anemoi_utils.egg-info/requires.txt

@@ -5,16 +5,19 @@ tqdm
 [all]
 gitpython
 nvsmi
-pyyaml
 requests
 termcolor
-tomli
-tqdm
 
 [dev]
 gitpython
+nbsphinx
 nvsmi
+pandoc
+pytest
 requests
+sphinx
+sphinx-argparse
+sphinx-rtd-theme
 termcolor
 tomli
 
@@ -35,5 +38,8 @@ requests
 gitpython
 nvsmi
 
+[tests]
+pytest
+
 [text]
 termcolor

anemoi_utils-0.3.0/tests/requirements.txt

@@ -1 +0,0 @@
-# Empty for now