esgpull 0.9.3__tar.gz → 0.9.5__tar.gz

{esgpull-0.9.3 → esgpull-0.9.5}/.github/workflows/ci.yml

@@ -9,6 +9,8 @@ on:
       - esgpull/**
       - tests/**
       - .github/workflows/ci.yml
+      - pyproject.toml
+      - uv.lock
   pull_request:
     branches:
       - main
@@ -17,6 +19,8 @@ on:
       - esgpull/**
       - tests/**
       - .github/workflows/ci.yml
+      - pyproject.toml
+      - uv.lock
 
 jobs:
   tests:

{esgpull-0.9.3 → esgpull-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.3
+Version: 0.9.5
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -31,13 +31,13 @@ Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=12.6.0
 Requires-Dist: setuptools>=65.4.1
-Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: sqlalchemy<2.1,>=2.0.0b2
 Requires-Dist: tomlkit>=0.11.5
 Description-Content-Type: text/markdown
 
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.3 → esgpull-0.9.5}/README.md

@@ -1,6 +1,6 @@
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.3 → esgpull-0.9.5}/esgpull/context.py

@@ -134,18 +134,46 @@ class Result:
         solr_terms: list[str] = []
         for name, values in self.query.selection.items():
             if index.is_bridge():
-                value_term = " ".join(quote_str(v) for v in values)
+                if name == "query":
+                    solr_terms.append(" ".join(values))
+                elif name.startswith("!"):
+                    value_term = " ".join(quote_str(v) for v in values)
+                    if len(values) > 1:
+                        value_term = f"({value_term})"
+                    solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                else:
+                    has_wildcard = any("*" in v for v in values)
+                    no_wildcard = any("*" not in v for v in values)
+
+                    if has_wildcard and no_wildcard:
+                        logger.warning(
+                            (
+                                f"Facet {name} has mixed wildcard/non-wildcard values. "
+                                "Non-wildcard values may match partially."
+                            )
+                        )
+                        value_term = " ".join(quote_str(v) for v in values)
+                        if len(values) > 1:
+                            value_term = f"({value_term})"
+                        solr_terms.append(f"{name}:{value_term}")
+                    elif has_wildcard:
+                        value_term = " ".join(quote_str(v) for v in values)
+                        if len(values) > 1:
+                            value_term = f"({value_term})"
+                        solr_terms.append(f"{name}:{value_term}")
+                    else:
+                        params[name] = ",".join(values)
             else:
                 value_term = " ".join(values)
-            if name == "query":  # freetext case
-                solr_terms.append(value_term)
-            else:
-                if len(values) > 1:
-                    value_term = f"({value_term})"
-                if name.startswith("!"):
-                    solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                if name == "query":
+                    solr_terms.append(value_term)
                 else:
-                    solr_terms.append(f"{name}:{value_term}")
+                    if len(values) > 1:
+                        value_term = f"({value_term})"
+                    if name.startswith("!"):
+                        solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                    else:
+                        solr_terms.append(f"{name}:{value_term}")
         if solr_terms:
             params["query"] = " AND ".join(solr_terms)
         for name, option in self.query.options.items(use_default=True):
@@ -562,6 +590,7 @@ class Context:
     ) -> list[File]:
         files: list[File] = []
         shas: set[str] = set()
+        file_ids: set[str] = set()
         async for result in self._fetch(*results):
             files_result = result.to(ResultFiles)
             files_result.process()
@@ -569,9 +598,13 @@ class Context:
                 for file in files_result.data:
                     if not keep_duplicates and file.sha in shas:
                         logger.debug(f"Duplicate file {file.file_id}")
-                    else:
-                        files.append(file)
-                        shas.add(file.sha)
+                        continue
+                    if not keep_duplicates and file.file_id in file_ids:
+                        logger.debug(f"Duplicate file_id {file.file_id}")
+                        continue
+                    files.append(file)
+                    shas.add(file.sha)
+                    file_ids.add(file.file_id)
         return files
 
     async def _search_as_queries(

{esgpull-0.9.3 → esgpull-0.9.5}/esgpull/esgpull.py

@@ -54,7 +54,7 @@ from esgpull.plugin import (
 )
 from esgpull.processor import Processor
 from esgpull.result import Err, Ok, Result
-from esgpull.tui import UI, DummyLive, Verbosity, logger
+from esgpull.tui import UI, DummyLive, ErrorCountColumn, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -387,6 +387,7 @@ class Esgpull:
             SpinnerColumn(),
             MofNCompleteColumn(),
             TimeRemainingColumn(compact=True, elapsed_when_finished=True),
+            ErrorCountColumn(),
         )
         file_columns: list[str | ProgressColumn] = [
             TextColumn("[cyan][{task.id}] [b blue]{task.fields[sha]}"),
@@ -434,7 +435,9 @@ class Esgpull:
         if use_db:
             self.db.add(*processor.files)
         queue_size = len(processor.tasks)
-        main_task_id = main_progress.add_task("", total=queue_size)
+        main_task_id = main_progress.add_task(
+            "", total=queue_size, nb_errors=0
+        )
         # TODO: rename ? installed/downloaded/completed/...
         files: list[File] = []
         errors: list[Err] = []
@@ -475,11 +478,13 @@ class Esgpull:
                                     )
                         case Err(_, err):
                             queue_size -= 1
-                            main_progress.update(
-                                main_task_id, total=queue_size
-                            )
                             result.data.file.status = FileStatus.Error
                             errors.append(result)
+                            main_progress.update(
+                                main_task_id,
+                                total=queue_size,
+                                nb_errors=len(errors),
+                            )
                             emit(
                                 Event.file_error,
                                 file=result.data.file,

esgpull-0.9.5/esgpull/migrations/versions/0.9.4_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.4
+Revises: 0.9.3
+Create Date: 2026-01-14 10:49:04.536961
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.4'
+down_revision = '0.9.3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull-0.9.5/esgpull/migrations/versions/0.9.5_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.5
+Revises: 0.9.4
+Create Date: 2026-02-04 15:52:02.010431
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.5'
+down_revision = '0.9.4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.9.3 → esgpull-0.9.5}/esgpull/tui.py

@@ -101,6 +101,17 @@ class ExceptionToDebugFilter(logging.Filter):
         return True
 
 
+class ErrorCountColumn(ProgressColumn):
+    def render(self, task):
+        nb_errors = task.fields.get("nb_errors", 0)
+
+        if nb_errors == 0:
+            return Text("")
+
+        suffix = "" if nb_errors == 1 else "s"
+        return Text(f"({nb_errors} download{suffix} failed)", style="red")
+
+
 @define
 class UI:
     path: Path = field(converter=Path)

{esgpull-0.9.3 → esgpull-0.9.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.9.3"
+version = "0.9.5"
 classifiers = [
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python :: 3",
@@ -27,7 +27,7 @@ dependencies = [
     "pyyaml>=6.0",
     "tomlkit>=0.11.5",
     "rich>=12.6.0",
-    "sqlalchemy>=2.0.0b2",
+    "sqlalchemy>=2.0.0b2,<2.1",
     "setuptools>=65.4.1",
     "aiostream>=0.4.5",
     "attrs>=22.1.0",

{esgpull-0.9.3 → esgpull-0.9.5}/requirements-dev.lock

@@ -14,24 +14,24 @@ aiofiles==25.1.0
     # via esgpull
 aiostream==0.7.1
     # via esgpull
-alembic==1.17.2
+alembic==1.18.3
     # via esgpull
 annotated-types==0.7.0
     # via pydantic
-anyio==4.11.0
+anyio==4.12.1
     # via httpx
 asttokens==3.0.1
     # via stack-data
 attrs==25.4.0
     # via cattrs
     # via esgpull
-babel==2.17.0
+babel==2.18.0
     # via mkdocs-material
 backrefs==6.1
     # via mkdocs-material
 cattrs==25.3.0
     # via esgpull
-certifi==2025.11.12
+certifi==2026.1.4
     # via httpcore
     # via httpx
     # via requests
@@ -49,18 +49,18 @@ colorama==0.4.6
     # via mkdocs-material
 comm==0.2.3
     # via ipykernel
-coverage==7.12.0
+coverage==7.13.3
     # via pytest-cov
-cryptography==46.0.3
+cryptography==46.0.4
     # via pyopenssl
-debugpy==1.8.17
+debugpy==1.8.20
     # via ipykernel
 decorator==5.2.1
     # via ipdb
     # via ipython
 deprecated==1.3.1
     # via click-params
-exceptiongroup==1.3.0
+exceptiongroup==1.3.1
     # via anyio
     # via cattrs
     # via ipython
@@ -69,11 +69,11 @@ execnet==2.1.2
     # via pytest-xdist
 executing==2.2.1
     # via stack-data
-filelock==3.20.0
+filelock==3.20.3
     # via pytest-mypy
 ghp-import==2.1.0
     # via mkdocs
-greenlet==3.2.4
+greenlet==3.3.1
     # via sqlalchemy
 h11==0.16.0
     # via httpcore
@@ -90,7 +90,7 @@ iniconfig==2.3.0
 ipdb==0.13.13
 ipykernel==7.1.0
     # via jupyter-console
-ipython==8.37.0
+ipython==8.38.0
     # via ipdb
     # via ipykernel
     # via jupyter-console
@@ -99,7 +99,7 @@ jedi==0.19.2
 jinja2==3.1.6
     # via mkdocs
     # via mkdocs-material
-jupyter-client==8.6.3
+jupyter-client==8.8.0
     # via ipykernel
     # via jupyter-console
 jupyter-console==6.6.3
@@ -107,9 +107,11 @@ jupyter-core==5.9.1
     # via ipykernel
     # via jupyter-client
     # via jupyter-console
+librt==0.7.8
+    # via mypy
 mako==1.3.10
     # via alembic
-markdown==3.10
+markdown==3.10.1
     # via mkdocs
     # via mkdocs-material
     # via pymdown-extensions
@@ -131,18 +133,18 @@ mkdocs==1.6.1
     # via mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.7.0
+mkdocs-material==9.7.1
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
-mypy==1.18.2
+mypy==1.19.1
     # via pytest-mypy
 mypy-extensions==1.1.0
     # via mypy
 nest-asyncio==1.6.0
     # via esgpull
     # via ipykernel
-orjson==3.11.4
-packaging==25.0
+orjson==3.11.7
+packaging==26.0
     # via esgpull
     # via ipykernel
     # via mkdocs
@@ -151,12 +153,12 @@ paginate==0.5.7
     # via mkdocs-material
 parso==0.8.5
     # via jedi
-pathspec==0.12.1
+pathspec==1.0.4
     # via mkdocs
     # via mypy
 pexpect==4.9.0
     # via ipython
-platformdirs==4.5.0
+platformdirs==4.5.1
     # via esgpull
     # via jupyter-core
     # via mkdocs-get-deps
@@ -166,15 +168,15 @@ pluggy==1.6.0
 prompt-toolkit==3.0.52
     # via ipython
     # via jupyter-console
-psutil==7.1.3
+psutil==7.2.2
     # via ipykernel
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.3
     # via stack-data
-pycparser==2.23
+pycparser==3.0
     # via cffi
-pydantic==2.12.4
+pydantic==2.12.5
     # via esgpull
     # via pydantic-settings
 pydantic-core==2.41.5
@@ -187,13 +189,13 @@ pygments==2.19.2
     # via mkdocs-material
     # via pytest
     # via rich
-pymdown-extensions==10.17.1
+pymdown-extensions==10.20.1
     # via mkdocs-material
 pyopenssl==25.3.0
     # via esgpull
-pyparsing==3.2.5
+pyparsing==3.3.2
     # via esgpull
-pytest==9.0.1
+pytest==9.0.2
     # via pytest-cov
     # via pytest-mypy
     # via pytest-xdist
@@ -219,28 +221,26 @@ pyzmq==27.1.0
     # via jupyter-console
 requests==2.32.5
     # via mkdocs-material
-rich==14.2.0
+rich==14.3.2
     # via esgpull
-setuptools==80.9.0
+setuptools==80.10.2
     # via esgpull
 six==1.17.0
     # via python-dateutil
-sniffio==1.3.1
-    # via anyio
-sqlalchemy==2.0.44
+sqlalchemy==2.0.46
     # via alembic
     # via esgpull
 stack-data==0.6.3
     # via ipython
-tomli==2.3.0
+tomli==2.4.0
     # via alembic
     # via coverage
     # via ipdb
     # via mypy
     # via pytest
-tomlkit==0.13.3
+tomlkit==0.14.0
     # via esgpull
-tornado==6.5.2
+tornado==6.5.4
     # via ipykernel
     # via jupyter-client
 traitlets==5.14.3
@@ -269,13 +269,13 @@ typing-extensions==4.15.0
 typing-inspection==0.4.2
     # via pydantic
     # via pydantic-settings
-urllib3==2.5.0
+urllib3==2.6.3
     # via requests
 validators==0.22.0
     # via click-params
 watchdog==6.0.0
     # via mkdocs
-wcwidth==0.2.14
+wcwidth==0.5.3
     # via prompt-toolkit
-wrapt==2.0.1
+wrapt==2.1.1
     # via deprecated

{esgpull-0.9.3 → esgpull-0.9.5}/requirements.lock

@@ -14,18 +14,18 @@ aiofiles==25.1.0
     # via esgpull
 aiostream==0.7.1
     # via esgpull
-alembic==1.17.2
+alembic==1.18.3
     # via esgpull
 annotated-types==0.7.0
     # via pydantic
-anyio==4.11.0
+anyio==4.12.1
     # via httpx
 attrs==25.4.0
     # via cattrs
     # via esgpull
 cattrs==25.3.0
     # via esgpull
-certifi==2025.11.12
+certifi==2026.1.4
     # via httpcore
     # via httpx
 cffi==2.0.0
@@ -35,14 +35,14 @@ click==8.3.1
     # via esgpull
 click-params==0.5.0
     # via esgpull
-cryptography==46.0.3
+cryptography==46.0.4
     # via pyopenssl
 deprecated==1.3.1
     # via click-params
-exceptiongroup==1.3.0
+exceptiongroup==1.3.1
     # via anyio
     # via cattrs
-greenlet==3.2.4
+greenlet==3.3.1
     # via sqlalchemy
 h11==0.16.0
     # via httpcore
@@ -63,13 +63,13 @@ mdurl==0.1.2
     # via markdown-it-py
 nest-asyncio==1.6.0
     # via esgpull
-packaging==25.0
+packaging==26.0
     # via esgpull
-platformdirs==4.5.0
+platformdirs==4.5.1
     # via esgpull
-pycparser==2.23
+pycparser==3.0
     # via cffi
-pydantic==2.12.4
+pydantic==2.12.5
     # via esgpull
     # via pydantic-settings
 pydantic-core==2.41.5
@@ -80,24 +80,22 @@ pygments==2.19.2
     # via rich
 pyopenssl==25.3.0
     # via esgpull
-pyparsing==3.2.5
+pyparsing==3.3.2
     # via esgpull
 python-dotenv==1.2.1
     # via pydantic-settings
 pyyaml==6.0.3
     # via esgpull
-rich==14.2.0
+rich==14.3.2
     # via esgpull
-setuptools==80.9.0
+setuptools==80.10.2
     # via esgpull
-sniffio==1.3.1
-    # via anyio
-sqlalchemy==2.0.44
+sqlalchemy==2.0.46
     # via alembic
     # via esgpull
-tomli==2.3.0
+tomli==2.4.0
     # via alembic
-tomlkit==0.13.3
+tomlkit==0.14.0
     # via esgpull
 typing-extensions==4.15.0
     # via aiostream
@@ -116,5 +114,5 @@ typing-inspection==0.4.2
     # via pydantic-settings
 validators==0.22.0
     # via click-params
-wrapt==2.0.1
+wrapt==2.1.1
     # via deprecated

{esgpull-0.9.3 → esgpull-0.9.5}/tests/cli/test_dataset.py

@@ -30,7 +30,7 @@ def test_update_displays_dataset_completion(root: Path, config: Config):
             "variable_id:areacella",
             "experiment_id:1pctCO2",
             "--distrib",
-            "false",
+            "true",
             "--track",
         ],
     )

esgpull-0.9.5/tests/cli/test_download.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from click.testing import CliRunner
+
+from esgpull import Esgpull
+from esgpull.cli.download import download
+from esgpull.config import Config
+from esgpull.models import File, FileStatus, Query
+
+
+def test_download_errors_in_progress(root: Path, config: Config):
+    # Configure the environment
+    config.generate(overwrite=True)
+    runner = CliRunner()
+    esg = Esgpull(root)
+
+    # Create a query without updating (no datasets will be created)
+    query = Query()
+    query.compute_sha()
+    query.track(query.options.default())
+
+    # Manually create files
+    failing_file = File(
+        file_id="failing_file",
+        dataset_id="test_dataset_id",
+        master_id="test_master_id",
+        url="https://nonexistent.path/to/file.nc",
+        version="1.0",
+        filename="failing.nc",
+        local_path="test/failing",
+        data_node="test_node",
+        checksum="12345",
+        checksum_type="SHA256",
+        size=1000,
+        status=FileStatus.Queued,
+    )
+    failing_file.compute_sha()
+
+    # Add files to query
+    query.files.append(failing_file)
+
+    # Add query to database
+    esg.db.add(query)
+
+    # Run the download instruction in the CLI
+    result_download = runner.invoke(download)
+
+    # Parse the output to extract
+    output = result_download.output
+    assert "download failed" in output

{esgpull-0.9.3 → esgpull-0.9.5}/tests/cli/test_remove.py

@@ -22,7 +22,7 @@ def test_update_after_remove(root: Path, config: Config):
             "experiment_id:1pctCO2",
             "source_id:CNRM-ESM2-1",
             "--distrib",
-            "false",
+            "true",
             "--track",
         ],
     )
@@ -47,7 +47,7 @@ def test_update_after_remove(root: Path, config: Config):
             "experiment_id:1pctCO2",
             "source_id:CNRM-ESM2-1",
             "--distrib",
-            "false",
+            "true",
             "--track",
         ],
     )

{esgpull-0.9.3 → esgpull-0.9.5}/tests/cli/test_update.py

@@ -48,7 +48,7 @@ def test_update_updates_timestamp(root: Path, config: Config):
             "variable_id:areacella",
             "experiment_id:1pctCO2",
             "--distrib",
-            "false",
+            "true",
             "--track",
         ],
     )