esgpull 0.9.4__tar.gz → 0.9.5__tar.gz

{esgpull-0.9.4 → esgpull-0.9.5}/.github/workflows/ci.yml

@@ -9,6 +9,8 @@ on:
       - esgpull/**
       - tests/**
       - .github/workflows/ci.yml
+      - pyproject.toml
+      - uv.lock
   pull_request:
     branches:
       - main
@@ -17,6 +19,8 @@ on:
       - esgpull/**
       - tests/**
       - .github/workflows/ci.yml
+      - pyproject.toml
+      - uv.lock
 
 jobs:
   tests:

{esgpull-0.9.4 → esgpull-0.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.4
+Version: 0.9.5
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -31,13 +31,13 @@ Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=12.6.0
 Requires-Dist: setuptools>=65.4.1
-Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: sqlalchemy<2.1,>=2.0.0b2
 Requires-Dist: tomlkit>=0.11.5
 Description-Content-Type: text/markdown
 
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.4 → esgpull-0.9.5}/README.md

@@ -1,6 +1,6 @@
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.4 → esgpull-0.9.5}/esgpull/context.py

@@ -590,6 +590,7 @@ class Context:
     ) -> list[File]:
         files: list[File] = []
         shas: set[str] = set()
+        file_ids: set[str] = set()
         async for result in self._fetch(*results):
             files_result = result.to(ResultFiles)
             files_result.process()
@@ -597,9 +598,13 @@ class Context:
                 for file in files_result.data:
                     if not keep_duplicates and file.sha in shas:
                         logger.debug(f"Duplicate file {file.file_id}")
-                    else:
-                        files.append(file)
-                        shas.add(file.sha)
+                        continue
+                    if not keep_duplicates and file.file_id in file_ids:
+                        logger.debug(f"Duplicate file_id {file.file_id}")
+                        continue
+                    files.append(file)
+                    shas.add(file.sha)
+                    file_ids.add(file.file_id)
         return files
 
     async def _search_as_queries(

{esgpull-0.9.4 → esgpull-0.9.5}/esgpull/esgpull.py

@@ -54,7 +54,7 @@ from esgpull.plugin import (
 )
 from esgpull.processor import Processor
 from esgpull.result import Err, Ok, Result
-from esgpull.tui import UI, DummyLive, Verbosity, logger
+from esgpull.tui import UI, DummyLive, ErrorCountColumn, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -387,6 +387,7 @@ class Esgpull:
             SpinnerColumn(),
             MofNCompleteColumn(),
             TimeRemainingColumn(compact=True, elapsed_when_finished=True),
+            ErrorCountColumn(),
         )
         file_columns: list[str | ProgressColumn] = [
             TextColumn("[cyan][{task.id}] [b blue]{task.fields[sha]}"),
@@ -434,7 +435,9 @@ class Esgpull:
         if use_db:
             self.db.add(*processor.files)
         queue_size = len(processor.tasks)
-        main_task_id = main_progress.add_task("", total=queue_size)
+        main_task_id = main_progress.add_task(
+            "", total=queue_size, nb_errors=0
+        )
         # TODO: rename ? installed/downloaded/completed/...
         files: list[File] = []
         errors: list[Err] = []
@@ -475,11 +478,13 @@ class Esgpull:
                                     )
                         case Err(_, err):
                             queue_size -= 1
-                            main_progress.update(
-                                main_task_id, total=queue_size
-                            )
                             result.data.file.status = FileStatus.Error
                             errors.append(result)
+                            main_progress.update(
+                                main_task_id,
+                                total=queue_size,
+                                nb_errors=len(errors),
+                            )
                             emit(
                                 Event.file_error,
                                 file=result.data.file,

esgpull-0.9.5/esgpull/migrations/versions/0.9.5_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.5
+Revises: 0.9.4
+Create Date: 2026-02-04 15:52:02.010431
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.5'
+down_revision = '0.9.4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.9.4 → esgpull-0.9.5}/esgpull/tui.py

@@ -101,6 +101,17 @@ class ExceptionToDebugFilter(logging.Filter):
         return True
 
 
+class ErrorCountColumn(ProgressColumn):
+    def render(self, task):
+        nb_errors = task.fields.get("nb_errors", 0)
+
+        if nb_errors == 0:
+            return Text("")
+
+        suffix = "" if nb_errors == 1 else "s"
+        return Text(f"({nb_errors} download{suffix} failed)", style="red")
+
+
 @define
 class UI:
     path: Path = field(converter=Path)

{esgpull-0.9.4 → esgpull-0.9.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.9.4"
+version = "0.9.5"
 classifiers = [
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python :: 3",
@@ -27,7 +27,7 @@ dependencies = [
     "pyyaml>=6.0",
     "tomlkit>=0.11.5",
     "rich>=12.6.0",
-    "sqlalchemy>=2.0.0b2",
+    "sqlalchemy>=2.0.0b2,<2.1",
     "setuptools>=65.4.1",
     "aiostream>=0.4.5",
     "attrs>=22.1.0",

{esgpull-0.9.4 → esgpull-0.9.5}/requirements-dev.lock

@@ -14,7 +14,7 @@ aiofiles==25.1.0
     # via esgpull
 aiostream==0.7.1
     # via esgpull
-alembic==1.18.0
+alembic==1.18.3
     # via esgpull
 annotated-types==0.7.0
     # via pydantic
@@ -25,7 +25,7 @@ asttokens==3.0.1
 attrs==25.4.0
     # via cattrs
     # via esgpull
-babel==2.17.0
+babel==2.18.0
     # via mkdocs-material
 backrefs==6.1
     # via mkdocs-material
@@ -49,11 +49,11 @@ colorama==0.4.6
     # via mkdocs-material
 comm==0.2.3
     # via ipykernel
-coverage==7.13.1
+coverage==7.13.3
     # via pytest-cov
-cryptography==46.0.3
+cryptography==46.0.4
     # via pyopenssl
-debugpy==1.8.19
+debugpy==1.8.20
     # via ipykernel
 decorator==5.2.1
     # via ipdb
@@ -73,7 +73,7 @@ filelock==3.20.3
     # via pytest-mypy
 ghp-import==2.1.0
     # via mkdocs
-greenlet==3.3.0
+greenlet==3.3.1
     # via sqlalchemy
 h11==0.16.0
     # via httpcore
@@ -107,11 +107,11 @@ jupyter-core==5.9.1
     # via ipykernel
     # via jupyter-client
     # via jupyter-console
-librt==0.7.7
+librt==0.7.8
     # via mypy
 mako==1.3.10
     # via alembic
-markdown==3.10
+markdown==3.10.1
     # via mkdocs
     # via mkdocs-material
     # via pymdown-extensions
@@ -143,8 +143,8 @@ mypy-extensions==1.1.0
 nest-asyncio==1.6.0
     # via esgpull
     # via ipykernel
-orjson==3.11.5
-packaging==25.0
+orjson==3.11.7
+packaging==26.0
     # via esgpull
     # via ipykernel
     # via mkdocs
@@ -153,7 +153,7 @@ paginate==0.5.7
     # via mkdocs-material
 parso==0.8.5
     # via jedi
-pathspec==1.0.3
+pathspec==1.0.4
     # via mkdocs
     # via mypy
 pexpect==4.9.0
@@ -168,13 +168,13 @@ pluggy==1.6.0
 prompt-toolkit==3.0.52
     # via ipython
     # via jupyter-console
-psutil==7.2.1
+psutil==7.2.2
     # via ipykernel
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.3
     # via stack-data
-pycparser==2.23
+pycparser==3.0
     # via cffi
 pydantic==2.12.5
     # via esgpull
@@ -189,11 +189,11 @@ pygments==2.19.2
     # via mkdocs-material
     # via pytest
     # via rich
-pymdown-extensions==10.20
+pymdown-extensions==10.20.1
     # via mkdocs-material
 pyopenssl==25.3.0
     # via esgpull
-pyparsing==3.3.1
+pyparsing==3.3.2
     # via esgpull
 pytest==9.0.2
     # via pytest-cov
@@ -221,13 +221,13 @@ pyzmq==27.1.0
     # via jupyter-console
 requests==2.32.5
     # via mkdocs-material
-rich==14.2.0
+rich==14.3.2
     # via esgpull
-setuptools==80.9.0
+setuptools==80.10.2
     # via esgpull
 six==1.17.0
     # via python-dateutil
-sqlalchemy==2.0.45
+sqlalchemy==2.0.46
     # via alembic
     # via esgpull
 stack-data==0.6.3
@@ -275,7 +275,7 @@ validators==0.22.0
     # via click-params
 watchdog==6.0.0
     # via mkdocs
-wcwidth==0.2.14
+wcwidth==0.5.3
     # via prompt-toolkit
-wrapt==2.0.1
+wrapt==2.1.1
     # via deprecated

{esgpull-0.9.4 → esgpull-0.9.5}/requirements.lock

@@ -14,7 +14,7 @@ aiofiles==25.1.0
     # via esgpull
 aiostream==0.7.1
     # via esgpull
-alembic==1.18.0
+alembic==1.18.3
     # via esgpull
 annotated-types==0.7.0
     # via pydantic
@@ -35,14 +35,14 @@ click==8.3.1
     # via esgpull
 click-params==0.5.0
     # via esgpull
-cryptography==46.0.3
+cryptography==46.0.4
     # via pyopenssl
 deprecated==1.3.1
     # via click-params
 exceptiongroup==1.3.1
     # via anyio
     # via cattrs
-greenlet==3.3.0
+greenlet==3.3.1
     # via sqlalchemy
 h11==0.16.0
     # via httpcore
@@ -63,11 +63,11 @@ mdurl==0.1.2
     # via markdown-it-py
 nest-asyncio==1.6.0
     # via esgpull
-packaging==25.0
+packaging==26.0
     # via esgpull
 platformdirs==4.5.1
     # via esgpull
-pycparser==2.23
+pycparser==3.0
     # via cffi
 pydantic==2.12.5
     # via esgpull
@@ -80,17 +80,17 @@ pygments==2.19.2
     # via rich
 pyopenssl==25.3.0
     # via esgpull
-pyparsing==3.3.1
+pyparsing==3.3.2
     # via esgpull
 python-dotenv==1.2.1
     # via pydantic-settings
 pyyaml==6.0.3
     # via esgpull
-rich==14.2.0
+rich==14.3.2
     # via esgpull
-setuptools==80.9.0
+setuptools==80.10.2
     # via esgpull
-sqlalchemy==2.0.45
+sqlalchemy==2.0.46
     # via alembic
     # via esgpull
 tomli==2.4.0
@@ -114,5 +114,5 @@ typing-inspection==0.4.2
     # via pydantic-settings
 validators==0.22.0
     # via click-params
-wrapt==2.0.1
+wrapt==2.1.1
     # via deprecated

esgpull-0.9.5/tests/cli/test_download.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from click.testing import CliRunner
+
+from esgpull import Esgpull
+from esgpull.cli.download import download
+from esgpull.config import Config
+from esgpull.models import File, FileStatus, Query
+
+
+def test_download_errors_in_progress(root: Path, config: Config):
+    # Configure the environment
+    config.generate(overwrite=True)
+    runner = CliRunner()
+    esg = Esgpull(root)
+
+    # Create a query without updating (no datasets will be created)
+    query = Query()
+    query.compute_sha()
+    query.track(query.options.default())
+
+    # Manually create files
+    failing_file = File(
+        file_id="failing_file",
+        dataset_id="test_dataset_id",
+        master_id="test_master_id",
+        url="https://nonexistent.path/to/file.nc",
+        version="1.0",
+        filename="failing.nc",
+        local_path="test/failing",
+        data_node="test_node",
+        checksum="12345",
+        checksum_type="SHA256",
+        size=1000,
+        status=FileStatus.Queued,
+    )
+    failing_file.compute_sha()
+
+    # Add files to query
+    query.files.append(failing_file)
+
+    # Add query to database
+    esg.db.add(query)
+
+    # Run the download instruction in the CLI
+    result_download = runner.invoke(download)
+
+    # Parse the output to extract
+    output = result_download.output
+    assert "download failed" in output

{esgpull-0.9.4 → esgpull-0.9.5}/tests/test_context.py

@@ -304,7 +304,11 @@ def test_hits_never_empty(
     ("index_node", "exc"),
     ## TODO: test bridge, but it is super slow
     [
-        (IPSL_NODE, does_not_raise()),
+        pytest.param(
+            IPSL_NODE,
+            does_not_raise(),
+            marks=pytest.mark.xfail(reason="unstable"),
+        ),
         (CEDA_NODE, does_not_raise()),
         ("https://github.com", pytest.raises(Exception)),
         ("not_a_real.url", pytest.raises(Exception)),
@@ -337,7 +341,7 @@ def test_bridge_exact_match_params(ctx):
 
 
 def test_bridge_wildcard_query_param(ctx):
-    query = Query(selection=dict(source_id='CESM*', variable_id='tas*'))
+    query = Query(selection=dict(source_id="CESM*", variable_id="tas*"))
     result = ctx.prepare_hits(
         query,
         file=False,
@@ -348,12 +352,12 @@ def test_bridge_wildcard_query_param(ctx):
     assert "query" in params
     assert "source_id" not in params
     assert "variable_id" not in params
-    assert 'source_id:CESM*' in params["query"]
-    assert 'variable_id:tas*' in params["query"]
+    assert "source_id:CESM*" in params["query"]
+    assert "variable_id:tas*" in params["query"]
 
 
 def test_bridge_mixed_exact_wildcard(ctx):
-    query = Query(selection=dict(source_id="CESM2", variable_id='tas*'))
+    query = Query(selection=dict(source_id="CESM2", variable_id="tas*"))
     result = ctx.prepare_hits(
         query,
         file=False,
@@ -364,7 +368,7 @@ def test_bridge_mixed_exact_wildcard(ctx):
     assert "source_id" in params
     assert params["source_id"] == "CESM2"
     assert "query" in params
-    assert 'variable_id:tas*' in params["query"]
+    assert "variable_id:tas*" in params["query"]
     assert "variable_id" not in params
 
 
@@ -424,6 +428,81 @@ def test_solr_unchanged(ctx):
     params = dict(result.request.url.params.items())
 
     assert "query" in params
-    assert params['query'] == 'source_id:CESM2 AND variable_id:tas'
+    assert params["query"] == "source_id:CESM2 AND variable_id:tas"
     assert "source_id" not in params
     assert "variable_id" not in params
+
+
+def test_files_skips_duplicate_file_ids(ctx):
+    """Test that _files skips duplicate file_ids to prevent DB constraint violations."""
+    import asyncio
+    from unittest.mock import patch, MagicMock
+    from esgpull.context import ResultFiles
+
+    # Create Solr response docs that will be deserialized by File.serialize()
+    # Two files with same file_id (dataset.v1.file.nc) but different checksums
+    solr_docs = [
+        {
+            "instance_id": "dataset.v1.file.nc|node1.com",
+            "dataset_id": "dataset.v1|node1.com",
+            "title": "file.nc",
+            "url": "https://node1.com/file.nc|application/netcdf",
+            "data_node": "node1.com",
+            "checksum": "abc123",
+            "checksum_type": "SHA256",
+            "size": 1000,
+            "directory_format_template_": "%(root)s/v1",  # Simple template
+        },
+        {
+            "instance_id": "dataset.v1.file.nc|node2.com",
+            "dataset_id": "dataset.v1|node2.com",
+            "title": "file.nc",
+            "url": "https://node2.com/file.nc|application/netcdf",
+            "data_node": "node2.com",
+            "checksum": "def456",  # Different checksum!
+            "checksum_type": "SHA256",
+            "size": 1000,
+            "directory_format_template_": "%(root)s/v1",  # Simple template
+        },
+        {
+            "instance_id": "dataset.v1.other.nc|node1.com",
+            "dataset_id": "dataset.v1|node1.com",
+            "title": "other.nc",
+            "url": "https://node1.com/other.nc|application/netcdf",
+            "data_node": "node1.com",
+            "checksum": "xyz789",
+            "checksum_type": "SHA256",
+            "size": 2000,
+            "directory_format_template_": "%(root)s/v1",  # Simple template
+        },
+    ]
+
+    # Create a ResultFiles with proper JSON that will be parsed by process()
+    query = Query()
+    result = ResultFiles(query=query, file=True)
+    result.request = MagicMock()
+    result.json = {"response": {"docs": solr_docs}}
+
+    # Mock _fetch to yield our result
+    async def mock_fetch(*results):
+        yield result
+
+    async def run_test():
+        with patch.object(ctx, "_fetch", mock_fetch):
+            return await ctx._files(result, keep_duplicates=False)
+
+    result_files = asyncio.run(run_test())
+
+    # Should only have 2 files (second one with duplicate file_id is skipped)
+    assert len(result_files) == 2
+
+    # Verify the file_ids in the result
+    file_ids = {f.file_id for f in result_files}
+    assert file_ids == {"dataset.v1.file.nc", "dataset.v1.other.nc"}
+
+    # The first file with dataset.v1.file.nc should be kept (abc123 checksum)
+    duplicate_files = [
+        f for f in result_files if f.file_id == "dataset.v1.file.nc"
+    ]
+    assert len(duplicate_files) == 1
+    assert duplicate_files[0].checksum == "abc123"

{esgpull-0.9.4 → esgpull-0.9.5}/uv.lock

@@ -379,7 +379,7 @@ wheels = [
 
 [[package]]
 name = "esgpull"
-version = "0.9.4"
+version = "0.9.5"
 source = { editable = "." }
 dependencies = [
     { name = "aiofiles" },
@@ -433,7 +433,7 @@ requires-dist = [
     { name = "pyyaml", specifier = ">=6.0" },
     { name = "rich", specifier = ">=12.6.0" },
     { name = "setuptools", specifier = ">=65.4.1" },
-    { name = "sqlalchemy", specifier = ">=2.0.0b2" },
+    { name = "sqlalchemy", specifier = ">=2.0.0b2,<2.1" },
     { name = "tomlkit", specifier = ">=0.11.5" },
 ]