esgpull 0.9.4__py3-none-any.whl → 0.9.5__py3-none-any.whl

esgpull/context.py

@@ -590,6 +590,7 @@ class Context:
     ) -> list[File]:
         files: list[File] = []
         shas: set[str] = set()
+        file_ids: set[str] = set()
         async for result in self._fetch(*results):
             files_result = result.to(ResultFiles)
             files_result.process()
@@ -597,9 +598,13 @@ class Context:
                 for file in files_result.data:
                     if not keep_duplicates and file.sha in shas:
                         logger.debug(f"Duplicate file {file.file_id}")
-                    else:
-                        files.append(file)
-                        shas.add(file.sha)
+                        continue
+                    if not keep_duplicates and file.file_id in file_ids:
+                        logger.debug(f"Duplicate file_id {file.file_id}")
+                        continue
+                    files.append(file)
+                    shas.add(file.sha)
+                    file_ids.add(file.file_id)
         return files
 
     async def _search_as_queries(

esgpull/esgpull.py

@@ -54,7 +54,7 @@ from esgpull.plugin import (
 )
 from esgpull.processor import Processor
 from esgpull.result import Err, Ok, Result
-from esgpull.tui import UI, DummyLive, Verbosity, logger
+from esgpull.tui import UI, DummyLive, ErrorCountColumn, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -387,6 +387,7 @@ class Esgpull:
             SpinnerColumn(),
             MofNCompleteColumn(),
             TimeRemainingColumn(compact=True, elapsed_when_finished=True),
+            ErrorCountColumn(),
         )
         file_columns: list[str | ProgressColumn] = [
             TextColumn("[cyan][{task.id}] [b blue]{task.fields[sha]}"),
@@ -434,7 +435,9 @@ class Esgpull:
         if use_db:
             self.db.add(*processor.files)
         queue_size = len(processor.tasks)
-        main_task_id = main_progress.add_task("", total=queue_size)
+        main_task_id = main_progress.add_task(
+            "", total=queue_size, nb_errors=0
+        )
         # TODO: rename ? installed/downloaded/completed/...
         files: list[File] = []
         errors: list[Err] = []
@@ -475,11 +478,13 @@ class Esgpull:
                                     )
                         case Err(_, err):
                             queue_size -= 1
-                            main_progress.update(
-                                main_task_id, total=queue_size
-                            )
                             result.data.file.status = FileStatus.Error
                             errors.append(result)
+                            main_progress.update(
+                                main_task_id,
+                                total=queue_size,
+                                nb_errors=len(errors),
+                            )
                             emit(
                                 Event.file_error,
                                 file=result.data.file,

esgpull/migrations/versions/0.9.5_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.5
+Revises: 0.9.4
+Create Date: 2026-02-04 15:52:02.010431
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.5'
+down_revision = '0.9.4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/tui.py

@@ -101,6 +101,17 @@ class ExceptionToDebugFilter(logging.Filter):
         return True
 
 
+class ErrorCountColumn(ProgressColumn):
+    def render(self, task):
+        nb_errors = task.fields.get("nb_errors", 0)
+
+        if nb_errors == 0:
+            return Text("")
+
+        suffix = "" if nb_errors == 1 else "s"
+        return Text(f"({nb_errors} download{suffix} failed)", style="red")
+
+
 @define
 class UI:
     path: Path = field(converter=Path)

{esgpull-0.9.4.dist-info → esgpull-0.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.4
+Version: 0.9.5
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -31,13 +31,13 @@ Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=12.6.0
 Requires-Dist: setuptools>=65.4.1
-Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: sqlalchemy<2.1,>=2.0.0b2
 Requires-Dist: tomlkit>=0.11.5
 Description-Content-Type: text/markdown
 
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.4.dist-info → esgpull-0.9.5.dist-info}/RECORD

@@ -1,10 +1,10 @@
 esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
 esgpull/config.py,sha256=APA3IqYyz_TY2FAm_TNSLsFIkOxmdGnbFLnd2H2YXIs,12042
 esgpull/constants.py,sha256=VlgRJDth5aDjyTopJ2W2JU83353XcRf18RIFAZt7yck,1263
-esgpull/context.py,sha256=LGGnWnKuk9xkUvf5EX3NLbKKRlGvtRe9blps7C-dkTo,26558
+esgpull/context.py,sha256=nF0ihtfLRw1mm6gIZHYLO_vK-Ga3vgr1I4Qy42Wi2Yo,26819
 esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
 esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
-esgpull/esgpull.py,sha256=mqvrXnyCzTSU3j_u5pYidpLs2j86TEFDPX6MhCnNfMs,19458
+esgpull/esgpull.py,sha256=2rBUuKuTBau8fpOA0mRwsFvmVrFAjXX42KKIyfMvpsM,19631
 esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
 esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
 esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
@@ -13,7 +13,7 @@ esgpull/plugin.py,sha256=t6iejis65UREQ71gnj1-gVFhlrGt1z1n3SpJ6KiP57E,18904
 esgpull/processor.py,sha256=ubhlKMMOY_ieZEcN9zcDPlmZxQDXqb2QVmeCwWo7zOE,5376
 esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
-esgpull/tui.py,sha256=XJs8AUm28YLWl3ephpxY8jHFvG7HrIr08u3RdYvABdE,11338
+esgpull/tui.py,sha256=7V3Qam7jOdB2CEI-j-ioMbWb28yvFEzqwexUGhZgolo,11639
 esgpull/utils.py,sha256=pLMQfY0p2oNIFyCZhHBD74BOAJtmt9QV6dTJ79VsfF8,1213
 esgpull/version.py,sha256=IHT4mKrIr8eV-C3HtmIVD85iGVH25n2ohoff31kaJ1A,93
 esgpull/cli/__init__.py,sha256=W0q_eyhvtPgyX5cXKSDDLUUrskGLzgNoJ9VCY9fJ6pY,1701
@@ -72,6 +72,7 @@ esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7E
 esgpull/migrations/versions/0.9.2_update_tables.py,sha256=aux--HPA_jUtvk7Zb2ZRXRhnRLT-whqZ0lLWt8Apyac,541
 esgpull/migrations/versions/0.9.3_update_tables.py,sha256=-k1CneOa7OTO6xPZ8uR0ccy-4B8TU3kM12-oYKFEOlA,541
 esgpull/migrations/versions/0.9.4_update_tables.py,sha256=tvvXvov7ggXO9A8B5399bixCuM9fSni1SHkJXa5f5II,541
+esgpull/migrations/versions/0.9.5_update_tables.py,sha256=pmLJeJus9-Dmj0geKCpHqrFxlEMLMhPZ_zB7oBLm5rQ,541
 esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
 esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
 esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
@@ -88,8 +89,8 @@ esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
 esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
 esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
 esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
-esgpull-0.9.4.dist-info/METADATA,sha256=MKviUWHQuaz6SVYBYuWPky-4BZ3LBRyWaET0A3i4JaA,3818
-esgpull-0.9.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-esgpull-0.9.4.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
-esgpull-0.9.4.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
-esgpull-0.9.4.dist-info/RECORD,,
+esgpull-0.9.5.dist-info/METADATA,sha256=KHIghvsCaNoJXZNOj6-MXNDZeQiGnLfI_-WesaZ8I7c,3833
+esgpull-0.9.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+esgpull-0.9.5.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
+esgpull-0.9.5.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
+esgpull-0.9.5.dist-info/RECORD,,