esgpull 0.9.3__py3-none-any.whl → 0.9.5__py3-none-any.whl

esgpull/context.py

@@ -134,18 +134,46 @@ class Result:
         solr_terms: list[str] = []
         for name, values in self.query.selection.items():
             if index.is_bridge():
-                value_term = " ".join(quote_str(v) for v in values)
+                if name == "query":
+                    solr_terms.append(" ".join(values))
+                elif name.startswith("!"):
+                    value_term = " ".join(quote_str(v) for v in values)
+                    if len(values) > 1:
+                        value_term = f"({value_term})"
+                    solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                else:
+                    has_wildcard = any("*" in v for v in values)
+                    no_wildcard = any("*" not in v for v in values)
+
+                    if has_wildcard and no_wildcard:
+                        logger.warning(
+                            (
+                                f"Facet {name} has mixed wildcard/non-wildcard values. "
+                                "Non-wildcard values may match partially."
+                            )
+                        )
+                        value_term = " ".join(quote_str(v) for v in values)
+                        if len(values) > 1:
+                            value_term = f"({value_term})"
+                        solr_terms.append(f"{name}:{value_term}")
+                    elif has_wildcard:
+                        value_term = " ".join(quote_str(v) for v in values)
+                        if len(values) > 1:
+                            value_term = f"({value_term})"
+                        solr_terms.append(f"{name}:{value_term}")
+                    else:
+                        params[name] = ",".join(values)
             else:
                 value_term = " ".join(values)
-            if name == "query":  # freetext case
-                solr_terms.append(value_term)
-            else:
-                if len(values) > 1:
-                    value_term = f"({value_term})"
-                if name.startswith("!"):
-                    solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                if name == "query":
+                    solr_terms.append(value_term)
                 else:
-                    solr_terms.append(f"{name}:{value_term}")
+                    if len(values) > 1:
+                        value_term = f"({value_term})"
+                    if name.startswith("!"):
+                        solr_terms.append(f"NOT ({name[1:]}:{value_term})")
+                    else:
+                        solr_terms.append(f"{name}:{value_term}")
         if solr_terms:
             params["query"] = " AND ".join(solr_terms)
         for name, option in self.query.options.items(use_default=True):
@@ -562,6 +590,7 @@ class Context:
     ) -> list[File]:
         files: list[File] = []
         shas: set[str] = set()
+        file_ids: set[str] = set()
         async for result in self._fetch(*results):
             files_result = result.to(ResultFiles)
             files_result.process()
@@ -569,9 +598,13 @@ class Context:
                 for file in files_result.data:
                     if not keep_duplicates and file.sha in shas:
                         logger.debug(f"Duplicate file {file.file_id}")
-                    else:
-                        files.append(file)
-                        shas.add(file.sha)
+                        continue
+                    if not keep_duplicates and file.file_id in file_ids:
+                        logger.debug(f"Duplicate file_id {file.file_id}")
+                        continue
+                    files.append(file)
+                    shas.add(file.sha)
+                    file_ids.add(file.file_id)
         return files
 
     async def _search_as_queries(

esgpull/esgpull.py

@@ -54,7 +54,7 @@ from esgpull.plugin import (
 )
 from esgpull.processor import Processor
 from esgpull.result import Err, Ok, Result
-from esgpull.tui import UI, DummyLive, Verbosity, logger
+from esgpull.tui import UI, DummyLive, ErrorCountColumn, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -387,6 +387,7 @@ class Esgpull:
             SpinnerColumn(),
             MofNCompleteColumn(),
             TimeRemainingColumn(compact=True, elapsed_when_finished=True),
+            ErrorCountColumn(),
         )
         file_columns: list[str | ProgressColumn] = [
             TextColumn("[cyan][{task.id}] [b blue]{task.fields[sha]}"),
@@ -434,7 +435,9 @@ class Esgpull:
         if use_db:
             self.db.add(*processor.files)
         queue_size = len(processor.tasks)
-        main_task_id = main_progress.add_task("", total=queue_size)
+        main_task_id = main_progress.add_task(
+            "", total=queue_size, nb_errors=0
+        )
         # TODO: rename ? installed/downloaded/completed/...
         files: list[File] = []
         errors: list[Err] = []
@@ -475,11 +478,13 @@ class Esgpull:
                                     )
                         case Err(_, err):
                             queue_size -= 1
-                            main_progress.update(
-                                main_task_id, total=queue_size
-                            )
                             result.data.file.status = FileStatus.Error
                             errors.append(result)
+                            main_progress.update(
+                                main_task_id,
+                                total=queue_size,
+                                nb_errors=len(errors),
+                            )
                             emit(
                                 Event.file_error,
                                 file=result.data.file,

esgpull/migrations/versions/0.9.4_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.4
+Revises: 0.9.3
+Create Date: 2026-01-14 10:49:04.536961
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.4'
+down_revision = '0.9.3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.9.5_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.5
+Revises: 0.9.4
+Create Date: 2026-02-04 15:52:02.010431
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.5'
+down_revision = '0.9.4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/tui.py

@@ -101,6 +101,17 @@ class ExceptionToDebugFilter(logging.Filter):
         return True
 
 
+class ErrorCountColumn(ProgressColumn):
+    def render(self, task):
+        nb_errors = task.fields.get("nb_errors", 0)
+
+        if nb_errors == 0:
+            return Text("")
+
+        suffix = "" if nb_errors == 1 else "s"
+        return Text(f"({nb_errors} download{suffix} failed)", style="red")
+
+
 @define
 class UI:
     path: Path = field(converter=Path)

{esgpull-0.9.3.dist-info → esgpull-0.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.3
+Version: 0.9.5
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -31,13 +31,13 @@ Requires-Dist: pyparsing>=3.0.9
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: rich>=12.6.0
 Requires-Dist: setuptools>=65.4.1
-Requires-Dist: sqlalchemy>=2.0.0b2
+Requires-Dist: sqlalchemy<2.1,>=2.0.0b2
 Requires-Dist: tomlkit>=0.11.5
 Description-Content-Type: text/markdown
 
 # esgpull - ESGF data management utility
 
-[![Rye](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/rye/main/artwork/badge.json)](https://rye.astral.sh)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
 `esgpull` is a tool that simplifies usage of the [ESGF Search API](https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html) for data discovery, and manages procedures related to downloading and storing files from ESGF.
 

{esgpull-0.9.3.dist-info → esgpull-0.9.5.dist-info}/RECORD

@@ -1,10 +1,10 @@
 esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
 esgpull/config.py,sha256=APA3IqYyz_TY2FAm_TNSLsFIkOxmdGnbFLnd2H2YXIs,12042
 esgpull/constants.py,sha256=VlgRJDth5aDjyTopJ2W2JU83353XcRf18RIFAZt7yck,1263
-esgpull/context.py,sha256=JEnIQGF1owl56n0azF8K3GNIU8TgeMu31NSfTqVAGQg,25123
+esgpull/context.py,sha256=nF0ihtfLRw1mm6gIZHYLO_vK-Ga3vgr1I4Qy42Wi2Yo,26819
 esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
 esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
-esgpull/esgpull.py,sha256=mqvrXnyCzTSU3j_u5pYidpLs2j86TEFDPX6MhCnNfMs,19458
+esgpull/esgpull.py,sha256=2rBUuKuTBau8fpOA0mRwsFvmVrFAjXX42KKIyfMvpsM,19631
 esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
 esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
 esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
@@ -13,7 +13,7 @@ esgpull/plugin.py,sha256=t6iejis65UREQ71gnj1-gVFhlrGt1z1n3SpJ6KiP57E,18904
 esgpull/processor.py,sha256=ubhlKMMOY_ieZEcN9zcDPlmZxQDXqb2QVmeCwWo7zOE,5376
 esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
-esgpull/tui.py,sha256=XJs8AUm28YLWl3ephpxY8jHFvG7HrIr08u3RdYvABdE,11338
+esgpull/tui.py,sha256=7V3Qam7jOdB2CEI-j-ioMbWb28yvFEzqwexUGhZgolo,11639
 esgpull/utils.py,sha256=pLMQfY0p2oNIFyCZhHBD74BOAJtmt9QV6dTJ79VsfF8,1213
 esgpull/version.py,sha256=IHT4mKrIr8eV-C3HtmIVD85iGVH25n2ohoff31kaJ1A,93
 esgpull/cli/__init__.py,sha256=W0q_eyhvtPgyX5cXKSDDLUUrskGLzgNoJ9VCY9fJ6pY,1701
@@ -71,6 +71,8 @@ esgpull/migrations/versions/0.9.0_update_tables.py,sha256=nXfPiyuseD5BXvu59zkeST
 esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7ENt_6KecG3z5xfRr_LwrtY,541
 esgpull/migrations/versions/0.9.2_update_tables.py,sha256=aux--HPA_jUtvk7Zb2ZRXRhnRLT-whqZ0lLWt8Apyac,541
 esgpull/migrations/versions/0.9.3_update_tables.py,sha256=-k1CneOa7OTO6xPZ8uR0ccy-4B8TU3kM12-oYKFEOlA,541
+esgpull/migrations/versions/0.9.4_update_tables.py,sha256=tvvXvov7ggXO9A8B5399bixCuM9fSni1SHkJXa5f5II,541
+esgpull/migrations/versions/0.9.5_update_tables.py,sha256=pmLJeJus9-Dmj0geKCpHqrFxlEMLMhPZ_zB7oBLm5rQ,541
 esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
 esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
 esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
@@ -87,8 +89,8 @@ esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
 esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
 esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
 esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
-esgpull-0.9.3.dist-info/METADATA,sha256=0wlDuo4TCJskBSO9v4FRuoLTAm7531jOPJDQAQJWBP4,3818
-esgpull-0.9.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-esgpull-0.9.3.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
-esgpull-0.9.3.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
-esgpull-0.9.3.dist-info/RECORD,,
+esgpull-0.9.5.dist-info/METADATA,sha256=KHIghvsCaNoJXZNOj6-MXNDZeQiGnLfI_-WesaZ8I7c,3833
+esgpull-0.9.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+esgpull-0.9.5.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
+esgpull-0.9.5.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
+esgpull-0.9.5.dist-info/RECORD,,

{esgpull-0.9.3.dist-info → esgpull-0.9.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any