esgpull 0.9.0__py3-none-any.whl → 0.9.1__py3-none-any.whl

esgpull/cli/remove.py

@@ -15,11 +15,13 @@ from esgpull.utils import format_size
 @args.query_id
 @opts.tag
 @opts.children
+@opts.yes
 @opts.verbosity
 def remove(
     query_id: str | None,
     tag: str | None,
     children: bool,
+    yes: bool,
     verbosity: Verbosity,
 ) -> None:
     """
@@ -49,7 +51,7 @@ def remove(
         graph.add(*queries)
         esg.ui.print(graph)
         msg = f"Remove {nb} quer{ies}?"
-        if not esg.ui.ask(msg, default=True):
+        if not yes and not esg.ui.ask(msg, default=True):
             raise Abort
         for query in queries:
             if query.has_files:
@@ -59,14 +61,18 @@ def remove(
                         f":stop_sign: {query.rich_name} is linked"
                         f" to {nb} downloaded files ({format_size(size)})."
                     )
-                    if not esg.ui.ask("Delete anyway?", default=False):
+                    if not yes and not esg.ui.ask(
+                        "Delete anyway?", default=False
+                    ):
                         raise Abort
             if not children and esg.graph.get_children(query.sha):
                 esg.ui.print(
                     ":stop_sign: Some queries block"
                     f" removal of {query.rich_name}."
                 )
-                if esg.ui.ask("Show blocking queries?", default=False):
+                if not yes and esg.ui.ask(
+                    "Show blocking queries?", default=False
+                ):
                     esg.ui.print(esg.graph.subgraph(query, children=True))
                 raise Exit(1)
         esg.db.delete(*queries)

esgpull/cli/self.py

@@ -199,7 +199,7 @@ def delete():
                 TempUI.print(f"Deleting {path} from config...")
                 TempUI.print("To remove all files from this install, run:\n")
                 config = Config.load(path=path)
-                for p in config.paths:
+                for p in config.paths.values():
                     if not p.is_relative_to(path):
                         TempUI.print(f"$ rm -rf {p}")
                 TempUI.print(f"$ rm -rf {path}")

esgpull/cli/update.py

@@ -11,7 +11,7 @@ from esgpull.cli.utils import get_queries, init_esgpull, valid_name_tag
 from esgpull.context import HintsDict, ResultSearch
 from esgpull.exceptions import UnsetOptionsError
 from esgpull.models import Dataset, File, FileStatus, Query
-from esgpull.tui import Verbosity, logger
+from esgpull.tui import Verbosity
 from esgpull.utils import format_size
 
 
@@ -177,26 +177,37 @@ def update(
             for qf, qf_files in zip(qfs, files):
                 qf.files = qf_files
         for qf in qfs:
-            new_files = [f for f in qf.files if f not in esg.db]
-            new_datasets = [d for d in qf.datasets if d not in esg.db]
-            nb_datasets = len(new_datasets)
-            nb_files = len(new_files)
             if not qf.query.tracked:
                 esg.db.add(qf.query)
                 continue
-            elif nb_datasets == nb_files == 0:
-                esg.ui.print(f"{qf.query.rich_name} is already up-to-date.")
-                continue
-            size = sum([file.size for file in new_files])
-            if size > 0:
-                queue_msg = " and send new files to download queue"
-            else:
-                queue_msg = ""
+            with esg.db.commit_context():
+                unregistered_datasets = [
+                    f for f in qf.datasets if f not in esg.db
+                ]
+                if len(unregistered_datasets) > 0:
+                    esg.ui.print(
+                        f"Adding {len(unregistered_datasets)} new datasets to database."
+                    )
+                    esg.db.session.add_all(unregistered_datasets)
+                files_from_db = [
+                    esg.db.get(File, f.sha) for f in qf.files if f in esg.db
+                ]
+                registered_files = [f for f in files_from_db if f is not None]
+                unregistered_files = [f for f in qf.files if f not in esg.db]
+                if len(unregistered_files) > 0:
+                    esg.ui.print(
+                        f"Adding {len(unregistered_files)} new files to database."
+                    )
+                    esg.db.session.add_all(unregistered_files)
+                files = registered_files + unregistered_files
+            not_done_files = [
+                file for file in files if file.status != FileStatus.Done
+            ]
+            download_size = sum(file.size for file in not_done_files)
             msg = (
-                f"\n{qf.query.rich_name}: {nb_files} new"
-                f" files, {nb_datasets} new datasets"
-                f" ({format_size(size)})."
-                f"\nUpdate the database{queue_msg}?"
+                f"\n{qf.query.rich_name}: {len(not_done_files)} "
+                f" files ({format_size(download_size)}) to download."
+                f"\nLink to query and send to download queue?"
             )
             if yes:
                 choice = "y"
@@ -212,30 +223,19 @@ def update(
             if choice == "y":
                 legacy = esg.legacy_query
                 has_legacy = legacy.state.persistent
+                applied_changes = False
                 with esg.db.commit_context():
-                    for dataset in esg.ui.track(
-                        new_datasets,
-                        description=f"{qf.query.rich_name} (datasets)",
-                    ):
-                        esg.db.session.add(dataset)
                     for file in esg.ui.track(
-                        new_files,
-                        description=f"{qf.query.rich_name} (files)",
+                        files,
+                        description=f"{qf.query.rich_name}",
                     ):
-                        file_db = esg.db.get(File, file.sha)
-                        if file_db is None:
-                            if esg.db.has_file_id(file):
-                                logger.error(
-                                    "File id already exists in database, "
-                                    "there might be an error with its checksum"
-                                    f"\n{file}"
-                                )
-                                continue
+                        if file.status != FileStatus.Done:
                             file.status = FileStatus.Queued
-                            esg.db.session.add(file)
-                        elif has_legacy and legacy in file_db.queries:
-                            esg.db.unlink(query=legacy, file=file_db)
-                        esg.db.link(query=qf.query, file=file)
-                    qf.query.updated_at = datetime.now(timezone.utc)
+                        if has_legacy and legacy in file.queries:
+                            _ = esg.db.unlink(query=legacy, file=file)
+                        changed = esg.db.link(query=qf.query, file=file)
+                        applied_changes = applied_changes or changed
+                    if applied_changes:
+                        qf.query.updated_at = datetime.now(timezone.utc)
                     esg.db.session.add(qf.query)
         esg.ui.raise_maybe_record(Exit(0))

esgpull/config.py

@@ -138,7 +138,7 @@ class Paths:
             root = InstallConfig.default
         return root / "plugins"
 
-    def __iter__(self) -> Iterator[Path]:
+    def values(self) -> Iterator[Path]:
         yield self.auth
         yield self.data
         yield self.db

esgpull/database.py

@@ -147,11 +147,19 @@ class Database:
         for item in items:
             make_transient(item)
 
-    def link(self, query: Query, file: File):
-        self.session.execute(sql.query_file.link(query, file))
+    def link(self, query: Query, file: File) -> bool:
+        if not self.session.scalar(sql.query_file.is_linked(query, file)):
+            self.session.execute(sql.query_file.link(query, file))
+            return True
+        else:
+            return False
 
-    def unlink(self, query: Query, file: File):
-        self.session.execute(sql.query_file.unlink(query, file))
+    def unlink(self, query: Query, file: File) -> bool:
+        if self.session.scalar(sql.query_file.is_linked(query, file)):
+            self.session.execute(sql.query_file.unlink(query, file))
+            return True
+        else:
+            return False
 
     def __contains__(self, item: Base | BaseNoSHA) -> bool:
         mapper = inspect(item.__class__)

esgpull/fs.py

@@ -10,7 +10,7 @@ from shutil import copyfile
 import aiofiles
 from aiofiles.threadpool.binary import AsyncBufferedIOBase
 
-from esgpull.config import Config
+from esgpull.config import Config, Paths
 from esgpull.models import File
 from esgpull.result import Err, Ok, Result
 from esgpull.tui import logger
@@ -63,45 +63,34 @@ class Digest:
 
 @dataclass
 class Filesystem:
-    auth: Path
-    data: Path
-    db: Path
-    log: Path
-    tmp: Path
+    paths: Paths
     disable_checksum: bool = False
     install: InitVar[bool] = True
 
     @staticmethod
     def from_config(config: Config, install: bool = False) -> Filesystem:
         return Filesystem(
-            auth=config.paths.auth,
-            data=config.paths.data,
-            db=config.paths.db,
-            log=config.paths.log,
-            tmp=config.paths.tmp,
+            paths=config.paths,
             disable_checksum=config.download.disable_checksum,
             install=install,
         )
 
     def __post_init__(self, install: bool = True) -> None:
         if install:
-            self.auth.mkdir(parents=True, exist_ok=True)
-            self.data.mkdir(parents=True, exist_ok=True)
-            self.db.mkdir(parents=True, exist_ok=True)
-            self.log.mkdir(parents=True, exist_ok=True)
-            self.tmp.mkdir(parents=True, exist_ok=True)
+            for path in self.paths.values():
+                path.mkdir(parents=True, exist_ok=True)
 
     def __getitem__(self, file: File) -> FilePath:
         if not isinstance(file, File):
             raise TypeError(file)
         return FilePath(
-            drs=self.data / file.local_path / file.filename,
-            tmp=self.tmp / f"{file.sha}.part",
+            drs=self.paths.data / file.local_path / file.filename,
+            tmp=self.paths.tmp / f"{file.sha}.part",
         )
 
     def glob_netcdf(self) -> Iterator[Path]:
-        for path in self.data.glob("**/*.nc"):
-            yield path.relative_to(self.data)
+        for path in self.paths.data.glob("**/*.nc"):
+            yield path.relative_to(self.paths.data)
 
     def open(self, file: File) -> FileObject:
         return FileObject(self[file])

esgpull/migrations/versions/0.9.1_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.1
+Revises: 0.9.0
+Create Date: 2025-08-08 10:38:14.204594
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.1'
+down_revision = '0.9.0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/models/sql.py

@@ -301,3 +301,11 @@ class query_file:
             .where(query_file_proxy.c.query_sha == query.sha)
             .where(query_file_proxy.c.file_sha == file.sha)
         )
+
+    @staticmethod
+    def is_linked(query: Query, file: File) -> sa.Select[tuple[bool]]:
+        return sa.select(
+            sa.exists()
+            .where(query_file_proxy.c.query_sha == query.sha)
+            .where(query_file_proxy.c.file_sha == file.sha)
+        )

{esgpull-0.9.0.dist-info → esgpull-0.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.0
+Version: 0.9.1
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -63,14 +63,29 @@ for dataset in datasets:
 
 ## Installation
 
-Install `esgpull` using pip or conda:
+`esgpull` is distributed via PyPI:
 
 ```shell
 pip install esgpull
+esgpull --help
+```
+
+For isolated installation, [`uv`](https://github.com/astral-sh/uv) or
+[`pipx`](https://github.com/pypa/pipx) are recommended:
+
+```shell
+# with uv
+uv tool install esgpull
+esgpull --help
+
+# alternatively, uvx enables running without explicit installation (comes with uv)
+uvx esgpull --help
 ```
 
 ```shell
-conda install -c conda-forge ipsl::esgpull
+# with pipx
+pipx install esgpull
+esgpull --help
 ```
 
 ## Usage

{esgpull-0.9.0.dist-info → esgpull-0.9.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
 esgpull/auth.py,sha256=QZ-l1ySLMP0fvuwYHRLv9FZYp1gqfju_eGaTMDByUxw,5205
-esgpull/config.py,sha256=34rxps0SduGj_d-4756KDI7suA13OYrOAZnYB0d4pbc,13425
+esgpull/config.py,sha256=9atYqxc3PiJKY1hfoEWZxf5Ba6U9SzMUs4QvcWWGFy0,13423
 esgpull/constants.py,sha256=WjG7xzMZNckOv5GhRehBtI7hoSwwLZvwkvEq5RG-dv4,1189
 esgpull/context.py,sha256=_XBAA9XM-fcJXLsqFlg1ziwQdndDoFxketdcFhi4oSM,23430
-esgpull/database.py,sha256=sPRxbUHVTKxUAyPMS5uMrYZ7LOZP-8MyISbBsW3x6Ec,6420
+esgpull/database.py,sha256=1wGeNbJp0gOLo5Q1N53JfiwVbZ8nfvZVkKlvEaJwOpU,6716
 esgpull/download.py,sha256=aR2c_SOuZtgX7tI2a9_N4Mn86ABq1k7Mxq_BdojFrP4,5600
 esgpull/esgpull.py,sha256=-OwaWvwmkur_mdVNKKxdxlASeK93tmOceV__jGRMwqo,19666
 esgpull/exceptions.py,sha256=wgLyhyIITdusNucPjnnURJX1Jxv1VVIr9PzJV_77qhg,3275
-esgpull/fs.py,sha256=O52QD7DVJImYYf9wWECMNWYrGt8EcHQ3z9TzroK6VAY,7623
+esgpull/fs.py,sha256=sc7Af2E3yh3V9KVuSPSXFBuFtlQ3L99UZmS1ZJuiBeM,7280
 esgpull/graph.py,sha256=Yl2VuF8PNn0R5xRyEK58Q1Xlx8B1PfhbTwt1JftFDro,15929
 esgpull/install_config.py,sha256=hzYpcHMtPMOK9fYcvVH-Hn_8zYsbs3yXlYgMumXo1zE,5598
 esgpull/plugin.py,sha256=wW19V55E1Fn2_ooZ5aTc58uJVSQ0WT43P29G3TlMD6A,18748
@@ -29,14 +29,14 @@ esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
 esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
 esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
 esgpull/cli/plugins.py,sha256=eTAwWqvrzDKs122_Wa3sOkuriSoInymxWySQQahrlHg,13357
-esgpull/cli/remove.py,sha256=p26hPhsgAgmAF4IsvjAI3jtzlIUq3k8Rxb0GTKYoEQM,2517
+esgpull/cli/remove.py,sha256=9fqE8NJdr1mHypu_N-TBJy_yl3elUBSfzEARxxTkqKg,2662
 esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
 esgpull/cli/search.py,sha256=c2OI0Xvw6h2zBnC8msjxidKJZ0KKUuWJK9FuAmQ15ZU,6369
-esgpull/cli/self.py,sha256=7nEEsK5W_Pth8IOSmvJHRlfwPPgXldhHAQK9yqf01S8,7932
+esgpull/cli/self.py,sha256=psgFcgkDyemquZEpoWp2cyjgampCgDzRc1QBvzqGs24,7941
 esgpull/cli/show.py,sha256=B-h7bKMrwgjnTHio2du8IPOLlKCaan56RQKAtzlQzcw,2822
 esgpull/cli/status.py,sha256=HEyj6QFABblADtYf1PWmSzghKX3fs33x9p5vpSqA514,2521
 esgpull/cli/track.py,sha256=Q9ZvvV5FFGzp6wQZflAd_OFmqhAWgl1JFBad2dCbEF0,3089
-esgpull/cli/update.py,sha256=dPF6LJtaF9r91xcXXYHG6VrOcwSTWXKEvTlOSqbQX0s,9371
+esgpull/cli/update.py,sha256=3JBf1HqglazUYZ63uHtm5y3ZAuS2XLGcwrsVYBkrmb0,9432
 esgpull/cli/utils.py,sha256=dE5dIH6tWmhItarLDrNldiUuuX5qUjnVpnu4KkE6V1g,7199
 esgpull/migrations/README,sha256=heMzebYwlGhnE8_4CWJ4LS74WoEZjBy-S-mIJRxAEKI,39
 esgpull/migrations/env.py,sha256=am2HhFrlIZNlXCaA5Ye7yKbIJ2MRSO5UFmUwB8l9fyE,2285
@@ -69,6 +69,7 @@ esgpull/migrations/versions/0.7.2_update_tables.py,sha256=cBWEhfYQlhGpE55NOY9eD2
 esgpull/migrations/versions/0.7.3_update_tables.py,sha256=4pNkM7VaGQqkLuIoLKuM0ftefLjDBVFRaV8Nkq5OL-Y,541
 esgpull/migrations/versions/0.8.0_update_tables.py,sha256=5rr3guWipnnuciFuviUxZUasdPw6x06WZCmY5nNCn5k,555
 esgpull/migrations/versions/0.9.0_update_tables.py,sha256=nXfPiyuseD5BXvu59zkeSTzb6EA7txpBudclU-MIyU0,555
+esgpull/migrations/versions/0.9.1_update_tables.py,sha256=ITewU2qgdCwhorsl2d_t7ENt_6KecG3z5xfRr_LwrtY,541
 esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py,sha256=MKqz0tfwGwRkgP4QDd-cpUmXCVr4tM_wlC2BfxqJ1_w,1031
 esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py,sha256=Al_o7fDmoRqc9vBCQgtgrNbSPIOBxdMZ5T-ztakqVeY,1033
 esgpull/migrations/versions/d14f179e553c_file_add_composite_index_dataset_id_.py,sha256=0vJvttugWmgKns4g-K4i3EU6eid2Z_K2e3H6Ktevf7c,860
@@ -81,12 +82,12 @@ esgpull/models/file.py,sha256=-8PPYtq7BWp-O_QtCDbkLdhTGTPhI1F1nodQacMnYGA,1517
 esgpull/models/options.py,sha256=BUWf3IthlSvroNn05eC3MtGRq7wHIYlr1dFI1Wg9acM,4744
 esgpull/models/query.py,sha256=le0BLncg7A8mG-DjaLjv1_rAPq6uJZzYJKajtgBIWlE,19659
 esgpull/models/selection.py,sha256=QCX_2eoCJcYB1ULll-J7UV5lCpkweis92FANQY8pH0o,5895
-esgpull/models/sql.py,sha256=-lrm2Xw-_DygboYWWkDambYcSzUhZeHefIhN4g-S0wI,8670
+esgpull/models/sql.py,sha256=K8Nre5HKFPjkRzUUW6p6Qk7aG8upbw8C3pnmCFlg7d8,8942
 esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
 esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
 esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
-esgpull-0.9.0.dist-info/METADATA,sha256=acC9Ldx5T3-f38Gxu04DkCL-cS6y0ysAITkxonwmBDs,3471
-esgpull-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-esgpull-0.9.0.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
-esgpull-0.9.0.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
-esgpull-0.9.0.dist-info/RECORD,,
+esgpull-0.9.1.dist-info/METADATA,sha256=Iy0nz7xstS0pywi5LNuLt-giW-5ng-X_lYb7WTrv8eE,3781
+esgpull-0.9.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+esgpull-0.9.1.dist-info/entry_points.txt,sha256=vyh7HvFrCp4iyMrTkDoSF3weaYrlNj2OJe0Fq5q4QB4,45
+esgpull-0.9.1.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
+esgpull-0.9.1.dist-info/RECORD,,