esgpull 0.9.0__tar.gz → 0.9.1__tar.gz

{esgpull-0.9.0 → esgpull-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esgpull
-Version: 0.9.0
+Version: 0.9.1
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -63,14 +63,29 @@ for dataset in datasets:
 
 ## Installation
 
-Install `esgpull` using pip or conda:
+`esgpull` is distributed via PyPI:
 
 ```shell
 pip install esgpull
+esgpull --help
+```
+
+For isolated installation, [`uv`](https://github.com/astral-sh/uv) or
+[`pipx`](https://github.com/pypa/pipx) are recommended:
+
+```shell
+# with uv
+uv tool install esgpull
+esgpull --help
+
+# alternatively, uvx enables running without explicit installation (comes with uv)
+uvx esgpull --help
 ```
 
 ```shell
-conda install -c conda-forge ipsl::esgpull
+# with pipx
+pipx install esgpull
+esgpull --help
 ```
 
 ## Usage

{esgpull-0.9.0 → esgpull-0.9.1}/README.md

@@ -27,14 +27,29 @@ for dataset in datasets:
 
 ## Installation
 
-Install `esgpull` using pip or conda:
+`esgpull` is distributed via PyPI:
 
 ```shell
 pip install esgpull
+esgpull --help
+```
+
+For isolated installation, [`uv`](https://github.com/astral-sh/uv) or
+[`pipx`](https://github.com/pypa/pipx) are recommended:
+
+```shell
+# with uv
+uv tool install esgpull
+esgpull --help
+
+# alternatively, uvx enables running without explicit installation (comes with uv)
+uvx esgpull --help
 ```
 
 ```shell
-conda install -c conda-forge ipsl::esgpull
+# with pipx
+pipx install esgpull
+esgpull --help
 ```
 
 ## Usage

{esgpull-0.9.0 → esgpull-0.9.1}/docs/docs/index.md

@@ -76,6 +76,10 @@ Have a look at the [Installation page](installation) for more ways to install.
 
 Jump directly to the [Quickstart guide](quickstart) to get to know how to use `esgpull`.
 
+## How to Cite
+
+If you use esgpull in your research, please cite it using the information provided in our [CITATION.cff](https://github.com/ESGF/esgf-download/blob/main/CITATION.cff) file.
+
 
 <!-- [ESGF portal]: https://esgf-node.ipsl.upmc.fr/search/cmip6-ipsl -->
 [ESGF Search API]: https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html

{esgpull-0.9.0 → esgpull-0.9.1}/docs/docs/installation.md

@@ -7,50 +7,31 @@ This document covers a few ways to install `esgpull`, a necessary first step int
 !!! note "Supporting lower python versions could be done with future releases, do not hesitate to ask for it."
 
 
-## Installing with conda / mamba
+## Installation
 
-To install `esgpull` in a new `conda` environment, run:
+`esgpull` is distributed via PyPI:
 
-```sh
-conda create --name my_env_name esgpull --channel ipsl --channel conda-forge
-```
-
-For `mamba` users:
-
-```sh
-mamba create --name my_env_name esgpull --channel ipsl --channel conda-forge
-```
-
-You can start using `esgpull` after activating the environment:
-
-```sh
-conda activate my_env_name
-esgpull --version
+```shell
+pip install esgpull
+esgpull --help
 ```
 
+For isolated installation, [`uv`](https://github.com/astral-sh/uv) or
+[`pipx`](https://github.com/pypa/pipx) are recommended:
 
-## Install with pip
-
-Make sure your python version meets the requirements (>=3.10), then you can run:
+```shell
+# with uv
+uv tool install esgpull
+esgpull --help
 
-```shell title="Install esgpull from pip"
-pip install git+https://github.com/ESGF/esgf-download
+# alternatively, uvx enables running without explicit installation (comes with uv)
+uvx esgpull --help
 ```
 
-
-## Install from source
-
-Esgpull is developed and maintained on GitHub, you can clone the public repository with:
-
 ```shell
-git clone https://github.com/ESGF/esgf-download
-```
-
-And then install with `pip`:
-
-```
-cd esg-pull
-python -m pip install .
+# with pipx
+pipx install esgpull
+esgpull --help
 ```
 
 
@@ -93,4 +74,4 @@ Installing `esgpull` is the first step to using it, but not the only one.
 
 ## Configuration
 
-`esgpull` is highly configurable, and it is recommended to take a look at the [configuration](configuration.md) page to learn more about it.
+`esgpull` is highly configurable, and it is recommended to take a look at the [configuration](configuration.md) page to learn more about it.

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/cli/remove.py

@@ -15,11 +15,13 @@ from esgpull.utils import format_size
 @args.query_id
 @opts.tag
 @opts.children
+@opts.yes
 @opts.verbosity
 def remove(
     query_id: str | None,
     tag: str | None,
     children: bool,
+    yes: bool,
     verbosity: Verbosity,
 ) -> None:
     """
@@ -49,7 +51,7 @@ def remove(
         graph.add(*queries)
         esg.ui.print(graph)
         msg = f"Remove {nb} quer{ies}?"
-        if not esg.ui.ask(msg, default=True):
+        if not yes and not esg.ui.ask(msg, default=True):
             raise Abort
         for query in queries:
             if query.has_files:
@@ -59,14 +61,18 @@ def remove(
                         f":stop_sign: {query.rich_name} is linked"
                         f" to {nb} downloaded files ({format_size(size)})."
                     )
-                    if not esg.ui.ask("Delete anyway?", default=False):
+                    if not yes and not esg.ui.ask(
+                        "Delete anyway?", default=False
+                    ):
                         raise Abort
             if not children and esg.graph.get_children(query.sha):
                 esg.ui.print(
                     ":stop_sign: Some queries block"
                     f" removal of {query.rich_name}."
                 )
-                if esg.ui.ask("Show blocking queries?", default=False):
+                if not yes and esg.ui.ask(
+                    "Show blocking queries?", default=False
+                ):
                     esg.ui.print(esg.graph.subgraph(query, children=True))
                 raise Exit(1)
         esg.db.delete(*queries)

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/cli/self.py

@@ -199,7 +199,7 @@ def delete():
                 TempUI.print(f"Deleting {path} from config...")
                 TempUI.print("To remove all files from this install, run:\n")
                 config = Config.load(path=path)
-                for p in config.paths:
+                for p in config.paths.values():
                     if not p.is_relative_to(path):
                         TempUI.print(f"$ rm -rf {p}")
                 TempUI.print(f"$ rm -rf {path}")

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/cli/update.py

@@ -11,7 +11,7 @@ from esgpull.cli.utils import get_queries, init_esgpull, valid_name_tag
 from esgpull.context import HintsDict, ResultSearch
 from esgpull.exceptions import UnsetOptionsError
 from esgpull.models import Dataset, File, FileStatus, Query
-from esgpull.tui import Verbosity, logger
+from esgpull.tui import Verbosity
 from esgpull.utils import format_size
 
 
@@ -177,26 +177,37 @@ def update(
             for qf, qf_files in zip(qfs, files):
                 qf.files = qf_files
         for qf in qfs:
-            new_files = [f for f in qf.files if f not in esg.db]
-            new_datasets = [d for d in qf.datasets if d not in esg.db]
-            nb_datasets = len(new_datasets)
-            nb_files = len(new_files)
             if not qf.query.tracked:
                 esg.db.add(qf.query)
                 continue
-            elif nb_datasets == nb_files == 0:
-                esg.ui.print(f"{qf.query.rich_name} is already up-to-date.")
-                continue
-            size = sum([file.size for file in new_files])
-            if size > 0:
-                queue_msg = " and send new files to download queue"
-            else:
-                queue_msg = ""
+            with esg.db.commit_context():
+                unregistered_datasets = [
+                    f for f in qf.datasets if f not in esg.db
+                ]
+                if len(unregistered_datasets) > 0:
+                    esg.ui.print(
+                        f"Adding {len(unregistered_datasets)} new datasets to database."
+                    )
+                    esg.db.session.add_all(unregistered_datasets)
+                files_from_db = [
+                    esg.db.get(File, f.sha) for f in qf.files if f in esg.db
+                ]
+                registered_files = [f for f in files_from_db if f is not None]
+                unregistered_files = [f for f in qf.files if f not in esg.db]
+                if len(unregistered_files) > 0:
+                    esg.ui.print(
+                        f"Adding {len(unregistered_files)} new files to database."
+                    )
+                    esg.db.session.add_all(unregistered_files)
+                files = registered_files + unregistered_files
+            not_done_files = [
+                file for file in files if file.status != FileStatus.Done
+            ]
+            download_size = sum(file.size for file in not_done_files)
             msg = (
-                f"\n{qf.query.rich_name}: {nb_files} new"
-                f" files, {nb_datasets} new datasets"
-                f" ({format_size(size)})."
-                f"\nUpdate the database{queue_msg}?"
+                f"\n{qf.query.rich_name}: {len(not_done_files)} "
+                f" files ({format_size(download_size)}) to download."
+                f"\nLink to query and send to download queue?"
             )
             if yes:
                 choice = "y"
@@ -212,30 +223,19 @@ def update(
             if choice == "y":
                 legacy = esg.legacy_query
                 has_legacy = legacy.state.persistent
+                applied_changes = False
                 with esg.db.commit_context():
-                    for dataset in esg.ui.track(
-                        new_datasets,
-                        description=f"{qf.query.rich_name} (datasets)",
-                    ):
-                        esg.db.session.add(dataset)
                     for file in esg.ui.track(
-                        new_files,
-                        description=f"{qf.query.rich_name} (files)",
+                        files,
+                        description=f"{qf.query.rich_name}",
                     ):
-                        file_db = esg.db.get(File, file.sha)
-                        if file_db is None:
-                            if esg.db.has_file_id(file):
-                                logger.error(
-                                    "File id already exists in database, "
-                                    "there might be an error with its checksum"
-                                    f"\n{file}"
-                                )
-                                continue
+                        if file.status != FileStatus.Done:
                             file.status = FileStatus.Queued
-                            esg.db.session.add(file)
-                        elif has_legacy and legacy in file_db.queries:
-                            esg.db.unlink(query=legacy, file=file_db)
-                        esg.db.link(query=qf.query, file=file)
-                    qf.query.updated_at = datetime.now(timezone.utc)
+                        if has_legacy and legacy in file.queries:
+                            _ = esg.db.unlink(query=legacy, file=file)
+                        changed = esg.db.link(query=qf.query, file=file)
+                        applied_changes = applied_changes or changed
+                    if applied_changes:
+                        qf.query.updated_at = datetime.now(timezone.utc)
                     esg.db.session.add(qf.query)
         esg.ui.raise_maybe_record(Exit(0))

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/config.py

@@ -138,7 +138,7 @@ class Paths:
             root = InstallConfig.default
         return root / "plugins"
 
-    def __iter__(self) -> Iterator[Path]:
+    def values(self) -> Iterator[Path]:
         yield self.auth
         yield self.data
         yield self.db

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/database.py

@@ -147,11 +147,19 @@ class Database:
         for item in items:
             make_transient(item)
 
-    def link(self, query: Query, file: File):
-        self.session.execute(sql.query_file.link(query, file))
+    def link(self, query: Query, file: File) -> bool:
+        if not self.session.scalar(sql.query_file.is_linked(query, file)):
+            self.session.execute(sql.query_file.link(query, file))
+            return True
+        else:
+            return False
 
-    def unlink(self, query: Query, file: File):
-        self.session.execute(sql.query_file.unlink(query, file))
+    def unlink(self, query: Query, file: File) -> bool:
+        if self.session.scalar(sql.query_file.is_linked(query, file)):
+            self.session.execute(sql.query_file.unlink(query, file))
+            return True
+        else:
+            return False
 
     def __contains__(self, item: Base | BaseNoSHA) -> bool:
         mapper = inspect(item.__class__)

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/fs.py

@@ -10,7 +10,7 @@ from shutil import copyfile
 import aiofiles
 from aiofiles.threadpool.binary import AsyncBufferedIOBase
 
-from esgpull.config import Config
+from esgpull.config import Config, Paths
 from esgpull.models import File
 from esgpull.result import Err, Ok, Result
 from esgpull.tui import logger
@@ -63,45 +63,34 @@ class Digest:
 
 @dataclass
 class Filesystem:
-    auth: Path
-    data: Path
-    db: Path
-    log: Path
-    tmp: Path
+    paths: Paths
     disable_checksum: bool = False
     install: InitVar[bool] = True
 
     @staticmethod
     def from_config(config: Config, install: bool = False) -> Filesystem:
         return Filesystem(
-            auth=config.paths.auth,
-            data=config.paths.data,
-            db=config.paths.db,
-            log=config.paths.log,
-            tmp=config.paths.tmp,
+            paths=config.paths,
             disable_checksum=config.download.disable_checksum,
             install=install,
         )
 
     def __post_init__(self, install: bool = True) -> None:
         if install:
-            self.auth.mkdir(parents=True, exist_ok=True)
-            self.data.mkdir(parents=True, exist_ok=True)
-            self.db.mkdir(parents=True, exist_ok=True)
-            self.log.mkdir(parents=True, exist_ok=True)
-            self.tmp.mkdir(parents=True, exist_ok=True)
+            for path in self.paths.values():
+                path.mkdir(parents=True, exist_ok=True)
 
     def __getitem__(self, file: File) -> FilePath:
         if not isinstance(file, File):
             raise TypeError(file)
         return FilePath(
-            drs=self.data / file.local_path / file.filename,
-            tmp=self.tmp / f"{file.sha}.part",
+            drs=self.paths.data / file.local_path / file.filename,
+            tmp=self.paths.tmp / f"{file.sha}.part",
         )
 
     def glob_netcdf(self) -> Iterator[Path]:
-        for path in self.data.glob("**/*.nc"):
-            yield path.relative_to(self.data)
+        for path in self.paths.data.glob("**/*.nc"):
+            yield path.relative_to(self.paths.data)
 
     def open(self, file: File) -> FileObject:
         return FileObject(self[file])

esgpull-0.9.1/esgpull/migrations/versions/0.9.1_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.9.1
+Revises: 0.9.0
+Create Date: 2025-08-08 10:38:14.204594
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.9.1'
+down_revision = '0.9.0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.9.0 → esgpull-0.9.1}/esgpull/models/sql.py

@@ -301,3 +301,11 @@ class query_file:
             .where(query_file_proxy.c.query_sha == query.sha)
             .where(query_file_proxy.c.file_sha == file.sha)
         )
+
+    @staticmethod
+    def is_linked(query: Query, file: File) -> sa.Select[tuple[bool]]:
+        return sa.select(
+            sa.exists()
+            .where(query_file_proxy.c.query_sha == query.sha)
+            .where(query_file_proxy.c.file_sha == file.sha)
+        )

{esgpull-0.9.0 → esgpull-0.9.1}/pyproject.toml

@@ -4,38 +4,38 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.9.0"
+version = "0.9.1"
 classifiers = [
-  "License :: OSI Approved :: BSD License",
-  "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.12"
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 description = "ESGF data discovery, download, replication tool"
-authors = [{name = "Sven Rodriguez", email = "srodriguez@ipsl.fr"}]
+authors = [{ name = "Sven Rodriguez", email = "srodriguez@ipsl.fr" }]
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-  "MyProxyClient>=2.1.0",
-  "aiofiles>=22.1.0",
-  "alembic>=1.8.1",
-  "click>=8.1.3",
-  "click-params>=0.4.0",
-  "httpx>=0.23.0",
-  "nest-asyncio>=1.5.6",
-  "pyOpenSSL>=22.1.0",
-  "pyyaml>=6.0",
-  "tomlkit>=0.11.5",
-  "rich>=12.6.0",
-  "sqlalchemy>=2.0.0b2",
-  "setuptools>=65.4.1",
-  "aiostream>=0.4.5",
-  "attrs>=22.1.0",
-  "cattrs>=22.2.0",
-  "platformdirs>=2.6.2",
-  "pyparsing>=3.0.9",
-  "packaging>=25.0",
+    "MyProxyClient>=2.1.0",
+    "aiofiles>=22.1.0",
+    "alembic>=1.8.1",
+    "click>=8.1.3",
+    "click-params>=0.4.0",
+    "httpx>=0.23.0",
+    "nest-asyncio>=1.5.6",
+    "pyOpenSSL>=22.1.0",
+    "pyyaml>=6.0",
+    "tomlkit>=0.11.5",
+    "rich>=12.6.0",
+    "sqlalchemy>=2.0.0b2",
+    "setuptools>=65.4.1",
+    "aiostream>=0.4.5",
+    "attrs>=22.1.0",
+    "cattrs>=22.2.0",
+    "platformdirs>=2.6.2",
+    "pyparsing>=3.0.9",
+    "packaging>=25.0",
 ]
 license.text = "BSD-3-Clause"
 
@@ -69,20 +69,14 @@ src_paths = ["esgpull", "tests", "migrations", "examples"]
 ignore_missing_imports = true
 
 [tool.pdm.scripts]
-doc = {shell = "cd docs && mkdocs serve", help = "Start doc server"}
+doc = { shell = "cd docs && mkdocs serve", help = "Start doc server" }
 
 [tool.pytest.ini_options]
 minversion = "6.2.4"
-filterwarnings = [
-  "ignore::DeprecationWarning"
-]
-markers = [
-  "slow: mark test as slow to run"
-]
+filterwarnings = ["ignore::DeprecationWarning"]
+markers = ["slow: mark test as slow to run"]
 addopts = "-r aR -n auto --cov=esgpull --cov-config=pyproject.toml --cov-report term-missing:skip-covered --mypy"
-testpaths = [
-  "tests/"
-]
+testpaths = ["tests/"]
 
 [tool.ruff]
 line-length = 79
@@ -94,22 +88,22 @@ extend-select = ["I"]
 [tool.rye]
 managed = true
 dev-dependencies = [
-  "jupyter-console>=6.6.3",
-  "typing-extensions>=4.12.2",
-  "pytest>=8.3.3",
-  "pytest-cov>=5.0.0",
-  "mkdocs-material>=9.5.34",
-  "mypy>=1.11.2",
-  "types-pyyaml>=6.0.12.20240808",
-  "types-aiofiles>=24.1.0.20240626",
-  "pytest-mypy>=0.10.3",
-  "pytest-xdist>=3.6.1",
-  "ipdb>=0.13.13",
-  "orjson>=3.10.7"
+    "jupyter-console>=6.6.3",
+    "typing-extensions>=4.12.2",
+    "pytest>=8.3.3",
+    "pytest-cov>=5.0.0",
+    "mkdocs-material>=9.5.34",
+    "mypy>=1.11.2",
+    "types-pyyaml>=6.0.12.20240808",
+    "types-aiofiles>=24.1.0.20240626",
+    "pytest-mypy>=0.10.3",
+    "pytest-xdist>=3.6.1",
+    "ipdb>=0.13.13",
+    "orjson>=3.10.7",
 ]
 
 [tool.rye.scripts]
-esg = {cmd = "esgpull", env-file = ".env"}
+esg = { cmd = "esgpull", env-file = ".env" }
 
 [dependency-groups]
 dev = [