esgpull 0.7.2__tar.gz → 0.8.0__tar.gz

esgpull-0.8.0/CITATION.cff

@@ -0,0 +1,33 @@
+# This CITATION.cff file was generated with cffinit.
+
+cff-version: 1.2.0
+title: esgpull
+message: ESGF command line download tool esgpull
+type: software
+authors:
+  - given-names: Sven
+    family-names: Rodriguez
+    email: sven.rodriguez@ipsl.fr
+    affiliation: Sorbonne University
+  - given-names: Atef
+    family-names: Ben Nasser
+    email: abennasser@ipsl.fr
+    affiliation: Centre National de Recherche Scientifique
+    orcid: 'https://orcid.org/0000-0001-6948-8735'
+  - given-names: Guillaume
+    family-names: Levavasseur
+    email: glipsl@ipsl.fr
+    affiliation: Sorbonne University
+    orcid: 'https://orcid.org/0000-0002-0801-0890'
+repository-code: 'https://github.com/ESGF/esgf-download.git'
+url: 'https://esgf.github.io/esgf-download/'
+abstract: >-
+  esgpull is a modern ESGF data management tool, bundled
+  with a custom asynchronous interface with the ESGF Search
+  API. It handles scanning, downloading and updating
+  datasets, files and queries from ESGF.
+license: BSD-3-Clause
+commit: 762494e
+version: 0.7.4
+doi: 10.5281/zenodo.14228984
+date-released: '2024-11-27'

{esgpull-0.7.2 → esgpull-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: esgpull
-Version: 0.7.2
+Version: 0.8.0
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -60,6 +60,18 @@ for dataset in datasets:
 - Command-line interface
 - HTTP download (async multi-file)
 
+## Installation
+
+Install `esgpull` using pip or conda:
+
+```shell
+pip install esgpull
+```
+
+```shell
+conda install -c conda-forge ipsl::esgpull
+```
+
 ## Usage
 
 ```console
@@ -87,3 +99,11 @@ Commands:
   untrack   Untrack queries
   update    Fetch files, link files <-> queries, send files to download...
 ```
+
+## Useful links
+* [ESGF Webinar: An Introduction to esgpull, A Replacement for Synda](https://www.youtube.com/watch?v=xv2RVMd1iCA)
+
+
+## Contributions
+
+You can use the common github workflow (through pull requests and issues) to contribute.

{esgpull-0.7.2 → esgpull-0.8.0}/README.md

@@ -25,6 +25,18 @@ for dataset in datasets:
 - Command-line interface
 - HTTP download (async multi-file)
 
+## Installation
+
+Install `esgpull` using pip or conda:
+
+```shell
+pip install esgpull
+```
+
+```shell
+conda install -c conda-forge ipsl::esgpull
+```
+
 ## Usage
 
 ```console
@@ -52,3 +64,11 @@ Commands:
   untrack   Untrack queries
   update    Fetch files, link files <-> queries, send files to download...
 ```
+
+## Useful links
+* [ESGF Webinar: An Introduction to esgpull, A Replacement for Synda](https://www.youtube.com/watch?v=xv2RVMd1iCA)
+
+
+## Contributions
+
+You can use the common github workflow (through pull requests and issues) to contribute.

{esgpull-0.7.2 → esgpull-0.8.0}/alembic.ini

@@ -3,12 +3,6 @@ script_location = esgpull/migrations
 prepend_sys_path = .
 version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
 
-[post_write_hooks]
-hooks = black
-black.type = console_scripts
-black.entrypoint = black
-black.options = -l 79 REVISION_SCRIPT_FILENAME
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/cli/show.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from datetime import datetime
+
 import click
 from click.exceptions import Abort, BadArgumentUsage, Exit
 
@@ -15,6 +17,18 @@ from esgpull.tui import Verbosity
 @groups.json_yaml
 @opts.files
 @opts.shas
+@click.option(
+    "--after",
+    type=click.DateTime(["%Y-%m-%d"]),
+    default=None,
+    help="Filter queries added after this date (YYYY-MM-DD)",
+)
+@click.option(
+    "--before",
+    type=click.DateTime(["%Y-%m-%d"]),
+    default=None,
+    help="Filter queries added before this date (YYYY-MM-DD)",
+)
 @opts.verbosity
 def show(
     query_id: str | None,
@@ -26,6 +40,8 @@ def show(
     json: bool,
     yaml: bool,
     shas: bool,
+    after: datetime | None,
+    before: datetime | None,
     verbosity: Verbosity,
 ) -> None:
     """
@@ -49,6 +65,19 @@ def show(
                 parents=parents,
                 keep_db=True,
             )
+
+        # Apply date filters if provided
+        if after or before:
+            filtered_queries = {}
+            for sha, query in graph.queries.items():
+                if after and query.added_at < after:
+                    continue
+                if before and query.added_at > before:
+                    continue
+                filtered_queries[sha] = query
+            graph.queries = filtered_queries
+            # Update the shas set to match the filtered queries
+            graph._shas = set(filtered_queries.keys())
         if tag is not None:
             tag_db = esg.graph.get_tag(tag)
             if tag_db is not None and tag_db.description is not None:

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/cli/status.py

@@ -44,8 +44,10 @@ def status(
             for status, count, total_size in status_count_size:
                 first_row = True
                 for query in esg.graph.queries.values():
-                    files = [f for f in query.files if f.status == status]
-                    if files:
+                    if not query.tracked:
+                        continue
+                    st_count, st_size = query.files_count_size(status)
+                    if st_count:
                         if first_row:
                             if first_line:
                                 first_line = False
@@ -61,7 +63,7 @@ def status(
                             first_row = False
                         table.add_row(
                             query.rich_name,
-                            str(len(files)),
-                            format_size(sum([f.size for f in files])),
+                            str(st_count),
+                            format_size(st_size),
                         )
         esg.ui.print(table)

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/cli/update.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 
 import click
 from click.exceptions import Abort, Exit
@@ -100,12 +101,20 @@ def update(
         #   It might be interesting for the special case where all files already
         #   exist in db, then the detailed fetch could be skipped.
         for qf in qfs:
-            qf_results = esg.context.prepare_search_distributed(
-                qf.expanded,
-                file=True,
-                hints=[qf.hints],
-                max_hits=None,
-            )
+            if esg.config.api.use_custom_distribution_algorithm:
+                qf_results = esg.context.prepare_search_distributed(
+                    qf.expanded,
+                    file=True,
+                    hints=[qf.hints],
+                    max_hits=None,
+                )
+            else:
+                qf_results = esg.context.prepare_search(
+                    qf.expanded,
+                    file=True,
+                    hits=[qf.hits],
+                    max_hits=None,
+                )
             nb_req = len(qf_results)
             if nb_req > 50:
                 msg = (
@@ -184,4 +193,6 @@ def update(
                         elif has_legacy and legacy in file_db.queries:
                             esg.db.unlink(query=legacy, file=file_db)
                         esg.db.link(query=qf.query, file=file)
+                    qf.query.updated_at = datetime.now(timezone.utc)
+                    esg.db.session.add(qf.query)
         esg.ui.raise_maybe_record(Exit(0))

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/cli/utils.py

@@ -15,7 +15,7 @@ from rich.text import Text
 from esgpull import Esgpull
 from esgpull.graph import Graph
 from esgpull.models import Dataset, File, Option, Options, Query, Selection
-from esgpull.tui import UI, TempUI, Verbosity
+from esgpull.tui import UI, TempUI, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -121,9 +121,15 @@ def totable(docs: list[OrderedDict[str, Any]]) -> Table:
     return table
 
 
+def safe_value(value: str) -> str:
+    if " " in value:
+        return f'"{value}"'
+    else:
+        return value
+
+
 def parse_facets(facets: list[str]) -> Selection:
     facet_dict: dict[str, list[str]] = {}
-    exact_terms: list[str] | None = None
     for facet in facets:
         match facet.split(":"):
             case [value]:
@@ -132,28 +138,9 @@ def parse_facets(facets: list[str]) -> Selection:
                 ...
             case _:
                 raise BadArgumentUsage(f"{facet!r} is not valid syntax.")
-        if value.startswith("/"):
-            if exact_terms is not None:
-                raise BadArgumentUsage("Nested exact string is forbidden.")
-            exact_terms = []
-        if exact_terms is not None:
-            if name != "query":
-                raise BadArgumentUsage(
-                    "Cannot use facet term inside an exact string."
-                )
-            exact_terms.append(value)
-            if value.endswith("/"):
-                final_exact_str = " ".join(exact_terms)
-                value = '"' + final_exact_str.strip("/") + '"'
-                exact_terms = None
-            else:
-                continue
-            facet_dict.setdefault(name, [])
-            facet_dict[name].append(value)
-        else:
-            values = value.split(",")
-            facet_dict.setdefault(name, [])
-            facet_dict[name].extend(values)
+        values = list(map(safe_value, value.split(",")))
+        facet_dict.setdefault(name, [])
+        facet_dict[name].extend(values)
     selection = Selection()
     for name, values in facet_dict.items():
         selection[name] = values
@@ -170,6 +157,13 @@ def parse_query(
     replica: str | None,
     retracted: str | None,
 ) -> Query:
+    logger.info(f"{facets=}")
+    logger.info(f"{tags=}")
+    logger.info(f"{require=}")
+    logger.info(f"{distrib=}")
+    logger.info(f"{latest=}")
+    logger.info(f"{replica=}")
+    logger.info(f"{retracted=}")
     options = Options(
         distrib=distrib or Option.notset,
         latest=latest or Option.notset,

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/config.py

@@ -126,6 +126,7 @@ class API:
     page_limit: int = 50
     default_options: DefaultOptions = Factory(DefaultOptions)
     default_query_id: str = ""
+    use_custom_distribution_algorithm: bool = False
 
 
 def fix_rename_search_api(doc: TOMLDocument) -> TOMLDocument:

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/context.py

@@ -18,7 +18,7 @@ from esgpull.config import Config
 from esgpull.exceptions import SolrUnstableQueryError
 from esgpull.models import Dataset, File, Query
 from esgpull.tui import logger
-from esgpull.utils import format_date, index2url, sync
+from esgpull.utils import format_date_iso, index2url, sync
 
 # workaround for notebooks with running event loop
 if asyncio.get_event_loop().is_running():
@@ -77,9 +77,9 @@ class Result:
         else:
             params["fields"] = "instance_id"
         if date_from is not None:
-            params["from"] = format_date(date_from)
+            params["from"] = format_date_iso(date_from)
         if date_to is not None:
-            params["to"] = format_date(date_to)
+            params["to"] = format_date_iso(date_to)
         if facets_param is not None:
             if len(set(facets_param) & DangerousFacets) > 0:
                 raise SolrUnstableQueryError(pretty_repr(self.query))
@@ -90,9 +90,9 @@ class Result:
             facets_star = False
         # [?]TODO: add nominal temporal constraints `to`
         # if "start" in facets:
-        #     query["start"] = format_date(str(facets.pop("start")))
+        #     query["start"] = format_date_iso(str(facets.pop("start")))
         # if "end" in facets:
-        #     query["end"] = format_date(str(facets.pop("end")))
+        #     query["end"] = format_date_iso(str(facets.pop("end")))
         solr_terms: list[str] = []
         for name, values in self.query.selection.items():
             value_term = " ".join(values)
@@ -282,7 +282,7 @@ class Context:
     #     # if since is None:
     #     #     self.since = since
     #     # else:
-    #     #     self.since = format_date(since)
+    #     #     self.since = format_date_iso(since)
 
     async def __aenter__(self) -> Context:
         if hasattr(self, "client"):

esgpull-0.8.0/esgpull/migrations/versions/0.7.3_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.7.3
+Revises: 0.7.2
+Create Date: 2024-09-20 12:22:11.989996
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.7.3'
+down_revision = '0.7.2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull-0.8.0/esgpull/migrations/versions/0.8.0_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.8.0
+Revises: 14c72daea083
+Create Date: 2025-05-15 11:28:10.755003
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.8.0'
+down_revision = '14c72daea083'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull-0.8.0/esgpull/migrations/versions/14c72daea083_query_add_column_updated_at.py

@@ -0,0 +1,36 @@
+"""query_add_column_updated_at
+
+Revision ID: 14c72daea083
+Revises: c7c8541fa741
+Create Date: 2025-05-07 14:49:43.993125
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '14c72daea083'
+down_revision = 'c7c8541fa741'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('updated_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True))
+
+    # Backfill nulls
+    op.execute('UPDATE query SET updated_at = CURRENT_TIMESTAMP WHERE updated_at IS NULL')
+
+    # Make non-nullable
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.alter_column('updated_at', nullable=False)
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.drop_column('updated_at')
+
+    # ### end Alembic commands ###

esgpull-0.8.0/esgpull/migrations/versions/c7c8541fa741_query_add_column_added_at.py

@@ -0,0 +1,37 @@
+"""query_add_column_added_at
+
+Revision ID: c7c8541fa741
+Revises: 0.7.3
+Create Date: 2025-05-05 16:14:57.140262
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c7c8541fa741'
+down_revision = '0.7.3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Add as nullable first
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('added_at', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=True))
+
+    # Backfill nulls
+    op.execute('UPDATE query SET added_at = CURRENT_TIMESTAMP WHERE added_at IS NULL')
+
+    # Make non-nullable
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.alter_column('added_at', nullable=False)
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('query', schema=None) as batch_op:
+        batch_op.drop_column('added_at')
+
+    # ### end Alembic commands ###

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/models/options.py

@@ -53,7 +53,7 @@ class Options(Base):
     replica: Mapped[Option] = mapped_column(sa.Enum(Option))
     retracted: Mapped[Option] = mapped_column(sa.Enum(Option))
 
-    _distrib_ = Option(False)
+    _distrib_ = Option(True)
     _latest_ = Option(True)
     _replica_ = Option(None)
     _retracted_ = Option(False)

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/models/query.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Iterator, MutableMapping, Sequence
+from datetime import datetime, timezone
 from typing import Any, Literal
 
 import sqlalchemy as sa
@@ -11,6 +12,7 @@ from rich.tree import Tree
 from sqlalchemy.orm import Mapped, mapped_column, object_session, relationship
 from typing_extensions import NotRequired, TypedDict
 
+from esgpull import utils
 from esgpull.exceptions import UntrackableQuery
 from esgpull.models.base import Base, Sha
 from esgpull.models.file import FileDict, FileStatus
@@ -24,7 +26,18 @@ from esgpull.models.utils import (
     rich_measure_impl,
     short_sha,
 )
-from esgpull.utils import format_size
+from esgpull.utils import format_date_iso, format_size
+
+QUERY_DATE_FMT = "%Y-%m-%d %H:%M:%S"
+
+
+def parse_date(d: datetime | str) -> datetime:
+    return utils.parse_date(d, fmt=QUERY_DATE_FMT)
+
+
+def format_date(d: datetime | str) -> str:
+    return utils.format_date(d, fmt=QUERY_DATE_FMT)
+
 
 query_file_proxy = sa.Table(
     "query_file",
@@ -152,6 +165,8 @@ class QueryDict(TypedDict):
     options: NotRequired[MutableMapping[str, bool | None]]
     selection: NotRequired[MutableMapping[str, FacetValues]]
     files: NotRequired[list[FileDict]]
+    added_at: NotRequired[str]
+    updated_at: NotRequired[str]
 
 
 class Query(Base):
@@ -181,6 +196,14 @@ class Query(Base):
         back_populates="queries",
         repr=False,
     )
+    added_at: Mapped[datetime] = mapped_column(
+        server_default=sa.func.now(),
+        default_factory=lambda: datetime.now(timezone.utc),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        server_default=sa.func.now(),
+        default_factory=lambda: datetime.now(timezone.utc),
+    )
 
     def __init__(
         self,
@@ -191,6 +214,8 @@ class Query(Base):
         options: Options | MutableMapping[str, bool | None] | None = None,
         selection: Selection | MutableMapping[str, FacetValues] | None = None,
         files: list[FileDict] | None = None,
+        added_at: datetime | str | None = None,
+        updated_at: datetime | str | None = None,
     ) -> None:
         self.tracked = tracked
         self.require = require
@@ -219,6 +244,14 @@ class Query(Base):
         if files is not None:
             for file in files:
                 self.files.append(File.fromdict(file))
+        if added_at is not None:
+            self.added_at = parse_date(added_at)
+        else:
+            self.added_at = datetime.now(timezone.utc)
+        if updated_at is not None:
+            self.updated_at = parse_date(updated_at)
+        else:
+            self.updated_at = datetime.now(timezone.utc)
 
     @property
     def has_files(self) -> bool:
@@ -313,6 +346,8 @@ class Query(Base):
             result["options"] = self.options.asdict()
         if self.selection:
             result["selection"] = self.selection.asdict()
+        result["added_at"] = format_date(self.added_at)
+        result["updated_at"] = format_date(self.updated_at)
         return result
 
     def clone(self, compute_sha: bool = True) -> Query:
@@ -409,6 +444,10 @@ class Query(Base):
         title = Text.from_markup(self.rich_name)
         if not self.tracked:
             title.append(" untracked", style="i red")
+        title.append(
+            f"\n│ added    {format_date_iso(self.added_at)}"
+            f"\n│ updated  {format_date_iso(self.updated_at)}"
+        )
         contents = Table.grid(padding=(0, 1))
         if not hasattr(self, "_rich_no_require") and self.require is not None:
             if len(self.require) == 40:

{esgpull-0.7.2 → esgpull-0.8.0}/esgpull/utils.py

@@ -31,7 +31,9 @@ def format_size(size: int) -> str:
     )
 
 
-def format_date(date: str | datetime.datetime, fmt: str = "%Y-%m-%d") -> str:
+def parse_date(
+    date: str | datetime.datetime, fmt: str = "%Y-%m-%d"
+) -> datetime.datetime:
     match date:
         case datetime.datetime():
             ...
@@ -39,7 +41,17 @@ def format_date(date: str | datetime.datetime, fmt: str = "%Y-%m-%d") -> str:
             date = datetime.datetime.strptime(date, fmt)
         case _:
             raise ValueError(date)
-    return date.replace(microsecond=0).isoformat() + "Z"
+    return date
+
+
+def format_date(date: str | datetime.datetime, fmt: str = "%Y-%m-%d") -> str:
+    return parse_date(date, fmt).strftime(fmt)
+
+
+def format_date_iso(
+    date: str | datetime.datetime, fmt: str = "%Y-%m-%d"
+) -> str:
+    return parse_date(date, fmt).replace(microsecond=0).isoformat() + "Z"
 
 
 def url2index(url: str) -> str:

{esgpull-0.7.2 → esgpull-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.7.2"
+version = "0.8.0"
 classifiers = [
   "License :: OSI Approved :: BSD License",
   "Programming Language :: Python :: 3",
@@ -102,5 +102,10 @@ dev-dependencies = [
   "types-pyyaml>=6.0.12.20240808",
   "types-aiofiles>=24.1.0.20240626",
   "pytest-mypy>=0.10.3",
-  "pytest-xdist>=3.6.1"
+  "pytest-xdist>=3.6.1",
+  "ipdb>=0.13.13",
+  "orjson>=3.10.7"
 ]
+
+[tool.rye.scripts]
+esg = {cmd = "esgpull", env-file = ".env"}