esgpull 0.7.1__tar.gz → 0.7.3__tar.gz

{esgpull-0.7.1 → esgpull-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: esgpull
-Version: 0.7.1
+Version: 0.7.3
 Summary: ESGF data discovery, download, replication tool
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
@@ -60,6 +60,18 @@ for dataset in datasets:
 - Command-line interface
 - HTTP download (async multi-file)
 
+## Installation
+
+Install `esgpull` using pip or conda:
+
+```shell
+pip install esgpull
+```
+
+```shell
+conda install -c conda-forge ipsl::esgpull
+```
+
 ## Usage
 
 ```console

{esgpull-0.7.1 → esgpull-0.7.3}/README.md

@@ -25,6 +25,18 @@ for dataset in datasets:
 - Command-line interface
 - HTTP download (async multi-file)
 
+## Installation
+
+Install `esgpull` using pip or conda:
+
+```shell
+pip install esgpull
+```
+
+```shell
+conda install -c conda-forge ipsl::esgpull
+```
+
 ## Usage
 
 ```console

{esgpull-0.7.1 → esgpull-0.7.3}/esgpull/cli/utils.py

@@ -15,7 +15,7 @@ from rich.text import Text
 from esgpull import Esgpull
 from esgpull.graph import Graph
 from esgpull.models import Dataset, File, Option, Options, Query, Selection
-from esgpull.tui import UI, TempUI, Verbosity
+from esgpull.tui import UI, TempUI, Verbosity, logger
 from esgpull.utils import format_size
 
 
@@ -121,9 +121,15 @@ def totable(docs: list[OrderedDict[str, Any]]) -> Table:
     return table
 
 
+def safe_value(value: str) -> str:
+    if " " in value:
+        return f'"{value}"'
+    else:
+        return value
+
+
 def parse_facets(facets: list[str]) -> Selection:
     facet_dict: dict[str, list[str]] = {}
-    exact_terms: list[str] | None = None
     for facet in facets:
         match facet.split(":"):
             case [value]:
@@ -132,28 +138,9 @@ def parse_facets(facets: list[str]) -> Selection:
                 ...
             case _:
                 raise BadArgumentUsage(f"{facet!r} is not valid syntax.")
-        if value.startswith("/"):
-            if exact_terms is not None:
-                raise BadArgumentUsage("Nested exact string is forbidden.")
-            exact_terms = []
-        if exact_terms is not None:
-            if name != "query":
-                raise BadArgumentUsage(
-                    "Cannot use facet term inside an exact string."
-                )
-            exact_terms.append(value)
-            if value.endswith("/"):
-                final_exact_str = " ".join(exact_terms)
-                value = '"' + final_exact_str.strip("/") + '"'
-                exact_terms = None
-            else:
-                continue
-            facet_dict.setdefault(name, [])
-            facet_dict[name].append(value)
-        else:
-            values = value.split(",")
-            facet_dict.setdefault(name, [])
-            facet_dict[name].extend(values)
+        values = list(map(safe_value, value.split(",")))
+        facet_dict.setdefault(name, [])
+        facet_dict[name].extend(values)
     selection = Selection()
     for name, values in facet_dict.items():
         selection[name] = values
@@ -170,6 +157,13 @@ def parse_query(
     replica: str | None,
     retracted: str | None,
 ) -> Query:
+    logger.info(f"{facets=}")
+    logger.info(f"{tags=}")
+    logger.info(f"{require=}")
+    logger.info(f"{distrib=}")
+    logger.info(f"{latest=}")
+    logger.info(f"{replica=}")
+    logger.info(f"{retracted=}")
     options = Options(
         distrib=distrib or Option.notset,
         latest=latest or Option.notset,

{esgpull-0.7.1 → esgpull-0.7.3}/esgpull/context.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import json
 import sys
 from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
 from dataclasses import dataclass, field
@@ -441,7 +442,9 @@ class Context:
             try:
                 resp = await self.client.send(result.request)
                 resp.raise_for_status()
-                result.json = resp.json()
+                result.json = json.loads(
+                    resp.content.decode(encoding="latin-1")
+                )
                 logger.info(f"✓ Fetched in {resp.elapsed}s {resp.url}")
             except HTTPError as exc:
                 result.exc = exc
@@ -464,6 +467,8 @@ class Context:
             group = BaseExceptionGroup("fetch", excs)
             if self.noraise:
                 logger.exception(group)
+                for exc in excs:
+                    logger.exception(exc)
             else:
                 raise group
 

esgpull-0.7.3/esgpull/migrations/versions/0.7.2_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.7.2
+Revises: 0.7.1
+Create Date: 2024-09-17 15:04:11.203421
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.7.2'
+down_revision = '0.7.1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull-0.7.3/esgpull/migrations/versions/0.7.3_update_tables.py

@@ -0,0 +1,28 @@
+"""update tables
+
+Revision ID: 0.7.3
+Revises: 0.7.2
+Create Date: 2024-09-20 12:22:11.989996
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0.7.3'
+down_revision = '0.7.2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.7.1 → esgpull-0.7.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "esgpull"
-version = "0.7.1"
+version = "0.7.3"
 classifiers = [
   "License :: OSI Approved :: BSD License",
   "Programming Language :: Python :: 3",
@@ -102,5 +102,10 @@ dev-dependencies = [
   "types-pyyaml>=6.0.12.20240808",
   "types-aiofiles>=24.1.0.20240626",
   "pytest-mypy>=0.10.3",
-  "pytest-xdist>=3.6.1"
+  "pytest-xdist>=3.6.1",
+  "ipdb>=0.13.13",
+  "orjson>=3.10.7"
 ]
+
+[tool.rye.scripts]
+esg = {cmd = "esgpull", env-file = ".env"}

{esgpull-0.7.1 → esgpull-0.7.3}/requirements-dev.lock

@@ -16,7 +16,7 @@ aiostream==0.6.2
     # via esgpull
 alembic==1.13.2
     # via esgpull
-anyio==4.4.0
+anyio==4.5.0
     # via httpx
 asttokens==2.4.1
     # via stack-data
@@ -53,6 +53,7 @@ cryptography==43.0.1
 debugpy==1.8.5
     # via ipykernel
 decorator==5.1.1
+    # via ipdb
     # via ipython
 deprecated==1.2.14
     # via click-params
@@ -65,7 +66,7 @@ execnet==2.1.1
     # via pytest-xdist
 executing==2.1.0
     # via stack-data
-filelock==3.16.0
+filelock==3.16.1
     # via pytest-mypy
 ghp-import==2.1.0
     # via mkdocs
@@ -77,15 +78,17 @@ httpcore==1.0.5
     # via httpx
 httpx==0.27.2
     # via esgpull
-idna==3.8
+idna==3.10
     # via anyio
     # via httpx
     # via requests
 iniconfig==2.0.0
     # via pytest
+ipdb==0.13.13
 ipykernel==6.29.5
     # via jupyter-console
 ipython==8.27.0
+    # via ipdb
     # via ipykernel
     # via jupyter-console
 jedi==0.19.1
@@ -93,7 +96,7 @@ jedi==0.19.1
 jinja2==3.1.4
     # via mkdocs
     # via mkdocs-material
-jupyter-client==8.6.2
+jupyter-client==8.6.3
     # via ipykernel
     # via jupyter-console
 jupyter-console==6.6.3
@@ -125,7 +128,7 @@ mkdocs==1.6.1
     # via mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.5.34
+mkdocs-material==9.5.35
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
 myproxyclient==2.1.1
@@ -137,6 +140,7 @@ mypy-extensions==1.0.0
 nest-asyncio==1.6.0
     # via esgpull
     # via ipykernel
+orjson==3.10.7
 packaging==24.1
     # via ipykernel
     # via mkdocs
@@ -149,7 +153,7 @@ pathspec==0.12.1
     # via mkdocs
 pexpect==4.9.0
     # via ipython
-platformdirs==4.3.2
+platformdirs==4.3.6
     # via esgpull
     # via jupyter-core
     # via mkdocs-get-deps
@@ -206,7 +210,7 @@ requests==2.32.3
     # via mkdocs-material
 rich==13.8.1
     # via esgpull
-setuptools==74.1.2
+setuptools==75.1.0
     # via esgpull
 six==1.16.0
     # via asttokens
@@ -215,13 +219,14 @@ six==1.16.0
 sniffio==1.3.1
     # via anyio
     # via httpx
-sqlalchemy==2.0.34
+sqlalchemy==2.0.35
     # via alembic
     # via esgpull
 stack-data==0.6.3
     # via ipython
 tomli==2.0.1
     # via coverage
+    # via ipdb
     # via mypy
     # via pytest
 tomlkit==0.13.2
@@ -238,7 +243,7 @@ traitlets==5.14.3
     # via jupyter-core
     # via matplotlib-inline
 types-aiofiles==24.1.0.20240626
-types-pyyaml==6.0.12.20240808
+types-pyyaml==6.0.12.20240917
 typing-extensions==4.12.2
     # via aiostream
     # via alembic

{esgpull-0.7.1 → esgpull-0.7.3}/requirements.lock

@@ -16,7 +16,7 @@ aiostream==0.6.2
     # via esgpull
 alembic==1.13.2
     # via esgpull
-anyio==4.4.0
+anyio==4.5.0
     # via httpx
 attrs==24.2.0
     # via cattrs
@@ -48,7 +48,7 @@ httpcore==1.0.5
     # via httpx
 httpx==0.27.2
     # via esgpull
-idna==3.8
+idna==3.10
     # via anyio
     # via httpx
 mako==1.3.5
@@ -63,7 +63,7 @@ myproxyclient==2.1.1
     # via esgpull
 nest-asyncio==1.6.0
     # via esgpull
-platformdirs==4.3.2
+platformdirs==4.3.6
     # via esgpull
 pycparser==2.22
     # via cffi
@@ -78,14 +78,14 @@ pyyaml==6.0.2
     # via esgpull
 rich==13.8.1
     # via esgpull
-setuptools==74.1.2
+setuptools==75.1.0
     # via esgpull
 six==1.16.0
     # via myproxyclient
 sniffio==1.3.1
     # via anyio
     # via httpx
-sqlalchemy==2.0.34
+sqlalchemy==2.0.35
     # via alembic
     # via esgpull
 tomlkit==0.13.2

esgpull-0.7.3/tests/cli/test_parse.py

@@ -0,0 +1,40 @@
+import pytest
+
+from esgpull.cli.utils import parse_facets
+from esgpull.models import Selection
+
+test_cases = [
+    (["ocean", "temperature"], Selection(query=["ocean", "temperature"])),
+    (["ocean temperature"], Selection(query='"ocean temperature"')),
+    (
+        [
+            "project:CMIP6",
+            "experiment_id:ssp126,ssp245,ssp585",
+            "variable_id:hurt,lai,pr,sfcWind,tasmax,tasmin,ts,tos",
+            "frequency:day",
+            "nominal_resolution:100 km",
+        ],
+        Selection(
+            project="CMIP6",
+            experiment_id=["ssp126", "ssp245", "ssp585"],
+            variable_id=[
+                "hurt",
+                "lai",
+                "pr",
+                "sfcWind",
+                "tasmax",
+                "tasmin",
+                "ts",
+                "tos",
+            ],
+            frequency="day",
+            nominal_resolution='"100 km"',
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize("facets, expected_query", test_cases)
+def test_parse_facets(facets, expected_query):
+    result = parse_facets(facets)
+    assert result == expected_query