esgpull 0.6.3__tar.gz → 0.6.4__tar.gz

{esgpull-0.6.3 → esgpull-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: esgpull
-Version: 0.6.3
+Version: 0.6.4
 Summary: ESGF data discovery, download, replication tool
 Author-Email: Sven Rodriguez <srodriguez@ipsl.fr>
 License: BSD 3-Clause License

esgpull-0.6.4/esgpull/migrations/versions/0.6.4_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.4
+Revises: 0.6.3
+Create Date: 2024-07-09 11:04:30.762439
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.4"
+down_revision = "0.6.3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

{esgpull-0.6.3 → esgpull-0.6.4}/esgpull/processor.py

@@ -41,7 +41,6 @@ class Task:
     def __init__(
         self,
         config: Config,
-        auth: Auth,
         fs: Filesystem,
         # *,
         # url: str | None = None,
@@ -49,7 +48,6 @@ class Task:
         start_callbacks: list[Callback] | None = None,
     ) -> None:
         self.config = config
-        self.auth = auth
         self.fs = fs
         self.ctx = DownloadCtx(file)
         if not self.config.download.disable_checksum:
@@ -61,16 +59,6 @@ class Task:
         # else:
         #     raise ValueError("no arguments")
         self.downloader = Simple()
-        msg: str | None = None
-        if not default_ssl_context_loaded:
-            msg = load_default_ssl_context()
-        self.ssl_context: ssl.SSLContext | bool
-        if self.config.download.disable_ssl:
-            self.ssl_context = False
-        else:
-            if msg is not None:
-                logger.info(msg)
-            self.ssl_context = default_ssl_context
         if start_callbacks is None:
             self.start_callbacks = []
         else:
@@ -93,20 +81,13 @@ class Task:
     #     raise ValueError(f"{url} is not valid")
 
     async def stream(
-        self, semaphore: asyncio.Semaphore
+        self,
+        semaphore: asyncio.Semaphore,
+        client: AsyncClient,
     ) -> AsyncIterator[Result]:
         ctx = self.ctx
         try:
-            async with (
-                semaphore,
-                self.fs.open(ctx.file) as file_obj,
-                AsyncClient(
-                    follow_redirects=True,
-                    cert=self.auth.cert,
-                    verify=self.ssl_context,
-                    timeout=self.config.download.http_timeout,
-                ) as client,
-            ):
+            async with semaphore, self.fs.open(ctx.file) as file_obj:
                 for callback in self.start_callbacks:
                     callback()
                 stream = self.downloader.stream(
@@ -117,6 +98,7 @@ class Task:
                 async for ctx in stream:
                     if ctx.chunk is not None:
                         await file_obj.write(ctx.chunk)
+                        ctx.chunk = None
                     if ctx.error:
                         err = DownloadSizeError(ctx.completed, ctx.file.size)
                         yield Err(ctx, err)
@@ -129,7 +111,8 @@ class Task:
             HTTPError,
             DownloadSizeError,
             GeneratorExit,
-            ssl.SSLError
+            ssl.SSLError,
+            FileNotFoundError,
             # KeyboardInterrupt,
         ) as err:
             yield Err(ctx, err)
@@ -145,13 +128,23 @@ class Processor:
         start_callbacks: dict[str, list[Callback]],
     ) -> None:
         self.config = config
+        self.auth = auth
         self.fs = fs
         self.files = list(filter(self.should_download, files))
         self.tasks = []
+        msg: str | None = None
+        if not default_ssl_context_loaded:
+            msg = load_default_ssl_context()
+        self.ssl_context: ssl.SSLContext | bool
+        if self.config.download.disable_ssl:
+            self.ssl_context = False
+        else:
+            if msg is not None:
+                logger.info(msg)
+            self.ssl_context = default_ssl_context
         for file in files:
             task = Task(
                 config=config,
-                auth=auth,
                 fs=fs,
                 file=file,
                 start_callbacks=start_callbacks[file.sha],
@@ -166,7 +159,13 @@ class Processor:
 
     async def process(self) -> AsyncIterator[Result]:
         semaphore = asyncio.Semaphore(self.config.download.max_concurrent)
-        streams = [task.stream(semaphore) for task in self.tasks]
-        async with merge(*streams).stream() as stream:
-            async for result in stream:
-                yield result
+        async with AsyncClient(
+            follow_redirects=True,
+            cert=self.auth.cert,
+            verify=self.ssl_context,
+            timeout=self.config.download.http_timeout,
+        ) as client:
+            streams = [task.stream(semaphore, client) for task in self.tasks]
+            async with merge(*streams).stream() as stream:
+                async for result in stream:
+                    yield result

esgpull-0.6.4/esgpull/version.py

@@ -0,0 +1 @@
+__version__ = "0.6.4"

{esgpull-0.6.3 → esgpull-0.6.4}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "platformdirs>=2.6.2",
     "pyparsing>=3.0.9",
 ]
-version = "0.6.3"
+version = "0.6.4"
 
 [project.license]
 file = "LICENSE"

{esgpull-0.6.3 → esgpull-0.6.4}/tests/test_processor.py

@@ -3,7 +3,6 @@ import asyncio
 import httpx
 import pytest
 
-from esgpull.auth import Auth
 from esgpull.fs import FileCheck, Filesystem
 from esgpull.models import File
 from esgpull.processor import Task
@@ -58,19 +57,15 @@ def fs(config):
 
 
 @pytest.fixture
-def auth(config):
-    return Auth.from_config(config)
-
-
-@pytest.fixture
-def task(config, auth, fs, smallfile):
-    return Task(config, auth, fs, file=smallfile)
+def task(config, fs, smallfile):
+    return Task(config, fs, file=smallfile)
 
 
 async def run_task(task_):
     semaphore = asyncio.Semaphore(1)
-    async for result in task_.stream(semaphore):
-        ...
+    async with httpx.AsyncClient() as client:
+        async for result in task_.stream(semaphore, client):
+            ...
     return result
 
 
@@ -78,7 +73,7 @@ async def run_task(task_):
     raises=(httpx.ConnectTimeout, httpx.ReadTimeout),
     reason="this is dependent on the IPSL data node's health (unstable)",
 )
-def test_task(auth, fs, smallfile, task):
+def test_task(fs, smallfile, task):
     result = asyncio.run(run_task(task))
     if not result.ok:
         raise result.err

esgpull-0.6.3/esgpull/cli/link.py

@@ -1,105 +0,0 @@
-# from __future__ import annotations
-
-# import click
-# from click.exceptions import Abort, Exit
-
-# from esgpull import Esgpull
-# from esgpull.cli.decorators import args, opts
-# from esgpull.cli.utils import get_queries, valid_name_tag
-# from esgpull.graph import Graph
-# from esgpull.tui import Verbosity
-
-
-# @click.command()
-# @args.query_id
-# @opts.tag
-# @opts.children
-# @opts.verbosity
-# def link(
-#     query_id: str | None,
-#     tag: str | None,
-#     children: bool,
-#     verbosity: Verbosity,
-# ) -> None:
-#     """
-#     Link known files to query/tag.
-#     Offline mode to avoid searching esgf ?
-#     """
-#     esg = Esgpull(verbosity=verbosity)
-#     with esg.ui.logging("remove", onraise=Abort):
-#         if query_id is None and tag is None:
-#             raise click.UsageError("No query or tag provided.")
-#         if not valid_name_tag(esg.graph, esg.ui, query_id, tag):
-#             raise Exit(1)
-#         queries = get_queries(
-#             esg.graph,
-#             query_id,
-#             tag,
-#             children=children,
-#         )
-#         nb = len(queries)
-#         ies = "ies" if nb > 1 else "y"
-#         graph = Graph(None)
-#         graph.add(*queries)
-#         esg.ui.print(graph)
-#         msg = f"Remove {nb} quer{ies}?"
-#         if not esg.ui.ask(msg, default=True):
-#             raise Abort
-#         for query in queries:
-#             if not children and esg.graph.get_children(query.sha):
-#                 esg.ui.print(
-#                     ":stop_sign: Some queries block "
-#                     f"removal of {query.rich_name}."
-#                 )
-#                 if esg.ui.ask("Show blocking queries?", default=False):
-#                     esg.ui.print(esg.graph.subgraph(query, children=True))
-#                 raise Exit(1)
-#         esg.db.delete(*queries)
-#         esg.ui.print(":+1:")
-
-
-# @click.command()
-# @args.query_id
-# @opts.tag
-# @opts.children
-# @opts.verbosity
-# def unlink(
-#     query_id: str | None,
-#     tag: str | None,
-#     children: bool,
-#     verbosity: Verbosity,
-# ) -> None:
-#     """
-#     Unlink files that were associated to query/tag
-#     """
-#     esg = Esgpull(verbosity=verbosity)
-#     with esg.ui.logging("remove", onraise=Abort):
-#         if query_id is None and tag is None:
-#             raise click.UsageError("No query or tag provided.")
-#         if not valid_name_tag(esg.graph, esg.ui, query_id, tag):
-#             raise Exit(1)
-#         queries = get_queries(
-#             esg.graph,
-#             query_id,
-#             tag,
-#             children=children,
-#         )
-#         nb = len(queries)
-#         ies = "ies" if nb > 1 else "y"
-#         graph = Graph(None)
-#         graph.add(*queries)
-#         esg.ui.print(graph)
-#         msg = f"Remove {nb} quer{ies}?"
-#         if not esg.ui.ask(msg, default=True):
-#             raise Abort
-#         for query in queries:
-#             if not children and esg.graph.get_children(query.sha):
-#                 esg.ui.print(
-#                     ":stop_sign: Some queries block "
-#                     f"removal of {query.rich_name}."
-#                 )
-#                 if esg.ui.ask("Show blocking queries?", default=False):
-#                     esg.ui.print(esg.graph.subgraph(query, children=True))
-#                 raise Exit(1)
-#         esg.db.delete(*queries)
-#         esg.ui.print(":+1:")

esgpull-0.6.3/esgpull/presets.py

@@ -1,13 +0,0 @@
-# from esgpull.models import Query
-
-
-# IPCC_SCENARIOS = Query(
-#     select=dict(
-#         query=(
-#             "experiment:(rcp26 rcp45 rcp60 rcp85) OR "
-#             "experiment_id:(ssp119 ssp126 ssp245 ssp370 ssp460 ssp585)"
-#         )
-#     )
-# )
-
-# TEMPERATURE = Query(select=dict(variable=["tas", "tos", "tasmin", "tasmax"]))

esgpull-0.6.3/esgpull/version.py

@@ -1 +0,0 @@
-__version__ = "0.6.3"