esgpull 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

esgpull/migrations/versions/0.6.4_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.4
+Revises: 0.6.3
+Create Date: 2024-07-09 11:04:30.762439
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.4"
+down_revision = "0.6.3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.6.5_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.5
+Revises: 0.6.4
+Create Date: 2024-07-09 16:29:09.389585
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.5"
+down_revision = "0.6.4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/processor.py

@@ -41,7 +41,6 @@ class Task:
     def __init__(
         self,
         config: Config,
-        auth: Auth,
         fs: Filesystem,
         # *,
         # url: str | None = None,
@@ -49,7 +48,6 @@ class Task:
         start_callbacks: list[Callback] | None = None,
     ) -> None:
         self.config = config
-        self.auth = auth
         self.fs = fs
         self.ctx = DownloadCtx(file)
         if not self.config.download.disable_checksum:
@@ -61,16 +59,6 @@ class Task:
         # else:
         #     raise ValueError("no arguments")
         self.downloader = Simple()
-        msg: str | None = None
-        if not default_ssl_context_loaded:
-            msg = load_default_ssl_context()
-        self.ssl_context: ssl.SSLContext | bool
-        if self.config.download.disable_ssl:
-            self.ssl_context = False
-        else:
-            if msg is not None:
-                logger.info(msg)
-            self.ssl_context = default_ssl_context
         if start_callbacks is None:
             self.start_callbacks = []
         else:
@@ -93,20 +81,13 @@ class Task:
     #     raise ValueError(f"{url} is not valid")
 
     async def stream(
-        self, semaphore: asyncio.Semaphore
+        self,
+        semaphore: asyncio.Semaphore,
+        client: AsyncClient,
     ) -> AsyncIterator[Result]:
         ctx = self.ctx
         try:
-            async with (
-                semaphore,
-                self.fs.open(ctx.file) as file_obj,
-                AsyncClient(
-                    follow_redirects=True,
-                    cert=self.auth.cert,
-                    verify=self.ssl_context,
-                    timeout=self.config.download.http_timeout,
-                ) as client,
-            ):
+            async with semaphore, self.fs.open(ctx.file) as file_obj:
                 for callback in self.start_callbacks:
                     callback()
                 stream = self.downloader.stream(
@@ -117,6 +98,7 @@ class Task:
                 async for ctx in stream:
                     if ctx.chunk is not None:
                         await file_obj.write(ctx.chunk)
+                        ctx.chunk = None
                     if ctx.error:
                         err = DownloadSizeError(ctx.completed, ctx.file.size)
                         yield Err(ctx, err)
@@ -129,7 +111,8 @@ class Task:
             HTTPError,
             DownloadSizeError,
             GeneratorExit,
-            ssl.SSLError
+            ssl.SSLError,
+            FileNotFoundError,
             # KeyboardInterrupt,
         ) as err:
             yield Err(ctx, err)
@@ -145,13 +128,23 @@ class Processor:
         start_callbacks: dict[str, list[Callback]],
     ) -> None:
         self.config = config
+        self.auth = auth
         self.fs = fs
         self.files = list(filter(self.should_download, files))
         self.tasks = []
+        msg: str | None = None
+        if not default_ssl_context_loaded:
+            msg = load_default_ssl_context()
+        self.ssl_context: ssl.SSLContext | bool
+        if self.config.download.disable_ssl:
+            self.ssl_context = False
+        else:
+            if msg is not None:
+                logger.info(msg)
+            self.ssl_context = default_ssl_context
         for file in files:
             task = Task(
                 config=config,
-                auth=auth,
                 fs=fs,
                 file=file,
                 start_callbacks=start_callbacks[file.sha],
@@ -166,7 +159,13 @@ class Processor:
 
     async def process(self) -> AsyncIterator[Result]:
         semaphore = asyncio.Semaphore(self.config.download.max_concurrent)
-        streams = [task.stream(semaphore) for task in self.tasks]
-        async with merge(*streams).stream() as stream:
-            async for result in stream:
-                yield result
+        async with AsyncClient(
+            follow_redirects=True,
+            cert=self.auth.cert,
+            verify=self.ssl_context,
+            timeout=self.config.download.http_timeout,
+        ) as client:
+            streams = [task.stream(semaphore, client) for task in self.tasks]
+            async with merge(*streams).stream() as stream:
+                async for result in stream:
+                    yield result

esgpull/version.py

@@ -1 +1 @@
-__version__ = "0.6.3"
+__version__ = "0.6.5"

{esgpull-0.6.3.dist-info → esgpull-0.6.5.dist-info}/METADATA

@@ -1,36 +1,14 @@
 Metadata-Version: 2.1
 Name: esgpull
-Version: 0.6.3
+Version: 0.6.5
 Summary: ESGF data discovery, download, replication tool
 Author-Email: Sven Rodriguez <srodriguez@ipsl.fr>
-License: BSD 3-Clause License
-        
-        Copyright (c) 2023, Institut Pierre-Simon Laplace (IPSL) and contributors
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions are met:
-        
-        1. Redistributions of source code must retain the above copyright notice, this
-           list of conditions and the following disclaimer.
-        
-        2. Redistributions in binary form must reproduce the above copyright notice,
-           this list of conditions and the following disclaimer in the documentation
-           and/or other materials provided with the distribution.
-        
-        3. Neither the name of the copyright holder nor the names of its
-           contributors may be used to endorse or promote products derived from
-           this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+License: BSD-3-Clause
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Project-URL: Repository, https://github.com/ESGF/esgf-download
 Project-URL: Documentation, https://esgf.github.io/esgf-download/
 Requires-Python: >=3.10

{esgpull-0.6.3.dist-info → esgpull-0.6.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-esgpull-0.6.3.dist-info/METADATA,sha256=8hz93j4mPbtVd1MXfDNuQKs98fJAHoJVeIfSa1-pqgI,4473
-esgpull-0.6.3.dist-info/WHEEL,sha256=vnE8JVcI2Wz7GRKorsPArnBdnW2SWKWGow5gu5tHlRU,90
-esgpull-0.6.3.dist-info/entry_points.txt,sha256=nfKsESeZyCiVD6dDCTCturf-vRnI-6GCuwqv9jxXCq8,46
-esgpull-0.6.3.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
+esgpull-0.6.5.dist-info/METADATA,sha256=vycdqEO7yU4ZlM-A_Db2By-LyaHl-QcqtGD3tnOd1Tc,2982
+esgpull-0.6.5.dist-info/WHEEL,sha256=mbxFTmdEUhG7evcdMkR3aBt9SWcoFBJ4CDwnfguNegA,90
+esgpull-0.6.5.dist-info/entry_points.txt,sha256=nfKsESeZyCiVD6dDCTCturf-vRnI-6GCuwqv9jxXCq8,46
+esgpull-0.6.5.dist-info/licenses/LICENSE,sha256=lUqGPGWDHHxjkUDuYgjLLY2XQXXn_EHU7fnrQWHGugc,1540
 esgpull/__init__.py,sha256=XItFDIMNmFUNNcKtUgXdfmGwUIWt4AAv0a4mZkfj5P8,240
 esgpull/auth.py,sha256=QZ-l1ySLMP0fvuwYHRLv9FZYp1gqfju_eGaTMDByUxw,5205
 esgpull/cli/__init__.py,sha256=boqxa_ku266IcZ6GY-7fVA2T9GLku1e01TXe3aqZmR4,1630
@@ -14,7 +14,6 @@ esgpull/cli/download.py,sha256=3_Fm8JJYBWKs63oQ1OLaHJCv9ccbeY2gIW8SDasaYNE,2356
 esgpull/cli/facet.py,sha256=V1u-DxNkhswwSt0qpXvuHrCI_tE8jAJGEe6_fMhYbaM,584
 esgpull/cli/get.py,sha256=2WXL01Ri0P_2Rf1xrp9bnsrrxir6whxkAC0SnohjFpg,678
 esgpull/cli/install.py,sha256=fd8nKwIFvOivgn_gOGn7XIk1BB9LXnhQB47KuIIy5AU,2880
-esgpull/cli/link.py,sha256=hZ3p3AySsOUIsiA7lIdxEctw3A7ocCJgbjWcNx2dRVU,3372
 esgpull/cli/login.py,sha256=FZ63SsB4fCDixwf7gMUR697Dk89W1OvpgeadKE4IqEU,2153
 esgpull/cli/remove.py,sha256=p26hPhsgAgmAF4IsvjAI3jtzlIUq3k8Rxb0GTKYoEQM,2517
 esgpull/cli/retry.py,sha256=UVpAjW_N7l6RTJ-T5qXojwcXPzzjT7sDKb_wBdvavrg,1310
@@ -58,6 +57,8 @@ esgpull/migrations/versions/0.6.0_update_tables.py,sha256=10Ioi84QwEWIB9Mqz-ItEG
 esgpull/migrations/versions/0.6.1_update_tables.py,sha256=L4mkKlZUz4ftINwNeVNsPhk32QYMeoTie4B-GXVeKXw,493
 esgpull/migrations/versions/0.6.2_update_tables.py,sha256=LFUVgDhxSll9FpvNFm-idD5t-SI3R30IO7cTzvShLVI,493
 esgpull/migrations/versions/0.6.3_update_tables.py,sha256=ut4M0b90OWVB9pKSGTJ4Bl5cucXFJzRD4-WA_ego0WY,493
+esgpull/migrations/versions/0.6.4_update_tables.py,sha256=PFALlSAjCFOqqoQgjq2TF0HjGc22fVdStMK2NNNa3Yk,493
+esgpull/migrations/versions/0.6.5_update_tables.py,sha256=NYO8vnS4h_g4Co4M1CJibB2WYLqmVAy6ZaApFk-do3c,493
 esgpull/models/__init__.py,sha256=rfa1yGLVDahFrnhq_8TPGzr7_oeBOFKNVD9EF0slUtY,725
 esgpull/models/base.py,sha256=3nbR2lYMHWfovWz4EiAJ2bIvKpMadRvYZDdMRQDvN7M,1237
 esgpull/models/dataset.py,sha256=1fOIVYIWKK5BivqvBpjfxrNpy9VfUHZng9Yc6ipPK1Q,905
@@ -70,11 +71,10 @@ esgpull/models/sql.py,sha256=T2whCvz_MFb0tY5Ij5xcFlZ4TxrS0JT1rRU_gd2qVsg,7041
 esgpull/models/synda_file.py,sha256=6o5unPhzVJGnbpA2MxcS0r-hrBwocHYVnLrqjSGtmuk,2387
 esgpull/models/tag.py,sha256=5CQDB9rAeCqog63ec9LPFN46HOFNkHPy-maY4gkBQ3E,461
 esgpull/models/utils.py,sha256=exwlIlIKYjhhfUE82w1kU_HeSQOSY97PTvpkhW0udMA,1631
-esgpull/presets.py,sha256=hMFY2QfcLY9icuer5SK_M7Oqbj_GSEGCYQiwhNhFqLA,336
-esgpull/processor.py,sha256=je2GS8eokqskpQx_qRh62BDr7lg8QfpjfM0jW9uJDDU,5444
+esgpull/processor.py,sha256=noenWExukON2P4klMN8Vt1ALzApmvc5gpsFqd7J5g78,5446
 esgpull/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 esgpull/result.py,sha256=f64l9gPFpFWgctmHVYrNWJvuXXB1sxpxXzJEIssLXxc,1020
 esgpull/tui.py,sha256=MG3VULh8uTrRspw_kFuhunmHqscO9Xd7PS7MG9WyvMY,10395
 esgpull/utils.py,sha256=eKipKCqj15dzCAqs1o7x5O1XClWPmDEM-qSvNwGZmXY,1233
-esgpull/version.py,sha256=zYiFHqR7JwbvdK9dvKrh-RTNfUqjHUwC4CTcFAPVYLc,22
-esgpull-0.6.3.dist-info/RECORD,,
+esgpull/version.py,sha256=KDgkBrBsBSUzbLgrOZ89YsNN06fU4j5bmcuEwo6q5pg,22
+esgpull-0.6.5.dist-info/RECORD,,

{esgpull-0.6.3.dist-info → esgpull-0.6.5.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.3.0)
+Generator: pdm-backend (2.3.2)
 Root-Is-Purelib: true
 Tag: py3-none-any

esgpull/cli/link.py

@@ -1,105 +0,0 @@
-# from __future__ import annotations
-
-# import click
-# from click.exceptions import Abort, Exit
-
-# from esgpull import Esgpull
-# from esgpull.cli.decorators import args, opts
-# from esgpull.cli.utils import get_queries, valid_name_tag
-# from esgpull.graph import Graph
-# from esgpull.tui import Verbosity
-
-
-# @click.command()
-# @args.query_id
-# @opts.tag
-# @opts.children
-# @opts.verbosity
-# def link(
-#     query_id: str | None,
-#     tag: str | None,
-#     children: bool,
-#     verbosity: Verbosity,
-# ) -> None:
-#     """
-#     Link known files to query/tag.
-#     Offline mode to avoid searching esgf ?
-#     """
-#     esg = Esgpull(verbosity=verbosity)
-#     with esg.ui.logging("remove", onraise=Abort):
-#         if query_id is None and tag is None:
-#             raise click.UsageError("No query or tag provided.")
-#         if not valid_name_tag(esg.graph, esg.ui, query_id, tag):
-#             raise Exit(1)
-#         queries = get_queries(
-#             esg.graph,
-#             query_id,
-#             tag,
-#             children=children,
-#         )
-#         nb = len(queries)
-#         ies = "ies" if nb > 1 else "y"
-#         graph = Graph(None)
-#         graph.add(*queries)
-#         esg.ui.print(graph)
-#         msg = f"Remove {nb} quer{ies}?"
-#         if not esg.ui.ask(msg, default=True):
-#             raise Abort
-#         for query in queries:
-#             if not children and esg.graph.get_children(query.sha):
-#                 esg.ui.print(
-#                     ":stop_sign: Some queries block "
-#                     f"removal of {query.rich_name}."
-#                 )
-#                 if esg.ui.ask("Show blocking queries?", default=False):
-#                     esg.ui.print(esg.graph.subgraph(query, children=True))
-#                 raise Exit(1)
-#         esg.db.delete(*queries)
-#         esg.ui.print(":+1:")
-
-
-# @click.command()
-# @args.query_id
-# @opts.tag
-# @opts.children
-# @opts.verbosity
-# def unlink(
-#     query_id: str | None,
-#     tag: str | None,
-#     children: bool,
-#     verbosity: Verbosity,
-# ) -> None:
-#     """
-#     Unlink files that were associated to query/tag
-#     """
-#     esg = Esgpull(verbosity=verbosity)
-#     with esg.ui.logging("remove", onraise=Abort):
-#         if query_id is None and tag is None:
-#             raise click.UsageError("No query or tag provided.")
-#         if not valid_name_tag(esg.graph, esg.ui, query_id, tag):
-#             raise Exit(1)
-#         queries = get_queries(
-#             esg.graph,
-#             query_id,
-#             tag,
-#             children=children,
-#         )
-#         nb = len(queries)
-#         ies = "ies" if nb > 1 else "y"
-#         graph = Graph(None)
-#         graph.add(*queries)
-#         esg.ui.print(graph)
-#         msg = f"Remove {nb} quer{ies}?"
-#         if not esg.ui.ask(msg, default=True):
-#             raise Abort
-#         for query in queries:
-#             if not children and esg.graph.get_children(query.sha):
-#                 esg.ui.print(
-#                     ":stop_sign: Some queries block "
-#                     f"removal of {query.rich_name}."
-#                 )
-#                 if esg.ui.ask("Show blocking queries?", default=False):
-#                     esg.ui.print(esg.graph.subgraph(query, children=True))
-#                 raise Exit(1)
-#         esg.db.delete(*queries)
-#         esg.ui.print(":+1:")

esgpull/presets.py

@@ -1,13 +0,0 @@
-# from esgpull.models import Query
-
-
-# IPCC_SCENARIOS = Query(
-#     select=dict(
-#         query=(
-#             "experiment:(rcp26 rcp45 rcp60 rcp85) OR "
-#             "experiment_id:(ssp119 ssp126 ssp245 ssp370 ssp460 ssp585)"
-#         )
-#     )
-# )
-
-# TEMPERATURE = Query(select=dict(variable=["tas", "tos", "tasmin", "tasmax"]))