cdsetool 0.2.11__tar.gz → 0.2.13__tar.gz

{cdsetool-0.2.11/src/cdsetool.egg-info → cdsetool-0.2.13}/PKG-INFO

@@ -1,26 +1,26 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.11
+Version: 0.2.13
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
-Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
-Project-URL: Bug Tracker, https://github.com/SDFIdk/CDSETool/issues
+Project-URL: Homepage, https://github.com/CDSETool/CDSETool
+Project-URL: Bug Tracker, https://github.com/CDSETool/CDSETool/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: typer<1,>=0.9
 Requires-Dist: rich<14,>=13.6
 Requires-Dist: requests<3,>=2.28.1
-Requires-Dist: pyjwt[crypto]~=2.8.0
+Requires-Dist: pyjwt[crypto]<2.10,>=2.8
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.4.2; extra == "test"
-Requires-Dist: pylint==3.2.2; extra == "test"
-Requires-Dist: pytest==8.2.1; extra == "test"
+Requires-Dist: pylint==3.3.1; extra == "test"
+Requires-Dist: pytest==8.3.3; extra == "test"
 Requires-Dist: pytest-cov==5.0.0; extra == "test"
 Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: ruff==0.6.6; extra == "test"
 Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.11
+pip install cdsetool==0.2.13
 ```
 
 ## Usage

{cdsetool-0.2.11 → cdsetool-0.2.13}/README.md

@@ -71,7 +71,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.11
+pip install cdsetool==0.2.13
 ```
 
 ## Usage

{cdsetool-0.2.11 → cdsetool-0.2.13}/pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdsetool"
-version = "0.2.11"
+version = "0.2.13"
 authors = [
   { name="Jacob Vejby", email="javej@sdfi.dk" },
 ]
 description = "Tools & CLI for interacting with CDSE product APIs"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 classifiers = [
   "Programming Language :: Python :: 3",
   "Operating System :: OS Independent",
@@ -19,22 +19,22 @@ dependencies = [
   "typer >= 0.9,< 1",
   "rich >= 13.6,< 14",
   "requests >= 2.28.1,< 3",
-  "pyjwt[crypto] ~= 2.8.0",
+  "pyjwt[crypto] >= 2.8,< 2.10",
   "geopandas >= 0.13.2",
 ]
 [project.optional-dependencies]
 test = [
-    "black==24.4.2",
-    "pylint==3.2.2",
-    "pytest==8.2.1",
+    "pylint==3.3.1",
+    "pytest==8.3.3",
     "pytest-cov==5.0.0",
     "requests-mock==1.12.1",
+    "ruff==0.6.6",
     "pytest-mock==3.14.0",
 ]
 
 [project.urls]
-"Homepage" = "https://github.com/SDFIdk/CDSETool"
-"Bug Tracker" = "https://github.com/SDFIdk/CDSETool/issues"
+"Homepage" = "https://github.com/CDSETool/CDSETool"
+"Bug Tracker" = "https://github.com/CDSETool/CDSETool/issues"
 
 [project.scripts]
 cdsetool = "cdsetool:cli.main"

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool/_processing.py

@@ -52,7 +52,6 @@ def _concurrent_process(
 
         # Continue until no more futures are queued
         while futures:
-
             # Wait for the first future(s) to complete
             done, not_done = wait(futures, return_when=FIRST_COMPLETED)
             futures = list(not_done)

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool/cli.py

@@ -12,9 +12,9 @@ from cdsetool.query import describe_collection, query_features
 from cdsetool.monitor import StatusMonitor
 from cdsetool.download import download_features
 
-app = typer.Typer()
+app = typer.Typer(no_args_is_help=True)
 
-query_app = typer.Typer()
+query_app = typer.Typer(no_args_is_help=True)
 app.add_typer(query_app, name="query")
 
 
@@ -120,3 +120,7 @@ def _to_dict(term_list: List[str]) -> Dict[str, str]:
         key, value = item.split("=")
         search_terms[key] = value
     return search_terms
+
+
+if __name__ == "__main__":
+    main()

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool/credentials.py

@@ -147,7 +147,7 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
             caller=self,
             authorization=False,
             max_retries=Retry(
-                total=2,
+                total=15,
                 backoff_factor=0.5,
                 allowed_methods=None,
                 raise_on_status=False,

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool/download.py

@@ -38,12 +38,13 @@ def download_feature(
     log = _get_logger(options)
     url = _get_feature_url(feature)
     title = feature.get("properties").get("title")
+    temp_dir_usr = _get_temp_dir(options)
 
     if not url or not title:
         log.debug(f"Bad URL ('{url}') or title ('{title}')")
         return None
 
-    filename = title.replace(".SAFE", ".zip")
+    filename = title + ".zip"
     result_path = os.path.join(path, filename)
 
     if not options.get("overwrite_existing", False) and os.path.exists(result_path):
@@ -58,19 +59,28 @@ def download_feature(
             # Always get a new session, credentials might have expired.
             try:
                 session = _get_credentials(options).get_session()
-            except (TokenClientConnectionError, TokenExpiredSignatureError) as e:
-                log.warning(e)
+            except TokenClientConnectionError:
+                log.warning("Token client connection failed, retrying..")
+                continue
+            except TokenExpiredSignatureError:
+                log.warning("Token signature expired, retrying..")
                 continue
             url = _follow_redirect(url, session)
-            with session.get(url, stream=True) as response:
+            name_dir_prefix = filename.replace(".zip", "____")
+            with (
+                session.get(url, stream=True) as response,
+                tempfile.TemporaryDirectory(
+                    prefix=name_dir_prefix, dir=temp_dir_usr
+                ) as temp_dir,
+            ):
                 if response.status_code != 200:
                     log.warning(f"Status code {response.status_code}, retrying..")
                     time.sleep(60 * (1 + (random.random() / 4)))
                     continue
 
                 status.set_filesize(int(response.headers["Content-Length"]))
-
-                with tempfile.NamedTemporaryFile() as file:
+                tmp_file = os.path.join(temp_dir, "download.zip")
+                with open(tmp_file, "wb") as file:
                     # Server might not send all bytes specified by the
                     # Content-Length header before closing connection.
                     # Log as a warning and try again.
@@ -85,8 +95,8 @@ def download_feature(
                     ) as e:
                         log.warning(e)
                         continue
-                    shutil.copy(file.name, result_path)
-
+                # Close file before copy so all buffers are flushed.
+                shutil.copy(tmp_file, result_path)
                 return filename
     log.error(f"Failed to download {filename}")
     return None
@@ -143,3 +153,7 @@ def _get_credentials(options: Dict) -> Credentials:
     return options.get("credentials") or Credentials(
         proxies=options.get("proxies", None)
     )
+
+
+def _get_temp_dir(options: Dict) -> Union[str, None]:
+    return options.get("tmpdir") or None

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool/query.py

@@ -9,8 +9,13 @@ from xml.etree import ElementTree
 from datetime import datetime, date
 import re
 import json
+from random import random
+from time import sleep
 import geopandas as gpd
+from requests.exceptions import ChunkedEncodingError
+from urllib3.exceptions import ProtocolError
 from cdsetool.credentials import Credentials
+from cdsetool.logger import NoopLogger
 
 
 class _FeatureIterator:
@@ -48,9 +53,11 @@ class FeatureQuery:
         collection: str,
         search_terms: Dict[str, Any],
         proxies: Union[Dict[str, str], None] = None,
+        options: Union[Dict[str, Any], None] = None,
     ) -> None:
         self.features = []
         self.proxies = proxies
+        self.log = (options or {}).get("logger") or NoopLogger()
         self.next_url = _query_url(
             collection, {**search_terms, "exactCount": "1"}, proxies=proxies
         )
@@ -76,16 +83,29 @@ class FeatureQuery:
         session = Credentials.make_session(
             None, False, Credentials.RETRIES, self.proxies
         )
-        with session.get(self.next_url) as response:
-            response.raise_for_status()
-            res = response.json()
-            self.features += res.get("features") or []
-
-            total_results = res.get("properties", {}).get("totalResults")
-            if total_results is not None:
-                self.total_results = total_results
-
-            self.__set_next_url(res)
+        attempts = 0
+        while attempts < 10:
+            attempts += 1
+            try:
+                with session.get(self.next_url) as response:
+                    if response.status_code != 200:
+                        self.log.warning(
+                            f"Status code {response.status_code}, retrying.."
+                        )
+                        sleep(60 * (1 + (random() / 4)))
+                        continue
+                    res = response.json()
+                    self.features += res.get("features") or []
+
+                    total_results = res.get("properties", {}).get("totalResults")
+                    if total_results is not None:
+                        self.total_results = total_results
+
+                    self.__set_next_url(res)
+                    return
+            except (ChunkedEncodingError, ConnectionResetError, ProtocolError) as e:
+                self.log.warning(e)
+                continue
 
     def __set_next_url(self, res) -> None:
         links = res.get("properties", {}).get("links") or []
@@ -101,11 +121,14 @@ def query_features(
     collection: str,
     search_terms: Dict[str, Any],
     proxies: Union[Dict[str, str], None] = None,
+    options: Union[Dict[str, Any], None] = None,
 ) -> FeatureQuery:
     """
     Returns an iterator over the features matching the search terms
     """
-    return FeatureQuery(collection, {"maxRecords": 2000, **search_terms}, proxies)
+    return FeatureQuery(
+        collection, {"maxRecords": 2000, **search_terms}, proxies, options
+    )
 
 
 def shape_to_wkt(shape: str) -> str:
@@ -156,7 +179,7 @@ def describe_collection(
     )
 
     parameters = {}
-    if not parameter_node_parent:
+    if parameter_node_parent is None:
         return parameters
     for parameter_node in parameter_node_parent:
         name = parameter_node.attrib.get("name")

{cdsetool-0.2.11 → cdsetool-0.2.13/src/cdsetool.egg-info}/PKG-INFO

@@ -1,26 +1,26 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.11
+Version: 0.2.13
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
-Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
-Project-URL: Bug Tracker, https://github.com/SDFIdk/CDSETool/issues
+Project-URL: Homepage, https://github.com/CDSETool/CDSETool
+Project-URL: Bug Tracker, https://github.com/CDSETool/CDSETool/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: typer<1,>=0.9
 Requires-Dist: rich<14,>=13.6
 Requires-Dist: requests<3,>=2.28.1
-Requires-Dist: pyjwt[crypto]~=2.8.0
+Requires-Dist: pyjwt[crypto]<2.10,>=2.8
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.4.2; extra == "test"
-Requires-Dist: pylint==3.2.2; extra == "test"
-Requires-Dist: pytest==8.2.1; extra == "test"
+Requires-Dist: pylint==3.3.1; extra == "test"
+Requires-Dist: pytest==8.3.3; extra == "test"
 Requires-Dist: pytest-cov==5.0.0; extra == "test"
 Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: ruff==0.6.6; extra == "test"
 Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.11
+pip install cdsetool==0.2.13
 ```
 
 ## Usage

{cdsetool-0.2.11 → cdsetool-0.2.13}/src/cdsetool.egg-info/requires.txt

@@ -1,13 +1,13 @@
 typer<1,>=0.9
 rich<14,>=13.6
 requests<3,>=2.28.1
-pyjwt[crypto]~=2.8.0
+pyjwt[crypto]<2.10,>=2.8
 geopandas>=0.13.2
 
 [test]
-black==24.4.2
-pylint==3.2.2
-pytest==8.2.1
+pylint==3.3.1
+pytest==8.3.3
 pytest-cov==5.0.0
 requests-mock==1.12.1
+ruff==0.6.6
 pytest-mock==3.14.0