cdsetool 0.2.7__tar.gz → 0.2.9__tar.gz

{cdsetool-0.2.7/src/cdsetool.egg-info → cdsetool-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.7
+Version: 0.2.9
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
 Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
@@ -10,18 +10,18 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer~=0.9.0
+Requires-Dist: typer<0.13,>=0.9
 Requires-Dist: rich<13.8,>=13.6
 Requires-Dist: requests<2.32.0,>=2.28.1
 Requires-Dist: pyjwt[crypto]~=2.8.0
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.2.0; extra == "test"
-Requires-Dist: pylint==3.0.3; extra == "test"
-Requires-Dist: pytest==8.1.1; extra == "test"
-Requires-Dist: pytest-cov==4.1.0; extra == "test"
-Requires-Dist: requests-mock==1.11.0; extra == "test"
-Requires-Dist: pytest-mock==3.12.0; extra == "test"
+Requires-Dist: black==24.4.2; extra == "test"
+Requires-Dist: pylint==3.1.0; extra == "test"
+Requires-Dist: pytest==8.2.0; extra == "test"
+Requires-Dist: pytest-cov==5.0.0; extra == "test"
+Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
 
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.7
+pip install cdsetool==0.2.9
 ```
 
 ## Usage

{cdsetool-0.2.7 → cdsetool-0.2.9}/README.md

@@ -71,7 +71,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.7
+pip install cdsetool==0.2.9
 ```
 
 ## Usage

{cdsetool-0.2.7 → cdsetool-0.2.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdsetool"
-version = "0.2.7"
+version = "0.2.9"
 authors = [
   { name="Jacob Vejby", email="javej@sdfi.dk" },
 ]
@@ -16,7 +16,7 @@ classifiers = [
   "Operating System :: OS Independent",
 ]
 dependencies = [
-  "typer ~= 0.9.0",
+  "typer >= 0.9,< 0.13",
   "rich >= 13.6,< 13.8",
   "requests >= 2.28.1,< 2.32.0",
   "pyjwt[crypto] ~= 2.8.0",
@@ -24,12 +24,12 @@ dependencies = [
 ]
 [project.optional-dependencies]
 test = [
-    "black==24.2.0",
-    "pylint==3.0.3",
-    "pytest==8.1.1",
-    "pytest-cov==4.1.0",
-    "requests-mock==1.11.0",
-    "pytest-mock==3.12.0",
+    "black==24.4.2",
+    "pylint==3.1.0",
+    "pytest==8.2.0",
+    "pytest-cov==5.0.0",
+    "requests-mock==1.12.1",
+    "pytest-mock==3.14.0",
 ]
 
 [project.urls]

cdsetool-0.2.9/src/cdsetool/_processing.py

@@ -0,0 +1,57 @@
+"""
+This module provides functions for processing data concurrently
+"""
+
+from concurrent.futures import wait, FIRST_COMPLETED
+from concurrent.futures import ThreadPoolExecutor
+
+
+def _concurrent_process(fun, iterable, workers=4):
+    """
+    Process items in an iterable concurrently
+
+    Items are taken from the iterable as soon as a worker becomes available
+
+    Returns an iterable of the results
+    """
+
+    # Futures are submitted in batches instead of all at once to avoid
+    # requesting too many items from the iterable at once, which is important if
+    # the iterable is a generator that is producing items on the fly.
+    #
+    # The 1.5 factor is a small overhead to keep jobs > workers at all times, instead
+    # of jobs == workers, which could cause the workers to be idle while waiting for
+    # the iterable to produce more items.
+    low_water_mark = int(workers * 1.5)
+    iterator = iter(iterable)
+
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = []
+
+        # Pluck an item from the iterable and submit it to the executor.
+        # If the iterable is exhausted, this function is a no-op.
+        def submit_item():
+            item = next(iterator, None)
+            if item is not None:
+                futures.append(executor.submit(fun, item))
+
+        # Fill the futures list up to the low water mark
+        def fill_futures():
+            for _ in range(low_water_mark - len(futures)):
+                submit_item()
+
+        # Submit the first batch of items
+        fill_futures()
+
+        # Continue until no more futures are queued
+        while futures:
+
+            # Wait for the first future(s) to complete
+            done, futures = wait(futures, return_when=FIRST_COMPLETED)
+            futures = list(futures)
+
+            for future in done:
+                yield future.result()
+
+            # Submit items to replace the ones that are done.
+            fill_futures()

{cdsetool-0.2.7 → cdsetool-0.2.9}/src/cdsetool/credentials.py

@@ -50,12 +50,26 @@ class TokenExchangeException(Exception):
     """
 
 
+class TokenClientConnectionError(Exception):
+    """
+    Raised when token connection fails.
+    """
+
+
+class TokenExpiredSignatureError(Exception):
+    """
+    Raised when token signature has expired.
+    """
+
+
 class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-instance-attributes
     """
     A class for handling credentials for the Copernicus Identity
     and Access Management (IAM) system
     """
 
+    RETRY_CODES = frozenset([413, 429, 500, 502, 503])
+
     def __init__(
         self,
         username=None,
@@ -71,7 +85,7 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
             total=5,
             backoff_factor=0.5,
             raise_on_status=False,
-            status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,
+            status_forcelist=self.RETRY_CODES,
         )
         self.__openid_conf = None
         self.__jwks = None
@@ -83,8 +97,8 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
 
         self.__access_token = None
         self.__refresh_token = None
-        self.__access_token_expires = datetime.now()
-        self.__refresh_token_expires = datetime.now()
+        self.__access_token_expires = datetime.now() - timedelta(hours=8)
+        self.__refresh_token_expires = self.__access_token_expires
 
         self.__lock = threading.Lock()
 
@@ -122,7 +136,7 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
                 backoff_factor=0.5,
                 allowed_methods=None,
                 raise_on_status=False,
-                status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,
+                status_forcelist=self.RETRY_CODES,
             ),
             proxies=self.__proxies,
         )
@@ -165,13 +179,19 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
                         "client_id": "cdse-public",
                     }
                 self.__token_exchange(data)
-            key = self.__jwk_client.get_signing_key_from_jwt(self.__access_token)
-            jwt.decode(
-                self.__access_token,
-                key=key.key,
-                algorithms=key._algorithms,  # pylint: disable=protected-access
-                options={"verify_aud": False},
-            )
+            try:
+                key = self.__jwk_client.get_signing_key_from_jwt(self.__access_token)
+            except jwt.PyJWKClientConnectionError as e:
+                raise TokenClientConnectionError from e
+            try:
+                jwt.decode(
+                    self.__access_token,
+                    key=key.key,
+                    algorithms=self.__id_token_signing_algos,  # pylint: disable=protected-access
+                    options={"verify_aud": False},
+                )
+            except jwt.ExpiredSignatureError as e:
+                raise TokenExpiredSignatureError from e
 
     def __read_credentials(self):
         try:
@@ -202,6 +222,10 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
     def __jwks_uri(self):
         return self.__openid_configuration["jwks_uri"]
 
+    @property
+    def __id_token_signing_algos(self):
+        return self.__openid_configuration["id_token_signing_alg_values_supported"]
+
     @property
     def __jwk_client(self):
         if self.__jwks:

{cdsetool-0.2.7 → cdsetool-0.2.9}/src/cdsetool/download.py

@@ -10,8 +10,14 @@ import random
 import tempfile
 import time
 import shutil
+from requests.exceptions import ChunkedEncodingError
+from urllib3.exceptions import ProtocolError
 from cdsetool._processing import _concurrent_process
-from cdsetool.credentials import Credentials
+from cdsetool.credentials import (
+    Credentials,
+    TokenClientConnectionError,
+    TokenExpiredSignatureError,
+)
 from cdsetool.logger import NoopLogger
 from cdsetool.monitor import NoopMonitor
 
@@ -39,34 +45,44 @@ def download_feature(feature, path, options=None):
         return filename
 
     with _get_monitor(options).status() as status:
-        (fd, tmp) = tempfile.mkstemp()  # pylint: disable=invalid-name
         status.set_filename(filename)
         attempts = 0
-        while attempts < 5:
+        while attempts < 10:
+            attempts += 1
             # Always get a new session, credentials might have expired.
-            session = _get_credentials(options).get_session()
+            try:
+                session = _get_credentials(options).get_session()
+            except (TokenClientConnectionError, TokenExpiredSignatureError) as e:
+                log.warning(e)
+                continue
             url = _follow_redirect(url, session)
             with session.get(url, stream=True) as response:
                 if response.status_code != 200:
                     log.warning(f"Status code {response.status_code}, retrying..")
-                    attempts += 1
                     time.sleep(60 * (1 + (random.random() / 4)))
                     continue
 
-                content_length = int(response.headers["Content-Length"])
+                status.set_filesize(int(response.headers["Content-Length"]))
+
+                with tempfile.NamedTemporaryFile() as file:
+                    # Server might not send all bytes specified by the
+                    # Content-Length header before closing connection.
+                    # Log as a warning and try again.
+                    try:
+                        for chunk in response.iter_content(chunk_size=1024 * 1024 * 5):
+                            file.write(chunk)
+                            status.add_progress(len(chunk))
+                    except (
+                        ChunkedEncodingError,
+                        ConnectionResetError,
+                        ProtocolError,
+                    ) as e:
+                        log.warning(e)
+                        continue
+                    shutil.copy(file.name, result_path)
 
-                status.set_filesize(content_length)
-
-                with open(fd, "wb") as file:
-                    for chunk in response.iter_content(chunk_size=1024 * 1024 * 5):
-                        file.write(chunk)
-                        status.add_progress(len(chunk))
-
-                shutil.move(tmp, result_path)
                 return filename
     log.error(f"Failed to download {filename}")
-    os.close(fd)
-    os.remove(tmp)
     return None
 
 
@@ -87,10 +103,9 @@ def download_features(features, path, options=None):
     def _download_feature(feature):
         return download_feature(feature, path, options)
 
-    for feature in _concurrent_process(
+    yield from _concurrent_process(
         _download_feature, features, options.get("concurrency", 1)
-    ):
-        yield feature
+    )
 
     options["monitor"].stop()
 

{cdsetool-0.2.7 → cdsetool-0.2.9}/src/cdsetool/query.py

@@ -65,8 +65,11 @@ class FeatureQuery:
         return self.features[index]
 
     def __fetch_features(self):
-        if self.next_url is not None:
-            res = requests.get(self.next_url, timeout=120, proxies=self.proxies).json()
+        if self.next_url is None:
+            return
+        with requests.get(self.next_url, timeout=120, proxies=self.proxies) as response:
+            response.raise_for_status()
+            res = response.json()
             self.features += res.get("features") or []
 
             total_results = res.get("properties", {}).get("totalResults")

{cdsetool-0.2.7 → cdsetool-0.2.9/src/cdsetool.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.7
+Version: 0.2.9
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
 Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
@@ -10,18 +10,18 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer~=0.9.0
+Requires-Dist: typer<0.13,>=0.9
 Requires-Dist: rich<13.8,>=13.6
 Requires-Dist: requests<2.32.0,>=2.28.1
 Requires-Dist: pyjwt[crypto]~=2.8.0
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.2.0; extra == "test"
-Requires-Dist: pylint==3.0.3; extra == "test"
-Requires-Dist: pytest==8.1.1; extra == "test"
-Requires-Dist: pytest-cov==4.1.0; extra == "test"
-Requires-Dist: requests-mock==1.11.0; extra == "test"
-Requires-Dist: pytest-mock==3.12.0; extra == "test"
+Requires-Dist: black==24.4.2; extra == "test"
+Requires-Dist: pylint==3.1.0; extra == "test"
+Requires-Dist: pytest==8.2.0; extra == "test"
+Requires-Dist: pytest-cov==5.0.0; extra == "test"
+Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
 
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.7
+pip install cdsetool==0.2.9
 ```
 
 ## Usage

cdsetool-0.2.9/src/cdsetool.egg-info/requires.txt

@@ -0,0 +1,13 @@
+typer<0.13,>=0.9
+rich<13.8,>=13.6
+requests<2.32.0,>=2.28.1
+pyjwt[crypto]~=2.8.0
+geopandas>=0.13.2
+
+[test]
+black==24.4.2
+pylint==3.1.0
+pytest==8.2.0
+pytest-cov==5.0.0
+requests-mock==1.12.1
+pytest-mock==3.14.0

cdsetool-0.2.7/src/cdsetool/_processing.py

@@ -1,38 +0,0 @@
-"""
-This module provides functions for processing data concurrently
-"""
-
-from concurrent.futures import wait, FIRST_COMPLETED
-from concurrent.futures import ThreadPoolExecutor
-
-
-def _concurrent_process(fun, iterable, workers=4):
-    """
-    Process items in an iterable concurrently
-
-    Items are taken from the iterable as soon as a worker becomes available
-
-    Returns an iterable of the results
-    """
-    low_water_mark = int(workers * 1.5)
-    iterator = iter(iterable)
-
-    with ThreadPoolExecutor(max_workers=workers) as executor:
-        futures = []
-
-        def submit_item():
-            item = next(iterator, None)
-            if item is not None:
-                futures.append(executor.submit(fun, item))
-
-        for _ in range(low_water_mark):
-            submit_item()
-
-        while futures:
-            done, futures = wait(futures, return_when=FIRST_COMPLETED)
-            futures = list(futures)
-            for future in done:
-                yield future.result()
-
-            if len(futures) < low_water_mark:
-                submit_item()

cdsetool-0.2.7/src/cdsetool.egg-info/requires.txt

@@ -1,13 +0,0 @@
-typer~=0.9.0
-rich<13.8,>=13.6
-requests<2.32.0,>=2.28.1
-pyjwt[crypto]~=2.8.0
-geopandas>=0.13.2
-
-[test]
-black==24.2.0
-pylint==3.0.3
-pytest==8.1.1
-pytest-cov==4.1.0
-requests-mock==1.11.0
-pytest-mock==3.12.0