cdsetool 0.2.8__tar.gz → 0.2.10__tar.gz

{cdsetool-0.2.8/src/cdsetool.egg-info → cdsetool-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.8
+Version: 0.2.10
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
 Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
@@ -10,18 +10,18 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer~=0.9.0
+Requires-Dist: typer<0.13,>=0.9
 Requires-Dist: rich<13.8,>=13.6
-Requires-Dist: requests<2.32.0,>=2.28.1
+Requires-Dist: requests<2.33.0,>=2.28.1
 Requires-Dist: pyjwt[crypto]~=2.8.0
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.3.0; extra == "test"
-Requires-Dist: pylint==3.1.0; extra == "test"
-Requires-Dist: pytest==8.1.1; extra == "test"
-Requires-Dist: pytest-cov==4.1.0; extra == "test"
-Requires-Dist: requests-mock==1.11.0; extra == "test"
-Requires-Dist: pytest-mock==3.12.0; extra == "test"
+Requires-Dist: black==24.4.2; extra == "test"
+Requires-Dist: pylint==3.2.2; extra == "test"
+Requires-Dist: pytest==8.2.0; extra == "test"
+Requires-Dist: pytest-cov==5.0.0; extra == "test"
+Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
 
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.8
+pip install cdsetool==0.2.10
 ```
 
 ## Usage

{cdsetool-0.2.8 → cdsetool-0.2.10}/README.md

@@ -71,7 +71,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.8
+pip install cdsetool==0.2.10
 ```
 
 ## Usage

{cdsetool-0.2.8 → cdsetool-0.2.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdsetool"
-version = "0.2.8"
+version = "0.2.10"
 authors = [
   { name="Jacob Vejby", email="javej@sdfi.dk" },
 ]
@@ -16,20 +16,20 @@ classifiers = [
   "Operating System :: OS Independent",
 ]
 dependencies = [
-  "typer ~= 0.9.0",
+  "typer >= 0.9,< 0.13",
   "rich >= 13.6,< 13.8",
-  "requests >= 2.28.1,< 2.32.0",
+  "requests >= 2.28.1,< 2.33.0",
   "pyjwt[crypto] ~= 2.8.0",
   "geopandas >= 0.13.2",
 ]
 [project.optional-dependencies]
 test = [
-    "black==24.3.0",
-    "pylint==3.1.0",
-    "pytest==8.1.1",
-    "pytest-cov==4.1.0",
-    "requests-mock==1.11.0",
-    "pytest-mock==3.12.0",
+    "black==24.4.2",
+    "pylint==3.2.2",
+    "pytest==8.2.0",
+    "pytest-cov==5.0.0",
+    "requests-mock==1.12.1",
+    "pytest-mock==3.14.0",
 ]
 
 [project.urls]

{cdsetool-0.2.8 → cdsetool-0.2.10}/src/cdsetool/credentials.py

@@ -56,12 +56,27 @@ class TokenClientConnectionError(Exception):
     """
 
 
+class TokenExpiredSignatureError(Exception):
+    """
+    Raised when token signature has expired.
+    """
+
+
 class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-instance-attributes
     """
     A class for handling credentials for the Copernicus Identity
     and Access Management (IAM) system
     """
 
+    RETRY_CODES = frozenset([413, 429, 500, 502, 503])
+
+    RETRIES = Retry(
+        total=5,
+        backoff_factor=0.5,
+        raise_on_status=False,
+        status_forcelist=RETRY_CODES,
+    )
+
     def __init__(
         self,
         username=None,
@@ -73,12 +88,6 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
         self.__password = password
 
         self.__proxies = proxies
-        self.__retries = Retry(
-            total=5,
-            backoff_factor=0.5,
-            raise_on_status=False,
-            status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,
-        )
         self.__openid_conf = None
         self.__jwks = None
         self.__openid_configuration_endpoint = (
@@ -89,8 +98,8 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
 
         self.__access_token = None
         self.__refresh_token = None
-        self.__access_token_expires = datetime.now()
-        self.__refresh_token_expires = datetime.now()
+        self.__access_token_expires = datetime.now() - timedelta(hours=8)
+        self.__refresh_token_expires = self.__access_token_expires
 
         self.__lock = threading.Lock()
 
@@ -103,11 +112,16 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
         """
         Returns a session with the credentials set as the Authorization header
         """
-        return self.__make_session(True, self.__retries, self.__proxies)
+        return self.make_session(self, True, self.RETRIES, self.__proxies)
 
-    def __make_session(self, authorization, max_retries, proxies):
+    @staticmethod
+    def make_session(caller, authorization, max_retries, proxies):
+        """
+        Creates a new session. Authorization is only available from callers
+        that are subclasses of Credentials.
+        """
         if authorization:
-            self.__ensure_tokens()
+            caller.__ensure_tokens()  # pylint: disable=protected-access
 
         session = requests.Session()
         session.mount("http://", HTTPAdapter(max_retries=max_retries))
@@ -115,24 +129,25 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
         if proxies is not None:
             session.proxies.update(proxies)
         if authorization:
-            session.headers.update({"Authorization": f"Bearer {self.__access_token}"})
+            token = caller.__access_token  # pylint: disable=protected-access
+            session.headers.update({"Authorization": f"Bearer {token}"})
         return session
 
     def __token_exchange(self, data):
         # Make a session that will retry post, respecting the retry-after
         # header when we get a 503 and a few other temporary failures.
-        session = self.__make_session(
+        session = self.make_session(
+            caller=self,
             authorization=False,
             max_retries=Retry(
                 total=2,
                 backoff_factor=0.5,
                 allowed_methods=None,
                 raise_on_status=False,
-                status_forcelist=Retry.RETRY_AFTER_STATUS_CODES,
+                status_forcelist=self.RETRY_CODES,
             ),
             proxies=self.__proxies,
         )
-        now = datetime.now()
         response = session.post(self.__token_endpoint, data=data, timeout=120)
 
         if response.status_code == 401:
@@ -149,13 +164,11 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
 
         self.__access_token = response["access_token"]
         self.__refresh_token = response["refresh_token"]
-        self.__access_token_expires = now + timedelta(seconds=response["expires_in"])
-        self.__refresh_token_expires = now + timedelta(
-            seconds=response["refresh_expires_in"]
-        )
+        return timedelta(seconds=response["refresh_expires_in"])
 
     def __ensure_tokens(self):
         with self.__lock:
+            refresh_expire_delta = None
             if self.__access_token_expires < datetime.now():
                 if self.__refresh_token_expires < datetime.now():
                     data = {
@@ -170,17 +183,25 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
                         "refresh_token": self.__refresh_token,
                         "client_id": "cdse-public",
                     }
-                self.__token_exchange(data)
+                refresh_expire_delta = self.__token_exchange(data)
             try:
                 key = self.__jwk_client.get_signing_key_from_jwt(self.__access_token)
             except jwt.PyJWKClientConnectionError as e:
                 raise TokenClientConnectionError from e
-            jwt.decode(
-                self.__access_token,
-                key=key.key,
-                algorithms=key._algorithms,  # pylint: disable=protected-access
-                options={"verify_aud": False},
-            )
+            try:
+                data = jwt.decode(
+                    self.__access_token,
+                    key=key.key,
+                    algorithms=self.__id_token_signing_algos,  # pylint: disable=protected-access
+                    options={"verify_aud": False},
+                )
+            except jwt.ExpiredSignatureError as e:
+                raise TokenExpiredSignatureError from e
+            if refresh_expire_delta is not None:
+                self.__access_token_expires = datetime.fromtimestamp(data["exp"])
+                self.__refresh_token_expires = (
+                    datetime.fromtimestamp(data["iat"]) + refresh_expire_delta
+                )
 
     def __read_credentials(self):
         try:
@@ -195,8 +216,11 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
         if self.__openid_conf:
             return self.__openid_conf
 
-        session = self.__make_session(
-            authorization=False, max_retries=self.__retries, proxies=self.__proxies
+        session = self.make_session(
+            caller=self,
+            authorization=False,
+            max_retries=self.RETRIES,
+            proxies=self.__proxies,
         )
         response = session.get(self.__openid_configuration_endpoint, timeout=120)
         response.raise_for_status()
@@ -211,6 +235,10 @@ class Credentials:  # pylint: disable=too-few-public-methods disable=too-many-in
     def __jwks_uri(self):
         return self.__openid_configuration["jwks_uri"]
 
+    @property
+    def __id_token_signing_algos(self):
+        return self.__openid_configuration["id_token_signing_alg_values_supported"]
+
     @property
     def __jwk_client(self):
         if self.__jwks:

{cdsetool-0.2.8 → cdsetool-0.2.10}/src/cdsetool/download.py

@@ -10,8 +10,14 @@ import random
 import tempfile
 import time
 import shutil
+from requests.exceptions import ChunkedEncodingError
+from urllib3.exceptions import ProtocolError
 from cdsetool._processing import _concurrent_process
-from cdsetool.credentials import Credentials, TokenClientConnectionError
+from cdsetool.credentials import (
+    Credentials,
+    TokenClientConnectionError,
+    TokenExpiredSignatureError,
+)
 from cdsetool.logger import NoopLogger
 from cdsetool.monitor import NoopMonitor
 
@@ -39,36 +45,44 @@ def download_feature(feature, path, options=None):
         return filename
 
     with _get_monitor(options).status() as status:
-        (fd, tmp) = tempfile.mkstemp()  # pylint: disable=invalid-name
         status.set_filename(filename)
         attempts = 0
-        while attempts < 5:
+        while attempts < 10:
+            attempts += 1
             # Always get a new session, credentials might have expired.
             try:
                 session = _get_credentials(options).get_session()
-            except TokenClientConnectionError as e:
+            except (TokenClientConnectionError, TokenExpiredSignatureError) as e:
                 log.warning(e)
                 continue
             url = _follow_redirect(url, session)
             with session.get(url, stream=True) as response:
                 if response.status_code != 200:
                     log.warning(f"Status code {response.status_code}, retrying..")
-                    attempts += 1
                     time.sleep(60 * (1 + (random.random() / 4)))
                     continue
 
                 status.set_filesize(int(response.headers["Content-Length"]))
 
-                with open(fd, "wb") as file:
-                    for chunk in response.iter_content(chunk_size=1024 * 1024 * 5):
-                        file.write(chunk)
-                        status.add_progress(len(chunk))
+                with tempfile.NamedTemporaryFile() as file:
+                    # Server might not send all bytes specified by the
+                    # Content-Length header before closing connection.
+                    # Log as a warning and try again.
+                    try:
+                        for chunk in response.iter_content(chunk_size=1024 * 1024 * 5):
+                            file.write(chunk)
+                            status.add_progress(len(chunk))
+                    except (
+                        ChunkedEncodingError,
+                        ConnectionResetError,
+                        ProtocolError,
+                    ) as e:
+                        log.warning(e)
+                        continue
+                    shutil.copy(file.name, result_path)
 
-                shutil.move(tmp, result_path)
                 return filename
     log.error(f"Failed to download {filename}")
-    os.close(fd)
-    os.remove(tmp)
     return None
 
 

{cdsetool-0.2.8 → cdsetool-0.2.10}/src/cdsetool/query.py

@@ -8,8 +8,8 @@ from xml.etree import ElementTree
 from datetime import datetime, date
 import re
 import json
-import requests
 import geopandas as gpd
+from cdsetool.credentials import Credentials
 
 
 class _FeatureIterator:
@@ -67,7 +67,10 @@ class FeatureQuery:
     def __fetch_features(self):
         if self.next_url is None:
             return
-        with requests.get(self.next_url, timeout=120, proxies=self.proxies) as response:
+        session = Credentials.make_session(
+            None, False, Credentials.RETRIES, self.proxies
+        )
+        with session.get(self.next_url) as response:
             response.raise_for_status()
             res = response.json()
             self.features += res.get("features") or []
@@ -236,18 +239,17 @@ _describe_docs = {}
 def _get_describe_doc(collection, proxies=None):
     if _describe_docs.get(collection):
         return _describe_docs.get(collection)
-    res = requests.get(
+    session = Credentials.make_session(None, False, Credentials.RETRIES, proxies)
+    with session.get(
         "https://catalogue.dataspace.copernicus.eu"
         + f"/resto/api/collections/{collection}/describe.xml",
-        timeout=120,
-        proxies=proxies,
-    )
-    assert res.status_code == 200, (
-        f"Unable to find collection with name {collection}. Please see "
-        + "https://documentation.dataspace.copernicus.eu"
-        + "/APIs/OpenSearch.html#collections "
-        + "for a list of available collections"
-    )
+    ) as res:
+        assert res.status_code == 200, (
+            f"Unable to find collection with name {collection}. Please see "
+            + "https://documentation.dataspace.copernicus.eu"
+            + "/APIs/OpenSearch.html#collections "
+            + "for a list of available collections"
+        )
 
-    _describe_docs[collection] = res.content
+        _describe_docs[collection] = res.content
     return _describe_docs.get(collection)

{cdsetool-0.2.8 → cdsetool-0.2.10/src/cdsetool.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cdsetool
-Version: 0.2.8
+Version: 0.2.10
 Summary: Tools & CLI for interacting with CDSE product APIs
 Author-email: Jacob Vejby <javej@sdfi.dk>
 Project-URL: Homepage, https://github.com/SDFIdk/CDSETool
@@ -10,18 +10,18 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer~=0.9.0
+Requires-Dist: typer<0.13,>=0.9
 Requires-Dist: rich<13.8,>=13.6
-Requires-Dist: requests<2.32.0,>=2.28.1
+Requires-Dist: requests<2.33.0,>=2.28.1
 Requires-Dist: pyjwt[crypto]~=2.8.0
 Requires-Dist: geopandas>=0.13.2
 Provides-Extra: test
-Requires-Dist: black==24.3.0; extra == "test"
-Requires-Dist: pylint==3.1.0; extra == "test"
-Requires-Dist: pytest==8.1.1; extra == "test"
-Requires-Dist: pytest-cov==4.1.0; extra == "test"
-Requires-Dist: requests-mock==1.11.0; extra == "test"
-Requires-Dist: pytest-mock==3.12.0; extra == "test"
+Requires-Dist: black==24.4.2; extra == "test"
+Requires-Dist: pylint==3.2.2; extra == "test"
+Requires-Dist: pytest==8.2.0; extra == "test"
+Requires-Dist: pytest-cov==5.0.0; extra == "test"
+Requires-Dist: requests-mock==1.12.1; extra == "test"
+Requires-Dist: pytest-mock==3.14.0; extra == "test"
 
 # CDSETool
 
@@ -96,7 +96,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
 Install `cdsetool` using pip:
 
 ```bash
-pip install cdsetool==0.2.8
+pip install cdsetool==0.2.10
 ```
 
 ## Usage

cdsetool-0.2.10/src/cdsetool.egg-info/requires.txt

@@ -0,0 +1,13 @@
+typer<0.13,>=0.9
+rich<13.8,>=13.6
+requests<2.33.0,>=2.28.1
+pyjwt[crypto]~=2.8.0
+geopandas>=0.13.2
+
+[test]
+black==24.4.2
+pylint==3.2.2
+pytest==8.2.0
+pytest-cov==5.0.0
+requests-mock==1.12.1
+pytest-mock==3.14.0

cdsetool-0.2.8/src/cdsetool.egg-info/requires.txt

@@ -1,13 +0,0 @@
-typer~=0.9.0
-rich<13.8,>=13.6
-requests<2.32.0,>=2.28.1
-pyjwt[crypto]~=2.8.0
-geopandas>=0.13.2
-
-[test]
-black==24.3.0
-pylint==3.1.0
-pytest==8.1.1
-pytest-cov==4.1.0
-requests-mock==1.11.0
-pytest-mock==3.12.0