eodag 3.0.1__py3-none-any.whl → 3.1.0b2__py3-none-any.whl

eodag/utils/requests.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 import logging
 import os
-from typing import Any, Optional, Tuple
+from typing import Any, Optional
 
 import requests
 
@@ -30,7 +30,7 @@ logger = logging.getLogger("eodag.utils.requests")
 
 
 def fetch_json(
-    file_url: str,
+    url: str,
     req_session: Optional[requests.Session] = None,
     auth: Optional[requests.auth.AuthBase] = None,
     timeout: float = HTTP_REQ_TIMEOUT,
@@ -38,32 +38,32 @@ def fetch_json(
     """
     Fetches http/distant or local json file
 
-    :param file_url: url from which the file can be fetched
+    :param url: url from which the file can be fetched
     :param req_session: (optional) requests session
     :param auth: (optional) authenticated object if request needs authentication
     :param timeout: (optional) authenticated object
     :returns: json file content
     """
     if req_session is None:
-        req_session = requests.Session()
+        req_session = requests.sessions.Session()
     try:
-        if not file_url.lower().startswith("http"):
-            file_url = path_to_uri(os.path.abspath(file_url))
+        if not url.lower().startswith("http"):
+            url = path_to_uri(os.path.abspath(url))
             req_session.mount("file://", LocalFileAdapter())
 
         headers = USER_AGENT
-        logger.debug(f"fetching {file_url}")
+        logger.debug(f"fetching {url}")
         res = req_session.get(
-            file_url,
+            url,
             headers=headers,
             auth=auth,
             timeout=timeout,
         )
         res.raise_for_status()
     except requests.exceptions.Timeout as exc:
-        raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc
+        raise TimeOutError(exc, timeout=timeout) from exc
     except requests.exceptions.RequestException as exc:
-        raise RequestError.from_error(exc, f"Unable to fetch {file_url}") from exc
+        raise RequestError.from_error(exc, f"Unable to fetch {url}") from exc
     else:
         return res.json()
 
@@ -75,7 +75,7 @@ class LocalFileAdapter(requests.adapters.BaseAdapter):
     """
 
     @staticmethod
-    def _chkpath(method: str, path: str) -> Tuple[int, str]:
+    def _chkpath(method: str, path: str) -> tuple[int, str]:
         """Return an HTTP status for the given filesystem path.
 
         :param method: method of the request
@@ -100,8 +100,8 @@ class LocalFileAdapter(requests.adapters.BaseAdapter):
     ) -> requests.Response:
         """Wraps a file, described in request, in a Response object.
 
-        :param req: The PreparedRequest being "sent".
-        :param kwargs: (not used) additionnal arguments of the request
+        :param request: The PreparedRequest being "sent".
+        :param kwargs: (not used) additional arguments of the request
         :returns: a Response object containing the file
         """
         response = requests.Response()

eodag/utils/rest.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 
 import datetime
 import re
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Optional
 
 import dateutil.parser
 from dateutil import tz
@@ -35,7 +35,7 @@ RFC3339_PATTERN = (
 )
 
 
-def get_datetime(arguments: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]:
+def get_datetime(arguments: dict[str, Any]) -> tuple[Optional[str], Optional[str]]:
     """Get start and end dates from a dict containing `/` separated dates in `datetime` item
 
     :param arguments: dict containing a single date or `/` separated dates in `datetime` item

eodag/utils/s3.py

@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import io
+import logging
+import os
+import zipfile
+from typing import TYPE_CHECKING, List, Optional
+from urllib.parse import urlparse
+
+import boto3
+import botocore
+
+from eodag.plugins.authentication.aws_auth import AwsAuth
+from eodag.utils import get_bucket_name_and_prefix, guess_file_type
+from eodag.utils.exceptions import (
+    AuthenticationError,
+    MisconfiguredError,
+    NotAvailableError,
+)
+
+if TYPE_CHECKING:
+    from zipfile import ZipInfo
+
+    from mypy_boto3_s3.client import S3Client
+
+    from eodag.api.product import EOProduct  # type: ignore
+
+logger = logging.getLogger("eodag.utils.s3")
+
+
+def fetch(
+    bucket_name: str, key_name: str, start: int, len: int, client_s3: S3Client
+) -> bytes:
+    """
+    Range-fetches a S3 key.
+
+    :param bucket_name: Bucket name of the object to fetch
+    :param key_name: Key name of the object to fetch
+    :param start: Bucket name to fetch
+    :param len: Bucket name to fetch
+    :param client_s3: s3 client used to fetch the object
+    :returns: Object bytes
+    """
+    end = start + len - 1
+    s3_object = client_s3.get_object(
+        Bucket=bucket_name, Key=key_name, Range="bytes=%d-%d" % (start, end)
+    )
+    return s3_object["Body"].read()
+
+
+def parse_int(bytes: bytes) -> int:
+    """
+    Parses 2 or 4 little-endian bits into their corresponding integer value.
+
+    :param bytes: bytes to parse
+    :returns: parsed int
+    """
+    val = (bytes[0]) + ((bytes[1]) << 8)
+    if len(bytes) > 3:
+        val += ((bytes[2]) << 16) + ((bytes[3]) << 24)
+    return val
+
+
+def list_files_in_s3_zipped_object(
+    bucket_name: str, key_name: str, client_s3: S3Client
+) -> List[ZipInfo]:
+    """
+    List files in s3 zipped object, without downloading it.
+
+    See https://stackoverflow.com/questions/41789176/how-to-count-files-inside-zip-in-aws-s3-without-downloading-it;
+    Based on https://stackoverflow.com/questions/51351000/read-zip-files-from-s3-without-downloading-the-entire-file
+
+    :param bucket_name: Bucket name of the object to fetch
+    :param key_name: Key name of the object to fetch
+    :param client_s3: s3 client used to fetch the object
+    :returns: List of files in zip
+    """
+    response = client_s3.head_object(Bucket=bucket_name, Key=key_name)
+    size = response["ContentLength"]
+
+    # End Of Central Directory bytes
+    eocd = fetch(bucket_name, key_name, size - 22, 22, client_s3)
+
+    # start offset and size of the central directory
+    cd_start = parse_int(eocd[16:20])
+    cd_size = parse_int(eocd[12:16])
+
+    # fetch central directory, append EOCD, and open as zipfile
+    cd = fetch(bucket_name, key_name, cd_start, cd_size, client_s3)
+    zip = zipfile.ZipFile(io.BytesIO(cd + eocd))
+
+    logger.debug("Found %s files in %s" % (len(zip.filelist), key_name))
+
+    return zip.filelist
+
+
+def update_assets_from_s3(
+    product: EOProduct,
+    auth: AwsAuth,
+    s3_endpoint: Optional[str] = None,
+    content_url: Optional[str] = None,
+) -> None:
+    """Update ``EOProduct.assets`` using content listed in its ``remote_location`` or given
+    ``content_url``.
+
+    If url points to a zipped archive, its content will also be be listed.
+
+    :param product: product to update
+    :param auth: Authentication plugin
+    :param s3_endpoint: s3 endpoint if not hosted on AWS
+    :param content_url: s3 URL pointing to the content that must be listed (defaults to
+                        ``product.remote_location`` if empty)
+    """
+    required_creds = ["aws_access_key_id", "aws_secret_access_key"]
+
+    if content_url is None:
+        content_url = product.remote_location
+
+    bucket, prefix = get_bucket_name_and_prefix(content_url)
+
+    if bucket is None or prefix is None:
+        logger.debug(f"No s3 prefix could guessed from {content_url}")
+        return None
+
+    try:
+        auth_dict = auth.authenticate()
+
+        if not all(x in auth_dict for x in required_creds):
+            raise MisconfiguredError(
+                f"Incomplete credentials for {product.provider}, missing "
+                f"{[x for x in required_creds if x not in auth_dict]}"
+            )
+        if not getattr(auth, "s3_client", None):
+            auth.s3_client = boto3.client(
+                service_name="s3",
+                endpoint_url=s3_endpoint,
+                aws_access_key_id=auth_dict.get("aws_access_key_id"),
+                aws_secret_access_key=auth_dict.get("aws_secret_access_key"),
+                aws_session_token=auth_dict.get("aws_session_token"),
+            )
+
+        logger.debug("Listing assets in %s", prefix)
+
+        if prefix.endswith(".zip"):
+            # List prefix zip content
+            assets_urls = [
+                f"zip+s3://{bucket}/{prefix}!{f.filename}"
+                for f in list_files_in_s3_zipped_object(bucket, prefix, auth.s3_client)
+            ]
+        else:
+            # List files in prefix
+            assets_urls = [
+                f"s3://{bucket}/{obj['Key']}"
+                for obj in auth.s3_client.list_objects(
+                    Bucket=bucket, Prefix=prefix, MaxKeys=300
+                ).get("Contents", [])
+            ]
+
+        for asset_url in assets_urls:
+            out_of_zip_url = asset_url.split("!")[-1]
+            key, roles = product.driver.guess_asset_key_and_roles(
+                out_of_zip_url, product
+            )
+            parsed_url = urlparse(out_of_zip_url)
+            title = os.path.basename(parsed_url.path)
+
+            if key and key not in product.assets:
+                product.assets[key] = {
+                    "title": title,
+                    "roles": roles,
+                    "href": asset_url,
+                }
+                if mime_type := guess_file_type(asset_url):
+                    product.assets[key]["type"] = mime_type
+
+        # sort assets
+        product.assets.data = dict(sorted(product.assets.data.items()))
+
+        # update driver
+        product.driver = product.get_driver()
+
+    except botocore.exceptions.ClientError as e:
+        if hasattr(auth.config, "auth_error_code") and str(
+            auth.config.auth_error_code
+        ) in str(e):
+            raise AuthenticationError(
+                f"Authentication failed on {s3_endpoint} s3"
+            ) from e
+        raise NotAvailableError(
+            f"assets for product {prefix} could not be found"
+        ) from e

eodag/utils/stac_reader.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 import re
 import socket
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Optional, Union
 from urllib.error import URLError
 from urllib.request import urlopen
 
@@ -108,7 +108,7 @@ def fetch_stac_items(
     max_connections: int = 100,
     timeout: int = HTTP_REQ_TIMEOUT,
     ssl_verify: bool = True,
-) -> List[Dict[str, Any]]:
+) -> list[dict[str, Any]]:
     """Fetch STAC item from a single item file or items from a catalog.
 
     :param stac_path: A STAC object filepath
@@ -142,13 +142,13 @@ def _fetch_stac_items_from_catalog(
     recursive: bool,
     max_connections: int,
     _text_opener: Callable[[str, bool], Any],
-) -> List[Any]:
+) -> list[Any]:
     """Fetch items from a STAC catalog"""
-    items: List[Dict[Any, Any]] = []
+    items: list[dict[Any, Any]] = []
 
     # pystac cannot yet return links from a single file catalog, see:
     # https://github.com/stac-utils/pystac/issues/256
-    extensions: Optional[Union[List[str], str]] = getattr(cat, "stac_extensions", None)
+    extensions: Optional[Union[list[str], str]] = getattr(cat, "stac_extensions", None)
     if extensions:
         extensions = extensions if isinstance(extensions, list) else [extensions]
         if "single-file-stac" in extensions:
@@ -157,7 +157,7 @@ def _fetch_stac_items_from_catalog(
 
     # Making the links absolutes allow for both relative and absolute links to be handled.
     if not recursive:
-        hrefs: List[Optional[str]] = [
+        hrefs: list[Optional[str]] = [
             link.get_absolute_href() for link in cat.get_item_links()
         ]
     else:
@@ -188,7 +188,7 @@ def fetch_stac_collections(
     max_connections: int = 100,
     timeout: int = HTTP_REQ_TIMEOUT,
     ssl_verify: bool = True,
-) -> List[Dict[str, Any]]:
+) -> list[dict[str, Any]]:
     """Fetch STAC collection(s) from a catalog.
 
     :param stac_path: A STAC object filepath
@@ -217,12 +217,12 @@ def _fetch_stac_collections_from_catalog(
     collection: Optional[str],
     max_connections: int,
     _text_opener: Callable[[str, bool], Any],
-) -> List[Any]:
+) -> list[Any]:
     """Fetch collections from a STAC catalog"""
-    collections: List[Dict[Any, Any]] = []
+    collections: list[dict[Any, Any]] = []
 
     # Making the links absolutes allow for both relative and absolute links to be handled.
-    hrefs: List[Optional[str]] = [
+    hrefs: list[Optional[str]] = [
         link.get_absolute_href()
         for link in cat.get_child_links()
         if collection is not None and link.title == collection

{eodag-3.0.1.dist-info → eodag-3.1.0b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: eodag
-Version: 3.0.1
+Version: 3.1.0b2
 Summary: Earth Observation Data Access Gateway
 Home-page: https://github.com/CS-SI/eodag
 Author: CS GROUP - France
@@ -36,95 +36,96 @@ Requires-Dist: boto3
 Requires-Dist: botocore
 Requires-Dist: click
 Requires-Dist: geojson
-Requires-Dist: jsonpath-ng <1.6.0
+Requires-Dist: jsonpath-ng<1.6.0
 Requires-Dist: lxml
-Requires-Dist: pydantic >=2.1.0
-Requires-Dist: pydantic-core
-Requires-Dist: PyJWT >=2.5.0
-Requires-Dist: pyproj >=2.1.0
+Requires-Dist: orjson<3.10.0; python_version >= "3.12" and platform_system == "Windows"
+Requires-Dist: orjson; python_version < "3.12" or platform_system != "Windows"
+Requires-Dist: pydantic!=2.10.0,>=2.1.0
+Requires-Dist: pydantic_core
+Requires-Dist: PyJWT[crypto]>=2.5.0
+Requires-Dist: pyproj>=2.1.0
 Requires-Dist: pyshp
-Requires-Dist: pystac >=1.0.0b1
+Requires-Dist: pystac>=1.0.0b1
 Requires-Dist: python-dateutil
 Requires-Dist: PyYAML
 Requires-Dist: requests
 Requires-Dist: setuptools
-Requires-Dist: shapely >=2.0.6
+Requires-Dist: shapely>=2.0.6
 Requires-Dist: stream-zip
 Requires-Dist: tqdm
-Requires-Dist: typing-extensions >=4.8.0
+Requires-Dist: typing_extensions>=4.8.0
 Requires-Dist: urllib3
 Requires-Dist: Whoosh
-Requires-Dist: orjson ; python_version < "3.12" or platform_system != "Windows"
-Requires-Dist: orjson <3.10.0 ; python_version >= "3.12" and platform_system == "Windows"
 Provides-Extra: all
-Requires-Dist: eodag[all-providers,csw,server,tutorials] ; extra == 'all'
+Requires-Dist: eodag[all-providers,csw,server,tutorials]; extra == "all"
 Provides-Extra: all-providers
-Requires-Dist: eodag[ecmwf,usgs] ; extra == 'all-providers'
+Requires-Dist: eodag[ecmwf,usgs]; extra == "all-providers"
 Provides-Extra: csw
-Requires-Dist: OWSLib >=0.27.1 ; extra == 'csw'
-Provides-Extra: dev
-Requires-Dist: eodag[all-providers,csw,server,stubs] ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
-Requires-Dist: pytest-cov ; extra == 'dev'
-Requires-Dist: py >=1.8.2 ; extra == 'dev'
-Requires-Dist: pytest-html <3.2.0 ; extra == 'dev'
-Requires-Dist: pytest-xdist ; extra == 'dev'
-Requires-Dist: pytest-socket ; extra == 'dev'
-Requires-Dist: pytest-instafail ; extra == 'dev'
-Requires-Dist: tox ; extra == 'dev'
-Requires-Dist: tox-uv ; extra == 'dev'
-Requires-Dist: faker ; extra == 'dev'
-Requires-Dist: moto ; extra == 'dev'
-Requires-Dist: twine ; extra == 'dev'
-Requires-Dist: wheel ; extra == 'dev'
-Requires-Dist: flake8 ; extra == 'dev'
-Requires-Dist: pre-commit ; extra == 'dev'
-Requires-Dist: responses <0.24.0 ; extra == 'dev'
-Requires-Dist: fastapi[all] ; extra == 'dev'
-Requires-Dist: stdlib-list ; extra == 'dev'
-Requires-Dist: mypy ; extra == 'dev'
-Provides-Extra: docs
-Requires-Dist: eodag[all,stubs] ; extra == 'docs'
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-book-theme >=1.0.0 ; extra == 'docs'
-Requires-Dist: sphinx-copybutton ; extra == 'docs'
-Requires-Dist: sphinx-tabs ; extra == 'docs'
-Requires-Dist: nbsphinx ; extra == 'docs'
-Requires-Dist: sphinx-autodoc-typehints ; extra == 'docs'
-Requires-Dist: sphinxemoji ; extra == 'docs'
+Requires-Dist: OWSLib>=0.27.1; extra == "csw"
 Provides-Extra: ecmwf
-Requires-Dist: ecmwf-api-client ; extra == 'ecmwf'
-Provides-Extra: notebook
-Requires-Dist: tqdm[notebook] ; extra == 'notebook'
+Requires-Dist: ecmwf-api-client; extra == "ecmwf"
+Provides-Extra: usgs
+Requires-Dist: usgs>=0.3.1; extra == "usgs"
 Provides-Extra: server
-Requires-Dist: fastapi >=0.93.0 ; extra == 'server'
-Requires-Dist: pygeofilter ; extra == 'server'
-Requires-Dist: starlette ; extra == 'server'
-Requires-Dist: uvicorn[standard] ; extra == 'server'
-Requires-Dist: pydantic-settings ; extra == 'server'
-Requires-Dist: cachetools ; extra == 'server'
-Provides-Extra: stubs
-Requires-Dist: boto3-stubs[essential] ; extra == 'stubs'
-Requires-Dist: types-lxml ; extra == 'stubs'
-Requires-Dist: types-cachetools ; extra == 'stubs'
-Requires-Dist: types-requests ; extra == 'stubs'
-Requires-Dist: types-python-dateutil ; extra == 'stubs'
-Requires-Dist: types-setuptools ; extra == 'stubs'
-Requires-Dist: types-tqdm ; extra == 'stubs'
-Requires-Dist: types-urllib3 ; extra == 'stubs'
+Requires-Dist: fastapi>=0.93.0; extra == "server"
+Requires-Dist: pygeofilter; extra == "server"
+Requires-Dist: starlette; extra == "server"
+Requires-Dist: uvicorn[standard]; extra == "server"
+Requires-Dist: pydantic-settings; extra == "server"
+Requires-Dist: cachetools; extra == "server"
+Provides-Extra: notebook
+Requires-Dist: tqdm[notebook]; extra == "notebook"
 Provides-Extra: tutorials
-Requires-Dist: eodag[ecmwf,notebook] ; extra == 'tutorials'
-Requires-Dist: eodag-cube >=0.2.0 ; extra == 'tutorials'
-Requires-Dist: jupyter ; extra == 'tutorials'
-Requires-Dist: ipyleaflet >=0.10.0 ; extra == 'tutorials'
-Requires-Dist: ipywidgets ; extra == 'tutorials'
-Requires-Dist: matplotlib ; extra == 'tutorials'
-Requires-Dist: folium ; extra == 'tutorials'
-Requires-Dist: imageio ; extra == 'tutorials'
-Requires-Dist: rasterio ; extra == 'tutorials'
-Requires-Dist: netcdf4 ; extra == 'tutorials'
-Provides-Extra: usgs
-Requires-Dist: usgs >=0.3.1 ; extra == 'usgs'
+Requires-Dist: eodag[ecmwf,notebook]; extra == "tutorials"
+Requires-Dist: eodag-cube>=0.6.0b1; extra == "tutorials"
+Requires-Dist: jupyter; extra == "tutorials"
+Requires-Dist: ipyleaflet>=0.10.0; extra == "tutorials"
+Requires-Dist: ipywidgets; extra == "tutorials"
+Requires-Dist: matplotlib; extra == "tutorials"
+Requires-Dist: folium; extra == "tutorials"
+Requires-Dist: imageio; extra == "tutorials"
+Requires-Dist: rasterio; extra == "tutorials"
+Requires-Dist: netcdf4; extra == "tutorials"
+Requires-Dist: cartopy; extra == "tutorials"
+Provides-Extra: dev
+Requires-Dist: eodag[all-providers,csw,server,stubs]; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: py>=1.8.2; extra == "dev"
+Requires-Dist: pytest-html<3.2.0; extra == "dev"
+Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: pytest-socket; extra == "dev"
+Requires-Dist: pytest-instafail; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: tox-uv; extra == "dev"
+Requires-Dist: faker; extra == "dev"
+Requires-Dist: moto>=5; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: wheel; extra == "dev"
+Requires-Dist: flake8; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: responses<0.24.0; extra == "dev"
+Requires-Dist: fastapi[all]; extra == "dev"
+Requires-Dist: stdlib-list; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: stubs
+Requires-Dist: boto3-stubs[essential]; extra == "stubs"
+Requires-Dist: types-lxml; extra == "stubs"
+Requires-Dist: types-cachetools; extra == "stubs"
+Requires-Dist: types-requests; extra == "stubs"
+Requires-Dist: types-python-dateutil; extra == "stubs"
+Requires-Dist: types-setuptools; extra == "stubs"
+Requires-Dist: types-tqdm; extra == "stubs"
+Requires-Dist: types-urllib3; extra == "stubs"
+Provides-Extra: docs
+Requires-Dist: eodag[all,stubs]; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-book-theme>=1.0.0; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+Requires-Dist: sphinx-tabs; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: sphinx_autodoc_typehints; extra == "docs"
+Requires-Dist: sphinxemoji; extra == "docs"
 
 .. image:: https://eodag.readthedocs.io/en/latest/_static/eodag_bycs.png
     :target: https://github.com/CS-SI/eodag
@@ -315,7 +316,7 @@ An eodag instance can be exposed through a STAC compliant REST api from the comm
 
 .. code-block:: bash
 
-    docker run -p 5000:5000 --rm csspace/eodag-server:3.0.1
+    docker run -p 5000:5000 --rm csspace/eodag-server:3.1.0b2
 
 You can also browse over your STAC API server using `STAC Browser <https://github.com/radiantearth/stac-browser>`_.
 Simply run: