eodag 2.12.0__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

eodag/plugins/search/cop_marine.py

@@ -0,0 +1,378 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import copy
+import logging
+import re
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+from urllib.parse import urlsplit
+
+import boto3
+import botocore
+import requests
+from dateutil.parser import isoparse
+from dateutil.tz import tzutc
+from dateutil.utils import today
+
+from eodag import EOProduct
+from eodag.api.product import AssetsDict
+from eodag.config import PluginConfig
+from eodag.plugins.search import PreparedSearch
+from eodag.plugins.search.static_stac_search import StaticStacSearch
+from eodag.utils import get_bucket_name_and_prefix
+from eodag.utils.exceptions import UnsupportedProductType, ValidationError
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3 import S3Client
+    from mypy_boto3_s3.type_defs import ListObjectsOutputTypeDef
+
+logger = logging.getLogger("eodag.search.cop_marine")
+
+
+def _get_date_from_yyyymmdd(date_str: str, item_key: str) -> Optional[datetime]:
+    year = date_str[:4]
+    month = date_str[4:6]
+    if len(date_str) > 6:
+        day = date_str[6:]
+    else:
+        day = "1"
+    try:
+        date = datetime(
+            int(year),
+            int(month),
+            int(day),
+            tzinfo=tzutc(),
+        )
+    except ValueError:
+        logger.error(f"{item_key}: {date_str} is not a valid date")
+        return None
+    else:
+        return date
+
+
+def _get_s3_client(endpoint_url: str) -> S3Client:
+    s3_session = boto3.Session()
+    return s3_session.client(
+        "s3",
+        config=botocore.config.Config(
+            # Configures to use subdomain/virtual calling format.
+            s3={"addressing_style": "virtual"},
+            signature_version=botocore.UNSIGNED,
+        ),
+        endpoint_url=endpoint_url,
+    )
+
+
+def _check_int_values_properties(properties: Dict[str, Any]):
+    # remove int values with a bit length of more than 64 from the properties
+    invalid = []
+    for prop, prop_value in properties.items():
+        if isinstance(prop_value, int) and prop_value.bit_length() > 64:
+            invalid.append(prop)
+        if isinstance(prop_value, dict):
+            _check_int_values_properties(prop_value)
+
+    for inv_key in invalid:
+        properties.pop(inv_key)
+
+
+class CopMarineSearch(StaticStacSearch):
+    """class that implements search for the Copernicus Marine provider"""
+
+    def __init__(self, provider: str, config: PluginConfig):
+        original_metadata_mapping = copy.deepcopy(config.metadata_mapping)
+        super().__init__(provider, config)
+        # reset to original metadata mapping from config (changed in super class init)
+        self.config.metadata_mapping = original_metadata_mapping
+
+    def _get_product_type_info(
+        self, product_type: str
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        """Fetch product type and associated datasets info"""
+
+        fetch_url = cast(
+            str,
+            self.config.discover_product_types["fetch_url"].format(
+                **self.config.__dict__
+            ),
+        )
+        logger.debug("fetch data for collection %s", product_type)
+        provider_product_type = self.config.products.get(product_type, {}).get(
+            "productType", None
+        )
+        if not provider_product_type:
+            provider_product_type = product_type
+        collection_url = (
+            fetch_url.replace("catalog.stac.json", provider_product_type)
+            + "/product.stac.json"
+        )
+        try:
+            collection_data = requests.get(collection_url).json()
+        except requests.RequestException:
+            logger.error("data for product %s could not be fetched", product_type)
+            raise UnsupportedProductType(product_type)
+
+        datasets = []
+        for link in [li for li in collection_data["links"] if li["rel"] == "item"]:
+            dataset_url = (
+                fetch_url.replace("catalog.stac.json", provider_product_type)
+                + "/"
+                + link["href"]
+            )
+            try:
+                dataset_item = requests.get(dataset_url).json()
+                datasets.append(dataset_item)
+            except requests.RequestException:
+                logger.error("data for dataset %s could not be fetched", link["title"])
+
+        return collection_data, datasets
+
+    def _get_product_by_id(
+        self,
+        collection_objects: ListObjectsOutputTypeDef,
+        product_id: str,
+        s3_url: str,
+        product_type: str,
+        dataset_item: Dict[str, Any],
+        collection_dict: Dict[str, Any],
+    ):
+        for obj in collection_objects["Contents"]:
+            if product_id in obj["Key"]:
+                return self._create_product(
+                    product_type, obj["Key"], s3_url, dataset_item, collection_dict
+                )
+        return None
+
+    def _create_product(
+        self,
+        product_type: str,
+        item_key: str,
+        s3_url: str,
+        dataset_item: Dict[str, Any],
+        collection_dict: Dict[str, Any],
+        use_dataset_dates: bool = False,
+    ) -> Optional[EOProduct]:
+
+        item_id = item_key.split("/")[-1].split(".")[0]
+        download_url = s3_url + "/" + item_key
+        properties = {
+            "id": item_id,
+            "title": item_id,
+            "geometry": self.config.metadata_mapping["defaultGeometry"],
+            "downloadLink": download_url,
+            "dataset": dataset_item["id"],
+        }
+        if use_dataset_dates:
+            if "start_datetime" in dataset_item:
+                properties["startTimeFromAscendingNode"] = dataset_item[
+                    "start_datetime"
+                ]
+                properties["completionTimeFromAscendingNode"] = dataset_item[
+                    "end_datetime"
+                ]
+            elif "datetime" in dataset_item:
+                properties["startTimeFromAscendingNode"] = dataset_item["datetime"]
+                properties["completionTimeFromAscendingNode"] = dataset_item["datetime"]
+        else:
+            item_dates = re.findall(r"\d{8}", item_key)
+            if not item_dates:
+                item_dates = re.findall(r"\d{6}", item_key)
+            item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
+            if not item_start:  # identified pattern was not a valid datetime
+                return None
+            if len(item_dates) > 2:  # start, end and created_at timestamps
+                item_end = _get_date_from_yyyymmdd(item_dates[1], item_key)
+            else:  # only date and created_at timestamps
+                item_end = item_start
+            properties["startTimeFromAscendingNode"] = item_start.strftime(
+                "%Y-%m-%dT%H:%M:%SZ"
+            )
+            properties["completionTimeFromAscendingNode"] = (
+                item_end or item_start
+            ).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        for key, value in collection_dict["properties"].items():
+            if key not in ["id", "title", "start_datetime", "end_datetime"]:
+                properties[key] = value
+        for key, value in dataset_item["properties"].items():
+            if key not in ["id", "title", "start_datetime", "end_datetime"]:
+                properties[key] = value
+        _check_int_values_properties(properties)
+
+        properties["thumbnail"] = collection_dict["assets"]["thumbnail"]["href"]
+        if "omiFigure" in collection_dict["assets"]:
+            properties["quicklook"] = collection_dict["assets"]["omiFigure"]["href"]
+        assets = {
+            "native": {
+                "title": "native",
+                "href": download_url,
+                "type": "application/x-netcdf",
+            }
+        }
+        product = EOProduct(self.provider, properties, productType=product_type)
+        product.assets = AssetsDict(product, assets)
+        return product
+
+    def query(
+        self,
+        prep: PreparedSearch = PreparedSearch(),
+        **kwargs: Any,
+    ) -> Tuple[List[EOProduct], Optional[int]]:
+        """
+        Implementation of search for the Copernicus Marine provider
+        :param prep: object containing search parameterds
+        :type prep: PreparedSearch
+        :param kwargs: additional search arguments
+        :returns: list of products and total number of products
+        :rtype: Tuple[List[EOProduct], Optional[int]]
+        """
+        page = prep.page
+        items_per_page = prep.items_per_page
+
+        # only return 1 page if pagination is disabled
+        if page > 1 and items_per_page <= 0:
+            return ([], 0) if prep.count else ([], None)
+
+        product_type = kwargs.get("productType", prep.product_type)
+        if not product_type:
+            raise ValidationError(
+                "parameter product type is required for search with cop_marine provider"
+            )
+        collection_dict, datasets_items_list = self._get_product_type_info(product_type)
+        products: List[EOProduct] = []
+        start_index = items_per_page * (page - 1) + 1
+        num_total = 0
+        for i, dataset_item in enumerate(datasets_items_list):
+            try:
+                logger.debug("searching data for dataset %s", dataset_item["id"])
+
+                # date bounds
+                if "startTimeFromAscendingNode" in kwargs:
+                    start_date = isoparse(kwargs["startTimeFromAscendingNode"])
+                elif "start_datetime" in dataset_item["properties"]:
+                    start_date = isoparse(dataset_item["properties"]["start_datetime"])
+                else:
+                    start_date = isoparse(dataset_item["properties"]["datetime"])
+                if not start_date.tzinfo:
+                    start_date = start_date.replace(tzinfo=tzutc())
+                if "completionTimeFromAscendingNode" in kwargs:
+                    end_date = isoparse(kwargs["completionTimeFromAscendingNode"])
+                elif "end_datetime" in dataset_item["properties"]:
+                    end_date = isoparse(dataset_item["properties"]["end_datetime"])
+                else:
+                    end_date = today(tzinfo=tzutc())
+                if not end_date.tzinfo:
+                    end_date = end_date.replace(tzinfo=tzutc())
+
+                # retrieve information about s3 from collection data
+                s3_url = dataset_item["assets"]["native"]["href"]
+            except KeyError as e:
+                logger.warning(
+                    f"Unable to extract info from {product_type} item #{i}: {str(e)}"
+                )
+                continue
+
+            url_parts = urlsplit(s3_url)
+            endpoint_url = url_parts.scheme + "://" + url_parts.hostname
+            bucket, collection_path = get_bucket_name_and_prefix(s3_url, 0)
+            if bucket is None or collection_path is None:
+                logger.warning(
+                    f"Unable to get bucket and prefix from {s3_url}, got {(bucket, collection_path)}"
+                )
+                continue
+
+            if ".nc" in collection_path:
+                num_total += 1
+                if num_total < start_index:
+                    continue
+                if len(products) < items_per_page or items_per_page < 0:
+                    product = self._create_product(
+                        product_type,
+                        collection_path,
+                        endpoint_url + "/" + bucket,
+                        dataset_item,
+                        collection_dict,
+                        True,
+                    )
+                    if product:
+                        products.append(product)
+                    continue
+
+            s3_client = _get_s3_client(endpoint_url)
+            stop_search = False
+            current_object = None
+            while not stop_search:
+                # list_objects returns max 1000 objects -> use marker to get next objects
+                if current_object:
+                    s3_objects = s3_client.list_objects(
+                        Bucket=bucket, Prefix=collection_path, Marker=current_object
+                    )
+                else:
+                    s3_objects = s3_client.list_objects(
+                        Bucket=bucket, Prefix=collection_path
+                    )
+                if "Contents" not in s3_objects:
+                    if len(products) == 0 and i == len(datasets_items_list) - 1:
+                        return ([], 0) if prep.count else ([], None)
+                    else:
+                        break
+
+                if "id" in kwargs:
+                    product = self._get_product_by_id(
+                        s3_objects,
+                        kwargs["id"],
+                        endpoint_url + "/" + bucket,
+                        product_type,
+                        dataset_item,
+                        collection_dict,
+                    )
+                    if product:
+                        return [product], 1
+                    current_object = s3_objects["Contents"][-1]["Key"]
+                    continue
+
+                for obj in s3_objects["Contents"]:
+                    item_key = obj["Key"]
+                    # filter according to date(s) in item id
+                    item_dates = re.findall(r"\d{8}", item_key)
+                    if not item_dates:
+                        item_dates = re.findall(r"\d{6}", item_key)
+                    item_start = _get_date_from_yyyymmdd(item_dates[0], item_key)
+                    if not item_start:  # identified pattern was not a valid datetime
+                        continue
+                    if item_start > end_date:
+                        stop_search = True
+                    if not item_dates or (start_date <= item_start <= end_date):
+                        num_total += 1
+                        if num_total < start_index:
+                            continue
+                        if len(products) < items_per_page or items_per_page < 0:
+                            product = self._create_product(
+                                product_type,
+                                item_key,
+                                endpoint_url + "/" + bucket,
+                                dataset_item,
+                                collection_dict,
+                            )
+                            if product:
+                                products.append(product)
+                    current_object = item_key
+
+        return products, num_total

eodag/plugins/search/creodias_s3.py

@@ -17,17 +17,18 @@
 # limitations under the License.
 import logging
 from types import MethodType
-from typing import Any, Dict, List
+from typing import Any, List
 
 import boto3
 import botocore
 from botocore.exceptions import BotoCoreError
 
-from eodag import EOProduct
-from eodag.api.product._assets import AssetsDict
+from eodag.api.product import AssetsDict, EOProduct  # type: ignore
+from eodag.api.search_result import RawSearchResult
 from eodag.config import PluginConfig
 from eodag.plugins.authentication.aws_auth import AwsAuth
-from eodag.plugins.search.qssearch import QueryStringSearch
+from eodag.plugins.search.qssearch import ODataV4Search
+from eodag.utils import guess_file_type
 from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, RequestError
 
 DATA_EXTENSIONS = ["jp2", "tiff", "nc", "grib"]
@@ -64,18 +65,19 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
         try:
             auth_dict = auth.authenticate()
             required_creds = ["aws_access_key_id", "aws_secret_access_key"]
-            if not all(getattr(auth, x) for x in required_creds):
+            if not all(x in auth_dict for x in required_creds):
                 raise MisconfiguredError(
                     f"Incomplete credentials for {product.provider}, missing "
-                    f"{[x for x in required_creds if not getattr(auth, x)]}"
+                    f"{[x for x in required_creds if x not in auth_dict]}"
                 )
             if not getattr(auth, "s3_client", None):
                 auth.s3_client = boto3.client(
                     "s3",
                     endpoint_url=config.base_uri,
-                    **auth_dict,
+                    aws_access_key_id=auth_dict["aws_access_key_id"],
+                    aws_secret_access_key=auth_dict["aws_secret_access_key"],
                 )
-            logger.debug(f"Listing assets in {prefix}")
+            logger.debug("Listing assets in %s", prefix)
             product.assets = AssetsDict(product)
             for asset in auth.s3_client.list_objects(
                 Bucket=config.s3_bucket, Prefix=prefix, MaxKeys=300
@@ -96,6 +98,8 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
                         "roles": [role],
                         "href": f"s3://{config.s3_bucket}/{asset['Key']}",
                     }
+                    if mime_type := guess_file_type(asset["Key"]):
+                        product.assets[asset_basename]["type"] = mime_type
             # update driver
             product.driver = product.get_driver()
 
@@ -104,13 +108,10 @@ def _update_assets(product: EOProduct, config: PluginConfig, auth: AwsAuth):
                 raise AuthenticationError(
                     f"Authentication failed on {config.base_uri} s3"
                 ) from e
-            else:
-                raise RequestError(
-                    "assets for product %s could not be found", prefix
-                ) from e
+            raise RequestError(f"assets for product {prefix} could not be found") from e
 
 
-class CreodiasS3Search(QueryStringSearch):
+class CreodiasS3Search(ODataV4Search):
     """
     Search on creodias and adapt results to s3
     """
@@ -119,7 +120,7 @@ class CreodiasS3Search(QueryStringSearch):
         super(CreodiasS3Search, self).__init__(provider, config)
 
     def normalize_results(
-        self, results: List[Dict[str, Any]], **kwargs: Any
+        self, results: RawSearchResult, **kwargs: Any
     ) -> List[EOProduct]:
         """Build EOProducts from provider results"""
 

eodag/plugins/search/csw.py

@@ -35,8 +35,9 @@ from shapely import geometry, wkt
 
 from eodag.api.product import EOProduct
 from eodag.api.product.metadata_mapping import properties_from_xml
+from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
-from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, DEFAULT_PROJ
+from eodag.utils import DEFAULT_PROJ
 from eodag.utils.import_system import patch_owslib_requests
 
 if TYPE_CHECKING:
@@ -64,16 +65,13 @@ class CSWSearch(Search):
 
     def query(
         self,
-        product_type: Optional[str] = None,
-        items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        page: int = DEFAULT_PAGE,
-        count: bool = True,
+        prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """Perform a search on a OGC/CSW-like interface"""
         product_type = kwargs.get("productType")
         if product_type is None:
-            return [], 0
+            return ([], 0) if prep.count else ([], None)
         auth = kwargs.get("auth")
         if auth:
             self.__init_catalog(**getattr(auth.config, "credentials", {}))
@@ -118,7 +116,7 @@ class CSWSearch(Search):
                 )
                 results.extend(partial_results)
         logger.info("Found %s overall results", len(results))
-        total_results = len(results) if count else None
+        total_results = len(results) if prep.count else None
         return results, total_results
 
     def __init_catalog(

eodag/plugins/search/data_request_search.py

@@ -19,8 +19,8 @@ from __future__ import annotations
 
 import logging
 import time
-from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
 
 import requests
 
@@ -30,10 +30,11 @@ from eodag.api.product.metadata_mapping import (
     mtd_cfg_as_conversion_and_querypath,
     properties_from_json,
 )
+from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
-from eodag.rest.stac import DEFAULT_MISSION_START_DATE
 from eodag.utils import (
     DEFAULT_ITEMS_PER_PAGE,
+    DEFAULT_MISSION_START_DATE,
     DEFAULT_PAGE,
     GENERIC_PRODUCT_TYPE,
     HTTP_REQ_TIMEOUT,
@@ -41,7 +42,12 @@ from eodag.utils import (
     deepcopy,
     string_to_jsonpath,
 )
-from eodag.utils.exceptions import NotAvailableError, RequestError, TimeOutError
+from eodag.utils.exceptions import (
+    NotAvailableError,
+    RequestError,
+    TimeOutError,
+    ValidationError,
+)
 
 if TYPE_CHECKING:
     from eodag.config import PluginConfig
@@ -57,6 +63,8 @@ class DataRequestSearch(Search):
         - if finished - fetch the result of the job
     """
 
+    data_request_id: Optional[str]
+
     def __init__(self, provider: str, config: PluginConfig) -> None:
         super(DataRequestSearch, self).__init__(provider, config)
         self.config.__dict__.setdefault("result_type", "json")
@@ -101,10 +109,10 @@ class DataRequestSearch(Search):
             self.config.pagination["next_page_url_key_path"] = string_to_jsonpath(
                 self.config.pagination.get("next_page_url_key_path", None)
             )
-        self.download_info = {}
+        self.download_info: Dict[str, Any] = {}
         self.data_request_id = None
 
-    def discover_product_types(self) -> Optional[Dict[str, Any]]:
+    def discover_product_types(self, **kwargs: Any) -> Optional[Dict[str, Any]]:
         """Fetch product types is disabled for `DataRequestSearch`
 
         :returns: empty dict
@@ -119,26 +127,30 @@ class DataRequestSearch(Search):
 
     def query(
         self,
-        product_type: Optional[str] = None,
-        items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        page: int = DEFAULT_PAGE,
-        count: bool = True,
+        prep: PreparedSearch = PreparedSearch(),
         **kwargs: Any,
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """
         performs the search for a provider where several steps are required to fetch the data
         """
+        if kwargs.get("sortBy"):
+            raise ValidationError(f"{self.provider} does not support sorting feature")
+
         product_type = kwargs.get("productType", None)
+
+        if product_type is None:
+            raise ValidationError("Required productType is missing")
+
         # replace "product_type" to "providerProductType" in search args if exists
         # for compatibility with DataRequestSearch method
         if kwargs.get("product_type"):
             kwargs["providerProductType"] = kwargs.pop("product_type", None)
-        provider_product_type = self._map_product_type(product_type or "")
+        provider_product_type = cast(str, self._map_product_type(product_type or ""))
         keywords = {k: v for k, v in kwargs.items() if k != "auth" and v is not None}
 
         if provider_product_type and provider_product_type != GENERIC_PRODUCT_TYPE:
             keywords["productType"] = provider_product_type
-        elif product_type:
+        else:
             keywords["productType"] = product_type
 
         # provider product type specific conf
@@ -185,7 +197,7 @@ class DataRequestSearch(Search):
             if not keywords.get("completionTimeFromAscendingNode", None):
                 keywords["completionTimeFromAscendingNode"] = getattr(
                     self.config, "product_type_config", {}
-                ).get("missionEndDate", datetime.utcnow().isoformat())
+                ).get("missionEndDate", datetime.now(timezone.utc).isoformat())
 
         # ask for data_request_id if not set (it must exist when iterating over pages)
         if not self.data_request_id:
@@ -246,16 +258,19 @@ class DataRequestSearch(Search):
         self, product_type: str, eodag_product_type: str, **kwargs: Any
     ) -> str:
         headers = getattr(self.auth, "headers", USER_AGENT)
+        ssl_verify = getattr(self.config.ssl_verify, "ssl_verify", True)
         try:
             url = self.config.data_request_url
-            request_body = format_query_params(
-                eodag_product_type, self.config, **kwargs
-            )
+            request_body = format_query_params(eodag_product_type, self.config, kwargs)
             logger.debug(
                 f"Sending search job request to {url} with {str(request_body)}"
             )
             request_job = requests.post(
-                url, json=request_body, headers=headers, timeout=HTTP_REQ_TIMEOUT
+                url,
+                json=request_body,
+                headers=headers,
+                timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             request_job.raise_for_status()
         except requests.exceptions.Timeout as exc:
@@ -271,9 +286,10 @@ class DataRequestSearch(Search):
     def _cancel_request(self, data_request_id: str) -> None:
         logger.info("deleting request job %s", data_request_id)
         delete_url = f"{self.config.data_request_url}/{data_request_id}"
+        headers = getattr(self.auth, "headers", USER_AGENT)
         try:
             delete_resp = requests.delete(
-                delete_url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
+                delete_url, headers=headers, timeout=HTTP_REQ_TIMEOUT
             )
             delete_resp.raise_for_status()
         except requests.exceptions.Timeout as exc:
@@ -284,9 +300,15 @@ class DataRequestSearch(Search):
     def _check_request_status(self, data_request_id: str) -> bool:
         logger.debug("checking status of request job %s", data_request_id)
         status_url = self.config.status_url + data_request_id
+        headers = getattr(self.auth, "headers", USER_AGENT)
+        ssl_verify = getattr(self.config, "ssl_verify", True)
+
         try:
             status_resp = requests.get(
-                status_url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
+                status_url,
+                headers=headers,
+                timeout=HTTP_REQ_TIMEOUT,
+                verify=ssl_verify,
             )
             status_resp.raise_for_status()
         except requests.exceptions.Timeout as exc:
@@ -315,9 +337,11 @@ class DataRequestSearch(Search):
         url = self.config.result_url.format(
             jobId=data_request_id, items_per_page=items_per_page, page=page
         )
+        ssl_verify = getattr(self.config, "ssl_verify", True)
+        headers = getattr(self.auth, "headers", USER_AGENT)
         try:
             return requests.get(
-                url, headers=self.auth.headers, timeout=HTTP_REQ_TIMEOUT
+                url, headers=headers, timeout=HTTP_REQ_TIMEOUT, verify=ssl_verify
             ).json()
         except requests.exceptions.Timeout as exc:
             raise TimeOutError(exc, timeout=HTTP_REQ_TIMEOUT) from exc