eodag 2.12.0__py3-none-any.whl → 3.0.0b1__py3-none-any.whl

eodag/rest/types/stac_search.py

@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+# Copyright 2023, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model describing a STAC search POST request"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
+
+import geojson
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PositiveInt,
+    StringConstraints,
+    field_serializer,
+    field_validator,
+    model_validator,
+)
+from shapely.geometry import (
+    GeometryCollection,
+    LineString,
+    MultiLineString,
+    MultiPoint,
+    MultiPolygon,
+    Point,
+    Polygon,
+    shape,
+)
+from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry
+from typing_extensions import Annotated
+
+from eodag.rest.utils.rfc3339 import rfc3339_str_to_datetime, str_to_interval
+from eodag.utils.exceptions import ValidationError
+
+if TYPE_CHECKING:
+    try:
+        from typing import Self
+    except ImportError:
+        from _typeshed import Self
+
+NumType = Union[float, int]
+
+BBox = Union[
+    Tuple[NumType, NumType, NumType, NumType],
+    Tuple[NumType, NumType, NumType, NumType, NumType, NumType],
+]
+
+Geometry = Union[
+    Point,
+    MultiPoint,
+    LineString,
+    MultiLineString,
+    Polygon,
+    MultiPolygon,
+    GeometryCollection,
+]
+
+
+Direction = Annotated[Literal["asc", "desc"], StringConstraints(min_length=1)]
+
+
+class Sortby(BaseModel):
+    """
+    A class representing a parameter with which we want to sort results and its sorting order in a
+    POST search
+
+    :param field: The name of the parameter with which we want to sort results
+    :type field: str
+    :param direction: The sorting order of the parameter
+    :type direction: str
+    """
+
+    __pydantic_config__ = ConfigDict(extra="forbid")
+
+    field: Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]
+    direction: Direction
+
+
+class SearchPostRequest(BaseModel):
+    """
+    class which describes the body of a search request
+
+    Overrides the validation for datetime and spatial filter from the base request model.
+    """
+
+    model_config = ConfigDict(populate_by_name=True, arbitrary_types_allowed=True)
+
+    provider: Optional[str] = None
+    collections: Optional[List[str]] = None
+    ids: Optional[List[str]] = None
+    bbox: Optional[BBox] = None
+    intersects: Optional[Geometry] = None
+    datetime: Optional[str] = None
+    limit: Optional[PositiveInt] = Field(  # type: ignore
+        default=None, description="Maximum number of items per page."
+    )
+    page: Optional[PositiveInt] = Field(  # type: ignore
+        default=None, description="Page number, must be a positive integer."
+    )
+    query: Optional[Dict[str, Any]] = None
+    filter: Optional[Dict[str, Any]] = None
+    filter_lang: Optional[str] = Field(
+        default=None,
+        alias="filter-lang",
+        description="The language used for filtering.",
+        validate_default=True,
+    )
+    sortby: Optional[List[Sortby]] = None
+    crunch: Optional[str] = None
+
+    @field_serializer("intersects")
+    def serialize_intersects(
+        self, intersects: Optional[Geometry]
+    ) -> Optional[Dict[str, Any]]:
+        """Serialize intersects from shapely to a proper dict"""
+        if intersects:
+            return geojson.loads(geojson.dumps(intersects))  # type: ignore
+        return None
+
+    @model_validator(mode="after")
+    def check_filter_lang(self) -> Self:
+        """Verify filter-lang has correct value"""
+        if not self.filter_lang and self.filter:
+            self.filter_lang = "cql2-json"
+        if self.filter_lang and not self.filter:
+            raise ValueError("filter-lang is set but filter is missing")
+        if self.filter_lang != "cql2-json" and self.filter:
+            raise ValueError('Only filter language "cql2-json" is accepted')
+        return self
+
+    @model_validator(mode="before")
+    @classmethod
+    def only_one_spatial(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Check bbox and intersects are not both supplied."""
+        if "intersects" in values and "bbox" in values:
+            raise ValueError("intersects and bbox parameters are mutually exclusive")
+        return values
+
+    @property
+    def start_date(self) -> Optional[str]:
+        """Extract the start date from the datetime string."""
+        start = str_to_interval(self.datetime)[0]
+        return start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" if start else None
+
+    @property
+    def end_date(self) -> Optional[str]:
+        """Extract the end date from the datetime string."""
+        end = str_to_interval(self.datetime)[1]
+        return end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" if end else None
+
+    @field_validator("ids", "collections", mode="before")
+    @classmethod
+    def str_to_str_list(cls, v: Union[str, List[str]]) -> List[str]:
+        """Convert ids and collections strings to list of strings"""
+        if isinstance(v, str):
+            return [i.strip() for i in v.split(",")]
+        return v
+
+    @field_validator("intersects", mode="before")
+    @classmethod
+    def validate_intersects(cls, v: Union[Dict[str, Any], Geometry]) -> Geometry:
+        """Verify format of intersects"""
+        if isinstance(v, BaseGeometry):
+            return v
+
+        if isinstance(v, dict) and v.get("type") in GEOMETRY_TYPES:  # type: ignore
+            return shape(v)
+
+        raise ValueError("Not a valid geometry")
+
+    @field_validator("bbox")
+    @classmethod
+    def validate_bbox(cls, v: BBox) -> BBox:
+        """Check order of supplied bbox coordinates."""
+        # Validate order
+        if len(v) == 4:
+            xmin, ymin, xmax, ymax = v
+        else:
+            xmin, ymin, min_elev, xmax, ymax, max_elev = v
+            if max_elev < min_elev:
+                raise ValueError(
+                    "Maximum elevation must greater than minimum elevation"
+                )
+
+        if xmax < xmin:
+            raise ValueError("Maximum longitude must be greater than minimum longitude")
+
+        if ymax < ymin:
+            raise ValueError("Maximum longitude must be greater than minimum longitude")
+
+        # Validate against WGS84
+        if xmin < -180 or ymin < -90 or xmax > 180 or ymax > 90:
+            raise ValueError("Bounding box must be within (-180, -90, 180, 90)")
+
+        return v
+
+    @field_validator("datetime")
+    @classmethod
+    def validate_datetime(cls, v: str) -> str:
+        """Validate datetime."""
+        if "/" in v:
+            values = v.split("/")
+        else:
+            # Single date is interpreted as end date
+            values = ["..", v]
+
+        dates: List[str] = []
+        for value in values:
+            if value == ".." or value == "":
+                dates.append("..")
+                continue
+
+            try:
+                dates.append(
+                    rfc3339_str_to_datetime(value).strftime("%Y-%m-%dT%H:%M:%SZ")
+                )
+            except ValidationError as e:
+                raise ValueError(e)
+
+        if dates[0] == ".." and dates[1] == "..":
+            raise ValueError(
+                "Invalid datetime range, both ends of range may not be open"
+            )
+
+        if ".." not in dates and dates[0] > dates[1]:
+            raise ValueError(
+                "Invalid datetime range, must match format (begin_date, end_date)"
+            )
+
+        return v
+
+    @property
+    def spatial_filter(self) -> Optional[Geometry]:
+        """Return a geojson-pydantic object representing the spatial filter for the search
+        request.
+
+        Check for both because the ``bbox`` and ``intersects`` parameters are
+        mutually exclusive.
+        """
+        if self.bbox:
+            return Polygon.from_bounds(*self.bbox)  # type: ignore
+
+        if self.intersects:
+            return self.intersects
+        return None
+
+
+def sortby2list(
+    v: Optional[str],
+) -> Optional[List[Sortby]]:
+    """
+    Convert sortby filter parameter GET syntax to POST syntax
+    """
+    if not v:
+        return None
+    sortby: List[Sortby] = []
+    for sortby_param in v.split(","):
+        sortby_param = sortby_param.strip()
+        direction: Direction = "desc" if sortby_param.startswith("-") else "asc"
+        field = sortby_param.lstrip("+-")
+        sortby.append(Sortby(field=field, direction=direction))
+    return sortby

eodag/rest/utils/__init__.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+# Copyright 2023, CS Systemes d'Information, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""EODAG REST utils"""
+from __future__ import annotations
+
+import glob
+import logging
+import os
+from io import BufferedReader
+from shutil import make_archive, rmtree
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    NamedTuple,
+    Optional,
+    Union,
+)
+from urllib.parse import unquote_plus, urlencode
+
+import orjson
+from fastapi import Request
+from pydantic import ValidationError as pydanticValidationError
+
+from eodag.plugins.crunch.filter_latest_intersect import FilterLatestIntersect
+from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
+from eodag.plugins.crunch.filter_overlap import FilterOverlap
+from eodag.utils import StreamResponse
+from eodag.utils.exceptions import ValidationError
+from eodag.utils.rest import get_date, get_datetime
+
+if TYPE_CHECKING:
+    from eodag.rest.types.stac_search import SearchPostRequest
+
+# exportable content
+__all__ = ["get_date", "get_datetime"]
+
+logger = logging.getLogger("eodag.rest.utils")
+
+
+class Cruncher(NamedTuple):
+    """Type hinted Cruncher namedTuple"""
+
+    clazz: Callable[..., Any]
+    config_params: List[str]
+
+
+crunchers = {
+    "latestIntersect": Cruncher(FilterLatestIntersect, []),
+    "latestByName": Cruncher(FilterLatestByName, ["name_pattern"]),
+    "overlap": Cruncher(FilterOverlap, ["minimum_overlap"]),
+}
+
+
+def format_pydantic_error(e: pydanticValidationError) -> str:
+    """Format Pydantic ValidationError
+
+    :param e: A Pydantic ValidationError object
+    :tyype e: pydanticValidationError
+    """
+    error_header = f"{e.error_count()} error(s). "
+
+    error_messages = [
+        f'{err["loc"][0]}: {err["msg"]}' if err["loc"] else err["msg"]
+        for err in e.errors()
+    ]
+    return error_header + "; ".join(set(error_messages))
+
+
+def is_dict_str_any(var: Any) -> bool:
+    """Verify whether the variable is of type dict[str, Any]"""
+    if isinstance(var, Dict):
+        return all(isinstance(k, str) for k in var.keys())  # type: ignore
+    return False
+
+
+def str2list(v: Optional[str]) -> Optional[List[str]]:
+    """Convert string to list base on , delimiter."""
+    if v:
+        return v.split(",")
+    return None
+
+
+def str2json(k: str, v: Optional[str] = None) -> Optional[Dict[str, Any]]:
+    """decoding a URL parameter and then parsing it as JSON."""
+    if not v:
+        return None
+    try:
+        return orjson.loads(unquote_plus(v))
+    except orjson.JSONDecodeError as e:
+        raise ValidationError(f"{k}: Incorrect JSON object") from e
+
+
+def flatten_list(nested_list: Union[Any, List[Any]]) -> List[Any]:
+    """Flatten a nested list structure into a single list."""
+    if not isinstance(nested_list, list):
+        return [nested_list]
+    else:
+        flattened: List[Any] = []
+        for element in nested_list:
+            flattened.extend(flatten_list(element))
+        return flattened
+
+
+def list_to_str_list(input_list: List[Any]) -> List[str]:
+    """Attempt to convert a list of any type to a list of strings."""
+    try:
+        # Try to convert each element to a string
+        return [str(element) for element in input_list]
+    except Exception as e:
+        # Raise an exception if any element cannot be converted
+        raise TypeError(f"Failed to convert to List[str]: {e}") from e
+
+
+def get_next_link(
+    request: Request,
+    search_request: SearchPostRequest,
+    total_results: Optional[int],
+    items_per_page: int,
+) -> Optional[Dict[str, Any]]:
+    """Generate next link URL and body"""
+    body = search_request.model_dump(exclude_none=True)
+    if "bbox" in body:
+        # bbox is tuple
+        body["bbox"] = list(body["bbox"])
+
+    params = dict(request.query_params)
+
+    page = int(body.get("page", 0) or params.get("page", 0)) or 1
+
+    if total_results is None or items_per_page * page >= total_results:
+        return None
+
+    url = str(request.state.url)
+    if request.method == "POST":
+        body["page"] = page + 1
+    else:
+        params["page"] = str(page + 1)
+        url += f"?{urlencode(params)}"
+
+    next: Dict[str, Any] = {
+        "rel": "next",
+        "href": url,
+        "title": "Next page",
+        "method": request.method,
+        "type": "application/geo+json",
+    }
+    if request.method == "POST":
+        next["body"] = body
+    return next
+
+
+def read_file_chunks_and_delete(
+    opened_file: BufferedReader, chunk_size: int = 64 * 1024
+) -> Iterator[bytes]:
+    """Yield file chunks and delete file when finished."""
+    while True:
+        data = opened_file.read(chunk_size)
+        if not data:
+            opened_file.close()
+            os.remove(opened_file.name)
+            logger.debug("%s deleted after streaming complete", opened_file.name)
+            break
+        yield data
+    yield data
+
+
+def file_to_stream(
+    file_path: str,
+) -> StreamResponse:
+    """Break a file into chunck and return it as a byte stream"""
+    if os.path.isdir(file_path):
+        # do not zip if dir contains only one file
+        all_filenames = [
+            f
+            for f in glob.glob(os.path.join(file_path, "**", "*"), recursive=True)
+            if os.path.isfile(f)
+        ]
+        if len(all_filenames) == 1:
+            filepath_to_stream = all_filenames[0]
+        else:
+            filepath_to_stream = f"{file_path}.zip"
+            logger.debug(
+                "Building archive for downloaded product path %s",
+                filepath_to_stream,
+            )
+            make_archive(file_path, "zip", file_path)
+            rmtree(file_path)
+    else:
+        filepath_to_stream = file_path
+
+    filename = os.path.basename(filepath_to_stream)
+    return StreamResponse(
+        content=read_file_chunks_and_delete(open(filepath_to_stream, "rb")),
+        headers={
+            "content-disposition": f"attachment; filename={filename}",
+        },
+    )

eodag/rest/utils/cql_evaluate.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+# Copyright 2023, CS Systemes d'Information, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from datetime import datetime as dt
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from pygeofilter import ast
+from pygeofilter.backends.evaluator import Evaluator, handle
+from pygeofilter.values import Geometry, Interval
+
+simpleNode = Union[ast.Attribute, str, int, complex, float, List[Any], Tuple[Any, ...]]
+
+
+class EodagEvaluator(Evaluator):
+    """
+    Evaluate a cql2 json expression and transform it to a STAC args object
+    """
+
+    @handle(ast.Attribute, str, int, complex, float, list, tuple)
+    def attribute(self, node: simpleNode, *_) -> simpleNode:
+        """handle attribute and literal"""
+        return node
+
+    @handle(Geometry)
+    def spatial(self, node: Geometry) -> Dict[str, Any]:
+        """handle geometry"""
+        return node.geometry
+
+    @handle(dt)
+    def temporal(self, node: dt) -> str:
+        """handle datetime"""
+        return node.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    @handle(Interval)
+    def interval(self, _, *interval: Any) -> List[Any]:
+        """handle datetime interval"""
+        return list(interval)
+
+    @handle(
+        ast.GeometryIntersects,
+        ast.Equal,
+        ast.LessEqual,
+        ast.GreaterEqual,
+        ast.TimeOverlaps,
+        ast.In,
+    )
+    def predicate(
+        self, node: ast.Predicate, lhs: Any, rhs: Any
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Handle predicates
+        Verify the property is first attribute in each predicate
+        """
+        if not isinstance(lhs, ast.Attribute):
+            raise ValueError(
+                f'invalid cql syntax, first argument in "{node.op.value}" must be a property'
+            )
+
+        if isinstance(node, ast.Equal) and not isinstance(
+            rhs, (int, float, complex, str)
+        ):
+            raise ValueError(
+                f'second argument in property "{lhs.name}" must be a string or a numeric value'
+            )
+
+        if isinstance(node, ast.GeometryIntersects) and not lhs.name == "geometry":
+            raise ValueError(
+                f'operator {node.op.value} is not supported for property "{lhs.name}"'
+            )
+
+        if isinstance(node, (ast.Equal, ast.GeometryIntersects)):
+            return {lhs.name: rhs}
+
+        if isinstance(node, ast.LessEqual):
+            if not isinstance(node.rhs, dt):
+                raise ValueError(
+                    f'operator "<=" is not supported for property "{lhs.name}"'
+                )
+            return {"end_datetime": rhs}
+
+        if isinstance(node, ast.GreaterEqual):
+            if not isinstance(node.rhs, dt):
+                raise ValueError(
+                    f'operator ">=" is not supported for property "{lhs.name}"'
+                )
+            return {"start_datetime": rhs}
+
+        if isinstance(node, ast.TimeOverlaps):
+            return {"start_datetime": rhs[0], "end_datetime": rhs[1]}
+
+        return None
+
+    @handle(ast.In)
+    def contains(self, node: ast.In, lhs: Any, *rhs: Any):
+        """handle in keyword"""
+        if not isinstance(node.sub_nodes, list):  # type: ignore
+            raise ValueError(
+                f'property "{lhs.name}" expects a value in list format with operator "in"'
+            )
+        return {lhs.name: list(rhs)}
+
+    @handle(ast.And)
+    def combination(self, _, lhs: Dict[str, str], rhs: Dict[str, str]):
+        """handle combinations"""
+        return {**lhs, **rhs}

eodag/rest/utils/rfc3339.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Copyright 2023, CS Systemes d'Information, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import datetime
+from typing import Optional, Tuple
+
+from eodag.utils.rest import rfc3339_str_to_datetime
+
+
+def str_to_interval(
+    interval: Optional[str],
+) -> Tuple[Optional[datetime.datetime], Optional[datetime.datetime]]:
+    """Extract a tuple of datetimes from an interval string.
+
+    Interval strings are defined by
+    OGC API - Features Part 1 for the datetime query parameter value. These follow the
+    form '1985-04-12T23:20:50.52Z/1986-04-12T23:20:50.52Z', and allow either the start
+    or end (but not both) to be open-ended with '..' or ''.
+
+    :param interval: The interval string to convert to a :class:`datetime.datetime`
+        tuple.
+    :type interval: str
+
+    :raises: :class:`ValueError`
+    """
+    if not interval:
+        return (None, None)
+
+    if "/" not in interval:
+        date = rfc3339_str_to_datetime(interval)
+        return (date, date)
+
+    values = interval.split("/")
+    if len(values) != 2:
+        raise ValueError(
+            f"Interval string '{interval}' contains more than one forward slash."
+        )
+
+    start = None
+    end = None
+    if values[0] not in ["..", ""]:
+        start = rfc3339_str_to_datetime(values[0])
+    if values[1] not in ["..", ""]:
+        end = rfc3339_str_to_datetime(values[1])
+
+    if start is None and end is None:
+        raise ValueError("Double open-ended intervals are not allowed.")
+    if start is not None and end is not None and start > end:
+        raise ValueError("Start datetime cannot be before end datetime.")
+    else:
+        return start, end