stac-fastapi-core 4.0.0a1__py3-none-any.whl

stac_fastapi/core/extensions/filter.py

@@ -0,0 +1,202 @@
+"""Filter extension logic for es conversion."""
+
+# """
+# Implements Filter Extension.
+
+# Basic CQL2 (AND, OR, NOT), comparison operators (=, <>, <, <=, >, >=), and IS NULL.
+# The comparison operators are allowed against string, numeric, boolean, date, and datetime types.
+
+# Advanced comparison operators (http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators)
+# defines the LIKE, IN, and BETWEEN operators.
+
+# Basic Spatial Operators (http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators)
+# defines the intersects operator (S_INTERSECTS).
+# """
+
+import re
+from enum import Enum
+from typing import Any, Dict
+
+_cql2_like_patterns = re.compile(r"\\.|[%_]|\\$")
+_valid_like_substitutions = {
+    "\\\\": "\\",
+    "\\%": "%",
+    "\\_": "_",
+    "%": "*",
+    "_": "?",
+}
+
+
+def _replace_like_patterns(match: re.Match) -> str:
+    pattern = match.group()
+    try:
+        return _valid_like_substitutions[pattern]
+    except KeyError:
+        raise ValueError(f"'{pattern}' is not a valid escape sequence")
+
+
+def cql2_like_to_es(string: str) -> str:
+    """
+    Convert CQL2 "LIKE" characters to Elasticsearch "wildcard" characters.
+
+    Args:
+        string (str): The string containing CQL2 wildcard characters.
+
+    Returns:
+        str: The converted string with Elasticsearch compatible wildcards.
+
+    Raises:
+        ValueError: If an invalid escape sequence is encountered.
+    """
+    return _cql2_like_patterns.sub(
+        repl=_replace_like_patterns,
+        string=string,
+    )
+
+
+class LogicalOp(str, Enum):
+    """Enumeration for logical operators used in constructing Elasticsearch queries."""
+
+    AND = "and"
+    OR = "or"
+    NOT = "not"
+
+
+class ComparisonOp(str, Enum):
+    """Enumeration for comparison operators used in filtering queries according to CQL2 standards."""
+
+    EQ = "="
+    NEQ = "<>"
+    LT = "<"
+    LTE = "<="
+    GT = ">"
+    GTE = ">="
+    IS_NULL = "isNull"
+
+
+class AdvancedComparisonOp(str, Enum):
+    """Enumeration for advanced comparison operators like 'like', 'between', and 'in'."""
+
+    LIKE = "like"
+    BETWEEN = "between"
+    IN = "in"
+
+
+class SpatialIntersectsOp(str, Enum):
+    """Enumeration for spatial intersection operator as per CQL2 standards."""
+
+    S_INTERSECTS = "s_intersects"
+
+
+queryables_mapping = {
+    "id": "id",
+    "collection": "collection",
+    "geometry": "geometry",
+    "datetime": "properties.datetime",
+    "created": "properties.created",
+    "updated": "properties.updated",
+    "cloud_cover": "properties.eo:cloud_cover",
+    "cloud_shadow_percentage": "properties.s2:cloud_shadow_percentage",
+    "nodata_pixel_percentage": "properties.s2:nodata_pixel_percentage",
+}
+
+
+def to_es_field(field: str) -> str:
+    """
+    Map a given field to its corresponding Elasticsearch field according to a predefined mapping.
+
+    Args:
+        field (str): The field name from a user query or filter.
+
+    Returns:
+        str: The mapped field name suitable for Elasticsearch queries.
+    """
+    return queryables_mapping.get(field, field)
+
+
+def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Transform a simplified CQL2 query structure to an Elasticsearch compatible query DSL.
+
+    Args:
+        query (Dict[str, Any]): The query dictionary containing 'op' and 'args'.
+
+    Returns:
+        Dict[str, Any]: The corresponding Elasticsearch query in the form of a dictionary.
+    """
+    if query["op"] in [LogicalOp.AND, LogicalOp.OR, LogicalOp.NOT]:
+        bool_type = {
+            LogicalOp.AND: "must",
+            LogicalOp.OR: "should",
+            LogicalOp.NOT: "must_not",
+        }[query["op"]]
+        return {"bool": {bool_type: [to_es(sub_query) for sub_query in query["args"]]}}
+
+    elif query["op"] in [
+        ComparisonOp.EQ,
+        ComparisonOp.NEQ,
+        ComparisonOp.LT,
+        ComparisonOp.LTE,
+        ComparisonOp.GT,
+        ComparisonOp.GTE,
+    ]:
+        range_op = {
+            ComparisonOp.LT: "lt",
+            ComparisonOp.LTE: "lte",
+            ComparisonOp.GT: "gt",
+            ComparisonOp.GTE: "gte",
+        }
+
+        field = to_es_field(query["args"][0]["property"])
+        value = query["args"][1]
+        if isinstance(value, dict) and "timestamp" in value:
+            value = value["timestamp"]
+            if query["op"] == ComparisonOp.EQ:
+                return {"range": {field: {"gte": value, "lte": value}}}
+            elif query["op"] == ComparisonOp.NEQ:
+                return {
+                    "bool": {
+                        "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
+                    }
+                }
+            else:
+                return {"range": {field: {range_op[query["op"]]: value}}}
+        else:
+            if query["op"] == ComparisonOp.EQ:
+                return {"term": {field: value}}
+            elif query["op"] == ComparisonOp.NEQ:
+                return {"bool": {"must_not": [{"term": {field: value}}]}}
+            else:
+                return {"range": {field: {range_op[query["op"]]: value}}}
+
+    elif query["op"] == ComparisonOp.IS_NULL:
+        field = to_es_field(query["args"][0]["property"])
+        return {"bool": {"must_not": {"exists": {"field": field}}}}
+
+    elif query["op"] == AdvancedComparisonOp.BETWEEN:
+        field = to_es_field(query["args"][0]["property"])
+        gte, lte = query["args"][1], query["args"][2]
+        if isinstance(gte, dict) and "timestamp" in gte:
+            gte = gte["timestamp"]
+        if isinstance(lte, dict) and "timestamp" in lte:
+            lte = lte["timestamp"]
+        return {"range": {field: {"gte": gte, "lte": lte}}}
+
+    elif query["op"] == AdvancedComparisonOp.IN:
+        field = to_es_field(query["args"][0]["property"])
+        values = query["args"][1]
+        if not isinstance(values, list):
+            raise ValueError(f"Arg {values} is not a list")
+        return {"terms": {field: values}}
+
+    elif query["op"] == AdvancedComparisonOp.LIKE:
+        field = to_es_field(query["args"][0]["property"])
+        pattern = cql2_like_to_es(query["args"][1])
+        return {"wildcard": {field: {"value": pattern, "case_insensitive": True}}}
+
+    elif query["op"] == SpatialIntersectsOp.S_INTERSECTS:
+        field = to_es_field(query["args"][0]["property"])
+        geometry = query["args"][1]
+        return {"geo_shape": {field: {"shape": geometry, "relation": "intersects"}}}
+
+    return {}

stac_fastapi/core/extensions/query.py

@@ -0,0 +1,79 @@
+"""STAC SQLAlchemy specific query search model.
+
+# TODO: replace with stac-pydantic
+"""
+
+import logging
+import operator
+from dataclasses import dataclass
+from enum import auto
+from types import DynamicClassAttribute
+from typing import Any, Callable, Dict, Optional
+
+from pydantic import BaseModel, root_validator
+from stac_pydantic.utils import AutoValueEnum
+
+from stac_fastapi.extensions.core.query import QueryExtension as QueryExtensionBase
+
+logger = logging.getLogger("uvicorn")
+logger.setLevel(logging.INFO)
+
+
+class Operator(str, AutoValueEnum):
+    """Defines the set of operators supported by the API."""
+
+    eq = auto()
+    ne = auto()
+    lt = auto()
+    lte = auto()
+    gt = auto()
+    gte = auto()
+
+    # TODO: These are defined in the spec but aren't currently implemented by the api
+    # startsWith = auto()
+    # endsWith = auto()
+    # contains = auto()
+    # in = auto()
+
+    @DynamicClassAttribute
+    def operator(self) -> Callable[[Any, Any], bool]:
+        """Return python operator."""
+        return getattr(operator, self._value_)
+
+
+class Queryables(str, AutoValueEnum):
+    """Queryable fields."""
+
+    ...
+
+
+@dataclass
+class QueryableTypes:
+    """Defines a set of queryable fields."""
+
+    ...
+
+
+class QueryExtensionPostRequest(BaseModel):
+    """Queryable validation.
+
+    Add queryables validation to the POST request
+    to raise errors for unsupported querys.
+    """
+
+    query: Optional[Dict[str, Dict[Operator, Any]]] = None
+
+    @root_validator(pre=True)
+    def validate_query_fields(cls, values: Dict) -> Dict:
+        """Validate query fields."""
+        ...
+
+
+class QueryExtension(QueryExtensionBase):
+    """Query Extenson.
+
+    Override the POST request model to add validation against
+    supported fields
+    """
+
+    POST = QueryExtensionPostRequest

stac_fastapi/core/models/__init__.py

@@ -0,0 +1 @@
+"""stac_fastapi.elasticsearch.models module."""

stac_fastapi/core/models/links.py

@@ -0,0 +1,205 @@
+"""link helpers."""
+
+from typing import Any, Dict, List, Optional
+from urllib.parse import ParseResult, parse_qs, urlencode, urljoin, urlparse
+
+import attr
+from stac_pydantic.links import Relations
+from stac_pydantic.shared import MimeTypes
+from starlette.requests import Request
+
+# Copied from pgstac links
+
+# These can be inferred from the item/collection, so they aren't included in the database
+# Instead they are dynamically generated when querying the database using the classes defined below
+INFERRED_LINK_RELS = {"self", "item", "parent", "collection", "root"}
+
+
+def merge_params(url: str, newparams: Dict) -> str:
+    """Merge url parameters."""
+    u = urlparse(url)
+    params = parse_qs(u.query)
+    params.update(newparams)
+    param_string = urlencode(params, True)
+
+    href = ParseResult(
+        scheme=u.scheme,
+        netloc=u.netloc,
+        path=u.path,
+        params=u.params,
+        query=param_string,
+        fragment=u.fragment,
+    ).geturl()
+    return href
+
+
+@attr.s
+class BaseLinks:
+    """Create inferred links common to collections and items."""
+
+    request: Request = attr.ib()
+
+    @property
+    def base_url(self):
+        """Get the base url."""
+        return str(self.request.base_url)
+
+    @property
+    def url(self):
+        """Get the current request url."""
+        return str(self.request.url)
+
+    def resolve(self, url):
+        """Resolve url to the current request url."""
+        return urljoin(str(self.base_url), str(url))
+
+    def link_self(self) -> Dict:
+        """Return the self link."""
+        return dict(rel=Relations.self.value, type=MimeTypes.json.value, href=self.url)
+
+    def link_root(self) -> Dict:
+        """Return the catalog root."""
+        return dict(
+            rel=Relations.root.value, type=MimeTypes.json.value, href=self.base_url
+        )
+
+    def create_links(self) -> List[Dict[str, Any]]:
+        """Return all inferred links."""
+        links = []
+        for name in dir(self):
+            if name.startswith("link_") and callable(getattr(self, name)):
+                link = getattr(self, name)()
+                if link is not None:
+                    links.append(link)
+        return links
+
+    async def get_links(
+        self, extra_links: Optional[List[Dict[str, Any]]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Generate all the links.
+
+        Get the links object for a stac resource by iterating through
+        available methods on this class that start with link_.
+        """
+        # TODO: Pass request.json() into function so this doesn't need to be coroutine
+        if self.request.method == "POST":
+            self.request.postbody = await self.request.json()
+        # join passed in links with generated links
+        # and update relative paths
+        links = self.create_links()
+
+        if extra_links:
+            # For extra links passed in,
+            # add links modified with a resolved href.
+            # Drop any links that are dynamically
+            # determined by the server (e.g. self, parent, etc.)
+            # Resolving the href allows for relative paths
+            # to be stored in pgstac and for the hrefs in the
+            # links of response STAC objects to be resolved
+            # to the request url.
+            links += [
+                {**link, "href": self.resolve(link["href"])}
+                for link in extra_links
+                if link["rel"] not in INFERRED_LINK_RELS
+            ]
+
+        return links
+
+
+@attr.s
+class CollectionLinks(BaseLinks):
+    """Create inferred links specific to collections."""
+
+    collection_id: str = attr.ib()
+    extensions: List[str] = attr.ib(default=attr.Factory(list))
+
+    def link_self(self) -> Dict:
+        """Return the self link."""
+        return dict(
+            rel=Relations.self.value,
+            type=MimeTypes.json.value,
+            href=urljoin(self.base_url, f"collections/{self.collection_id}"),
+        )
+
+    def link_parent(self) -> Dict[str, Any]:
+        """Create the `parent` link."""
+        return dict(rel=Relations.parent, type=MimeTypes.json.value, href=self.base_url)
+
+    def link_items(self) -> Dict[str, Any]:
+        """Create the `items` link."""
+        return dict(
+            rel="items",
+            type=MimeTypes.geojson.value,
+            href=urljoin(self.base_url, f"collections/{self.collection_id}/items"),
+        )
+
+    def link_queryables(self) -> Dict[str, Any]:
+        """Create the `queryables` link."""
+        if "FilterExtension" in self.extensions:
+            return dict(
+                rel="queryables",
+                type=MimeTypes.json.value,
+                href=urljoin(
+                    self.base_url, f"collections/{self.collection_id}/queryables"
+                ),
+            )
+        else:
+            return None
+
+    def link_aggregate(self) -> Dict[str, Any]:
+        """Create the `aggregate` link."""
+        if "AggregationExtension" in self.extensions:
+            return dict(
+                rel="aggregate",
+                type=MimeTypes.json.value,
+                href=urljoin(
+                    self.base_url, f"collections/{self.collection_id}/aggregate"
+                ),
+            )
+        else:
+            return None
+
+    def link_aggregations(self) -> Dict[str, Any]:
+        """Create the `aggregations` link."""
+        if "AggregationExtension" in self.extensions:
+            return dict(
+                rel="aggregations",
+                type=MimeTypes.json.value,
+                href=urljoin(
+                    self.base_url, f"collections/{self.collection_id}/aggregations"
+                ),
+            )
+        else:
+            return None
+
+
+@attr.s
+class PagingLinks(BaseLinks):
+    """Create links for paging."""
+
+    next: Optional[str] = attr.ib(kw_only=True, default=None)
+
+    def link_next(self) -> Optional[Dict[str, Any]]:
+        """Create link for next page."""
+        if self.next is not None:
+            method = self.request.method
+            if method == "GET":
+                href = merge_params(self.url, {"token": self.next})
+                link = dict(
+                    rel=Relations.next.value,
+                    type=MimeTypes.json.value,
+                    method=method,
+                    href=href,
+                )
+                return link
+            if method == "POST":
+                return {
+                    "rel": Relations.next,
+                    "type": MimeTypes.json,
+                    "method": method,
+                    "href": f"{self.request.url}",
+                    "body": {**self.request.postbody, "token": self.next},
+                }
+
+        return None

stac_fastapi/core/models/search.py

@@ -0,0 +1 @@
+"""Unused search model."""

stac_fastapi/core/rate_limit.py

@@ -0,0 +1,44 @@
+"""Rate limiting middleware."""
+
+import logging
+import os
+from typing import Optional
+
+from fastapi import FastAPI, Request
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
+from slowapi.middleware import SlowAPIMiddleware
+from slowapi.util import get_remote_address
+
+logger = logging.getLogger(__name__)
+
+
+def get_limiter(key_func=get_remote_address):
+    """Create and return a Limiter instance for rate limiting."""
+    return Limiter(key_func=key_func)
+
+
+def setup_rate_limit(
+    app: FastAPI, rate_limit: Optional[str] = None, key_func=get_remote_address
+):
+    """Set up rate limiting middleware."""
+    RATE_LIMIT = rate_limit or os.getenv("STAC_FASTAPI_RATE_LIMIT")
+
+    if not RATE_LIMIT:
+        logger.info("Rate limiting is disabled")
+        return
+
+    logger.info(f"Setting up rate limit with RATE_LIMIT={RATE_LIMIT}")
+
+    limiter = get_limiter(key_func)
+    app.state.limiter = limiter
+    app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
+    app.add_middleware(SlowAPIMiddleware)
+
+    @app.middleware("http")
+    @limiter.limit(RATE_LIMIT)
+    async def rate_limit_middleware(request: Request, call_next):
+        response = await call_next(request)
+        return response
+
+    logger.info("Rate limit setup complete")