stac-fastapi-core 4.0.0a1__py3-none-any.whl

stac_fastapi/core/database_logic.py

@@ -0,0 +1,226 @@
+"""Database logic core."""
+
+import os
+from functools import lru_cache
+from typing import Any, Dict, List, Optional, Protocol
+
+from stac_fastapi.types.stac import Item
+
+
+# stac_pydantic classes extend _GeometryBase, which doesn't have a type field,
+# So create our own Protocol for typing
+# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection]
+class Geometry(Protocol):  # noqa
+    type: str
+    coordinates: Any
+
+
+COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections")
+ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_")
+
+ES_INDEX_NAME_UNSUPPORTED_CHARS = {
+    "\\",
+    "/",
+    "*",
+    "?",
+    '"',
+    "<",
+    ">",
+    "|",
+    " ",
+    ",",
+    "#",
+    ":",
+}
+
+_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE = str.maketrans(
+    "", "", "".join(ES_INDEX_NAME_UNSUPPORTED_CHARS)
+)
+
+ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*"
+
+DEFAULT_SORT = {
+    "properties.datetime": {"order": "desc"},
+    "id": {"order": "desc"},
+    "collection": {"order": "desc"},
+}
+
+ES_ITEMS_SETTINGS = {
+    "index": {
+        "sort.field": list(DEFAULT_SORT.keys()),
+        "sort.order": [v["order"] for v in DEFAULT_SORT.values()],
+    }
+}
+
+ES_MAPPINGS_DYNAMIC_TEMPLATES = [
+    # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
+    {
+        "descriptions": {
+            "match_mapping_type": "string",
+            "match": "description",
+            "mapping": {"type": "text"},
+        }
+    },
+    {
+        "titles": {
+            "match_mapping_type": "string",
+            "match": "title",
+            "mapping": {"type": "text"},
+        }
+    },
+    # Projection Extension https://github.com/stac-extensions/projection
+    {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}},
+    {
+        "proj_projjson": {
+            "match": "proj:projjson",
+            "mapping": {"type": "object", "enabled": False},
+        }
+    },
+    {
+        "proj_centroid": {
+            "match": "proj:centroid",
+            "mapping": {"type": "geo_point"},
+        }
+    },
+    {
+        "proj_geometry": {
+            "match": "proj:geometry",
+            "mapping": {"type": "object", "enabled": False},
+        }
+    },
+    {
+        "no_index_href": {
+            "match": "href",
+            "mapping": {"type": "text", "index": False},
+        }
+    },
+    # Default all other strings not otherwise specified to keyword
+    {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}},
+    {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}},
+]
+
+ES_ITEMS_MAPPINGS = {
+    "numeric_detection": False,
+    "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,
+    "properties": {
+        "id": {"type": "keyword"},
+        "collection": {"type": "keyword"},
+        "geometry": {"type": "geo_shape"},
+        "assets": {"type": "object", "enabled": False},
+        "links": {"type": "object", "enabled": False},
+        "properties": {
+            "type": "object",
+            "properties": {
+                # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md
+                "datetime": {"type": "date"},
+                "start_datetime": {"type": "date"},
+                "end_datetime": {"type": "date"},
+                "created": {"type": "date"},
+                "updated": {"type": "date"},
+                # Satellite Extension https://github.com/stac-extensions/sat
+                "sat:absolute_orbit": {"type": "integer"},
+                "sat:relative_orbit": {"type": "integer"},
+            },
+        },
+    },
+}
+
+ES_COLLECTIONS_MAPPINGS = {
+    "numeric_detection": False,
+    "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES,
+    "properties": {
+        "id": {"type": "keyword"},
+        "extent.spatial.bbox": {"type": "long"},
+        "extent.temporal.interval": {"type": "date"},
+        "providers": {"type": "object", "enabled": False},
+        "links": {"type": "object", "enabled": False},
+        "item_assets": {"type": "object", "enabled": False},
+    },
+}
+
+
+@lru_cache(256)
+def index_by_collection_id(collection_id: str) -> str:
+    """
+    Translate a collection id into an Elasticsearch index name.
+
+    Args:
+        collection_id (str): The collection id to translate into an index name.
+
+    Returns:
+        str: The index name derived from the collection id.
+    """
+    cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+    return (
+        f"{ITEMS_INDEX_PREFIX}{cleaned.lower()}_{collection_id.encode('utf-8').hex()}"
+    )
+
+
+@lru_cache(256)
+def index_alias_by_collection_id(collection_id: str) -> str:
+    """
+    Translate a collection id into an Elasticsearch index alias.
+
+    Args:
+        collection_id (str): The collection id to translate into an index alias.
+
+    Returns:
+        str: The index alias derived from the collection id.
+    """
+    cleaned = collection_id.translate(_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE)
+    return f"{ITEMS_INDEX_PREFIX}{cleaned}"
+
+
+def indices(collection_ids: Optional[List[str]]) -> str:
+    """
+    Get a comma-separated string of index names for a given list of collection ids.
+
+    Args:
+        collection_ids: A list of collection ids.
+
+    Returns:
+        A string of comma-separated index names. If `collection_ids` is empty, returns the default indices.
+    """
+    return (
+        ",".join(map(index_alias_by_collection_id, collection_ids))
+        if collection_ids
+        else ITEM_INDICES
+    )
+
+
+def mk_item_id(item_id: str, collection_id: str) -> str:
+    """Create the document id for an Item in Elasticsearch.
+
+    Args:
+        item_id (str): The id of the Item.
+        collection_id (str): The id of the Collection that the Item belongs to.
+
+    Returns:
+        str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character.
+    """
+    return f"{item_id}|{collection_id}"
+
+
+def mk_actions(collection_id: str, processed_items: List[Item]) -> List[Dict[str, Any]]:
+    """Create Elasticsearch bulk actions for a list of processed items.
+
+    Args:
+        collection_id (str): The identifier for the collection the items belong to.
+        processed_items (List[Item]): The list of processed items to be bulk indexed.
+
+    Returns:
+        List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed,
+        each action being a dictionary with the following keys:
+        - `_index`: the index to store the document in.
+        - `_id`: the document's identifier.
+        - `_source`: the source of the document.
+    """
+    index_alias = index_alias_by_collection_id(collection_id)
+    return [
+        {
+            "_index": index_alias,
+            "_id": mk_item_id(item["id"], item["collection"]),
+            "_source": item,
+        }
+        for item in processed_items
+    ]

stac_fastapi/core/datetime_utils.py

@@ -0,0 +1,39 @@
+"""A few datetime methods."""
+from datetime import datetime, timezone
+
+
+# Borrowed from pystac - https://github.com/stac-utils/pystac/blob/f5e4cf4a29b62e9ef675d4a4dac7977b09f53c8f/pystac/utils.py#L370-L394
+def datetime_to_str(dt: datetime, timespec: str = "auto") -> str:
+    """Convert a :class:`datetime.datetime` instance to an ISO8601 string in the `RFC 3339, section 5.6.
+
+    <https://datatracker.ietf.org/doc/html/rfc3339#section-5.6>`__ format required by
+    the :stac-spec:`STAC Spec <master/item-spec/common-metadata.md#date-and-time>`.
+
+    Args:
+        dt : The datetime to convert.
+        timespec: An optional argument that specifies the number of additional
+            terms of the time to include. Valid options are 'auto', 'hours',
+            'minutes', 'seconds', 'milliseconds' and 'microseconds'. The default value
+            is 'auto'.
+    Returns:
+        str: The ISO8601 (RFC 3339) formatted string representing the datetime.
+    """
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+
+    timestamp = dt.isoformat(timespec=timespec)
+    zulu = "+00:00"
+    if timestamp.endswith(zulu):
+        timestamp = f"{timestamp[: -len(zulu)]}Z"
+
+    return timestamp
+
+
+def now_in_utc() -> datetime:
+    """Return a datetime value of now with the UTC timezone applied."""
+    return datetime.now(timezone.utc)
+
+
+def now_to_rfc3339_str() -> str:
+    """Return an RFC 3339 string representing now."""
+    return datetime_to_str(now_in_utc())

stac_fastapi/core/extensions/__init__.py

@@ -0,0 +1,5 @@
+"""elasticsearch extensions modifications."""
+
+from .query import Operator, QueryableTypes, QueryExtension
+
+__all__ = ["Operator", "QueryableTypes", "QueryExtension"]