stac-fastapi-core 6.5.0__py3-none-any.whl → 6.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,8 @@
3
3
  import abc
4
4
  from typing import Any, Dict, Iterable, List, Optional, Tuple
5
5
 
6
+ from stac_pydantic.shared import BBox
7
+
6
8
 
7
9
  class BaseDatabaseLogic(abc.ABC):
8
10
  """
@@ -19,7 +21,12 @@ class BaseDatabaseLogic(abc.ABC):
19
21
  limit: int,
20
22
  request: Any = None,
21
23
  sort: Optional[List[Dict[str, Any]]] = None,
22
- ) -> Tuple[List[Dict[str, Any]], Optional[str]]:
24
+ bbox: Optional[BBox] = None,
25
+ q: Optional[List[str]] = None,
26
+ filter: Optional[Dict[str, Any]] = None,
27
+ query: Optional[Dict[str, Dict[str, Any]]] = None,
28
+ datetime: Optional[str] = None,
29
+ ) -> Tuple[List[Dict[str, Any]], Optional[str], Optional[int]]:
23
30
  """Retrieve a list of collections from the database, supporting pagination.
24
31
 
25
32
  Args:
@@ -27,9 +34,14 @@ class BaseDatabaseLogic(abc.ABC):
27
34
  limit (int): The number of results to return.
28
35
  request (Any, optional): The FastAPI request object. Defaults to None.
29
36
  sort (Optional[List[Dict[str, Any]]], optional): Optional sort parameter. Defaults to None.
37
+ bbox (Optional[BBox], optional): Bounding box to filter collections by spatial extent. Defaults to None.
38
+ q (Optional[List[str]], optional): Free text search terms. Defaults to None.
39
+ filter (Optional[Dict[str, Any]], optional): Structured query in CQL2 format. Defaults to None.
40
+ query (Optional[Dict[str, Dict[str, Any]]], optional): Query extension parameters. Defaults to None.
41
+ datetime (Optional[str], optional): Temporal filter. Defaults to None.
30
42
 
31
43
  Returns:
32
- A tuple of (collections, next pagination token if any).
44
+ A tuple of (collections, next pagination token if any, optional count).
33
45
  """
34
46
  pass
35
47
 
stac_fastapi/core/core.py CHANGED
@@ -201,17 +201,6 @@ class CoreClient(AsyncBaseCoreClient):
201
201
  ]
202
202
  )
203
203
 
204
- collections = await self.all_collections(request=kwargs["request"])
205
- for collection in collections["collections"]:
206
- landing_page["links"].append(
207
- {
208
- "rel": Relations.child.value,
209
- "type": MimeTypes.json.value,
210
- "title": collection.get("title") or collection.get("id"),
211
- "href": urljoin(base_url, f"collections/{collection['id']}"),
212
- }
213
- )
214
-
215
204
  # Add OpenAPI URL
216
205
  landing_page["links"].append(
217
206
  {
@@ -240,65 +229,69 @@ class CoreClient(AsyncBaseCoreClient):
240
229
 
241
230
  async def all_collections(
242
231
  self,
243
- datetime: Optional[str] = None,
244
232
  limit: Optional[int] = None,
233
+ bbox: Optional[BBox] = None,
234
+ datetime: Optional[str] = None,
245
235
  fields: Optional[List[str]] = None,
246
236
  sortby: Optional[Union[str, List[str]]] = None,
247
237
  filter_expr: Optional[str] = None,
248
238
  filter_lang: Optional[str] = None,
249
239
  q: Optional[Union[str, List[str]]] = None,
250
240
  query: Optional[str] = None,
241
+ request: Request = None,
242
+ token: Optional[str] = None,
251
243
  **kwargs,
252
244
  ) -> stac_types.Collections:
253
245
  """Read all collections from the database.
254
246
 
255
247
  Args:
256
- datetime (Optional[str]): Filter collections by datetime range.
257
248
  limit (Optional[int]): Maximum number of collections to return.
249
+ bbox (Optional[BBox]): Bounding box to filter collections by spatial extent.
250
+ datetime (Optional[str]): Filter collections by datetime range.
258
251
  fields (Optional[List[str]]): Fields to include or exclude from the results.
259
- sortby (Optional[str]): Sorting options for the results.
252
+ sortby (Optional[Union[str, List[str]]]): Sorting options for the results.
260
253
  filter_expr (Optional[str]): Structured filter expression in CQL2 JSON or CQL2-text format.
261
- query (Optional[str]): Legacy query parameter (deprecated).
262
254
  filter_lang (Optional[str]): Must be 'cql2-json' or 'cql2-text' if specified, other values will result in an error.
263
255
  q (Optional[Union[str, List[str]]]): Free text search terms.
256
+ query (Optional[str]): Legacy query parameter (deprecated).
257
+ request (Request): FastAPI Request object.
258
+ token (Optional[str]): Pagination token for retrieving the next page of results.
264
259
  **kwargs: Keyword arguments from the request.
265
260
 
266
261
  Returns:
267
262
  A Collections object containing all the collections in the database and links to various resources.
268
263
  """
269
- request = kwargs["request"]
270
264
  base_url = str(request.base_url)
271
265
 
272
- # Get the global limit from environment variable
273
- global_limit = None
274
- env_limit = os.getenv("STAC_ITEM_LIMIT")
275
- if env_limit:
276
- try:
277
- global_limit = int(env_limit)
278
- except ValueError:
279
- # Handle invalid integer in environment variable
280
- pass
281
-
282
- # Apply global limit if it exists
283
- if global_limit is not None:
284
- # If a limit was provided, use the smaller of the two
285
- if limit is not None:
286
- limit = min(limit, global_limit)
287
- else:
288
- limit = global_limit
266
+ global_max_limit = (
267
+ int(os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT"))
268
+ if os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT")
269
+ else None
270
+ )
271
+ query_limit = request.query_params.get("limit")
272
+ default_limit = int(os.getenv("STAC_DEFAULT_COLLECTION_LIMIT", 300))
273
+
274
+ body_limit = None
275
+ try:
276
+ if request.method == "POST" and request.body():
277
+ body_data = await request.json()
278
+ body_limit = body_data.get("limit")
279
+ except Exception:
280
+ pass
281
+
282
+ if body_limit is not None:
283
+ limit = int(body_limit)
284
+ elif query_limit:
285
+ limit = int(query_limit)
289
286
  else:
290
- # No global limit, use provided limit or default
291
- if limit is None:
292
- query_limit = request.query_params.get("limit")
293
- if query_limit:
294
- try:
295
- limit = int(query_limit)
296
- except ValueError:
297
- limit = 10
298
- else:
299
- limit = 10
287
+ limit = default_limit
300
288
 
301
- token = request.query_params.get("token")
289
+ if global_max_limit is not None:
290
+ limit = min(limit, global_max_limit)
291
+
292
+ # Get token from query params only if not already provided (for GET requests)
293
+ if token is None:
294
+ token = request.query_params.get("token")
302
295
 
303
296
  # Process fields parameter for filtering collection properties
304
297
  includes, excludes = set(), set()
@@ -398,6 +391,7 @@ class CoreClient(AsyncBaseCoreClient):
398
391
  limit=limit,
399
392
  request=request,
400
393
  sort=sort,
394
+ bbox=bbox,
401
395
  q=q_list,
402
396
  filter=parsed_filter,
403
397
  query=parsed_query,
@@ -499,6 +493,11 @@ class CoreClient(AsyncBaseCoreClient):
499
493
  # Pass all parameters from search_request to all_collections
500
494
  return await self.all_collections(
501
495
  limit=search_request.limit if hasattr(search_request, "limit") else None,
496
+ bbox=search_request.bbox if hasattr(search_request, "bbox") else None,
497
+ datetime=search_request.datetime
498
+ if hasattr(search_request, "datetime")
499
+ else None,
500
+ token=search_request.token if hasattr(search_request, "token") else None,
502
501
  fields=fields,
503
502
  sortby=sortby,
504
503
  filter_expr=search_request.filter
@@ -562,7 +561,7 @@ class CoreClient(AsyncBaseCoreClient):
562
561
  request (Request): FastAPI Request object.
563
562
  bbox (Optional[BBox]): Optional bounding box filter.
564
563
  datetime (Optional[str]): Optional datetime or interval filter.
565
- limit (Optional[int]): Optional page size. Defaults to env ``STAC_ITEM_LIMIT`` when unset.
564
+ limit (Optional[int]): Optional page size. Defaults to env `STAC_DEFAULT_ITEM_LIMIT` when unset.
566
565
  sortby (Optional[str]): Optional sort specification. Accepts repeated values
567
566
  like ``sortby=-properties.datetime`` or ``sortby=+id``. Bare fields (e.g. ``sortby=id``)
568
567
  imply ascending order.
@@ -653,15 +652,12 @@ class CoreClient(AsyncBaseCoreClient):
653
652
  q (Optional[List[str]]): Free text query to filter the results.
654
653
  intersects (Optional[str]): GeoJSON geometry to search in.
655
654
  kwargs: Additional parameters to be passed to the API.
656
-
657
655
  Returns:
658
656
  ItemCollection: Collection of `Item` objects representing the search results.
659
657
 
660
658
  Raises:
661
659
  HTTPException: If any error occurs while searching the catalog.
662
660
  """
663
- limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
664
-
665
661
  base_args = {
666
662
  "collections": collections,
667
663
  "ids": ids,
@@ -736,6 +732,34 @@ class CoreClient(AsyncBaseCoreClient):
736
732
  Raises:
737
733
  HTTPException: If there is an error with the cql2_json filter.
738
734
  """
735
+ global_max_limit = (
736
+ int(os.getenv("STAC_GLOBAL_ITEM_MAX_LIMIT"))
737
+ if os.getenv("STAC_GLOBAL_ITEM_MAX_LIMIT")
738
+ else None
739
+ )
740
+ query_limit = request.query_params.get("limit")
741
+ default_limit = int(os.getenv("STAC_DEFAULT_ITEM_LIMIT", 10))
742
+
743
+ body_limit = None
744
+ try:
745
+ if request.method == "POST" and request.body():
746
+ body_data = await request.json()
747
+ body_limit = body_data.get("limit")
748
+ except Exception:
749
+ pass
750
+
751
+ if body_limit is not None:
752
+ limit = int(body_limit)
753
+ elif query_limit:
754
+ limit = int(query_limit)
755
+ else:
756
+ limit = default_limit
757
+
758
+ if global_max_limit:
759
+ limit = min(limit, global_max_limit)
760
+
761
+ search_request.limit = limit
762
+
739
763
  base_url = str(request.base_url)
740
764
 
741
765
  search = self.database.make_search()
@@ -812,7 +836,6 @@ class CoreClient(AsyncBaseCoreClient):
812
836
  if hasattr(search_request, "sortby") and getattr(search_request, "sortby"):
813
837
  sort = self.database.populate_sort(getattr(search_request, "sortby"))
814
838
 
815
- limit = 10
816
839
  if search_request.limit:
817
840
  limit = search_request.limit
818
841
 
@@ -18,6 +18,10 @@ class CollectionsSearchRequest(ExtendedSearch):
18
18
  """Extended search model for collections with free text search support."""
19
19
 
20
20
  q: Optional[Union[str, List[str]]] = None
21
+ token: Optional[str] = None
22
+ query: Optional[
23
+ str
24
+ ] = None # Legacy query extension (deprecated but still supported)
21
25
 
22
26
 
23
27
  class CollectionsSearchEndpointExtension(ApiExtension):
@@ -1,6 +1,7 @@
1
1
  """Serializers."""
2
2
 
3
3
  import abc
4
+ import logging
4
5
  from copy import deepcopy
5
6
  from typing import Any, List, Optional
6
7
 
@@ -13,6 +14,8 @@ from stac_fastapi.core.utilities import get_bool_env
13
14
  from stac_fastapi.types import stac as stac_types
14
15
  from stac_fastapi.types.links import ItemLinks, resolve_links
15
16
 
17
+ logger = logging.getLogger(__name__)
18
+
16
19
 
17
20
  @attr.s
18
21
  class Serializer(abc.ABC):
@@ -168,6 +171,9 @@ class CollectionSerializer(Serializer):
168
171
  # Avoid modifying the input dict in-place ... doing so breaks some tests
169
172
  collection = deepcopy(collection)
170
173
 
174
+ # Remove internal bbox_shape field (not part of STAC spec)
175
+ collection.pop("bbox_shape", None)
176
+
171
177
  # Set defaults
172
178
  collection_id = collection.get("id")
173
179
  collection.setdefault("type", "Collection")
@@ -10,15 +10,7 @@ from typing import Any, Dict, List, Optional, Set, Union
10
10
 
11
11
  from stac_fastapi.types.stac import Item
12
12
 
13
-
14
- def get_max_limit():
15
- """
16
- Retrieve a MAX_LIMIT value from an environment variable.
17
-
18
- Returns:
19
- int: The int value parsed from the environment variable.
20
- """
21
- return int(os.getenv("ENV_MAX_LIMIT", 10000))
13
+ MAX_LIMIT = 10000
22
14
 
23
15
 
24
16
  def get_bool_env(name: str, default: Union[bool, str] = False) -> bool:
@@ -1,2 +1,2 @@
1
1
  """library version."""
2
- __version__ = "6.5.0"
2
+ __version__ = "6.6.0"
@@ -0,0 +1,53 @@
1
+ Metadata-Version: 2.4
2
+ Name: stac_fastapi_core
3
+ Version: 6.6.0
4
+ Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
5
+ Project-URL: Homepage, https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
+ License: MIT
7
+ Keywords: Elasticsearch,FastAPI,Opensearch,STAC,STAC-API,stac-fastapi
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: Intended Audience :: Information Technology
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Requires-Python: >=3.9
19
+ Requires-Dist: attrs>=23.2.0
20
+ Requires-Dist: fastapi~=0.109.0
21
+ Requires-Dist: geojson-pydantic~=1.0.0
22
+ Requires-Dist: jsonschema~=4.0.0
23
+ Requires-Dist: orjson~=3.11.0
24
+ Requires-Dist: overrides~=7.4.0
25
+ Requires-Dist: pydantic<3.0.0,>=2.4.1
26
+ Requires-Dist: pygeofilter~=0.3.1
27
+ Requires-Dist: slowapi~=0.1.9
28
+ Requires-Dist: stac-fastapi-api==6.0.0
29
+ Requires-Dist: stac-fastapi-extensions==6.0.0
30
+ Requires-Dist: stac-fastapi-types==6.0.0
31
+ Requires-Dist: stac-pydantic~=3.3.0
32
+ Description-Content-Type: text/markdown
33
+
34
+ # stac-fastapi-core
35
+
36
+ Core functionality for stac-fastapi. For full documentation, please see the [main README](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/blob/main/README.md).
37
+
38
+ ## Package Information
39
+
40
+ - **Package name**: stac-fastapi-core
41
+ - **Description**: Core functionality for STAC API implementations.
42
+ - **Documentation**: [https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/)
43
+ - **Source**: [GitHub Repository](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/)
44
+
45
+ ## Installation
46
+
47
+ ```bash
48
+ pip install stac-fastapi-core
49
+ ```
50
+
51
+ ## Quick Start
52
+
53
+ For detailed usage and examples, please refer to the [main documentation](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/).
@@ -1,25 +1,24 @@
1
1
  stac_fastapi/core/__init__.py,sha256=8izV3IWRGdXmDOK1hIPQAanbWs9EI04PJCGgqG1ZGIs,20
2
- stac_fastapi/core/base_database_logic.py,sha256=nhj0CZNur_SRs4GtXTER-Zjq8JPub5zINiCKbCjw0Bs,3814
2
+ stac_fastapi/core/base_database_logic.py,sha256=3_XJ_j06ogQHE-Tcjkv5Vye_zNDn9OEU9lNYU03am1k,4618
3
3
  stac_fastapi/core/base_settings.py,sha256=R3_Sx7n5XpGMs3zAwFJD7y008WvGU_uI2xkaabm82Kg,239
4
4
  stac_fastapi/core/basic_auth.py,sha256=RhFv3RVSHF6OaqnaaU2DO4ncJ_S5nB1q8UNpnVJJsrk,2155
5
- stac_fastapi/core/core.py,sha256=8eGoIcEZVh-4JGZ9PSrY-e8ypEqfmIBxk_napP1HRL4,48014
5
+ stac_fastapi/core/core.py,sha256=f1-TJSH7Hl5MYTtdgjBUGUvFi7gFLex3MxU4ubYBqhI,48838
6
6
  stac_fastapi/core/datetime_utils.py,sha256=TrTgbU7AKNC-ic4a3HptfE5XAc9tHR7uJasZyhOuwnc,2633
7
7
  stac_fastapi/core/rate_limit.py,sha256=Gu8dAaJReGsj1L91U6m2tflU6RahpXDRs2-AYSKoybA,1318
8
8
  stac_fastapi/core/route_dependencies.py,sha256=hdtuMkv-zY1vg0YxiCz1aKP0SbBcORqDGEKDGgEazW8,5482
9
- stac_fastapi/core/serializers.py,sha256=HU7sVSMa6w_F_qs_gdAeIFZ18GW-6t8ZHFmgI4-1uNw,7455
9
+ stac_fastapi/core/serializers.py,sha256=ZW5hPgq-mftk6zxJeZGur-1Qxn7YGc3fJYFLsd-SYwM,7619
10
10
  stac_fastapi/core/session.py,sha256=aXqu4LXfVbAAsChMVXd9gAhczA2bZPne6HqPeklAwMY,474
11
- stac_fastapi/core/utilities.py,sha256=WbspaJey_Cs-7TrBKasdqq7yjB7vjKiU01KyJM0m8_E,7506
12
- stac_fastapi/core/version.py,sha256=KQjuGSR03-CXgF6wsaZ8qsni161S2BjhOn3wTX8JAMw,45
11
+ stac_fastapi/core/utilities.py,sha256=xXWO5oJCNDi7_C5jPYlHZD0B-DL-FN66eEUBUSW-cXw,7296
12
+ stac_fastapi/core/version.py,sha256=zPQp-GtLh4R45hT8V_KAWzwZcP4-jyw7Xjms_eNMZBc,45
13
13
  stac_fastapi/core/extensions/__init__.py,sha256=zSIAqou8jnakWPbkh4Ddcx1-oazZVBOs7U2PAakAdU0,291
14
14
  stac_fastapi/core/extensions/aggregation.py,sha256=v1hUHqlYuMqfQ554g3cTp16pUyRYucQxPERbHPAFtf8,1878
15
- stac_fastapi/core/extensions/collections_search.py,sha256=bneJQuu-OIsW5P0IbfL6Wz-iowSdAmyz6C0X6vzMn9M,6366
15
+ stac_fastapi/core/extensions/collections_search.py,sha256=q7eRBykEqNRCiTfkmM_TobqKkxA3n1zQ7dYo37juE6s,6503
16
16
  stac_fastapi/core/extensions/fields.py,sha256=NCT5XHvfaf297eDPNaIFsIzvJnbbUTpScqF0otdx0NA,1066
17
17
  stac_fastapi/core/extensions/filter.py,sha256=-NQGME7rR_ereuDx-LAa1M5JhEXFaKiTtkH2asraYHE,2998
18
18
  stac_fastapi/core/extensions/query.py,sha256=Xmo8pfZEZKPudZEjjozv3R0wLOP0ayjC9E67sBOXqWY,1803
19
19
  stac_fastapi/core/models/__init__.py,sha256=g-D1DiGfmC9Bg27DW9JzkN6fAvscv75wyhyiZ6NzvIk,48
20
20
  stac_fastapi/core/models/links.py,sha256=0dWSEMt3aa7NCISlHwo11zLBeIV1LwXG3JGjrXC3dZI,6672
21
21
  stac_fastapi/core/models/search.py,sha256=7SgAUyzHGXBXSqB4G6cwq9FMwoAS00momb7jvBkjyow,27
22
- stac_fastapi_core-6.5.0.dist-info/METADATA,sha256=bTSmvE848AqW76bda8EzNHRORPTXp08pcig6HdGw7ZI,41746
23
- stac_fastapi_core-6.5.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
24
- stac_fastapi_core-6.5.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
25
- stac_fastapi_core-6.5.0.dist-info/RECORD,,
22
+ stac_fastapi_core-6.6.0.dist-info/METADATA,sha256=h54wZw2qk4ZgGH7kFlwIFmxR3wgEHwoXJ3p69w-iOFs,2163
23
+ stac_fastapi_core-6.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
24
+ stac_fastapi_core-6.6.0.dist-info/RECORD,,
@@ -1,5 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.45.1)
2
+ Generator: hatchling 1.27.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
-
@@ -1,726 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: stac-fastapi-core
3
- Version: 6.5.0
4
- Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
5
- Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
- License: MIT
7
- Classifier: Intended Audience :: Developers
8
- Classifier: Intended Audience :: Information Technology
9
- Classifier: Intended Audience :: Science/Research
10
- Classifier: Programming Language :: Python :: 3.9
11
- Classifier: Programming Language :: Python :: 3.10
12
- Classifier: Programming Language :: Python :: 3.11
13
- Classifier: Programming Language :: Python :: 3.12
14
- Classifier: Programming Language :: Python :: 3.13
15
- Classifier: License :: OSI Approved :: MIT License
16
- Requires-Python: >=3.9
17
- Description-Content-Type: text/markdown
18
- Requires-Dist: fastapi~=0.109.0
19
- Requires-Dist: attrs>=23.2.0
20
- Requires-Dist: pydantic<3.0.0,>=2.4.1
21
- Requires-Dist: stac-pydantic~=3.3.0
22
- Requires-Dist: stac-fastapi.types==6.0.0
23
- Requires-Dist: stac-fastapi.api==6.0.0
24
- Requires-Dist: stac-fastapi.extensions==6.0.0
25
- Requires-Dist: orjson~=3.9.0
26
- Requires-Dist: overrides~=7.4.0
27
- Requires-Dist: geojson-pydantic~=1.0.0
28
- Requires-Dist: pygeofilter~=0.3.1
29
- Requires-Dist: jsonschema~=4.0.0
30
- Requires-Dist: slowapi~=0.1.9
31
-
32
- # stac-fastapi-elasticsearch-opensearch
33
-
34
- <!-- markdownlint-disable MD033 MD041 -->
35
-
36
-
37
- <p align="left">
38
- <img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
39
- </p>
40
-
41
- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents)
42
-
43
- [![Downloads](https://static.pepy.tech/badge/stac-fastapi-core?color=blue)](https://pepy.tech/project/stac-fastapi-core)
44
- [![GitHub contributors](https://img.shields.io/github/contributors/stac-utils/stac-fastapi-elasticsearch-opensearch?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/graphs/contributors)
45
- [![GitHub stars](https://img.shields.io/github/stars/stac-utils/stac-fastapi-elasticsearch-opensearch.svg?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/stargazers)
46
- [![GitHub forks](https://img.shields.io/github/forks/stac-utils/stac-fastapi-elasticsearch-opensearch.svg?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/network/members)
47
- [![PyPI version](https://img.shields.io/pypi/v/stac-fastapi-elasticsearch.svg?color=blue)](https://pypi.org/project/stac-fastapi-elasticsearch/)
48
- [![STAC](https://img.shields.io/badge/STAC-1.1.0-blue.svg)](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
49
- [![stac-fastapi](https://img.shields.io/badge/stac--fastapi-6.0.0-blue.svg)](https://github.com/stac-utils/stac-fastapi)
50
-
51
- ## Sponsors & Supporters
52
-
53
- The following organizations have contributed time and/or funding to support the development of this project:
54
-
55
- <p align="left">
56
- <a href="https://healy-hyperspatial.github.io/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/hh-logo-blue.png" alt="Healy Hyperspatial" height="100" hspace="20"></a>
57
- <a href="https://atomicmaps.io/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/am-logo-black.png" alt="Atomic Maps" height="100" hspace="20"></a>
58
- <a href="https://remotesensing.vito.be/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/VITO.png" alt="VITO Remote Sensing" height="100" hspace="20"></a>
59
- </p>
60
-
61
- ## Project Introduction - What is SFEOS?
62
-
63
- SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable API implementation for serving SpatioTemporal Asset Catalog (STAC) data - an enhanced GeoJSON format designed specifically for geospatial assets like satellite imagery, aerial photography, and other Earth observation data. This project enables organizations to:
64
-
65
- - **Efficiently catalog and search geospatial data** such as satellite imagery, aerial photography, DEMs, and other geospatial assets using Elasticsearch or OpenSearch as the database backend
66
- - **Implement standardized STAC APIs** that support complex spatial, temporal, and property-based queries across large collections of geospatial data
67
- - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities
68
- - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries
69
- - **Perform geospatial aggregations** to analyze data distribution across space and time
70
- - **Enhanced collection search capabilities** with support for sorting and field selection
71
-
72
- This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs.
73
-
74
- ## Common Deployment Patterns
75
-
76
- stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs:
77
-
78
- - **Containerized Application**: Run as a Docker container with connections to Elasticsearch/OpenSearch databases
79
- - **Serverless Function**: Deploy as AWS Lambda or similar serverless function with API Gateway
80
- - **Traditional Server**: Run on virtual machines or bare metal servers in your infrastructure
81
- - **Kubernetes**: Deploy as part of a larger microservices architecture with container orchestration
82
-
83
- The implementation is flexible and can scale from small local deployments to large production environments serving millions of geospatial assets.
84
-
85
- ## Technologies
86
-
87
- This project is built on the following technologies: STAC, stac-fastapi, FastAPI, Elasticsearch, Python, OpenSearch
88
-
89
- <p align="left">
90
- <a href="https://stacspec.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/STAC-01.png" alt="STAC" height="100" hspace="10"></a>
91
- <a href="https://www.python.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/python.png" alt="Python" height="80" hspace="10"></a>
92
- <a href="https://fastapi.tiangolo.com/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/fastapi.svg" alt="FastAPI" height="80" hspace="10"></a>
93
- <a href="https://www.elastic.co/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/elasticsearch.png" alt="Elasticsearch" height="80" hspace="10"></a>
94
- <a href="https://opensearch.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/opensearch.svg" alt="OpenSearch" height="80" hspace="10"></a>
95
- </p>
96
-
97
- ## Table of Contents
98
-
99
- - [stac-fastapi-elasticsearch-opensearch](#stac-fastapi-elasticsearch-opensearch)
100
- - [Sponsors & Supporters](#sponsors--supporters)
101
- - [Project Introduction - What is SFEOS?](#project-introduction---what-is-sfeos)
102
- - [Common Deployment Patterns](#common-deployment-patterns)
103
- - [Technologies](#technologies)
104
- - [Table of Contents](#table-of-contents)
105
- - [Collection Search Extensions](#collection-search-extensions)
106
- - [Documentation & Resources](#documentation--resources)
107
- - [Package Structure](#package-structure)
108
- - [Examples](#examples)
109
- - [Performance](#performance)
110
- - [Direct Response Mode](#direct-response-mode)
111
- - [Quick Start](#quick-start)
112
- - [Installation](#installation)
113
- - [Running Locally](#running-locally)
114
- - [Using Pre-built Docker Images](#using-pre-built-docker-images)
115
- - [Using Docker Compose](#using-docker-compose)
116
- - [Configuration Reference](#configuration-reference)
117
- - [Datetime-Based Index Management](#datetime-based-index-management)
118
- - [Overview](#overview)
119
- - [When to Use](#when-to-use)
120
- - [Configuration](#configuration)
121
- - [Enabling Datetime-Based Indexing](#enabling-datetime-based-indexing)
122
- - [Related Configuration Variables](#related-configuration-variables)
123
- - [How Datetime-Based Indexing Works](#how-datetime-based-indexing-works)
124
- - [Index and Alias Naming Convention](#index-and-alias-naming-convention)
125
- - [Index Size Management](#index-size-management)
126
- - [Interacting with the API](#interacting-with-the-api)
127
- - [Configure the API](#configure-the-api)
128
- - [Collection Pagination](#collection-pagination)
129
- - [Ingesting Sample Data CLI Tool](#ingesting-sample-data-cli-tool)
130
- - [Elasticsearch Mappings](#elasticsearch-mappings)
131
- - [Managing Elasticsearch Indices](#managing-elasticsearch-indices)
132
- - [Snapshots](#snapshots)
133
- - [Reindexing](#reindexing)
134
- - [Auth](#auth)
135
- - [Aggregation](#aggregation)
136
- - [Rate Limiting](#rate-limiting)
137
-
138
- ## Documentation & Resources
139
-
140
- - **Online Documentation**: [https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/)
141
- - **Source Code**: [https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch)
142
- - **API Examples**: [Postman Documentation](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) - Examples of how to use the API endpoints
143
- - **Community**:
144
- - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions
145
- - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers
146
-
147
- ## Collection Search Extensions
148
-
149
- SFEOS provides enhanced collection search capabilities through two primary routes:
150
- - **GET/POST `/collections`**: The standard STAC endpoint with extended query parameters
151
- - **GET/POST `/collections-search`**: A custom endpoint that supports the same parameters, created to avoid conflicts with the STAC Transactions extension if enabled (which uses POST `/collections` for collection creation)
152
-
153
- These endpoints support advanced collection discovery features including:
154
-
155
- - **Sorting**: Sort collections by sortable fields using the `sortby` parameter
156
- - Example: `/collections?sortby=+id` (ascending sort by ID)
157
- - Example: `/collections?sortby=-id` (descending sort by ID)
158
- - Example: `/collections?sortby=-temporal` (descending sort by temporal extent)
159
-
160
- - **Field Selection**: Request only specific fields to be returned using the `fields` parameter
161
- - Example: `/collections?fields=id,title,description`
162
- - This helps reduce payload size when only certain fields are needed
163
-
164
- - **Free Text Search**: Search across collection text fields using the `q` parameter
165
- - Example: `/collections?q=landsat`
166
- - Searches across multiple text fields including title, description, and keywords
167
- - Supports partial word matching and relevance-based sorting
168
-
169
- - **Structured Filtering**: Filter collections using CQL2 expressions
170
- - JSON format: `/collections?filter={"op":"=","args":[{"property":"id"},"sentinel-2"]}&filter-lang=cql2-json`
171
- - Text format: `/collections?filter=id='sentinel-2'&filter-lang=cql2-text` (note: string values must be quoted)
172
- - Advanced text format: `/collections?filter=id LIKE '%sentinel%'&filter-lang=cql2-text` (supports LIKE, BETWEEN, etc.)
173
- - Supports both CQL2 JSON and CQL2 text formats with various operators
174
- - Enables precise filtering on any collection property
175
-
176
- - **Datetime Filtering**: Filter collections by their temporal extent using the `datetime` parameter
177
- - Example: `/collections?datetime=2020-01-01T00:00:00Z/2020-12-31T23:59:59Z` (finds collections with temporal extents that overlap this range)
178
- - Example: `/collections?datetime=2020-06-15T12:00:00Z` (finds collections whose temporal extent includes this specific time)
179
- - Example: `/collections?datetime=2020-01-01T00:00:00Z/..` (finds collections with temporal extents that extend to or beyond January 1, 2020)
180
- - Example: `/collections?datetime=../2020-12-31T23:59:59Z` (finds collections with temporal extents that begin on or before December 31, 2020)
181
- - Collections are matched if their temporal extent overlaps with the provided datetime parameter
182
- - This allows for efficient discovery of collections based on time periods
183
-
184
- These extensions make it easier to build user interfaces that display and navigate through collections efficiently.
185
-
186
- > **Configuration**: Collection search extensions (sorting, field selection, free text search, structured filtering, and datetime filtering) for the `/collections` endpoint can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled.
187
- >
188
- > **Configuration**: The custom `/collections-search` endpoint can be enabled by setting the `ENABLE_COLLECTIONS_SEARCH_ROUTE` environment variable to `true`. By default, this endpoint is **disabled**.
189
-
190
- > **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on:
191
- > - `id` (keyword field)
192
- > - `extent.temporal.interval` (date field)
193
- > - `temporal` (alias to extent.temporal.interval)
194
- >
195
- > Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings.
196
- >
197
- > **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable.
198
-
199
-
200
- ## Package Structure
201
-
202
- This project is organized into several packages, each with a specific purpose:
203
-
204
- - **stac_fastapi_core**: Core functionality that's database-agnostic, including API models, extensions, and shared utilities. This package provides the foundation for building STAC API implementations with any database backend. See [stac-fastapi-mongo](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo) for a working example.
205
-
206
- - **sfeos_helpers**: Shared helper functions and utilities used by both the Elasticsearch and OpenSearch backends. This package includes:
207
- - `database`: Specialized modules for index, document, and database utility operations
208
- - `aggregation`: Elasticsearch/OpenSearch-specific aggregation functionality
209
- - Shared logic and utilities that improve code reuse between backends
210
-
211
- - **stac_fastapi_elasticsearch**: Complete implementation of the STAC API using Elasticsearch as the backend database. This package depends on both `stac_fastapi_core` and `sfeos_helpers`.
212
-
213
- - **stac_fastapi_opensearch**: Complete implementation of the STAC API using OpenSearch as the backend database. This package depends on both `stac_fastapi_core` and `sfeos_helpers`.
214
-
215
- ## Examples
216
-
217
- The `/examples` directory contains several useful examples and reference implementations:
218
-
219
- - **pip_docker**: Examples of running stac-fastapi-elasticsearch from PyPI in Docker without needing any code from the repository
220
- - **auth**: Authentication examples including:
221
- - Basic authentication
222
- - OAuth2 with Keycloak
223
- - Route dependencies configuration
224
- - **rate_limit**: Example of implementing rate limiting for API requests
225
- - **postman_collections**: Postman collection files you can import for testing API endpoints
226
-
227
- These examples provide practical reference implementations for various deployment scenarios and features.
228
-
229
- ## Performance
230
-
231
- ### Direct Response Mode
232
-
233
- - The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0.
234
- - **Control via environment variable**: Set `ENABLE_DIRECT_RESPONSE=true` to enable this feature.
235
- - **How it works**: When enabled, endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance.
236
- - **Important limitation**: All FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes when this mode is enabled.
237
- - **Best use case**: This mode is best suited for public or read-only APIs where authentication and custom logic are not required.
238
- - **Default setting**: `false` for safety.
239
- - **More information**: See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) for background and implementation details.
240
-
241
- ## Quick Start
242
-
243
- This section helps you get up and running with stac-fastapi-elasticsearch-opensearch quickly.
244
-
245
- ### Installation
246
-
247
- - **For versions 4.0.0a1 and newer** (PEP 625 compliant naming):
248
- ```bash
249
- pip install stac-fastapi-elasticsearch # Elasticsearch backend
250
- pip install stac-fastapi-opensearch # Opensearch backend
251
- pip install stac-fastapi-core # Core library
252
- ```
253
-
254
- - **For versions 4.0.0a0 and older**:
255
- ```bash
256
- pip install stac-fastapi.elasticsearch # Elasticsearch backend
257
- pip install stac-fastapi.opensearch # Opensearch backend
258
- pip install stac-fastapi.core # Core library
259
- ```
260
-
261
- > **Important Note:** Starting with version 4.0.0a1, package names have changed from using periods (e.g., `stac-fastapi.core`) to using hyphens (e.g., `stac-fastapi-core`) to comply with PEP 625. The internal package structure uses underscores, but users should install with hyphens as shown above. Please update your requirements files accordingly.
262
-
263
- ### Running Locally
264
-
265
- There are two main ways to run the API locally:
266
-
267
- #### Using Pre-built Docker Images
268
-
269
- - We provide ready-to-use Docker images through GitHub Container Registry:
270
- - [ElasticSearch backend](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-es)
271
- - [OpenSearch backend](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-os)
272
-
273
- - **Pull and run the images**:
274
- ```shell
275
- # For Elasticsearch backend
276
- docker pull ghcr.io/stac-utils/stac-fastapi-es:latest
277
-
278
- # For OpenSearch backend
279
- docker pull ghcr.io/stac-utils/stac-fastapi-os:latest
280
- ```
281
-
282
- #### Using Docker Compose
283
-
284
- - **Prerequisites**: Ensure [Docker Compose](https://docs.docker.com/compose/install/) or [Podman Compose](https://podman-desktop.io/docs/compose) is installed on your machine.
285
-
286
- - **Start the API**:
287
- ```shell
288
- docker compose up elasticsearch app-elasticsearch
289
- ```
290
-
291
- - **Configuration**: By default, Docker Compose uses Elasticsearch 8.x and OpenSearch 2.11.1. To use different versions, create a `.env` file:
292
- ```shell
293
- ELASTICSEARCH_VERSION=8.11.0
294
- OPENSEARCH_VERSION=2.11.1
295
- ENABLE_DIRECT_RESPONSE=false
296
- ```
297
-
298
- - **Compatibility**: The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
299
-
300
-
301
-
302
- ## Configuration Reference
303
-
304
- You can customize additional settings in your `.env` file:
305
-
306
- | Variable | Description | Default | Required |
307
- |------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------|
308
- | `ES_HOST` | Hostname for external Elasticsearch/OpenSearch. | `localhost` | Optional |
309
- | `ES_PORT` | Port for Elasticsearch/OpenSearch. | `9200` (ES) / `9202` (OS)| Optional |
310
- | `ES_USE_SSL` | Use SSL for connecting to Elasticsearch/OpenSearch. | `true` | Optional |
311
- | `ES_VERIFY_CERTS` | Verify SSL certificates when connecting. | `true` | Optional |
312
- | `ES_API_KEY` | API Key for external Elasticsearch/OpenSearch. | N/A | Optional |
313
- | `ES_TIMEOUT` | Client timeout for Elasticsearch/OpenSearch. | DB client default | Optional |
314
- | `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-<backend>` | Optional |
315
- | `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional |
316
- | `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional |
317
- | `STAC_FASTAPI_LANDING_PAGE_ID` | Landing page ID | `stac-fastapi` | Optional |
318
- | `APP_HOST` | Server bind address. | `0.0.0.0` | Optional |
319
- | `APP_PORT` | Server port. | `8000` | Optional |
320
- | `ENVIRONMENT` | Runtime environment. | `local` | Optional |
321
- | `WEB_CONCURRENCY` | Number of worker processes. | `10` | Optional |
322
- | `RELOAD` | Enable auto-reload for development. | `true` | Optional |
323
- | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional |
324
- | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
325
- | `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
326
- | `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional |
327
- | `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
328
- | `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
329
- | `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
330
- | `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields, free text search, structured filtering, and datetime filtering) on the core `/collections` endpoint. | `true` | Optional |
331
- | `ENABLE_COLLECTIONS_SEARCH_ROUTE` | Enable the custom `/collections-search` endpoint (both GET and POST methods). When disabled, the custom endpoint will not be available, but collection search extensions will still be available on the core `/collections` endpoint if `ENABLE_COLLECTIONS_SEARCH` is true. | `false` | Optional |
332
- | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. This is useful for deployments where mutating the catalog via the API should be prevented. If set to `true`, the POST `/collections` route for search will be unavailable in the API. | `true` | Optional |
333
- | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
334
- | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
335
- | `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
336
- | `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional |
337
-
338
- > [!NOTE]
339
- > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
340
-
341
- ## Datetime-Based Index Management
342
-
343
- ### Overview
344
-
345
- SFEOS supports two indexing strategies for managing STAC items:
346
-
347
- 1. **Simple Indexing** (default) - One index per collection
348
- 2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
349
-
350
- The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
351
-
352
- ### When to Use
353
-
354
- **Recommended for:**
355
- - Systems with large collections containing millions of items
356
- - Systems requiring high-performance temporal searching
357
-
358
- **Pros:**
359
- - Multiple times faster queries with datetime filter
360
- - Reduced database load - only relevant indexes are searched
361
-
362
- **Cons:**
363
- - Slightly longer item indexing time (automatic index management)
364
- - Greater management complexity
365
-
366
- ### Configuration
367
-
368
- #### Enabling Datetime-Based Indexing
369
-
370
- Enable datetime-based indexing by setting the following environment variable:
371
-
372
- ```bash
373
- ENABLE_DATETIME_INDEX_FILTERING=true
374
- ```
375
-
376
- ### Related Configuration Variables
377
-
378
- | Variable | Description | Default | Example |
379
- |----------|-------------|---------|---------|
380
- | `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
381
- | `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
382
- | `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
383
-
384
- ## How Datetime-Based Indexing Works
385
-
386
- ### Index and Alias Naming Convention
387
-
388
- The system uses a precise naming convention:
389
-
390
- **Physical indexes:**
391
- ```
392
- {ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
393
- ```
394
-
395
- **Aliases:**
396
- ```
397
- {ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
398
- {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
399
- {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
400
- ```
401
-
402
- **Example:**
403
-
404
- *Physical indexes:*
405
- - `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
406
-
407
- *Aliases:*
408
- - `items_sentinel-2-l2a` - main collection alias
409
- - `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
410
- - `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
411
-
412
- ### Index Size Management
413
-
414
- **Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
415
-
416
- ## Interacting with the API
417
-
418
- - **Creating a Collection**:
419
- ```shell
420
- curl -X "POST" "http://localhost:8080/collections" \
421
- -H 'Content-Type: application/json; charset=utf-8' \
422
- -d $'{
423
- "id": "my_collection"
424
- }'
425
- ```
426
-
427
- - **Adding an Item to a Collection**:
428
- ```shell
429
- curl -X "POST" "http://localhost:8080/collections/my_collection/items" \
430
- -H 'Content-Type: application/json; charset=utf-8' \
431
- -d @item.json
432
- ```
433
-
434
- - **Searching for Items**:
435
- ```shell
436
- curl -X "GET" "http://localhost:8080/search" \
437
- -H 'Content-Type: application/json; charset=utf-8' \
438
- -d $'{
439
- "collections": ["my_collection"],
440
- "limit": 10
441
- }'
442
- ```
443
-
444
- - **Filtering by Bbox**:
445
- ```shell
446
- curl -X "GET" "http://localhost:8080/search" \
447
- -H 'Content-Type: application/json; charset=utf-8' \
448
- -d $'{
449
- "collections": ["my_collection"],
450
- "bbox": [-180, -90, 180, 90]
451
- }'
452
- ```
453
-
454
- - **Filtering by Datetime**:
455
- ```shell
456
- curl -X "GET" "http://localhost:8080/search" \
457
- -H 'Content-Type: application/json; charset=utf-8' \
458
- -d $'{
459
- "collections": ["my_collection"],
460
- "datetime": "2020-01-01T00:00:00Z/2020-12-31T23:59:59Z"
461
- }'
462
- ```
463
-
464
- ## Configure the API
465
-
466
- - **API Title and Description**: By default set to `stac-fastapi-<backend>`. Customize these by setting:
467
- - `STAC_FASTAPI_TITLE`: Changes the API title in the documentation
468
- - `STAC_FASTAPI_DESCRIPTION`: Changes the API description in the documentation
469
-
470
- - **Database Indices**: By default, the API reads from and writes to:
471
- - `collections` index for collections
472
- - `items_<collection name>` indices for items
473
- - Customize with `STAC_COLLECTIONS_INDEX` and `STAC_ITEMS_INDEX_PREFIX` environment variables
474
-
475
- - **Root Path Configuration**: The application root path is the base URL by default.
476
- - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`)
477
-
478
- - **Feature Configuration**: Control which features are enabled:
479
- - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable.
480
- - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable.
481
-
482
- ## Collection Pagination
483
-
484
- - **Overview**: The collections route supports pagination through optional query parameters.
485
- - **Parameters**:
486
- - `limit`: Controls the number of collections returned per page
487
- - `token`: Used to retrieve subsequent pages of results
488
- - **Response Structure**: The `links` field in the response contains a `next` link with the token for the next page of results.
489
- - **Example Usage**:
490
- ```shell
491
- curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
492
- ```
493
-
494
- ## Ingesting Sample Data CLI Tool
495
-
496
- - **Overview**: The `data_loader.py` script provides a convenient way to load STAC items into the database.
497
-
498
- - **Usage**:
499
- ```shell
500
- python3 data_loader.py --base-url http://localhost:8080
501
- ```
502
-
503
- - **Options**:
504
- ```
505
- --base-url TEXT Base URL of the STAC API [required]
506
- --collection-id TEXT ID of the collection to which items are added
507
- --use-bulk Use bulk insert method for items
508
- --data-dir PATH Directory containing collection.json and feature
509
- collection file
510
- --help Show this message and exit.
511
- ```
512
-
513
- - **Example Workflows**:
514
- - **Loading Sample Data**:
515
- ```shell
516
- python3 data_loader.py --base-url http://localhost:8080
517
- ```
518
- - **Loading Data to a Specific Collection**:
519
- ```shell
520
- python3 data_loader.py --base-url http://localhost:8080 --collection-id my-collection
521
- ```
522
- - **Using Bulk Insert for Performance**:
523
- ```shell
524
- python3 data_loader.py --base-url http://localhost:8080 --use-bulk
525
- ```
526
-
527
- ## Elasticsearch Mappings
528
-
529
- - **Overview**: Mappings apply to search index, not source data. They define how documents and their fields are stored and indexed.
530
- - **Implementation**:
531
- - Mappings are stored in index templates that are created on application startup
532
- - These templates are automatically applied when creating new Collection and Item indices
533
- - The `sfeos_helpers` package contains shared mapping definitions used by both Elasticsearch and OpenSearch backends
534
- - **Customization**: Custom mappings can be defined by extending the base mapping templates.
535
-
536
- ## Managing Elasticsearch Indices
537
-
538
- ### Snapshots
539
-
540
- - **Overview**: Snapshots provide a way to backup and restore your indices.
541
-
542
- - **Creating a Snapshot Repository**:
543
- ```shell
544
- curl -X "PUT" "http://localhost:9200/_snapshot/my_fs_backup" \
545
- -H 'Content-Type: application/json; charset=utf-8' \
546
- -d $'{
547
- "type": "fs",
548
- "settings": {
549
- "location": "/usr/share/elasticsearch/snapshots/my_fs_backup"
550
- }
551
- }'
552
- ```
553
- - This creates a snapshot repository that stores files in the elasticsearch/snapshots directory in this git repo clone
554
- - The elasticsearch.yml and compose files create a mapping from that directory to /usr/share/elasticsearch/snapshots within the Elasticsearch container and grant permissions for using it
555
-
556
- - **Creating a Snapshot**:
557
- ```shell
558
- curl -X "PUT" "http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2?wait_for_completion=true" \
559
- -H 'Content-Type: application/json; charset=utf-8' \
560
- -d $'{
561
- "metadata": {
562
- "taken_because": "dump of all items",
563
- "taken_by": "pvarner"
564
- },
565
- "include_global_state": false,
566
- "ignore_unavailable": false,
567
- "indices": "items_my-collection"
568
- }'
569
- ```
570
- - This creates a snapshot named my_snapshot_2 and waits for the action to be completed before returning
571
- - This can also be done asynchronously by omitting the wait_for_completion parameter, and queried for status later
572
- - The indices parameter determines which indices are snapshotted, and can include wildcards
573
-
574
- - **Viewing Snapshots**:
575
- ```shell
576
- # View a specific snapshot
577
- curl http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2
578
-
579
- # View all snapshots
580
- curl http://localhost:9200/_snapshot/my_fs_backup/_all
581
- ```
582
- - These commands allow you to check the status and details of your snapshots
583
-
584
- - **Restoring a Snapshot**:
585
- ```shell
586
- curl -X "POST" "http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2/_restore?wait_for_completion=true" \
587
- -H 'Content-Type: application/json; charset=utf-8' \
588
- -d $'{
589
- "include_aliases": false,
590
- "include_global_state": false,
591
- "ignore_unavailable": true,
592
- "rename_replacement": "items_$1-copy",
593
- "indices": "items_*",
594
- "rename_pattern": "items_(.+)"
595
- }'
596
- ```
597
- - This specific command will restore any indices that match items_* and rename them so that the new index name will be suffixed with -copy
598
- - The rename_pattern and rename_replacement parameters allow you to restore indices under new names
599
-
600
- - **Updating Collection References**:
601
- ```shell
602
- curl -X "POST" "http://localhost:9200/items_my-collection-copy/_update_by_query" \
603
- -H 'Content-Type: application/json; charset=utf-8' \
604
- -d $'{
605
- "query": {
606
- "match_all": {}
607
- },
608
- "script": {
609
- "lang": "painless",
610
- "params": {
611
- "collection": "my-collection-copy"
612
- },
613
- "source": "ctx._source.collection = params.collection"
614
- }
615
- }'
616
- ```
617
- - After restoring, the item documents have been restored in the new index (e.g., my-collection-copy), but the value of the collection field in those documents is still the original value of my-collection
618
- - This command updates these values to match the new collection name using Elasticsearch's Update By Query feature
619
-
620
- - **Creating a New Collection**:
621
- ```shell
622
- curl -X "POST" "http://localhost:8080/collections" \
623
- -H 'Content-Type: application/json' \
624
- -d $'{
625
- "id": "my-collection-copy"
626
- }'
627
- ```
628
- - The final step is to create a new collection through the API with the new name for each of the restored indices
629
- - This gives you a copy of the collection that has a resource URI (/collections/my-collection-copy) and can be correctly queried by collection name
630
-
631
- ### Reindexing
632
-
633
- - **Overview**: Reindexing allows you to copy documents from one index to another, optionally transforming them in the process.
634
-
635
- - **Use Cases**:
636
- - Apply changes to documents
637
- - Correct dynamically generated mappings
638
- - Transform data (e.g., lowercase identifiers)
639
- - The index templates will make sure that manually created indices will also have the correct mappings and settings
640
-
641
- - **Example: Reindexing with Transformation**:
642
- ```shell
643
- curl -X "POST" "http://localhost:9200/_reindex" \
644
- -H 'Content-Type: application/json' \
645
- -d $'{
646
- "source": {
647
- "index": "items_my-collection-lower_my-collection-hex-000001"
648
- },
649
- "dest": {
650
- "index": "items_my-collection-lower_my-collection-hex-000002"
651
- },
652
- "script": {
653
- "source": "ctx._source.id = ctx._source.id.toLowerCase()",
654
- "lang": "painless"
655
- }
656
- }'
657
- ```
658
- - In this example, we make a copy of an existing Item index but change the Item identifier to be lowercase
659
- - The script parameter allows you to transform documents during the reindexing process
660
-
661
- - **Updating Aliases**:
662
- ```shell
663
- curl -X "POST" "http://localhost:9200/_aliases" \
664
- -H 'Content-Type: application/json' \
665
- -d $'{
666
- "actions": [
667
- {
668
- "remove": {
669
- "index": "*",
670
- "alias": "items_my-collection"
671
- }
672
- },
673
- {
674
- "add": {
675
- "index": "items_my-collection-lower_my-collection-hex-000002",
676
- "alias": "items_my-collection"
677
- }
678
- }
679
- ]
680
- }'
681
- ```
682
- - If you are happy with the data in the newly created index, you can move the alias items_my-collection to the new index
683
- - This makes the modified Items with lowercase identifiers visible to users accessing my-collection in the STAC API
684
- - Using aliases allows you to switch between different index versions without changing the API endpoint
685
-
686
- ## Auth
687
-
688
- - **Overview**: Authentication is an optional feature that can be enabled through Route Dependencies.
689
- - **Implementation Options**:
690
- - Basic authentication
691
- - OAuth2 with Keycloak
692
- - Custom route dependencies
693
- - **Configuration**: Authentication can be configured using the `STAC_FASTAPI_ROUTE_DEPENDENCIES` environment variable.
694
- - **Examples and Documentation**: Detailed examples and implementation guides can be found in the [examples/auth](examples/auth) directory.
695
-
696
- ## Aggregation
697
-
698
- - **Supported Aggregations**:
699
- - Spatial aggregations of points and geometries
700
- - Frequency distribution aggregation of any property including dates
701
- - Temporal distribution of datetime values
702
-
703
- - **Endpoint Locations**:
704
- - Root Catalog level: `/aggregations`
705
- - Collection level: `/<collection_id>/aggregations`
706
-
707
- - **Implementation Details**: The `sfeos_helpers.aggregation` package provides specialized functionality for both Elasticsearch and OpenSearch backends.
708
-
709
- - **Documentation**: Detailed information about supported aggregations can be found in [the aggregation docs](./docs/src/aggregation.md).
710
-
711
-
712
- ## Rate Limiting
713
-
714
- - **Overview**: Rate limiting is an optional security feature that controls API request frequency on a remote address basis.
715
-
716
- - **Configuration**: Enabled by setting the `STAC_FASTAPI_RATE_LIMIT` environment variable:
717
- ```
718
- STAC_FASTAPI_RATE_LIMIT=500/minute
719
- ```
720
-
721
- - **Functionality**:
722
- - Limits each client to a specified number of requests per time period (e.g., 500 requests per minute)
723
- - Helps prevent API abuse and maintains system stability
724
- - Ensures fair resource allocation among all clients
725
-
726
- - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
@@ -1 +0,0 @@
1
- stac_fastapi