stac-fastapi-elasticsearch 6.5.1__py3-none-any.whl → 6.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ from starlette.requests import Request
17
17
 
18
18
  from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
19
19
  from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
20
- from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit
20
+ from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, get_bool_env
21
21
  from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
22
22
  from stac_fastapi.elasticsearch.config import (
23
23
  ElasticsearchSettings as SyncElasticsearchSettings,
@@ -29,13 +29,15 @@ from stac_fastapi.extensions.core.transaction.request import (
29
29
  )
30
30
  from stac_fastapi.sfeos_helpers import filter as filter_module
31
31
  from stac_fastapi.sfeos_helpers.database import (
32
+ add_bbox_shape_to_collection,
33
+ apply_collections_bbox_filter_shared,
34
+ apply_collections_datetime_filter_shared,
32
35
  apply_free_text_filter_shared,
33
36
  apply_intersects_filter_shared,
34
37
  create_index_templates_shared,
35
38
  delete_item_index_shared,
36
39
  get_queryables_mapping_shared,
37
40
  index_alias_by_collection_id,
38
- index_by_collection_id,
39
41
  mk_actions,
40
42
  mk_item_id,
41
43
  populate_sort_shared,
@@ -99,26 +101,6 @@ async def create_collection_index() -> None:
99
101
  await client.close()
100
102
 
101
103
 
102
- async def create_item_index(collection_id: str):
103
- """
104
- Create the index for Items. The settings of the index template will be used implicitly.
105
-
106
- Args:
107
- collection_id (str): Collection identifier.
108
-
109
- Returns:
110
- None
111
-
112
- """
113
- client = AsyncElasticsearchSettings().create_client
114
-
115
- await client.options(ignore_status=400).indices.create(
116
- index=f"{index_by_collection_id(collection_id)}-000001",
117
- body={"aliases": {index_alias_by_collection_id(collection_id): {}}},
118
- )
119
- await client.close()
120
-
121
-
122
104
  async def delete_item_index(collection_id: str):
123
105
  """Delete the index for items in a collection.
124
106
 
@@ -175,6 +157,7 @@ class DatabaseLogic(BaseDatabaseLogic):
175
157
  limit: int,
176
158
  request: Request,
177
159
  sort: Optional[List[Dict[str, Any]]] = None,
160
+ bbox: Optional[List[float]] = None,
178
161
  q: Optional[List[str]] = None,
179
162
  filter: Optional[Dict[str, Any]] = None,
180
163
  query: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -187,6 +170,7 @@ class DatabaseLogic(BaseDatabaseLogic):
187
170
  limit (int): The number of results to return.
188
171
  request (Request): The FastAPI request object.
189
172
  sort (Optional[List[Dict[str, Any]]]): Optional sort parameter from the request.
173
+ bbox (Optional[List[float]]): Bounding box to filter collections by spatial extent.
190
174
  q (Optional[List[str]]): Free text search terms.
191
175
  query (Optional[Dict[str, Dict[str, Any]]]): Query extension parameters.
192
176
  filter (Optional[Dict[str, Any]]): Structured query in CQL2 format.
@@ -314,12 +298,15 @@ class DatabaseLogic(BaseDatabaseLogic):
314
298
  query_parts.append({"bool": {"must_not": {"match_all": {}}}})
315
299
  raise
316
300
 
317
- # Combine all query parts with AND logic if there are multiple
318
- datetime_filter = None
319
- if datetime:
320
- datetime_filter = self._apply_collection_datetime_filter(datetime)
321
- if datetime_filter:
322
- query_parts.append(datetime_filter)
301
+ # Apply bbox filter if provided
302
+ bbox_filter = apply_collections_bbox_filter_shared(bbox)
303
+ if bbox_filter:
304
+ query_parts.append(bbox_filter)
305
+
306
+ # Apply datetime filter if provided
307
+ datetime_filter = apply_collections_datetime_filter_shared(datetime)
308
+ if datetime_filter:
309
+ query_parts.append(datetime_filter)
323
310
 
324
311
  # Combine all query parts with AND logic
325
312
  if query_parts:
@@ -329,12 +316,6 @@ class DatabaseLogic(BaseDatabaseLogic):
329
316
  else {"bool": {"must": query_parts}}
330
317
  )
331
318
 
332
- # Create a copy of the body for count query (without pagination and sorting)
333
- count_body = body.copy()
334
- if "search_after" in count_body:
335
- del count_body["search_after"]
336
- count_body["size"] = 0
337
-
338
319
  # Create async tasks for both search and count
339
320
  search_task = asyncio.create_task(
340
321
  self.client.search(
@@ -384,41 +365,6 @@ class DatabaseLogic(BaseDatabaseLogic):
384
365
 
385
366
  return collections, next_token, matched
386
367
 
387
- @staticmethod
388
- def _apply_collection_datetime_filter(
389
- datetime_str: Optional[str],
390
- ) -> Optional[Dict[str, Any]]:
391
- """Create a temporal filter for collections based on their extent."""
392
- if not datetime_str:
393
- return None
394
-
395
- # Parse the datetime string into start and end
396
- if "/" in datetime_str:
397
- start, end = datetime_str.split("/")
398
- # Replace open-ended ranges with concrete dates
399
- if start == "..":
400
- # For open-ended start, use a very early date
401
- start = "1800-01-01T00:00:00Z"
402
- if end == "..":
403
- # For open-ended end, use a far future date
404
- end = "2999-12-31T23:59:59Z"
405
- else:
406
- # If it's just a single date, use it for both start and end
407
- start = end = datetime_str
408
-
409
- return {
410
- "bool": {
411
- "must": [
412
- # Check if any date in the array is less than or equal to the query end date
413
- # This will match if the collection's start date is before or equal to the query end date
414
- {"range": {"extent.temporal.interval": {"lte": end}}},
415
- # Check if any date in the array is greater than or equal to the query start date
416
- # This will match if the collection's end date is after or equal to the query start date
417
- {"range": {"extent.temporal.interval": {"gte": start}}},
418
- ]
419
- }
420
- }
421
-
422
368
  async def get_one_item(self, collection_id: str, item_id: str) -> Dict:
423
369
  """Retrieve a single item from the database.
424
370
 
@@ -816,7 +762,7 @@ class DatabaseLogic(BaseDatabaseLogic):
816
762
  index_param = ITEM_INDICES
817
763
  query = add_collections_to_body(collection_ids, query)
818
764
 
819
- max_result_window = get_max_limit()
765
+ max_result_window = MAX_LIMIT
820
766
 
821
767
  size_limit = min(limit + 1, max_result_window)
822
768
 
@@ -1386,7 +1332,7 @@ class DatabaseLogic(BaseDatabaseLogic):
1386
1332
  None
1387
1333
 
1388
1334
  Notes:
1389
- A new index is created for the items in the Collection using the `create_item_index` function.
1335
+ A new index is created for the items in the Collection if the index insertion strategy requires it.
1390
1336
  """
1391
1337
  collection_id = collection["id"]
1392
1338
 
@@ -1404,6 +1350,12 @@ class DatabaseLogic(BaseDatabaseLogic):
1404
1350
  if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
1405
1351
  raise ConflictError(f"Collection {collection_id} already exists")
1406
1352
 
1353
+ if get_bool_env("ENABLE_COLLECTIONS_SEARCH") or get_bool_env(
1354
+ "ENABLE_COLLECTIONS_SEARCH_ROUTE"
1355
+ ):
1356
+ # Convert bbox to bbox_shape for geospatial queries (ES/OS specific)
1357
+ add_bbox_shape_to_collection(collection)
1358
+
1407
1359
  # Index the collection in the database
1408
1360
  await self.client.index(
1409
1361
  index=COLLECTIONS_INDEX,
@@ -1507,6 +1459,12 @@ class DatabaseLogic(BaseDatabaseLogic):
1507
1459
  await self.delete_collection(collection_id)
1508
1460
 
1509
1461
  else:
1462
+ if get_bool_env("ENABLE_COLLECTIONS_SEARCH") or get_bool_env(
1463
+ "ENABLE_COLLECTIONS_SEARCH_ROUTE"
1464
+ ):
1465
+ # Convert bbox to bbox_shape for geospatial queries (ES/OS specific)
1466
+ add_bbox_shape_to_collection(collection)
1467
+
1510
1468
  # Update the existing collection
1511
1469
  await self.client.index(
1512
1470
  index=COLLECTIONS_INDEX,
@@ -1,2 +1,2 @@
1
1
  """library version."""
2
- __version__ = "6.5.1"
2
+ __version__ = "6.6.0"
@@ -0,0 +1,58 @@
1
+ Metadata-Version: 2.4
2
+ Name: stac_fastapi_elasticsearch
3
+ Version: 6.6.0
4
+ Summary: An implementation of STAC API based on the FastAPI framework with Elasticsearch.
5
+ Project-URL: Homepage, https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
+ License: MIT
7
+ Keywords: Elasticsearch,FastAPI,STAC,STAC-API,stac-fastapi
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: Intended Audience :: Information Technology
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Requires-Python: >=3.9
19
+ Requires-Dist: elasticsearch[async]~=8.19.1
20
+ Requires-Dist: sfeos-helpers==6.6.0
21
+ Requires-Dist: stac-fastapi-core==6.6.0
22
+ Requires-Dist: starlette<0.36.0,>=0.35.0
23
+ Requires-Dist: uvicorn~=0.23.0
24
+ Provides-Extra: dev
25
+ Requires-Dist: ciso8601~=2.3.0; extra == 'dev'
26
+ Requires-Dist: httpx<0.28.0,>=0.24.0; extra == 'dev'
27
+ Requires-Dist: pre-commit~=3.0.0; extra == 'dev'
28
+ Requires-Dist: pytest-asyncio~=0.21.0; extra == 'dev'
29
+ Requires-Dist: pytest-cov~=4.0.0; extra == 'dev'
30
+ Requires-Dist: pytest~=8.0; extra == 'dev'
31
+ Provides-Extra: docs
32
+ Requires-Dist: mkdocs-material~=9.0.0; extra == 'docs'
33
+ Requires-Dist: mkdocs~=1.4.0; extra == 'docs'
34
+ Requires-Dist: pdocs~=1.2.0; extra == 'docs'
35
+ Provides-Extra: server
36
+ Requires-Dist: uvicorn[standard]~=0.23.0; extra == 'server'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # stac-fastapi-elasticsearch
40
+
41
+ This is the Elasticsearch backend for stac-fastapi. For full documentation, please see the [main README](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/blob/main/README.md).
42
+
43
+ ## Package Information
44
+
45
+ - **Package name**: stac-fastapi-elasticsearch
46
+ - **Description**: An implementation of STAC API based on the FastAPI framework with Elasticsearch.
47
+ - **Documentation**: [https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/)
48
+ - **Source**: [GitHub Repository](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/)
49
+
50
+ ## Installation
51
+
52
+ ```bash
53
+ pip install stac-fastapi-elasticsearch
54
+ ```
55
+
56
+ ## Quick Start
57
+
58
+ For detailed usage and examples, please refer to the [main documentation](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/).
@@ -0,0 +1,9 @@
1
+ stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
2
+ stac_fastapi/elasticsearch/app.py,sha256=024U5xvXmSWUJABS9SekbqqamhsNSpqPBII2NBWktz8,9979
3
+ stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
4
+ stac_fastapi/elasticsearch/database_logic.py,sha256=mQgWkqcsVwxUHGzD8dh8n5KHSh83oLuX9DNx6m-YqDo,68730
5
+ stac_fastapi/elasticsearch/version.py,sha256=zPQp-GtLh4R45hT8V_KAWzwZcP4-jyw7Xjms_eNMZBc,45
6
+ stac_fastapi_elasticsearch-6.6.0.dist-info/METADATA,sha256=gW-QOtinAj-Xkx0JnuFWP8cVv5aCQG9x3EaEOcMK_OM,2541
7
+ stac_fastapi_elasticsearch-6.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
8
+ stac_fastapi_elasticsearch-6.6.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
9
+ stac_fastapi_elasticsearch-6.6.0.dist-info/RECORD,,
@@ -1,5 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.45.1)
2
+ Generator: hatchling 1.27.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
-
@@ -1,731 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: stac-fastapi-elasticsearch
3
- Version: 6.5.1
4
- Summary: An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.
5
- Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
- License: MIT
7
- Classifier: Intended Audience :: Developers
8
- Classifier: Intended Audience :: Information Technology
9
- Classifier: Intended Audience :: Science/Research
10
- Classifier: Programming Language :: Python :: 3.9
11
- Classifier: Programming Language :: Python :: 3.10
12
- Classifier: Programming Language :: Python :: 3.11
13
- Classifier: Programming Language :: Python :: 3.12
14
- Classifier: Programming Language :: Python :: 3.13
15
- Classifier: License :: OSI Approved :: MIT License
16
- Requires-Python: >=3.9
17
- Description-Content-Type: text/markdown
18
- Requires-Dist: stac-fastapi-core==6.5.1
19
- Requires-Dist: sfeos-helpers==6.5.1
20
- Requires-Dist: elasticsearch[async]~=8.18.0
21
- Requires-Dist: uvicorn~=0.23.0
22
- Requires-Dist: starlette<0.36.0,>=0.35.0
23
- Provides-Extra: dev
24
- Requires-Dist: pytest~=7.0.0; extra == "dev"
25
- Requires-Dist: pytest-cov~=4.0.0; extra == "dev"
26
- Requires-Dist: pytest-asyncio~=0.21.0; extra == "dev"
27
- Requires-Dist: pre-commit~=3.0.0; extra == "dev"
28
- Requires-Dist: ciso8601~=2.3.0; extra == "dev"
29
- Requires-Dist: httpx<0.28.0,>=0.24.0; extra == "dev"
30
- Provides-Extra: docs
31
- Requires-Dist: mkdocs~=1.4.0; extra == "docs"
32
- Requires-Dist: mkdocs-material~=9.0.0; extra == "docs"
33
- Requires-Dist: pdocs~=1.2.0; extra == "docs"
34
- Provides-Extra: server
35
- Requires-Dist: uvicorn[standard]~=0.23.0; extra == "server"
36
-
37
- # stac-fastapi-elasticsearch-opensearch
38
-
39
- <!-- markdownlint-disable MD033 MD041 -->
40
-
41
-
42
- <p align="left">
43
- <img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
44
- </p>
45
-
46
- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents)
47
-
48
- [![Downloads](https://static.pepy.tech/badge/stac-fastapi-core?color=blue)](https://pepy.tech/project/stac-fastapi-core)
49
- [![GitHub contributors](https://img.shields.io/github/contributors/stac-utils/stac-fastapi-elasticsearch-opensearch?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/graphs/contributors)
50
- [![GitHub stars](https://img.shields.io/github/stars/stac-utils/stac-fastapi-elasticsearch-opensearch.svg?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/stargazers)
51
- [![GitHub forks](https://img.shields.io/github/forks/stac-utils/stac-fastapi-elasticsearch-opensearch.svg?color=blue)](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/network/members)
52
- [![PyPI version](https://img.shields.io/pypi/v/stac-fastapi-elasticsearch.svg?color=blue)](https://pypi.org/project/stac-fastapi-elasticsearch/)
53
- [![STAC](https://img.shields.io/badge/STAC-1.1.0-blue.svg)](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
54
- [![stac-fastapi](https://img.shields.io/badge/stac--fastapi-6.0.0-blue.svg)](https://github.com/stac-utils/stac-fastapi)
55
-
56
- ## Sponsors & Supporters
57
-
58
- The following organizations have contributed time and/or funding to support the development of this project:
59
-
60
- <p align="left">
61
- <a href="https://healy-hyperspatial.github.io/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/hh-logo-blue.png" alt="Healy Hyperspatial" height="100" hspace="20"></a>
62
- <a href="https://atomicmaps.io/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/am-logo-black.png" alt="Atomic Maps" height="100" hspace="20"></a>
63
- <a href="https://remotesensing.vito.be/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/VITO.png" alt="VITO Remote Sensing" height="100" hspace="20"></a>
64
- </p>
65
-
66
- ## Project Introduction - What is SFEOS?
67
-
68
- SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable API implementation for serving SpatioTemporal Asset Catalog (STAC) data - an enhanced GeoJSON format designed specifically for geospatial assets like satellite imagery, aerial photography, and other Earth observation data. This project enables organizations to:
69
-
70
- - **Efficiently catalog and search geospatial data** such as satellite imagery, aerial photography, DEMs, and other geospatial assets using Elasticsearch or OpenSearch as the database backend
71
- - **Implement standardized STAC APIs** that support complex spatial, temporal, and property-based queries across large collections of geospatial data
72
- - **Scale to millions of geospatial assets** with fast search performance through optimized spatial indexing and query capabilities
73
- - **Support OGC-compliant filtering** including spatial operations (intersects, contains, etc.) and temporal queries
74
- - **Perform geospatial aggregations** to analyze data distribution across space and time
75
- - **Enhanced collection search capabilities** with support for sorting and field selection
76
-
77
- This implementation builds on the STAC-FastAPI framework, providing a production-ready solution specifically optimized for Elasticsearch and OpenSearch databases. It's ideal for organizations managing large geospatial data catalogs who need efficient discovery and access capabilities through standardized APIs.
78
-
79
- ## Common Deployment Patterns
80
-
81
- stac-fastapi-elasticsearch-opensearch can be deployed in several ways depending on your needs:
82
-
83
- - **Containerized Application**: Run as a Docker container with connections to Elasticsearch/OpenSearch databases
84
- - **Serverless Function**: Deploy as AWS Lambda or similar serverless function with API Gateway
85
- - **Traditional Server**: Run on virtual machines or bare metal servers in your infrastructure
86
- - **Kubernetes**: Deploy as part of a larger microservices architecture with container orchestration
87
-
88
- The implementation is flexible and can scale from small local deployments to large production environments serving millions of geospatial assets.
89
-
90
- ## Technologies
91
-
92
- This project is built on the following technologies: STAC, stac-fastapi, FastAPI, Elasticsearch, Python, OpenSearch
93
-
94
- <p align="left">
95
- <a href="https://stacspec.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/STAC-01.png" alt="STAC" height="100" hspace="10"></a>
96
- <a href="https://www.python.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/python.png" alt="Python" height="80" hspace="10"></a>
97
- <a href="https://fastapi.tiangolo.com/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/fastapi.svg" alt="FastAPI" height="80" hspace="10"></a>
98
- <a href="https://www.elastic.co/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/elasticsearch.png" alt="Elasticsearch" height="80" hspace="10"></a>
99
- <a href="https://opensearch.org/"><img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/opensearch.svg" alt="OpenSearch" height="80" hspace="10"></a>
100
- </p>
101
-
102
- ## Table of Contents
103
-
104
- - [stac-fastapi-elasticsearch-opensearch](#stac-fastapi-elasticsearch-opensearch)
105
- - [Sponsors & Supporters](#sponsors--supporters)
106
- - [Project Introduction - What is SFEOS?](#project-introduction---what-is-sfeos)
107
- - [Common Deployment Patterns](#common-deployment-patterns)
108
- - [Technologies](#technologies)
109
- - [Table of Contents](#table-of-contents)
110
- - [Collection Search Extensions](#collection-search-extensions)
111
- - [Documentation & Resources](#documentation--resources)
112
- - [Package Structure](#package-structure)
113
- - [Examples](#examples)
114
- - [Performance](#performance)
115
- - [Direct Response Mode](#direct-response-mode)
116
- - [Quick Start](#quick-start)
117
- - [Installation](#installation)
118
- - [Running Locally](#running-locally)
119
- - [Using Pre-built Docker Images](#using-pre-built-docker-images)
120
- - [Using Docker Compose](#using-docker-compose)
121
- - [Configuration Reference](#configuration-reference)
122
- - [Datetime-Based Index Management](#datetime-based-index-management)
123
- - [Overview](#overview)
124
- - [When to Use](#when-to-use)
125
- - [Configuration](#configuration)
126
- - [Enabling Datetime-Based Indexing](#enabling-datetime-based-indexing)
127
- - [Related Configuration Variables](#related-configuration-variables)
128
- - [How Datetime-Based Indexing Works](#how-datetime-based-indexing-works)
129
- - [Index and Alias Naming Convention](#index-and-alias-naming-convention)
130
- - [Index Size Management](#index-size-management)
131
- - [Interacting with the API](#interacting-with-the-api)
132
- - [Configure the API](#configure-the-api)
133
- - [Collection Pagination](#collection-pagination)
134
- - [Ingesting Sample Data CLI Tool](#ingesting-sample-data-cli-tool)
135
- - [Elasticsearch Mappings](#elasticsearch-mappings)
136
- - [Managing Elasticsearch Indices](#managing-elasticsearch-indices)
137
- - [Snapshots](#snapshots)
138
- - [Reindexing](#reindexing)
139
- - [Auth](#auth)
140
- - [Aggregation](#aggregation)
141
- - [Rate Limiting](#rate-limiting)
142
-
143
- ## Documentation & Resources
144
-
145
- - **Online Documentation**: [https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch](https://stac-utils.github.io/stac-fastapi-elasticsearch-opensearch/)
146
- - **Source Code**: [https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch)
147
- - **API Examples**: [Postman Documentation](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) - Examples of how to use the API endpoints
148
- - **Community**:
149
- - [Gitter Chat](https://app.gitter.im/#/room/#stac-fastapi-elasticsearch_community:gitter.im) - For real-time discussions
150
- - [GitHub Discussions](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/discussions) - For longer-form questions and answers
151
-
152
- ## Collection Search Extensions
153
-
154
- SFEOS provides enhanced collection search capabilities through two primary routes:
155
- - **GET/POST `/collections`**: The standard STAC endpoint with extended query parameters
156
- - **GET/POST `/collections-search`**: A custom endpoint that supports the same parameters, created to avoid conflicts with the STAC Transactions extension if enabled (which uses POST `/collections` for collection creation)
157
-
158
- These endpoints support advanced collection discovery features including:
159
-
160
- - **Sorting**: Sort collections by sortable fields using the `sortby` parameter
161
- - Example: `/collections?sortby=+id` (ascending sort by ID)
162
- - Example: `/collections?sortby=-id` (descending sort by ID)
163
- - Example: `/collections?sortby=-temporal` (descending sort by temporal extent)
164
-
165
- - **Field Selection**: Request only specific fields to be returned using the `fields` parameter
166
- - Example: `/collections?fields=id,title,description`
167
- - This helps reduce payload size when only certain fields are needed
168
-
169
- - **Free Text Search**: Search across collection text fields using the `q` parameter
170
- - Example: `/collections?q=landsat`
171
- - Searches across multiple text fields including title, description, and keywords
172
- - Supports partial word matching and relevance-based sorting
173
-
174
- - **Structured Filtering**: Filter collections using CQL2 expressions
175
- - JSON format: `/collections?filter={"op":"=","args":[{"property":"id"},"sentinel-2"]}&filter-lang=cql2-json`
176
- - Text format: `/collections?filter=id='sentinel-2'&filter-lang=cql2-text` (note: string values must be quoted)
177
- - Advanced text format: `/collections?filter=id LIKE '%sentinel%'&filter-lang=cql2-text` (supports LIKE, BETWEEN, etc.)
178
- - Supports both CQL2 JSON and CQL2 text formats with various operators
179
- - Enables precise filtering on any collection property
180
-
181
- - **Datetime Filtering**: Filter collections by their temporal extent using the `datetime` parameter
182
- - Example: `/collections?datetime=2020-01-01T00:00:00Z/2020-12-31T23:59:59Z` (finds collections with temporal extents that overlap this range)
183
- - Example: `/collections?datetime=2020-06-15T12:00:00Z` (finds collections whose temporal extent includes this specific time)
184
- - Example: `/collections?datetime=2020-01-01T00:00:00Z/..` (finds collections with temporal extents that extend to or beyond January 1, 2020)
185
- - Example: `/collections?datetime=../2020-12-31T23:59:59Z` (finds collections with temporal extents that begin on or before December 31, 2020)
186
- - Collections are matched if their temporal extent overlaps with the provided datetime parameter
187
- - This allows for efficient discovery of collections based on time periods
188
-
189
- These extensions make it easier to build user interfaces that display and navigate through collections efficiently.
190
-
191
- > **Configuration**: Collection search extensions (sorting, field selection, free text search, structured filtering, and datetime filtering) for the `/collections` endpoint can be disabled by setting the `ENABLE_COLLECTIONS_SEARCH` environment variable to `false`. By default, these extensions are enabled.
192
- >
193
- > **Configuration**: The custom `/collections-search` endpoint can be enabled by setting the `ENABLE_COLLECTIONS_SEARCH_ROUTE` environment variable to `true`. By default, this endpoint is **disabled**.
194
-
195
- > **Note**: Sorting is only available on fields that are indexed for sorting in Elasticsearch/OpenSearch. With the default mappings, you can sort on:
196
- > - `id` (keyword field)
197
- > - `extent.temporal.interval` (date field)
198
- > - `temporal` (alias to extent.temporal.interval)
199
- >
200
- > Text fields like `title` and `description` are not sortable by default as they use text analysis for better search capabilities. Attempting to sort on these fields will result in a user-friendly error message explaining which fields are sortable and how to make additional fields sortable by updating the mappings.
201
- >
202
- > **Important**: Adding keyword fields to make text fields sortable can significantly increase the index size, especially for large text fields. Consider the storage implications when deciding which fields to make sortable.
203
-
204
-
205
- ## Package Structure
206
-
207
- This project is organized into several packages, each with a specific purpose:
208
-
209
- - **stac_fastapi_core**: Core functionality that's database-agnostic, including API models, extensions, and shared utilities. This package provides the foundation for building STAC API implementations with any database backend. See [stac-fastapi-mongo](https://github.com/Healy-Hyperspatial/stac-fastapi-mongo) for a working example.
210
-
211
- - **sfeos_helpers**: Shared helper functions and utilities used by both the Elasticsearch and OpenSearch backends. This package includes:
212
- - `database`: Specialized modules for index, document, and database utility operations
213
- - `aggregation`: Elasticsearch/OpenSearch-specific aggregation functionality
214
- - Shared logic and utilities that improve code reuse between backends
215
-
216
- - **stac_fastapi_elasticsearch**: Complete implementation of the STAC API using Elasticsearch as the backend database. This package depends on both `stac_fastapi_core` and `sfeos_helpers`.
217
-
218
- - **stac_fastapi_opensearch**: Complete implementation of the STAC API using OpenSearch as the backend database. This package depends on both `stac_fastapi_core` and `sfeos_helpers`.
219
-
220
- ## Examples
221
-
222
- The `/examples` directory contains several useful examples and reference implementations:
223
-
224
- - **pip_docker**: Examples of running stac-fastapi-elasticsearch from PyPI in Docker without needing any code from the repository
225
- - **auth**: Authentication examples including:
226
- - Basic authentication
227
- - OAuth2 with Keycloak
228
- - Route dependencies configuration
229
- - **rate_limit**: Example of implementing rate limiting for API requests
230
- - **postman_collections**: Postman collection files you can import for testing API endpoints
231
-
232
- These examples provide practical reference implementations for various deployment scenarios and features.
233
-
234
- ## Performance
235
-
236
- ### Direct Response Mode
237
-
238
- - The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0.
239
- - **Control via environment variable**: Set `ENABLE_DIRECT_RESPONSE=true` to enable this feature.
240
- - **How it works**: When enabled, endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance.
241
- - **Important limitation**: All FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes when this mode is enabled.
242
- - **Best use case**: This mode is best suited for public or read-only APIs where authentication and custom logic are not required.
243
- - **Default setting**: `false` for safety.
244
- - **More information**: See [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347) for background and implementation details.
245
-
246
- ## Quick Start
247
-
248
- This section helps you get up and running with stac-fastapi-elasticsearch-opensearch quickly.
249
-
250
- ### Installation
251
-
252
- - **For versions 4.0.0a1 and newer** (PEP 625 compliant naming):
253
- ```bash
254
- pip install stac-fastapi-elasticsearch # Elasticsearch backend
255
- pip install stac-fastapi-opensearch # Opensearch backend
256
- pip install stac-fastapi-core # Core library
257
- ```
258
-
259
- - **For versions 4.0.0a0 and older**:
260
- ```bash
261
- pip install stac-fastapi.elasticsearch # Elasticsearch backend
262
- pip install stac-fastapi.opensearch # Opensearch backend
263
- pip install stac-fastapi.core # Core library
264
- ```
265
-
266
- > **Important Note:** Starting with version 4.0.0a1, package names have changed from using periods (e.g., `stac-fastapi.core`) to using hyphens (e.g., `stac-fastapi-core`) to comply with PEP 625. The internal package structure uses underscores, but users should install with hyphens as shown above. Please update your requirements files accordingly.
267
-
268
- ### Running Locally
269
-
270
- There are two main ways to run the API locally:
271
-
272
- #### Using Pre-built Docker Images
273
-
274
- - We provide ready-to-use Docker images through GitHub Container Registry:
275
- - [ElasticSearch backend](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-es)
276
- - [OpenSearch backend](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-os)
277
-
278
- - **Pull and run the images**:
279
- ```shell
280
- # For Elasticsearch backend
281
- docker pull ghcr.io/stac-utils/stac-fastapi-es:latest
282
-
283
- # For OpenSearch backend
284
- docker pull ghcr.io/stac-utils/stac-fastapi-os:latest
285
- ```
286
-
287
- #### Using Docker Compose
288
-
289
- - **Prerequisites**: Ensure [Docker Compose](https://docs.docker.com/compose/install/) or [Podman Compose](https://podman-desktop.io/docs/compose) is installed on your machine.
290
-
291
- - **Start the API**:
292
- ```shell
293
- docker compose up elasticsearch app-elasticsearch
294
- ```
295
-
296
- - **Configuration**: By default, Docker Compose uses Elasticsearch 8.x and OpenSearch 2.11.1. To use different versions, create a `.env` file:
297
- ```shell
298
- ELASTICSEARCH_VERSION=8.11.0
299
- OPENSEARCH_VERSION=2.11.1
300
- ENABLE_DIRECT_RESPONSE=false
301
- ```
302
-
303
- - **Compatibility**: The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
304
-
305
-
306
-
307
- ## Configuration Reference
308
-
309
- You can customize additional settings in your `.env` file:
310
-
311
- | Variable | Description | Default | Required |
312
- |------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------|
313
- | `ES_HOST` | Hostname for external Elasticsearch/OpenSearch. | `localhost` | Optional |
314
- | `ES_PORT` | Port for Elasticsearch/OpenSearch. | `9200` (ES) / `9202` (OS)| Optional |
315
- | `ES_USE_SSL` | Use SSL for connecting to Elasticsearch/OpenSearch. | `true` | Optional |
316
- | `ES_VERIFY_CERTS` | Verify SSL certificates when connecting. | `true` | Optional |
317
- | `ES_API_KEY` | API Key for external Elasticsearch/OpenSearch. | N/A | Optional |
318
- | `ES_TIMEOUT` | Client timeout for Elasticsearch/OpenSearch. | DB client default | Optional |
319
- | `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-<backend>` | Optional |
320
- | `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional |
321
- | `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional |
322
- | `STAC_FASTAPI_LANDING_PAGE_ID` | Landing page ID | `stac-fastapi` | Optional |
323
- | `APP_HOST` | Server bind address. | `0.0.0.0` | Optional |
324
- | `APP_PORT` | Server port. | `8000` | Optional |
325
- | `ENVIRONMENT` | Runtime environment. | `local` | Optional |
326
- | `WEB_CONCURRENCY` | Number of worker processes. | `10` | Optional |
327
- | `RELOAD` | Enable auto-reload for development. | `true` | Optional |
328
- | `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional |
329
- | `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
330
- | `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
331
- | `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional |
332
- | `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
333
- | `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
334
- | `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
335
- | `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields, free text search, structured filtering, and datetime filtering) on the core `/collections` endpoint. | `true` | Optional |
336
- | `ENABLE_COLLECTIONS_SEARCH_ROUTE` | Enable the custom `/collections-search` endpoint (both GET and POST methods). When disabled, the custom endpoint will not be available, but collection search extensions will still be available on the core `/collections` endpoint if `ENABLE_COLLECTIONS_SEARCH` is true. | `false` | Optional |
337
- | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. This is useful for deployments where mutating the catalog via the API should be prevented. If set to `true`, the POST `/collections` route for search will be unavailable in the API. | `true` | Optional |
338
- | `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
339
- | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
340
- | `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional |
341
- | `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional |
342
-
343
- > [!NOTE]
344
- > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
345
-
346
- ## Datetime-Based Index Management
347
-
348
- ### Overview
349
-
350
- SFEOS supports two indexing strategies for managing STAC items:
351
-
352
- 1. **Simple Indexing** (default) - One index per collection
353
- 2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
354
-
355
- The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
356
-
357
- ### When to Use
358
-
359
- **Recommended for:**
360
- - Systems with large collections containing millions of items
361
- - Systems requiring high-performance temporal searching
362
-
363
- **Pros:**
364
- - Multiple times faster queries with datetime filter
365
- - Reduced database load - only relevant indexes are searched
366
-
367
- **Cons:**
368
- - Slightly longer item indexing time (automatic index management)
369
- - Greater management complexity
370
-
371
- ### Configuration
372
-
373
- #### Enabling Datetime-Based Indexing
374
-
375
- Enable datetime-based indexing by setting the following environment variable:
376
-
377
- ```bash
378
- ENABLE_DATETIME_INDEX_FILTERING=true
379
- ```
380
-
381
- ### Related Configuration Variables
382
-
383
- | Variable | Description | Default | Example |
384
- |----------|-------------|---------|---------|
385
- | `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
386
- | `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
387
- | `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
388
-
389
- ## How Datetime-Based Indexing Works
390
-
391
- ### Index and Alias Naming Convention
392
-
393
- The system uses a precise naming convention:
394
-
395
- **Physical indexes:**
396
- ```
397
- {ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
398
- ```
399
-
400
- **Aliases:**
401
- ```
402
- {ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
403
- {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
404
- {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
405
- ```
406
-
407
- **Example:**
408
-
409
- *Physical indexes:*
410
- - `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
411
-
412
- *Aliases:*
413
- - `items_sentinel-2-l2a` - main collection alias
414
- - `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
415
- - `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
416
-
417
- ### Index Size Management
418
-
419
- **Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
420
-
421
- ## Interacting with the API
422
-
423
- - **Creating a Collection**:
424
- ```shell
425
- curl -X "POST" "http://localhost:8080/collections" \
426
- -H 'Content-Type: application/json; charset=utf-8' \
427
- -d $'{
428
- "id": "my_collection"
429
- }'
430
- ```
431
-
432
- - **Adding an Item to a Collection**:
433
- ```shell
434
- curl -X "POST" "http://localhost:8080/collections/my_collection/items" \
435
- -H 'Content-Type: application/json; charset=utf-8' \
436
- -d @item.json
437
- ```
438
-
439
- - **Searching for Items**:
440
- ```shell
441
- curl -X "GET" "http://localhost:8080/search" \
442
- -H 'Content-Type: application/json; charset=utf-8' \
443
- -d $'{
444
- "collections": ["my_collection"],
445
- "limit": 10
446
- }'
447
- ```
448
-
449
- - **Filtering by Bbox**:
450
- ```shell
451
- curl -X "GET" "http://localhost:8080/search" \
452
- -H 'Content-Type: application/json; charset=utf-8' \
453
- -d $'{
454
- "collections": ["my_collection"],
455
- "bbox": [-180, -90, 180, 90]
456
- }'
457
- ```
458
-
459
- - **Filtering by Datetime**:
460
- ```shell
461
- curl -X "GET" "http://localhost:8080/search" \
462
- -H 'Content-Type: application/json; charset=utf-8' \
463
- -d $'{
464
- "collections": ["my_collection"],
465
- "datetime": "2020-01-01T00:00:00Z/2020-12-31T23:59:59Z"
466
- }'
467
- ```
468
-
469
- ## Configure the API
470
-
471
- - **API Title and Description**: By default set to `stac-fastapi-<backend>`. Customize these by setting:
472
- - `STAC_FASTAPI_TITLE`: Changes the API title in the documentation
473
- - `STAC_FASTAPI_DESCRIPTION`: Changes the API description in the documentation
474
-
475
- - **Database Indices**: By default, the API reads from and writes to:
476
- - `collections` index for collections
477
- - `items_<collection name>` indices for items
478
- - Customize with `STAC_COLLECTIONS_INDEX` and `STAC_ITEMS_INDEX_PREFIX` environment variables
479
-
480
- - **Root Path Configuration**: The application root path is the base URL by default.
481
- - For AWS Lambda with Gateway API: Set `STAC_FASTAPI_ROOT_PATH` to match the Gateway API stage name (e.g., `/v1`)
482
-
483
- - **Feature Configuration**: Control which features are enabled:
484
- - `ENABLE_COLLECTIONS_SEARCH`: Set to `true` (default) to enable collection search extensions (sort, fields). Set to `false` to disable.
485
- - `ENABLE_TRANSACTIONS_EXTENSIONS`: Set to `true` (default) to enable transaction extensions. Set to `false` to disable.
486
-
487
- ## Collection Pagination
488
-
489
- - **Overview**: The collections route supports pagination through optional query parameters.
490
- - **Parameters**:
491
- - `limit`: Controls the number of collections returned per page
492
- - `token`: Used to retrieve subsequent pages of results
493
- - **Response Structure**: The `links` field in the response contains a `next` link with the token for the next page of results.
494
- - **Example Usage**:
495
- ```shell
496
- curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
497
- ```
498
-
499
- ## Ingesting Sample Data CLI Tool
500
-
501
- - **Overview**: The `data_loader.py` script provides a convenient way to load STAC items into the database.
502
-
503
- - **Usage**:
504
- ```shell
505
- python3 data_loader.py --base-url http://localhost:8080
506
- ```
507
-
508
- - **Options**:
509
- ```
510
- --base-url TEXT Base URL of the STAC API [required]
511
- --collection-id TEXT ID of the collection to which items are added
512
- --use-bulk Use bulk insert method for items
513
- --data-dir PATH Directory containing collection.json and feature
514
- collection file
515
- --help Show this message and exit.
516
- ```
517
-
518
- - **Example Workflows**:
519
- - **Loading Sample Data**:
520
- ```shell
521
- python3 data_loader.py --base-url http://localhost:8080
522
- ```
523
- - **Loading Data to a Specific Collection**:
524
- ```shell
525
- python3 data_loader.py --base-url http://localhost:8080 --collection-id my-collection
526
- ```
527
- - **Using Bulk Insert for Performance**:
528
- ```shell
529
- python3 data_loader.py --base-url http://localhost:8080 --use-bulk
530
- ```
531
-
532
- ## Elasticsearch Mappings
533
-
534
- - **Overview**: Mappings apply to search index, not source data. They define how documents and their fields are stored and indexed.
535
- - **Implementation**:
536
- - Mappings are stored in index templates that are created on application startup
537
- - These templates are automatically applied when creating new Collection and Item indices
538
- - The `sfeos_helpers` package contains shared mapping definitions used by both Elasticsearch and OpenSearch backends
539
- - **Customization**: Custom mappings can be defined by extending the base mapping templates.
540
-
541
- ## Managing Elasticsearch Indices
542
-
543
- ### Snapshots
544
-
545
- - **Overview**: Snapshots provide a way to backup and restore your indices.
546
-
547
- - **Creating a Snapshot Repository**:
548
- ```shell
549
- curl -X "PUT" "http://localhost:9200/_snapshot/my_fs_backup" \
550
- -H 'Content-Type: application/json; charset=utf-8' \
551
- -d $'{
552
- "type": "fs",
553
- "settings": {
554
- "location": "/usr/share/elasticsearch/snapshots/my_fs_backup"
555
- }
556
- }'
557
- ```
558
- - This creates a snapshot repository that stores files in the elasticsearch/snapshots directory in this git repo clone
559
- - The elasticsearch.yml and compose files create a mapping from that directory to /usr/share/elasticsearch/snapshots within the Elasticsearch container and grant permissions for using it
560
-
561
- - **Creating a Snapshot**:
562
- ```shell
563
- curl -X "PUT" "http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2?wait_for_completion=true" \
564
- -H 'Content-Type: application/json; charset=utf-8' \
565
- -d $'{
566
- "metadata": {
567
- "taken_because": "dump of all items",
568
- "taken_by": "pvarner"
569
- },
570
- "include_global_state": false,
571
- "ignore_unavailable": false,
572
- "indices": "items_my-collection"
573
- }'
574
- ```
575
- - This creates a snapshot named my_snapshot_2 and waits for the action to be completed before returning
576
- - This can also be done asynchronously by omitting the wait_for_completion parameter, and queried for status later
577
- - The indices parameter determines which indices are snapshotted, and can include wildcards
578
-
579
- - **Viewing Snapshots**:
580
- ```shell
581
- # View a specific snapshot
582
- curl http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2
583
-
584
- # View all snapshots
585
- curl http://localhost:9200/_snapshot/my_fs_backup/_all
586
- ```
587
- - These commands allow you to check the status and details of your snapshots
588
-
589
- - **Restoring a Snapshot**:
590
- ```shell
591
- curl -X "POST" "http://localhost:9200/_snapshot/my_fs_backup/my_snapshot_2/_restore?wait_for_completion=true" \
592
- -H 'Content-Type: application/json; charset=utf-8' \
593
- -d $'{
594
- "include_aliases": false,
595
- "include_global_state": false,
596
- "ignore_unavailable": true,
597
- "rename_replacement": "items_$1-copy",
598
- "indices": "items_*",
599
- "rename_pattern": "items_(.+)"
600
- }'
601
- ```
602
- - This specific command will restore any indices that match items_* and rename them so that the new index name will be suffixed with -copy
603
- - The rename_pattern and rename_replacement parameters allow you to restore indices under new names
604
-
605
- - **Updating Collection References**:
606
- ```shell
607
- curl -X "POST" "http://localhost:9200/items_my-collection-copy/_update_by_query" \
608
- -H 'Content-Type: application/json; charset=utf-8' \
609
- -d $'{
610
- "query": {
611
- "match_all": {}
612
- },
613
- "script": {
614
- "lang": "painless",
615
- "params": {
616
- "collection": "my-collection-copy"
617
- },
618
- "source": "ctx._source.collection = params.collection"
619
- }
620
- }'
621
- ```
622
- - After restoring, the item documents have been restored in the new index (e.g., my-collection-copy), but the value of the collection field in those documents is still the original value of my-collection
623
- - This command updates these values to match the new collection name using Elasticsearch's Update By Query feature
624
-
625
- - **Creating a New Collection**:
626
- ```shell
627
- curl -X "POST" "http://localhost:8080/collections" \
628
- -H 'Content-Type: application/json' \
629
- -d $'{
630
- "id": "my-collection-copy"
631
- }'
632
- ```
633
- - The final step is to create a new collection through the API with the new name for each of the restored indices
634
- - This gives you a copy of the collection that has a resource URI (/collections/my-collection-copy) and can be correctly queried by collection name
635
-
636
- ### Reindexing
637
-
638
- - **Overview**: Reindexing allows you to copy documents from one index to another, optionally transforming them in the process.
639
-
640
- - **Use Cases**:
641
- - Apply changes to documents
642
- - Correct dynamically generated mappings
643
- - Transform data (e.g., lowercase identifiers)
644
- - The index templates will make sure that manually created indices will also have the correct mappings and settings
645
-
646
- - **Example: Reindexing with Transformation**:
647
- ```shell
648
- curl -X "POST" "http://localhost:9200/_reindex" \
649
- -H 'Content-Type: application/json' \
650
- -d $'{
651
- "source": {
652
- "index": "items_my-collection-lower_my-collection-hex-000001"
653
- },
654
- "dest": {
655
- "index": "items_my-collection-lower_my-collection-hex-000002"
656
- },
657
- "script": {
658
- "source": "ctx._source.id = ctx._source.id.toLowerCase()",
659
- "lang": "painless"
660
- }
661
- }'
662
- ```
663
- - In this example, we make a copy of an existing Item index but change the Item identifier to be lowercase
664
- - The script parameter allows you to transform documents during the reindexing process
665
-
666
- - **Updating Aliases**:
667
- ```shell
668
- curl -X "POST" "http://localhost:9200/_aliases" \
669
- -H 'Content-Type: application/json' \
670
- -d $'{
671
- "actions": [
672
- {
673
- "remove": {
674
- "index": "*",
675
- "alias": "items_my-collection"
676
- }
677
- },
678
- {
679
- "add": {
680
- "index": "items_my-collection-lower_my-collection-hex-000002",
681
- "alias": "items_my-collection"
682
- }
683
- }
684
- ]
685
- }'
686
- ```
687
- - If you are happy with the data in the newly created index, you can move the alias items_my-collection to the new index
688
- - This makes the modified Items with lowercase identifiers visible to users accessing my-collection in the STAC API
689
- - Using aliases allows you to switch between different index versions without changing the API endpoint
690
-
691
- ## Auth
692
-
693
- - **Overview**: Authentication is an optional feature that can be enabled through Route Dependencies.
694
- - **Implementation Options**:
695
- - Basic authentication
696
- - OAuth2 with Keycloak
697
- - Custom route dependencies
698
- - **Configuration**: Authentication can be configured using the `STAC_FASTAPI_ROUTE_DEPENDENCIES` environment variable.
699
- - **Examples and Documentation**: Detailed examples and implementation guides can be found in the [examples/auth](examples/auth) directory.
700
-
701
- ## Aggregation
702
-
703
- - **Supported Aggregations**:
704
- - Spatial aggregations of points and geometries
705
- - Frequency distribution aggregation of any property including dates
706
- - Temporal distribution of datetime values
707
-
708
- - **Endpoint Locations**:
709
- - Root Catalog level: `/aggregations`
710
- - Collection level: `/<collection_id>/aggregations`
711
-
712
- - **Implementation Details**: The `sfeos_helpers.aggregation` package provides specialized functionality for both Elasticsearch and OpenSearch backends.
713
-
714
- - **Documentation**: Detailed information about supported aggregations can be found in [the aggregation docs](./docs/src/aggregation.md).
715
-
716
-
717
- ## Rate Limiting
718
-
719
- - **Overview**: Rate limiting is an optional security feature that controls API request frequency on a remote address basis.
720
-
721
- - **Configuration**: Enabled by setting the `STAC_FASTAPI_RATE_LIMIT` environment variable:
722
- ```
723
- STAC_FASTAPI_RATE_LIMIT=500/minute
724
- ```
725
-
726
- - **Functionality**:
727
- - Limits each client to a specified number of requests per time period (e.g., 500 requests per minute)
728
- - Helps prevent API abuse and maintains system stability
729
- - Ensures fair resource allocation among all clients
730
-
731
- - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
@@ -1,10 +0,0 @@
1
- stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
2
- stac_fastapi/elasticsearch/app.py,sha256=024U5xvXmSWUJABS9SekbqqamhsNSpqPBII2NBWktz8,9979
3
- stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
4
- stac_fastapi/elasticsearch/database_logic.py,sha256=eNPzK_dBhNJgAgqEpF3hHmqep_7_KGpLVEQA-jusEAY,70183
5
- stac_fastapi/elasticsearch/version.py,sha256=FuGC3fKnAmD4Wk95swJ6qCVBs5mZiShrlRKuSH-voyE,45
6
- stac_fastapi_elasticsearch-6.5.1.dist-info/METADATA,sha256=JFd5dRXZznzO6uqbUGS1wVL5YbkyNorypBCQ2T-KVA8,42084
7
- stac_fastapi_elasticsearch-6.5.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
8
- stac_fastapi_elasticsearch-6.5.1.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
9
- stac_fastapi_elasticsearch-6.5.1.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
10
- stac_fastapi_elasticsearch-6.5.1.dist-info/RECORD,,
@@ -1 +0,0 @@
1
- stac_fastapi