stac-fastapi-elasticsearch 6.1.0__py3-none-any.whl → 6.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stac_fastapi/elasticsearch/app.py +1 -1
- stac_fastapi/elasticsearch/database_logic.py +102 -48
- stac_fastapi/elasticsearch/version.py +1 -1
- {stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.1.dist-info}/METADATA +91 -6
- stac_fastapi_elasticsearch-6.2.1.dist-info/RECORD +10 -0
- {stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.1.dist-info}/WHEEL +1 -1
- stac_fastapi_elasticsearch-6.1.0.dist-info/RECORD +0 -10
- {stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.1.dist-info}/entry_points.txt +0 -0
- {stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.1.dist-info}/top_level.txt +0 -0
|
@@ -117,7 +117,7 @@ post_request_model = create_post_request_model(search_extensions)
|
|
|
117
117
|
app_config = {
|
|
118
118
|
"title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"),
|
|
119
119
|
"description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"),
|
|
120
|
-
"api_version": os.getenv("STAC_FASTAPI_VERSION", "6.1
|
|
120
|
+
"api_version": os.getenv("STAC_FASTAPI_VERSION", "6.2.1"),
|
|
121
121
|
"settings": settings,
|
|
122
122
|
"extensions": extensions,
|
|
123
123
|
"client": CoreClient(
|
|
@@ -4,7 +4,7 @@ import asyncio
|
|
|
4
4
|
import logging
|
|
5
5
|
from base64 import urlsafe_b64decode, urlsafe_b64encode
|
|
6
6
|
from copy import deepcopy
|
|
7
|
-
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
|
|
7
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
|
|
8
8
|
|
|
9
9
|
import attr
|
|
10
10
|
import elasticsearch.helpers as helpers
|
|
@@ -27,7 +27,7 @@ from stac_fastapi.extensions.core.transaction.request import (
|
|
|
27
27
|
PartialItem,
|
|
28
28
|
PatchOperation,
|
|
29
29
|
)
|
|
30
|
-
from stac_fastapi.sfeos_helpers import filter
|
|
30
|
+
from stac_fastapi.sfeos_helpers import filter as filter_module
|
|
31
31
|
from stac_fastapi.sfeos_helpers.database import (
|
|
32
32
|
apply_free_text_filter_shared,
|
|
33
33
|
apply_intersects_filter_shared,
|
|
@@ -36,7 +36,6 @@ from stac_fastapi.sfeos_helpers.database import (
|
|
|
36
36
|
get_queryables_mapping_shared,
|
|
37
37
|
index_alias_by_collection_id,
|
|
38
38
|
index_by_collection_id,
|
|
39
|
-
indices,
|
|
40
39
|
mk_actions,
|
|
41
40
|
mk_item_id,
|
|
42
41
|
populate_sort_shared,
|
|
@@ -59,9 +58,14 @@ from stac_fastapi.sfeos_helpers.mappings import (
|
|
|
59
58
|
ITEMS_INDEX_PREFIX,
|
|
60
59
|
Geometry,
|
|
61
60
|
)
|
|
61
|
+
from stac_fastapi.sfeos_helpers.search_engine import (
|
|
62
|
+
BaseIndexInserter,
|
|
63
|
+
BaseIndexSelector,
|
|
64
|
+
IndexInsertionFactory,
|
|
65
|
+
IndexSelectorFactory,
|
|
66
|
+
)
|
|
62
67
|
from stac_fastapi.types.errors import ConflictError, NotFoundError
|
|
63
68
|
from stac_fastapi.types.links import resolve_links
|
|
64
|
-
from stac_fastapi.types.rfc3339 import DateTimeType
|
|
65
69
|
from stac_fastapi.types.stac import Collection, Item
|
|
66
70
|
|
|
67
71
|
logger = logging.getLogger(__name__)
|
|
@@ -139,6 +143,8 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
139
143
|
sync_settings: SyncElasticsearchSettings = attr.ib(
|
|
140
144
|
factory=SyncElasticsearchSettings
|
|
141
145
|
)
|
|
146
|
+
async_index_selector: BaseIndexSelector = attr.ib(init=False)
|
|
147
|
+
async_index_inserter: BaseIndexInserter = attr.ib(init=False)
|
|
142
148
|
|
|
143
149
|
client = attr.ib(init=False)
|
|
144
150
|
sync_client = attr.ib(init=False)
|
|
@@ -147,6 +153,10 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
147
153
|
"""Initialize clients after the class is instantiated."""
|
|
148
154
|
self.client = self.async_settings.create_client
|
|
149
155
|
self.sync_client = self.sync_settings.create_client
|
|
156
|
+
self.async_index_inserter = IndexInsertionFactory.create_insertion_strategy(
|
|
157
|
+
self.client
|
|
158
|
+
)
|
|
159
|
+
self.async_index_selector = IndexSelectorFactory.create_selector(self.client)
|
|
150
160
|
|
|
151
161
|
item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
|
|
152
162
|
collection_serializer: Type[CollectionSerializer] = attr.ib(
|
|
@@ -216,15 +226,23 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
216
226
|
with the index for the Collection as the target index and the combined `mk_item_id` as the document id.
|
|
217
227
|
"""
|
|
218
228
|
try:
|
|
219
|
-
|
|
229
|
+
response = await self.client.search(
|
|
220
230
|
index=index_alias_by_collection_id(collection_id),
|
|
221
|
-
|
|
231
|
+
body={
|
|
232
|
+
"query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
|
|
233
|
+
"size": 1,
|
|
234
|
+
},
|
|
222
235
|
)
|
|
236
|
+
if response["hits"]["total"]["value"] == 0:
|
|
237
|
+
raise NotFoundError(
|
|
238
|
+
f"Item {item_id} does not exist inside Collection {collection_id}"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
return response["hits"]["hits"][0]["_source"]
|
|
223
242
|
except ESNotFoundError:
|
|
224
243
|
raise NotFoundError(
|
|
225
244
|
f"Item {item_id} does not exist inside Collection {collection_id}"
|
|
226
245
|
)
|
|
227
|
-
return item["_source"]
|
|
228
246
|
|
|
229
247
|
async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
|
|
230
248
|
"""Retrieve mapping of Queryables for search.
|
|
@@ -260,31 +278,21 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
260
278
|
|
|
261
279
|
@staticmethod
|
|
262
280
|
def apply_datetime_filter(
|
|
263
|
-
search: Search,
|
|
264
|
-
) -> Search:
|
|
281
|
+
search: Search, datetime: Optional[str]
|
|
282
|
+
) -> Tuple[Search, Dict[str, Optional[str]]]:
|
|
265
283
|
"""Apply a filter to search on datetime, start_datetime, and end_datetime fields.
|
|
266
284
|
|
|
267
285
|
Args:
|
|
268
286
|
search: The search object to filter.
|
|
269
|
-
|
|
270
|
-
- A single datetime string (e.g., "2023-01-01T12:00:00")
|
|
271
|
-
- A datetime range string (e.g., "2023-01-01/2023-12-31")
|
|
272
|
-
- A datetime object
|
|
273
|
-
- A tuple of (start_datetime, end_datetime)
|
|
287
|
+
datetime: Optional[str]
|
|
274
288
|
|
|
275
289
|
Returns:
|
|
276
290
|
The filtered search object.
|
|
277
291
|
"""
|
|
278
|
-
|
|
279
|
-
return search
|
|
292
|
+
datetime_search = return_date(datetime)
|
|
280
293
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
datetime_search = return_date(interval)
|
|
284
|
-
except (ValueError, TypeError) as e:
|
|
285
|
-
# Handle invalid interval formats if return_date fails
|
|
286
|
-
logger.error(f"Invalid interval format: {interval}, error: {e}")
|
|
287
|
-
return search
|
|
294
|
+
if not datetime_search:
|
|
295
|
+
return search, datetime_search
|
|
288
296
|
|
|
289
297
|
if "eq" in datetime_search:
|
|
290
298
|
# For exact matches, include:
|
|
@@ -351,7 +359,10 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
351
359
|
),
|
|
352
360
|
]
|
|
353
361
|
|
|
354
|
-
return
|
|
362
|
+
return (
|
|
363
|
+
search.query(Q("bool", should=should, minimum_should_match=1)),
|
|
364
|
+
datetime_search,
|
|
365
|
+
)
|
|
355
366
|
|
|
356
367
|
@staticmethod
|
|
357
368
|
def apply_bbox_filter(search: Search, bbox: List):
|
|
@@ -466,7 +477,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
466
477
|
otherwise the original Search object.
|
|
467
478
|
"""
|
|
468
479
|
if _filter is not None:
|
|
469
|
-
es_query =
|
|
480
|
+
es_query = filter_module.to_es(await self.get_queryables_mapping(), _filter)
|
|
470
481
|
search = search.query(es_query)
|
|
471
482
|
|
|
472
483
|
return search
|
|
@@ -493,6 +504,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
493
504
|
token: Optional[str],
|
|
494
505
|
sort: Optional[Dict[str, Dict[str, str]]],
|
|
495
506
|
collection_ids: Optional[List[str]],
|
|
507
|
+
datetime_search: Dict[str, Optional[str]],
|
|
496
508
|
ignore_unavailable: bool = True,
|
|
497
509
|
) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]:
|
|
498
510
|
"""Execute a search query with limit and other optional parameters.
|
|
@@ -503,6 +515,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
503
515
|
token (Optional[str]): The token used to return the next set of results.
|
|
504
516
|
sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted.
|
|
505
517
|
collection_ids (Optional[List[str]]): The collection ids to search.
|
|
518
|
+
datetime_search (Dict[str, Optional[str]]): Datetime range used for index selection.
|
|
506
519
|
ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True.
|
|
507
520
|
|
|
508
521
|
Returns:
|
|
@@ -523,7 +536,9 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
523
536
|
|
|
524
537
|
query = search.query.to_dict() if search.query else None
|
|
525
538
|
|
|
526
|
-
index_param =
|
|
539
|
+
index_param = await self.async_index_selector.select_indexes(
|
|
540
|
+
collection_ids, datetime_search
|
|
541
|
+
)
|
|
527
542
|
if len(index_param) > ES_MAX_URL_LENGTH - 300:
|
|
528
543
|
index_param = ITEM_INDICES
|
|
529
544
|
query = add_collections_to_body(collection_ids, query)
|
|
@@ -590,6 +605,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
590
605
|
geometry_geohash_grid_precision: int,
|
|
591
606
|
geometry_geotile_grid_precision: int,
|
|
592
607
|
datetime_frequency_interval: str,
|
|
608
|
+
datetime_search,
|
|
593
609
|
ignore_unavailable: Optional[bool] = True,
|
|
594
610
|
):
|
|
595
611
|
"""Return aggregations of STAC Items."""
|
|
@@ -625,7 +641,10 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
625
641
|
if k in aggregations
|
|
626
642
|
}
|
|
627
643
|
|
|
628
|
-
index_param =
|
|
644
|
+
index_param = await self.async_index_selector.select_indexes(
|
|
645
|
+
collection_ids, datetime_search
|
|
646
|
+
)
|
|
647
|
+
|
|
629
648
|
search_task = asyncio.create_task(
|
|
630
649
|
self.client.search(
|
|
631
650
|
index=index_param,
|
|
@@ -667,14 +686,21 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
667
686
|
|
|
668
687
|
"""
|
|
669
688
|
await self.check_collection_exists(collection_id=item["collection"])
|
|
689
|
+
alias = index_alias_by_collection_id(item["collection"])
|
|
690
|
+
doc_id = mk_item_id(item["id"], item["collection"])
|
|
670
691
|
|
|
671
|
-
if not exist_ok
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
692
|
+
if not exist_ok:
|
|
693
|
+
alias_exists = await self.client.indices.exists_alias(name=alias)
|
|
694
|
+
|
|
695
|
+
if alias_exists:
|
|
696
|
+
alias_info = await self.client.indices.get_alias(name=alias)
|
|
697
|
+
indices = list(alias_info.keys())
|
|
698
|
+
|
|
699
|
+
for index in indices:
|
|
700
|
+
if await self.client.exists(index=index, id=doc_id):
|
|
701
|
+
raise ConflictError(
|
|
702
|
+
f"Item {item['id']} in collection {item['collection']} already exists"
|
|
703
|
+
)
|
|
678
704
|
|
|
679
705
|
return self.item_serializer.stac_to_db(item, base_url)
|
|
680
706
|
|
|
@@ -805,7 +831,6 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
805
831
|
# Extract item and collection IDs
|
|
806
832
|
item_id = item["id"]
|
|
807
833
|
collection_id = item["collection"]
|
|
808
|
-
|
|
809
834
|
# Ensure kwargs is a dictionary
|
|
810
835
|
kwargs = kwargs or {}
|
|
811
836
|
|
|
@@ -823,9 +848,12 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
823
848
|
item=item, base_url=base_url, exist_ok=exist_ok
|
|
824
849
|
)
|
|
825
850
|
|
|
851
|
+
target_index = await self.async_index_inserter.get_target_index(
|
|
852
|
+
collection_id, item
|
|
853
|
+
)
|
|
826
854
|
# Index the item in the database
|
|
827
855
|
await self.client.index(
|
|
828
|
-
index=
|
|
856
|
+
index=target_index,
|
|
829
857
|
id=mk_item_id(item_id, collection_id),
|
|
830
858
|
document=item,
|
|
831
859
|
refresh=refresh,
|
|
@@ -858,6 +886,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
858
886
|
item_id=item_id,
|
|
859
887
|
operations=operations,
|
|
860
888
|
base_url=base_url,
|
|
889
|
+
create_nest=True,
|
|
861
890
|
refresh=refresh,
|
|
862
891
|
)
|
|
863
892
|
|
|
@@ -867,6 +896,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
867
896
|
item_id: str,
|
|
868
897
|
operations: List[PatchOperation],
|
|
869
898
|
base_url: str,
|
|
899
|
+
create_nest: bool = False,
|
|
870
900
|
refresh: bool = True,
|
|
871
901
|
) -> Item:
|
|
872
902
|
"""Database logic for json patching an item following RF6902.
|
|
@@ -901,16 +931,31 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
901
931
|
else:
|
|
902
932
|
script_operations.append(operation)
|
|
903
933
|
|
|
904
|
-
script = operations_to_script(script_operations)
|
|
934
|
+
script = operations_to_script(script_operations, create_nest=create_nest)
|
|
905
935
|
|
|
906
936
|
try:
|
|
907
|
-
await self.client.
|
|
937
|
+
search_response = await self.client.search(
|
|
908
938
|
index=index_alias_by_collection_id(collection_id),
|
|
939
|
+
body={
|
|
940
|
+
"query": {"term": {"_id": mk_item_id(item_id, collection_id)}},
|
|
941
|
+
"size": 1,
|
|
942
|
+
},
|
|
943
|
+
)
|
|
944
|
+
if search_response["hits"]["total"]["value"] == 0:
|
|
945
|
+
raise NotFoundError(
|
|
946
|
+
f"Item {item_id} does not exist inside Collection {collection_id}"
|
|
947
|
+
)
|
|
948
|
+
document_index = search_response["hits"]["hits"][0]["_index"]
|
|
949
|
+
await self.client.update(
|
|
950
|
+
index=document_index,
|
|
909
951
|
id=mk_item_id(item_id, collection_id),
|
|
910
952
|
script=script,
|
|
911
953
|
refresh=True,
|
|
912
954
|
)
|
|
913
|
-
|
|
955
|
+
except ESNotFoundError:
|
|
956
|
+
raise NotFoundError(
|
|
957
|
+
f"Item {item_id} does not exist inside Collection {collection_id}"
|
|
958
|
+
)
|
|
914
959
|
except BadRequestError as exc:
|
|
915
960
|
raise HTTPException(
|
|
916
961
|
status_code=400, detail=exc.info["error"]["caused_by"]
|
|
@@ -921,7 +966,9 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
921
966
|
if new_collection_id:
|
|
922
967
|
await self.client.reindex(
|
|
923
968
|
body={
|
|
924
|
-
"dest": {
|
|
969
|
+
"dest": {
|
|
970
|
+
"index": f"{ITEMS_INDEX_PREFIX}{new_collection_id}"
|
|
971
|
+
}, # # noqa
|
|
925
972
|
"source": {
|
|
926
973
|
"index": f"{ITEMS_INDEX_PREFIX}{collection_id}",
|
|
927
974
|
"query": {"term": {"id": {"value": item_id}}},
|
|
@@ -929,8 +976,8 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
929
976
|
"script": {
|
|
930
977
|
"lang": "painless",
|
|
931
978
|
"source": (
|
|
932
|
-
f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');"""
|
|
933
|
-
f"""ctx._source.collection = '{new_collection_id}';"""
|
|
979
|
+
f"""ctx._id = ctx._id.replace('{collection_id}', '{new_collection_id}');""" # noqa
|
|
980
|
+
f"""ctx._source.collection = '{new_collection_id}';""" # noqa
|
|
934
981
|
),
|
|
935
982
|
},
|
|
936
983
|
},
|
|
@@ -990,9 +1037,9 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
990
1037
|
|
|
991
1038
|
try:
|
|
992
1039
|
# Perform the delete operation
|
|
993
|
-
await self.client.
|
|
1040
|
+
await self.client.delete_by_query(
|
|
994
1041
|
index=index_alias_by_collection_id(collection_id),
|
|
995
|
-
|
|
1042
|
+
body={"query": {"term": {"_id": mk_item_id(item_id, collection_id)}}},
|
|
996
1043
|
refresh=refresh,
|
|
997
1044
|
)
|
|
998
1045
|
except ESNotFoundError:
|
|
@@ -1092,8 +1139,10 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1092
1139
|
refresh=refresh,
|
|
1093
1140
|
)
|
|
1094
1141
|
|
|
1095
|
-
|
|
1096
|
-
|
|
1142
|
+
if self.async_index_inserter.should_create_collection_index():
|
|
1143
|
+
await self.async_index_inserter.create_simple_index(
|
|
1144
|
+
self.client, collection_id
|
|
1145
|
+
)
|
|
1097
1146
|
|
|
1098
1147
|
async def find_collection(self, collection_id: str) -> Collection:
|
|
1099
1148
|
"""Find and return a collection from the database.
|
|
@@ -1218,6 +1267,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1218
1267
|
collection_id=collection_id,
|
|
1219
1268
|
operations=operations,
|
|
1220
1269
|
base_url=base_url,
|
|
1270
|
+
create_nest=True,
|
|
1221
1271
|
refresh=refresh,
|
|
1222
1272
|
)
|
|
1223
1273
|
|
|
@@ -1226,6 +1276,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1226
1276
|
collection_id: str,
|
|
1227
1277
|
operations: List[PatchOperation],
|
|
1228
1278
|
base_url: str,
|
|
1279
|
+
create_nest: bool = False,
|
|
1229
1280
|
refresh: bool = True,
|
|
1230
1281
|
) -> Collection:
|
|
1231
1282
|
"""Database logic for json patching a collection following RF6902.
|
|
@@ -1253,7 +1304,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1253
1304
|
else:
|
|
1254
1305
|
script_operations.append(operation)
|
|
1255
1306
|
|
|
1256
|
-
script = operations_to_script(script_operations)
|
|
1307
|
+
script = operations_to_script(script_operations, create_nest=create_nest)
|
|
1257
1308
|
|
|
1258
1309
|
try:
|
|
1259
1310
|
await self.client.update(
|
|
@@ -1367,9 +1418,12 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1367
1418
|
|
|
1368
1419
|
# Perform the bulk insert
|
|
1369
1420
|
raise_on_error = self.async_settings.raise_on_bulk_error
|
|
1421
|
+
actions = await self.async_index_inserter.prepare_bulk_actions(
|
|
1422
|
+
collection_id, processed_items
|
|
1423
|
+
)
|
|
1370
1424
|
success, errors = await helpers.async_bulk(
|
|
1371
1425
|
self.client,
|
|
1372
|
-
|
|
1426
|
+
actions,
|
|
1373
1427
|
refresh=refresh,
|
|
1374
1428
|
raise_on_error=raise_on_error,
|
|
1375
1429
|
)
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""library version."""
|
|
2
|
-
__version__ = "6.1
|
|
2
|
+
__version__ = "6.2.1"
|
{stac_fastapi_elasticsearch-6.1.0.dist-info → stac_fastapi_elasticsearch-6.2.1.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version: 6.1
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stac_fastapi_elasticsearch
|
|
3
|
+
Version: 6.2.1
|
|
4
4
|
Summary: An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -15,8 +15,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Requires-Python: >=3.9
|
|
17
17
|
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: stac-fastapi-core==6.1
|
|
19
|
-
Requires-Dist: sfeos-helpers==6.1
|
|
18
|
+
Requires-Dist: stac-fastapi-core==6.2.1
|
|
19
|
+
Requires-Dist: sfeos-helpers==6.2.1
|
|
20
20
|
Requires-Dist: elasticsearch[async]~=8.18.0
|
|
21
21
|
Requires-Dist: uvicorn~=0.23.0
|
|
22
22
|
Requires-Dist: starlette<0.36.0,>=0.35.0
|
|
@@ -33,6 +33,15 @@ Requires-Dist: mkdocs-material~=9.0.0; extra == "docs"
|
|
|
33
33
|
Requires-Dist: pdocs~=1.2.0; extra == "docs"
|
|
34
34
|
Provides-Extra: server
|
|
35
35
|
Requires-Dist: uvicorn[standard]~=0.23.0; extra == "server"
|
|
36
|
+
Dynamic: classifier
|
|
37
|
+
Dynamic: description
|
|
38
|
+
Dynamic: description-content-type
|
|
39
|
+
Dynamic: home-page
|
|
40
|
+
Dynamic: license
|
|
41
|
+
Dynamic: provides-extra
|
|
42
|
+
Dynamic: requires-dist
|
|
43
|
+
Dynamic: requires-python
|
|
44
|
+
Dynamic: summary
|
|
36
45
|
|
|
37
46
|
# stac-fastapi-elasticsearch-opensearch
|
|
38
47
|
|
|
@@ -121,6 +130,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
121
130
|
- [Auth](#auth)
|
|
122
131
|
- [Aggregation](#aggregation)
|
|
123
132
|
- [Rate Limiting](#rate-limiting)
|
|
133
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
124
134
|
|
|
125
135
|
## Documentation & Resources
|
|
126
136
|
|
|
@@ -262,10 +272,86 @@ You can customize additional settings in your `.env` file:
|
|
|
262
272
|
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
263
273
|
| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
|
|
264
274
|
| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
|
|
275
|
+
| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
|
|
265
276
|
|
|
266
277
|
> [!NOTE]
|
|
267
278
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
268
279
|
|
|
280
|
+
## Datetime-Based Index Management
|
|
281
|
+
|
|
282
|
+
### Overview
|
|
283
|
+
|
|
284
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
285
|
+
|
|
286
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
287
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
288
|
+
|
|
289
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
290
|
+
|
|
291
|
+
### When to Use
|
|
292
|
+
|
|
293
|
+
**Recommended for:**
|
|
294
|
+
- Systems with large collections containing millions of items
|
|
295
|
+
- Systems requiring high-performance temporal searching
|
|
296
|
+
|
|
297
|
+
**Pros:**
|
|
298
|
+
- Multiple times faster queries with datetime filter
|
|
299
|
+
- Reduced database load - only relevant indexes are searched
|
|
300
|
+
|
|
301
|
+
**Cons:**
|
|
302
|
+
- Slightly longer item indexing time (automatic index management)
|
|
303
|
+
- Greater management complexity
|
|
304
|
+
|
|
305
|
+
### Configuration
|
|
306
|
+
|
|
307
|
+
#### Enabling Datetime-Based Indexing
|
|
308
|
+
|
|
309
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
### Related Configuration Variables
|
|
316
|
+
|
|
317
|
+
| Variable | Description | Default | Example |
|
|
318
|
+
|----------|-------------|---------|---------|
|
|
319
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
320
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
321
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
322
|
+
|
|
323
|
+
## How Datetime-Based Indexing Works
|
|
324
|
+
|
|
325
|
+
### Index and Alias Naming Convention
|
|
326
|
+
|
|
327
|
+
The system uses a precise naming convention:
|
|
328
|
+
|
|
329
|
+
**Physical indexes:**
|
|
330
|
+
```
|
|
331
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
**Aliases:**
|
|
335
|
+
```
|
|
336
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
337
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
338
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
**Example:**
|
|
342
|
+
|
|
343
|
+
*Physical indexes:*
|
|
344
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
345
|
+
|
|
346
|
+
*Aliases:*
|
|
347
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
348
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
349
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
350
|
+
|
|
351
|
+
### Index Size Management
|
|
352
|
+
|
|
353
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
354
|
+
|
|
269
355
|
## Interacting with the API
|
|
270
356
|
|
|
271
357
|
- **Creating a Collection**:
|
|
@@ -574,4 +660,3 @@ You can customize additional settings in your `.env` file:
|
|
|
574
660
|
- Ensures fair resource allocation among all clients
|
|
575
661
|
|
|
576
662
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
577
|
-
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
|
|
2
|
+
stac_fastapi/elasticsearch/app.py,sha256=vhP5jhguGI419Q7MRVJ6WBbxtDun8wcWQLIIqkP6Ilc,5662
|
|
3
|
+
stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
|
|
4
|
+
stac_fastapi/elasticsearch/database_logic.py,sha256=N6HDLxr4GMIxREy4F_hESJZTUvpjCcj95c6V7di7i24,58478
|
|
5
|
+
stac_fastapi/elasticsearch/version.py,sha256=TvDysD5xP1CNnI8XDtkkz5sggBM1uuxjhZCv120q3AU,45
|
|
6
|
+
stac_fastapi_elasticsearch-6.2.1.dist-info/METADATA,sha256=MuvyyPQzwYR5iNMpET3p1Y-TALqPl6dA73U3igERncg,35411
|
|
7
|
+
stac_fastapi_elasticsearch-6.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
8
|
+
stac_fastapi_elasticsearch-6.2.1.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
|
|
9
|
+
stac_fastapi_elasticsearch-6.2.1.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
+
stac_fastapi_elasticsearch-6.2.1.dist-info/RECORD,,
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
|
|
2
|
-
stac_fastapi/elasticsearch/app.py,sha256=EK0H5qiRX-frbSHkBkZ_4Lzo0Jn8hBzJckSLCYbQlKs,5662
|
|
3
|
-
stac_fastapi/elasticsearch/config.py,sha256=itvPYr4TiOg9pWQrycgGaQxQ_Vc2KKP3aHdtH0OUZvw,5322
|
|
4
|
-
stac_fastapi/elasticsearch/database_logic.py,sha256=8P-eUGkDQAABOJWaYd_NTHpKpKug7o2c6BMDWYHH5m4,56201
|
|
5
|
-
stac_fastapi/elasticsearch/version.py,sha256=7IrY7mbr0cGVqZsk6wmCeITxZjDgz_mPHUswrziX5ME,45
|
|
6
|
-
stac_fastapi_elasticsearch-6.1.0.dist-info/METADATA,sha256=a2Z6VUi-YBORUsPZZDBJt1HuLqPeEPKtFgE2PqKEXj8,32286
|
|
7
|
-
stac_fastapi_elasticsearch-6.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
8
|
-
stac_fastapi_elasticsearch-6.1.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
|
|
9
|
-
stac_fastapi_elasticsearch-6.1.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
-
stac_fastapi_elasticsearch-6.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|