stac-fastapi-elasticsearch 4.1.0__py3-none-any.whl → 4.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stac_fastapi/elasticsearch/app.py +34 -15
- stac_fastapi/elasticsearch/config.py +24 -2
- stac_fastapi/elasticsearch/database_logic.py +232 -35
- stac_fastapi/elasticsearch/version.py +1 -1
- {stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/METADATA +8 -5
- stac_fastapi_elasticsearch-4.2.0.dist-info/RECORD +10 -0
- stac_fastapi_elasticsearch-4.1.0.dist-info/RECORD +0 -10
- {stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/WHEEL +0 -0
- {stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/entry_points.txt +0 -0
- {stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""FastAPI application."""
|
|
2
2
|
|
|
3
|
+
import logging
|
|
3
4
|
import os
|
|
4
5
|
from contextlib import asynccontextmanager
|
|
5
6
|
|
|
@@ -23,6 +24,7 @@ from stac_fastapi.core.extensions.fields import FieldsExtension
|
|
|
23
24
|
from stac_fastapi.core.rate_limit import setup_rate_limit
|
|
24
25
|
from stac_fastapi.core.route_dependencies import get_route_dependencies
|
|
25
26
|
from stac_fastapi.core.session import Session
|
|
27
|
+
from stac_fastapi.core.utilities import get_bool_env
|
|
26
28
|
from stac_fastapi.elasticsearch.config import ElasticsearchSettings
|
|
27
29
|
from stac_fastapi.elasticsearch.database_logic import (
|
|
28
30
|
DatabaseLogic,
|
|
@@ -39,6 +41,12 @@ from stac_fastapi.extensions.core import (
|
|
|
39
41
|
)
|
|
40
42
|
from stac_fastapi.extensions.third_party import BulkTransactionExtension
|
|
41
43
|
|
|
44
|
+
logging.basicConfig(level=logging.INFO)
|
|
45
|
+
logger = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
TRANSACTIONS_EXTENSIONS = get_bool_env("ENABLE_TRANSACTIONS_EXTENSIONS", default=True)
|
|
48
|
+
logger.info("TRANSACTIONS_EXTENSIONS is set to %s", TRANSACTIONS_EXTENSIONS)
|
|
49
|
+
|
|
42
50
|
settings = ElasticsearchSettings()
|
|
43
51
|
session = Session.create_from_settings(settings)
|
|
44
52
|
|
|
@@ -60,19 +68,6 @@ aggregation_extension.POST = EsAggregationExtensionPostRequest
|
|
|
60
68
|
aggregation_extension.GET = EsAggregationExtensionGetRequest
|
|
61
69
|
|
|
62
70
|
search_extensions = [
|
|
63
|
-
TransactionExtension(
|
|
64
|
-
client=TransactionsClient(
|
|
65
|
-
database=database_logic, session=session, settings=settings
|
|
66
|
-
),
|
|
67
|
-
settings=settings,
|
|
68
|
-
),
|
|
69
|
-
BulkTransactionExtension(
|
|
70
|
-
client=BulkTransactionsClient(
|
|
71
|
-
database=database_logic,
|
|
72
|
-
session=session,
|
|
73
|
-
settings=settings,
|
|
74
|
-
)
|
|
75
|
-
),
|
|
76
71
|
FieldsExtension(),
|
|
77
72
|
QueryExtension(),
|
|
78
73
|
SortExtension(),
|
|
@@ -81,6 +76,27 @@ search_extensions = [
|
|
|
81
76
|
FreeTextExtension(),
|
|
82
77
|
]
|
|
83
78
|
|
|
79
|
+
if TRANSACTIONS_EXTENSIONS:
|
|
80
|
+
search_extensions.insert(
|
|
81
|
+
0,
|
|
82
|
+
TransactionExtension(
|
|
83
|
+
client=TransactionsClient(
|
|
84
|
+
database=database_logic, session=session, settings=settings
|
|
85
|
+
),
|
|
86
|
+
settings=settings,
|
|
87
|
+
),
|
|
88
|
+
)
|
|
89
|
+
search_extensions.insert(
|
|
90
|
+
1,
|
|
91
|
+
BulkTransactionExtension(
|
|
92
|
+
client=BulkTransactionsClient(
|
|
93
|
+
database=database_logic,
|
|
94
|
+
session=session,
|
|
95
|
+
settings=settings,
|
|
96
|
+
)
|
|
97
|
+
),
|
|
98
|
+
)
|
|
99
|
+
|
|
84
100
|
extensions = [aggregation_extension] + search_extensions
|
|
85
101
|
|
|
86
102
|
database_logic.extensions = [type(ext).__name__ for ext in extensions]
|
|
@@ -90,11 +106,14 @@ post_request_model = create_post_request_model(search_extensions)
|
|
|
90
106
|
api = StacApi(
|
|
91
107
|
title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"),
|
|
92
108
|
description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"),
|
|
93
|
-
api_version=os.getenv("STAC_FASTAPI_VERSION", "4.
|
|
109
|
+
api_version=os.getenv("STAC_FASTAPI_VERSION", "4.2.0"),
|
|
94
110
|
settings=settings,
|
|
95
111
|
extensions=extensions,
|
|
96
112
|
client=CoreClient(
|
|
97
|
-
database=database_logic,
|
|
113
|
+
database=database_logic,
|
|
114
|
+
session=session,
|
|
115
|
+
post_request_model=post_request_model,
|
|
116
|
+
landing_page_id=os.getenv("STAC_FASTAPI_LANDING_PAGE_ID", "stac-fastapi"),
|
|
98
117
|
),
|
|
99
118
|
search_get_request_model=create_get_request_model(search_extensions),
|
|
100
119
|
search_post_request_model=post_request_model,
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
import logging
|
|
4
4
|
import os
|
|
5
5
|
import ssl
|
|
6
|
-
from typing import Any, Dict, Set
|
|
6
|
+
from typing import Any, Dict, Set, Union
|
|
7
7
|
|
|
8
8
|
import certifi
|
|
9
9
|
from elasticsearch._async.client import AsyncElasticsearch
|
|
10
10
|
|
|
11
11
|
from elasticsearch import Elasticsearch # type: ignore[attr-defined]
|
|
12
12
|
from stac_fastapi.core.base_settings import ApiBaseSettings
|
|
13
|
-
from stac_fastapi.core.utilities import get_bool_env
|
|
13
|
+
from stac_fastapi.core.utilities import get_bool_env, validate_refresh
|
|
14
14
|
from stac_fastapi.types.config import ApiSettings
|
|
15
15
|
|
|
16
16
|
|
|
@@ -88,6 +88,17 @@ class ElasticsearchSettings(ApiSettings, ApiBaseSettings):
|
|
|
88
88
|
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
89
89
|
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
90
90
|
|
|
91
|
+
@property
|
|
92
|
+
def database_refresh(self) -> Union[bool, str]:
|
|
93
|
+
"""
|
|
94
|
+
Get the value of the DATABASE_REFRESH environment variable.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
Union[bool, str]: The value of DATABASE_REFRESH, which can be True, False, or "wait_for".
|
|
98
|
+
"""
|
|
99
|
+
value = os.getenv("DATABASE_REFRESH", "false")
|
|
100
|
+
return validate_refresh(value)
|
|
101
|
+
|
|
91
102
|
@property
|
|
92
103
|
def create_client(self):
|
|
93
104
|
"""Create es client."""
|
|
@@ -109,6 +120,17 @@ class AsyncElasticsearchSettings(ApiSettings, ApiBaseSettings):
|
|
|
109
120
|
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
110
121
|
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
111
122
|
|
|
123
|
+
@property
|
|
124
|
+
def database_refresh(self) -> Union[bool, str]:
|
|
125
|
+
"""
|
|
126
|
+
Get the value of the DATABASE_REFRESH environment variable.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
Union[bool, str]: The value of DATABASE_REFRESH, which can be True, False, or "wait_for".
|
|
130
|
+
"""
|
|
131
|
+
value = os.getenv("DATABASE_REFRESH", "false")
|
|
132
|
+
return validate_refresh(value)
|
|
133
|
+
|
|
112
134
|
@property
|
|
113
135
|
def create_client(self):
|
|
114
136
|
"""Create async elasticsearch client."""
|
|
@@ -31,7 +31,7 @@ from stac_fastapi.core.database_logic import (
|
|
|
31
31
|
)
|
|
32
32
|
from stac_fastapi.core.extensions import filter
|
|
33
33
|
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
|
|
34
|
-
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
|
|
34
|
+
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, validate_refresh
|
|
35
35
|
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
|
|
36
36
|
from stac_fastapi.elasticsearch.config import (
|
|
37
37
|
ElasticsearchSettings as SyncElasticsearchSettings,
|
|
@@ -290,6 +290,34 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
290
290
|
)
|
|
291
291
|
return item["_source"]
|
|
292
292
|
|
|
293
|
+
async def get_queryables_mapping(self, collection_id: str = "*") -> dict:
|
|
294
|
+
"""Retrieve mapping of Queryables for search.
|
|
295
|
+
|
|
296
|
+
Args:
|
|
297
|
+
collection_id (str, optional): The id of the Collection the Queryables
|
|
298
|
+
belongs to. Defaults to "*".
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
dict: A dictionary containing the Queryables mappings.
|
|
302
|
+
"""
|
|
303
|
+
queryables_mapping = {}
|
|
304
|
+
|
|
305
|
+
mappings = await self.client.indices.get_mapping(
|
|
306
|
+
index=f"{ITEMS_INDEX_PREFIX}{collection_id}",
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
for mapping in mappings.values():
|
|
310
|
+
fields = mapping["mappings"].get("properties", {})
|
|
311
|
+
properties = fields.pop("properties", {}).get("properties", {}).keys()
|
|
312
|
+
|
|
313
|
+
for field_key in fields:
|
|
314
|
+
queryables_mapping[field_key] = field_key
|
|
315
|
+
|
|
316
|
+
for property_key in properties:
|
|
317
|
+
queryables_mapping[property_key] = f"properties.{property_key}"
|
|
318
|
+
|
|
319
|
+
return queryables_mapping
|
|
320
|
+
|
|
293
321
|
@staticmethod
|
|
294
322
|
def make_search():
|
|
295
323
|
"""Database logic to create a Search instance."""
|
|
@@ -518,8 +546,9 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
518
546
|
|
|
519
547
|
return search
|
|
520
548
|
|
|
521
|
-
|
|
522
|
-
|
|
549
|
+
async def apply_cql2_filter(
|
|
550
|
+
self, search: Search, _filter: Optional[Dict[str, Any]]
|
|
551
|
+
):
|
|
523
552
|
"""
|
|
524
553
|
Apply a CQL2 filter to an Elasticsearch Search object.
|
|
525
554
|
|
|
@@ -539,7 +568,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
539
568
|
otherwise the original Search object.
|
|
540
569
|
"""
|
|
541
570
|
if _filter is not None:
|
|
542
|
-
es_query = filter.to_es(_filter)
|
|
571
|
+
es_query = filter.to_es(await self.get_queryables_mapping(), _filter)
|
|
543
572
|
search = search.query(es_query)
|
|
544
573
|
|
|
545
574
|
return search
|
|
@@ -845,15 +874,19 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
845
874
|
async def create_item(
|
|
846
875
|
self,
|
|
847
876
|
item: Item,
|
|
848
|
-
refresh: bool = False,
|
|
849
877
|
base_url: str = "",
|
|
850
878
|
exist_ok: bool = False,
|
|
879
|
+
**kwargs: Any,
|
|
851
880
|
):
|
|
852
881
|
"""Database logic for creating one item.
|
|
853
882
|
|
|
854
883
|
Args:
|
|
855
884
|
item (Item): The item to be created.
|
|
856
|
-
|
|
885
|
+
base_url (str, optional): The base URL for the item. Defaults to an empty string.
|
|
886
|
+
exist_ok (bool, optional): Whether to allow the item to exist already. Defaults to False.
|
|
887
|
+
**kwargs: Additional keyword arguments.
|
|
888
|
+
- refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
|
|
889
|
+
- refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
|
|
857
890
|
|
|
858
891
|
Raises:
|
|
859
892
|
ConflictError: If the item already exists in the database.
|
|
@@ -861,12 +894,28 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
861
894
|
Returns:
|
|
862
895
|
None
|
|
863
896
|
"""
|
|
864
|
-
#
|
|
897
|
+
# Extract item and collection IDs
|
|
865
898
|
item_id = item["id"]
|
|
866
899
|
collection_id = item["collection"]
|
|
900
|
+
|
|
901
|
+
# Ensure kwargs is a dictionary
|
|
902
|
+
kwargs = kwargs or {}
|
|
903
|
+
|
|
904
|
+
# Resolve the `refresh` parameter
|
|
905
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
906
|
+
refresh = validate_refresh(refresh)
|
|
907
|
+
|
|
908
|
+
# Log the creation attempt
|
|
909
|
+
logger.info(
|
|
910
|
+
f"Creating item {item_id} in collection {collection_id} with refresh={refresh}"
|
|
911
|
+
)
|
|
912
|
+
|
|
913
|
+
# Prepare the item for insertion
|
|
867
914
|
item = await self.async_prep_create_item(
|
|
868
915
|
item=item, base_url=base_url, exist_ok=exist_ok
|
|
869
916
|
)
|
|
917
|
+
|
|
918
|
+
# Index the item in the database
|
|
870
919
|
await self.client.index(
|
|
871
920
|
index=index_alias_by_collection_id(collection_id),
|
|
872
921
|
id=mk_item_id(item_id, collection_id),
|
|
@@ -874,26 +923,43 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
874
923
|
refresh=refresh,
|
|
875
924
|
)
|
|
876
925
|
|
|
877
|
-
async def delete_item(
|
|
878
|
-
self, item_id: str, collection_id: str, refresh: bool = False
|
|
879
|
-
):
|
|
926
|
+
async def delete_item(self, item_id: str, collection_id: str, **kwargs: Any):
|
|
880
927
|
"""Delete a single item from the database.
|
|
881
928
|
|
|
882
929
|
Args:
|
|
883
930
|
item_id (str): The id of the Item to be deleted.
|
|
884
931
|
collection_id (str): The id of the Collection that the Item belongs to.
|
|
885
|
-
|
|
932
|
+
**kwargs: Additional keyword arguments.
|
|
933
|
+
- refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
|
|
934
|
+
- refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
|
|
886
935
|
|
|
887
936
|
Raises:
|
|
888
937
|
NotFoundError: If the Item does not exist in the database.
|
|
938
|
+
|
|
939
|
+
Returns:
|
|
940
|
+
None
|
|
889
941
|
"""
|
|
942
|
+
# Ensure kwargs is a dictionary
|
|
943
|
+
kwargs = kwargs or {}
|
|
944
|
+
|
|
945
|
+
# Resolve the `refresh` parameter
|
|
946
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
947
|
+
refresh = validate_refresh(refresh)
|
|
948
|
+
|
|
949
|
+
# Log the deletion attempt
|
|
950
|
+
logger.info(
|
|
951
|
+
f"Deleting item {item_id} from collection {collection_id} with refresh={refresh}"
|
|
952
|
+
)
|
|
953
|
+
|
|
890
954
|
try:
|
|
955
|
+
# Perform the delete operation
|
|
891
956
|
await self.client.delete(
|
|
892
957
|
index=index_alias_by_collection_id(collection_id),
|
|
893
958
|
id=mk_item_id(item_id, collection_id),
|
|
894
959
|
refresh=refresh,
|
|
895
960
|
)
|
|
896
961
|
except ESNotFoundError:
|
|
962
|
+
# Raise a custom NotFoundError if the item does not exist
|
|
897
963
|
raise NotFoundError(
|
|
898
964
|
f"Item {item_id} in collection {collection_id} not found"
|
|
899
965
|
)
|
|
@@ -916,24 +982,41 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
916
982
|
except ESNotFoundError:
|
|
917
983
|
raise NotFoundError(f"Mapping for index {index_name} not found")
|
|
918
984
|
|
|
919
|
-
async def create_collection(self, collection: Collection,
|
|
985
|
+
async def create_collection(self, collection: Collection, **kwargs: Any):
|
|
920
986
|
"""Create a single collection in the database.
|
|
921
987
|
|
|
922
988
|
Args:
|
|
923
989
|
collection (Collection): The Collection object to be created.
|
|
924
|
-
|
|
990
|
+
**kwargs: Additional keyword arguments.
|
|
991
|
+
- refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
|
|
992
|
+
- refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
|
|
925
993
|
|
|
926
994
|
Raises:
|
|
927
995
|
ConflictError: If a Collection with the same id already exists in the database.
|
|
928
996
|
|
|
997
|
+
Returns:
|
|
998
|
+
None
|
|
999
|
+
|
|
929
1000
|
Notes:
|
|
930
1001
|
A new index is created for the items in the Collection using the `create_item_index` function.
|
|
931
1002
|
"""
|
|
932
1003
|
collection_id = collection["id"]
|
|
933
1004
|
|
|
1005
|
+
# Ensure kwargs is a dictionary
|
|
1006
|
+
kwargs = kwargs or {}
|
|
1007
|
+
|
|
1008
|
+
# Resolve the `refresh` parameter
|
|
1009
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
1010
|
+
refresh = validate_refresh(refresh)
|
|
1011
|
+
|
|
1012
|
+
# Log the creation attempt
|
|
1013
|
+
logger.info(f"Creating collection {collection_id} with refresh={refresh}")
|
|
1014
|
+
|
|
1015
|
+
# Check if the collection already exists
|
|
934
1016
|
if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
|
|
935
1017
|
raise ConflictError(f"Collection {collection_id} already exists")
|
|
936
1018
|
|
|
1019
|
+
# Index the collection in the database
|
|
937
1020
|
await self.client.index(
|
|
938
1021
|
index=COLLECTIONS_INDEX,
|
|
939
1022
|
id=collection_id,
|
|
@@ -941,6 +1024,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
941
1024
|
refresh=refresh,
|
|
942
1025
|
)
|
|
943
1026
|
|
|
1027
|
+
# Create the item index for the collection
|
|
944
1028
|
await create_item_index(collection_id)
|
|
945
1029
|
|
|
946
1030
|
async def find_collection(self, collection_id: str) -> Collection:
|
|
@@ -970,29 +1054,52 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
970
1054
|
return collection["_source"]
|
|
971
1055
|
|
|
972
1056
|
async def update_collection(
|
|
973
|
-
self, collection_id: str, collection: Collection,
|
|
1057
|
+
self, collection_id: str, collection: Collection, **kwargs: Any
|
|
974
1058
|
):
|
|
975
|
-
"""Update a collection
|
|
1059
|
+
"""Update a collection in the database.
|
|
976
1060
|
|
|
977
1061
|
Args:
|
|
978
|
-
self: The instance of the object calling this function.
|
|
979
1062
|
collection_id (str): The ID of the collection to be updated.
|
|
980
1063
|
collection (Collection): The Collection object to be used for the update.
|
|
1064
|
+
**kwargs: Additional keyword arguments.
|
|
1065
|
+
- refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
|
|
1066
|
+
- refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
|
|
1067
|
+
Returns:
|
|
1068
|
+
None
|
|
981
1069
|
|
|
982
1070
|
Raises:
|
|
983
|
-
NotFoundError: If the collection with the given `collection_id` is not
|
|
984
|
-
|
|
1071
|
+
NotFoundError: If the collection with the given `collection_id` is not found in the database.
|
|
1072
|
+
ConflictError: If a conflict occurs during the update.
|
|
985
1073
|
|
|
986
1074
|
Notes:
|
|
987
1075
|
This function updates the collection in the database using the specified
|
|
988
|
-
`collection_id` and
|
|
989
|
-
|
|
1076
|
+
`collection_id` and the provided `Collection` object. If the collection ID
|
|
1077
|
+
changes, the function creates a new collection, reindexes the items, and deletes
|
|
1078
|
+
the old collection.
|
|
990
1079
|
"""
|
|
1080
|
+
# Ensure kwargs is a dictionary
|
|
1081
|
+
kwargs = kwargs or {}
|
|
1082
|
+
|
|
1083
|
+
# Resolve the `refresh` parameter
|
|
1084
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
1085
|
+
refresh = validate_refresh(refresh)
|
|
1086
|
+
|
|
1087
|
+
# Log the update attempt
|
|
1088
|
+
logger.info(f"Updating collection {collection_id} with refresh={refresh}")
|
|
1089
|
+
|
|
1090
|
+
# Ensure the collection exists
|
|
991
1091
|
await self.find_collection(collection_id=collection_id)
|
|
992
1092
|
|
|
1093
|
+
# Handle collection ID change
|
|
993
1094
|
if collection_id != collection["id"]:
|
|
1095
|
+
logger.info(
|
|
1096
|
+
f"Collection ID change detected: {collection_id} -> {collection['id']}"
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
# Create the new collection
|
|
994
1100
|
await self.create_collection(collection, refresh=refresh)
|
|
995
1101
|
|
|
1102
|
+
# Reindex items from the old collection to the new collection
|
|
996
1103
|
await self.client.reindex(
|
|
997
1104
|
body={
|
|
998
1105
|
"dest": {"index": f"{ITEMS_INDEX_PREFIX}{collection['id']}"},
|
|
@@ -1006,9 +1113,11 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1006
1113
|
refresh=refresh,
|
|
1007
1114
|
)
|
|
1008
1115
|
|
|
1116
|
+
# Delete the old collection
|
|
1009
1117
|
await self.delete_collection(collection_id)
|
|
1010
1118
|
|
|
1011
1119
|
else:
|
|
1120
|
+
# Update the existing collection
|
|
1012
1121
|
await self.client.index(
|
|
1013
1122
|
index=COLLECTIONS_INDEX,
|
|
1014
1123
|
id=collection_id,
|
|
@@ -1016,33 +1125,57 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1016
1125
|
refresh=refresh,
|
|
1017
1126
|
)
|
|
1018
1127
|
|
|
1019
|
-
async def delete_collection(self, collection_id: str,
|
|
1128
|
+
async def delete_collection(self, collection_id: str, **kwargs: Any):
|
|
1020
1129
|
"""Delete a collection from the database.
|
|
1021
1130
|
|
|
1022
1131
|
Parameters:
|
|
1023
|
-
self: The instance of the object calling this function.
|
|
1024
1132
|
collection_id (str): The ID of the collection to be deleted.
|
|
1025
|
-
|
|
1133
|
+
kwargs (Any, optional): Additional keyword arguments, including `refresh`.
|
|
1134
|
+
- refresh (str): Whether to refresh the index after the operation. Can be "true", "false", or "wait_for".
|
|
1135
|
+
- refresh (bool): Whether to refresh the index after the operation. Defaults to the value in `self.async_settings.database_refresh`.
|
|
1026
1136
|
|
|
1027
1137
|
Raises:
|
|
1028
1138
|
NotFoundError: If the collection with the given `collection_id` is not found in the database.
|
|
1029
1139
|
|
|
1140
|
+
Returns:
|
|
1141
|
+
None
|
|
1142
|
+
|
|
1030
1143
|
Notes:
|
|
1031
1144
|
This function first verifies that the collection with the specified `collection_id` exists in the database, and then
|
|
1032
|
-
deletes the collection. If `refresh` is set to
|
|
1033
|
-
function also calls `delete_item_index` to delete the index for the items in the collection.
|
|
1145
|
+
deletes the collection. If `refresh` is set to "true", "false", or "wait_for", the index is refreshed accordingly after
|
|
1146
|
+
the deletion. Additionally, this function also calls `delete_item_index` to delete the index for the items in the collection.
|
|
1034
1147
|
"""
|
|
1148
|
+
# Ensure kwargs is a dictionary
|
|
1149
|
+
kwargs = kwargs or {}
|
|
1150
|
+
|
|
1151
|
+
# Verify that the collection exists
|
|
1035
1152
|
await self.find_collection(collection_id=collection_id)
|
|
1153
|
+
|
|
1154
|
+
# Resolve the `refresh` parameter
|
|
1155
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
1156
|
+
refresh = validate_refresh(refresh)
|
|
1157
|
+
|
|
1158
|
+
# Log the deletion attempt
|
|
1159
|
+
logger.info(f"Deleting collection {collection_id} with refresh={refresh}")
|
|
1160
|
+
|
|
1161
|
+
# Delete the collection from the database
|
|
1036
1162
|
await self.client.delete(
|
|
1037
1163
|
index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh
|
|
1038
1164
|
)
|
|
1039
|
-
|
|
1165
|
+
|
|
1166
|
+
# Delete the item index for the collection
|
|
1167
|
+
try:
|
|
1168
|
+
await delete_item_index(collection_id)
|
|
1169
|
+
except Exception as e:
|
|
1170
|
+
logger.error(
|
|
1171
|
+
f"Failed to delete item index for collection {collection_id}: {e}"
|
|
1172
|
+
)
|
|
1040
1173
|
|
|
1041
1174
|
async def bulk_async(
|
|
1042
1175
|
self,
|
|
1043
1176
|
collection_id: str,
|
|
1044
1177
|
processed_items: List[Item],
|
|
1045
|
-
|
|
1178
|
+
**kwargs: Any,
|
|
1046
1179
|
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1047
1180
|
"""
|
|
1048
1181
|
Perform a bulk insert of items into the database asynchronously.
|
|
@@ -1050,7 +1183,12 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1050
1183
|
Args:
|
|
1051
1184
|
collection_id (str): The ID of the collection to which the items belong.
|
|
1052
1185
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
1053
|
-
|
|
1186
|
+
**kwargs (Any): Additional keyword arguments, including:
|
|
1187
|
+
- refresh (str, optional): Whether to refresh the index after the bulk insert.
|
|
1188
|
+
Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
|
|
1189
|
+
- refresh (bool, optional): Whether to refresh the index after the bulk insert.
|
|
1190
|
+
- raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
|
|
1191
|
+
Defaults to the value of `self.async_settings.raise_on_bulk_error`.
|
|
1054
1192
|
|
|
1055
1193
|
Returns:
|
|
1056
1194
|
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
@@ -1059,10 +1197,31 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1059
1197
|
|
|
1060
1198
|
Notes:
|
|
1061
1199
|
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1062
|
-
The insert is performed
|
|
1063
|
-
The `mk_actions` function is called to generate a list of actions for the bulk insert.
|
|
1064
|
-
the index is refreshed after the bulk insert
|
|
1200
|
+
The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
1201
|
+
completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
|
|
1202
|
+
parameter determines whether the index is refreshed after the bulk insert:
|
|
1203
|
+
- "true": Forces an immediate refresh of the index.
|
|
1204
|
+
- "false": Does not refresh the index immediately (default behavior).
|
|
1205
|
+
- "wait_for": Waits for the next refresh cycle to make the changes visible.
|
|
1065
1206
|
"""
|
|
1207
|
+
# Ensure kwargs is a dictionary
|
|
1208
|
+
kwargs = kwargs or {}
|
|
1209
|
+
|
|
1210
|
+
# Resolve the `refresh` parameter
|
|
1211
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
1212
|
+
refresh = validate_refresh(refresh)
|
|
1213
|
+
|
|
1214
|
+
# Log the bulk insert attempt
|
|
1215
|
+
logger.info(
|
|
1216
|
+
f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
|
|
1217
|
+
)
|
|
1218
|
+
|
|
1219
|
+
# Handle empty processed_items
|
|
1220
|
+
if not processed_items:
|
|
1221
|
+
logger.warning(f"No items to insert for collection {collection_id}")
|
|
1222
|
+
return 0, []
|
|
1223
|
+
|
|
1224
|
+
# Perform the bulk insert
|
|
1066
1225
|
raise_on_error = self.async_settings.raise_on_bulk_error
|
|
1067
1226
|
success, errors = await helpers.async_bulk(
|
|
1068
1227
|
self.client,
|
|
@@ -1070,13 +1229,19 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1070
1229
|
refresh=refresh,
|
|
1071
1230
|
raise_on_error=raise_on_error,
|
|
1072
1231
|
)
|
|
1232
|
+
|
|
1233
|
+
# Log the result
|
|
1234
|
+
logger.info(
|
|
1235
|
+
f"Bulk insert completed for collection {collection_id}: {success} successes, {len(errors)} errors"
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1073
1238
|
return success, errors
|
|
1074
1239
|
|
|
1075
1240
|
def bulk_sync(
|
|
1076
1241
|
self,
|
|
1077
1242
|
collection_id: str,
|
|
1078
1243
|
processed_items: List[Item],
|
|
1079
|
-
|
|
1244
|
+
**kwargs: Any,
|
|
1080
1245
|
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1081
1246
|
"""
|
|
1082
1247
|
Perform a bulk insert of items into the database synchronously.
|
|
@@ -1084,7 +1249,12 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1084
1249
|
Args:
|
|
1085
1250
|
collection_id (str): The ID of the collection to which the items belong.
|
|
1086
1251
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
1087
|
-
|
|
1252
|
+
**kwargs (Any): Additional keyword arguments, including:
|
|
1253
|
+
- refresh (str, optional): Whether to refresh the index after the bulk insert.
|
|
1254
|
+
Can be "true", "false", or "wait_for". Defaults to the value of `self.sync_settings.database_refresh`.
|
|
1255
|
+
- refresh (bool, optional): Whether to refresh the index after the bulk insert.
|
|
1256
|
+
- raise_on_error (bool, optional): Whether to raise an error if any of the bulk operations fail.
|
|
1257
|
+
Defaults to the value of `self.async_settings.raise_on_bulk_error`.
|
|
1088
1258
|
|
|
1089
1259
|
Returns:
|
|
1090
1260
|
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
@@ -1094,9 +1264,30 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1094
1264
|
Notes:
|
|
1095
1265
|
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1096
1266
|
The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
1097
|
-
completed. The `mk_actions` function is called to generate a list of actions for the bulk insert.
|
|
1098
|
-
|
|
1267
|
+
completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. The `refresh`
|
|
1268
|
+
parameter determines whether the index is refreshed after the bulk insert:
|
|
1269
|
+
- "true": Forces an immediate refresh of the index.
|
|
1270
|
+
- "false": Does not refresh the index immediately (default behavior).
|
|
1271
|
+
- "wait_for": Waits for the next refresh cycle to make the changes visible.
|
|
1099
1272
|
"""
|
|
1273
|
+
# Ensure kwargs is a dictionary
|
|
1274
|
+
kwargs = kwargs or {}
|
|
1275
|
+
|
|
1276
|
+
# Resolve the `refresh` parameter
|
|
1277
|
+
refresh = kwargs.get("refresh", self.async_settings.database_refresh)
|
|
1278
|
+
refresh = validate_refresh(refresh)
|
|
1279
|
+
|
|
1280
|
+
# Log the bulk insert attempt
|
|
1281
|
+
logger.info(
|
|
1282
|
+
f"Performing bulk insert for collection {collection_id} with refresh={refresh}"
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
# Handle empty processed_items
|
|
1286
|
+
if not processed_items:
|
|
1287
|
+
logger.warning(f"No items to insert for collection {collection_id}")
|
|
1288
|
+
return 0, []
|
|
1289
|
+
|
|
1290
|
+
# Perform the bulk insert
|
|
1100
1291
|
raise_on_error = self.sync_settings.raise_on_bulk_error
|
|
1101
1292
|
success, errors = helpers.bulk(
|
|
1102
1293
|
self.sync_client,
|
|
@@ -1104,6 +1295,12 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
1104
1295
|
refresh=refresh,
|
|
1105
1296
|
raise_on_error=raise_on_error,
|
|
1106
1297
|
)
|
|
1298
|
+
|
|
1299
|
+
# Log the result
|
|
1300
|
+
logger.info(
|
|
1301
|
+
f"Bulk insert completed for collection {collection_id}: {success} successes, {len(errors)} errors"
|
|
1302
|
+
)
|
|
1303
|
+
|
|
1107
1304
|
return success, errors
|
|
1108
1305
|
|
|
1109
1306
|
# DANGER
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""library version."""
|
|
2
|
-
__version__ = "4.
|
|
2
|
+
__version__ = "4.2.0"
|
{stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stac-fastapi-elasticsearch
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.2.0
|
|
4
4
|
Summary: An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Requires-Python: >=3.9
|
|
17
17
|
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: stac-fastapi-core==4.
|
|
18
|
+
Requires-Dist: stac-fastapi-core==4.2.0
|
|
19
19
|
Requires-Dist: elasticsearch[async]~=8.18.0
|
|
20
20
|
Requires-Dist: uvicorn~=0.23.0
|
|
21
21
|
Requires-Dist: starlette<0.36.0,>=0.35.0
|
|
@@ -140,6 +140,7 @@ You can customize additional settings in your `.env` file:
|
|
|
140
140
|
| `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-elasticsearch` or `stac-fastapi-opensearch` | Optional |
|
|
141
141
|
| `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional |
|
|
142
142
|
| `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional |
|
|
143
|
+
| `STAC_FASTAPI_LANDING_PAGE_ID` | Landing page ID | `stac-fastapi` | Optional |
|
|
143
144
|
| `APP_HOST` | Server bind address. | `0.0.0.0` | Optional |
|
|
144
145
|
| `APP_PORT` | Server port. | `8080` | Optional |
|
|
145
146
|
| `ENVIRONMENT` | Runtime environment. | `local` | Optional |
|
|
@@ -147,10 +148,12 @@ You can customize additional settings in your `.env` file:
|
|
|
147
148
|
| `RELOAD` | Enable auto-reload for development. | `true` | Optional |
|
|
148
149
|
| `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional |
|
|
149
150
|
| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
|
|
150
|
-
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
|
|
151
|
-
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
|
|
151
|
+
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional | |
|
|
152
152
|
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
153
|
-
| `
|
|
153
|
+
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional
|
|
154
|
+
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` Optional |
|
|
155
|
+
| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
|
|
156
|
+
| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
|
|
154
157
|
|
|
155
158
|
> [!NOTE]
|
|
156
159
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
|
|
2
|
+
stac_fastapi/elasticsearch/app.py,sha256=JQRKE9gfr0mTgkG5QdyNLIOFb8G6NXpQ_HklYHJpocs,4983
|
|
3
|
+
stac_fastapi/elasticsearch/config.py,sha256=y2b7BP3lQfUgTIga1Uk39kk8RJZ6nzymKKOcdiBu49M,5134
|
|
4
|
+
stac_fastapi/elasticsearch/database_logic.py,sha256=gE8hi7wPNMbi6Xknw6h73pCmzaBWBn4SYEVCM-4qqbU,51111
|
|
5
|
+
stac_fastapi/elasticsearch/version.py,sha256=XAYr-IO1hoDdSshTkYzWFp3wj4AdjSQwUik30pTEaAo,45
|
|
6
|
+
stac_fastapi_elasticsearch-4.2.0.dist-info/METADATA,sha256=GU_EroLaZkNatiKaKCp1Hl5R8NhyF9Lr4npSNJJ062Y,21906
|
|
7
|
+
stac_fastapi_elasticsearch-4.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
8
|
+
stac_fastapi_elasticsearch-4.2.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
|
|
9
|
+
stac_fastapi_elasticsearch-4.2.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
+
stac_fastapi_elasticsearch-4.2.0.dist-info/RECORD,,
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
stac_fastapi/elasticsearch/__init__.py,sha256=w_MZutYLreNV372sCuO46bPb0TngmPs4u8737ueS0wE,31
|
|
2
|
-
stac_fastapi/elasticsearch/app.py,sha256=TWxbf_xm7r48tCiGwrZshmOB4iH1sZBTUSoJbBBt-hM,4397
|
|
3
|
-
stac_fastapi/elasticsearch/config.py,sha256=1ZNbl29yglppkv_EOxz_5H2siQCMjXb3bUS3QLoGWjA,4363
|
|
4
|
-
stac_fastapi/elasticsearch/database_logic.py,sha256=LMJI47FDTyT6cuBbNo9Cp0DqqaqHRBxyc54RbNEf5ec,42608
|
|
5
|
-
stac_fastapi/elasticsearch/version.py,sha256=3xU5aBmxgAcPizRlef_YROCW9ULsGOA4flSyd9AQog4,45
|
|
6
|
-
stac_fastapi_elasticsearch-4.1.0.dist-info/METADATA,sha256=9Lh4WFz3_hrzYpeLwPNO5RLXBmOpbL1EGo3MBuvhtJE,20881
|
|
7
|
-
stac_fastapi_elasticsearch-4.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
8
|
-
stac_fastapi_elasticsearch-4.1.0.dist-info/entry_points.txt,sha256=aCKixki0LpUl64UPsPMtiNvfdyq-QsTCxVjJ54VF6Jk,82
|
|
9
|
-
stac_fastapi_elasticsearch-4.1.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
-
stac_fastapi_elasticsearch-4.1.0.dist-info/RECORD,,
|
{stac_fastapi_elasticsearch-4.1.0.dist-info → stac_fastapi_elasticsearch-4.2.0.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|