stac-fastapi-opensearch 4.0.0a1__py3-none-any.whl → 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stac_fastapi/opensearch/app.py +15 -8
- stac_fastapi/opensearch/config.py +37 -8
- stac_fastapi/opensearch/database_logic.py +267 -88
- stac_fastapi/opensearch/version.py +1 -1
- {stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/METADATA +21 -8
- stac_fastapi_opensearch-4.1.0.dist-info/RECORD +10 -0
- stac_fastapi_opensearch-4.0.0a1.dist-info/RECORD +0 -10
- {stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/WHEEL +0 -0
- {stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/entry_points.txt +0 -0
- {stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/top_level.txt +0 -0
stac_fastapi/opensearch/app.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
"""FastAPI application."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
|
|
6
|
+
from fastapi import FastAPI
|
|
4
7
|
|
|
5
8
|
from stac_fastapi.api.app import StacApi
|
|
6
9
|
from stac_fastapi.api.models import create_get_request_model, create_post_request_model
|
|
@@ -87,7 +90,7 @@ post_request_model = create_post_request_model(search_extensions)
|
|
|
87
90
|
api = StacApi(
|
|
88
91
|
title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"),
|
|
89
92
|
description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"),
|
|
90
|
-
api_version=os.getenv("STAC_FASTAPI_VERSION", "
|
|
93
|
+
api_version=os.getenv("STAC_FASTAPI_VERSION", "4.1.0"),
|
|
91
94
|
settings=settings,
|
|
92
95
|
extensions=extensions,
|
|
93
96
|
client=CoreClient(
|
|
@@ -97,17 +100,21 @@ api = StacApi(
|
|
|
97
100
|
search_post_request_model=post_request_model,
|
|
98
101
|
route_dependencies=get_route_dependencies(),
|
|
99
102
|
)
|
|
100
|
-
app = api.app
|
|
101
|
-
app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "")
|
|
102
|
-
|
|
103
|
-
# Add rate limit
|
|
104
|
-
setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT"))
|
|
105
103
|
|
|
106
104
|
|
|
107
|
-
@
|
|
108
|
-
async def
|
|
105
|
+
@asynccontextmanager
|
|
106
|
+
async def lifespan(app: FastAPI):
|
|
107
|
+
"""Lifespan handler for FastAPI app. Initializes index templates and collections at startup."""
|
|
109
108
|
await create_index_templates()
|
|
110
109
|
await create_collection_index()
|
|
110
|
+
yield
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
app = api.app
|
|
114
|
+
app.router.lifespan_context = lifespan
|
|
115
|
+
app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "")
|
|
116
|
+
# Add rate limit
|
|
117
|
+
setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT"))
|
|
111
118
|
|
|
112
119
|
|
|
113
120
|
def run() -> None:
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"""API configuration."""
|
|
2
|
+
import logging
|
|
2
3
|
import os
|
|
3
4
|
import ssl
|
|
4
5
|
from typing import Any, Dict, Set
|
|
@@ -7,12 +8,13 @@ import certifi
|
|
|
7
8
|
from opensearchpy import AsyncOpenSearch, OpenSearch
|
|
8
9
|
|
|
9
10
|
from stac_fastapi.core.base_settings import ApiBaseSettings
|
|
11
|
+
from stac_fastapi.core.utilities import get_bool_env
|
|
10
12
|
from stac_fastapi.types.config import ApiSettings
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def _es_config() -> Dict[str, Any]:
|
|
14
16
|
# Determine the scheme (http or https)
|
|
15
|
-
use_ssl =
|
|
17
|
+
use_ssl = get_bool_env("ES_USE_SSL", default=True)
|
|
16
18
|
scheme = "https" if use_ssl else "http"
|
|
17
19
|
|
|
18
20
|
# Configure the hosts parameter with the correct scheme
|
|
@@ -33,7 +35,7 @@ def _es_config() -> Dict[str, Any]:
|
|
|
33
35
|
"headers": {"accept": "application/json", "Content-Type": "application/json"},
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
http_compress =
|
|
38
|
+
http_compress = get_bool_env("ES_HTTP_COMPRESS", default=True)
|
|
37
39
|
if http_compress:
|
|
38
40
|
config["http_compress"] = True
|
|
39
41
|
|
|
@@ -42,8 +44,8 @@ def _es_config() -> Dict[str, Any]:
|
|
|
42
44
|
return config
|
|
43
45
|
|
|
44
46
|
# Include SSL settings if using https
|
|
45
|
-
config["ssl_version"] = ssl.PROTOCOL_SSLv23
|
|
46
|
-
config["verify_certs"] =
|
|
47
|
+
config["ssl_version"] = ssl.PROTOCOL_SSLv23
|
|
48
|
+
config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True)
|
|
47
49
|
|
|
48
50
|
# Include CA Certificates if verifying certs
|
|
49
51
|
if config["verify_certs"]:
|
|
@@ -69,11 +71,19 @@ _forbidden_fields: Set[str] = {"type"}
|
|
|
69
71
|
|
|
70
72
|
|
|
71
73
|
class OpensearchSettings(ApiSettings, ApiBaseSettings):
|
|
72
|
-
"""
|
|
74
|
+
"""
|
|
75
|
+
API settings.
|
|
76
|
+
|
|
77
|
+
Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable.
|
|
78
|
+
If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled.
|
|
79
|
+
Default is False for safety.
|
|
80
|
+
"""
|
|
73
81
|
|
|
74
|
-
# Fields which are defined by STAC but not included in the database model
|
|
75
82
|
forbidden_fields: Set[str] = _forbidden_fields
|
|
76
83
|
indexed_fields: Set[str] = {"datetime"}
|
|
84
|
+
enable_response_models: bool = False
|
|
85
|
+
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
86
|
+
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
77
87
|
|
|
78
88
|
@property
|
|
79
89
|
def create_client(self):
|
|
@@ -82,13 +92,32 @@ class OpensearchSettings(ApiSettings, ApiBaseSettings):
|
|
|
82
92
|
|
|
83
93
|
|
|
84
94
|
class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings):
|
|
85
|
-
"""
|
|
95
|
+
"""
|
|
96
|
+
API settings.
|
|
97
|
+
|
|
98
|
+
Set enable_direct_response via the ENABLE_DIRECT_RESPONSE environment variable.
|
|
99
|
+
If enabled, all API routes use direct response for maximum performance, but ALL FastAPI dependencies (including authentication, custom status codes, and validation) are disabled.
|
|
100
|
+
Default is False for safety.
|
|
101
|
+
"""
|
|
86
102
|
|
|
87
|
-
# Fields which are defined by STAC but not included in the database model
|
|
88
103
|
forbidden_fields: Set[str] = _forbidden_fields
|
|
89
104
|
indexed_fields: Set[str] = {"datetime"}
|
|
105
|
+
enable_response_models: bool = False
|
|
106
|
+
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
107
|
+
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
90
108
|
|
|
91
109
|
@property
|
|
92
110
|
def create_client(self):
|
|
93
111
|
"""Create async elasticsearch client."""
|
|
94
112
|
return AsyncOpenSearch(**_es_config())
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# Warn at import if direct response is enabled (applies to either settings class)
|
|
116
|
+
if (
|
|
117
|
+
OpensearchSettings().enable_direct_response
|
|
118
|
+
or AsyncOpensearchSettings().enable_direct_response
|
|
119
|
+
):
|
|
120
|
+
logging.basicConfig(level=logging.WARNING)
|
|
121
|
+
logging.warning(
|
|
122
|
+
"ENABLE_DIRECT_RESPONSE is True: All FastAPI dependencies (including authentication) are DISABLED for all routes!"
|
|
123
|
+
)
|
|
@@ -9,12 +9,10 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
|
|
|
9
9
|
|
|
10
10
|
import attr
|
|
11
11
|
from opensearchpy import exceptions, helpers
|
|
12
|
-
from opensearchpy.exceptions import TransportError
|
|
13
12
|
from opensearchpy.helpers.query import Q
|
|
14
13
|
from opensearchpy.helpers.search import Search
|
|
15
14
|
from starlette.requests import Request
|
|
16
15
|
|
|
17
|
-
from stac_fastapi.core import serializers
|
|
18
16
|
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
|
|
19
17
|
from stac_fastapi.core.database_logic import (
|
|
20
18
|
COLLECTIONS_INDEX,
|
|
@@ -32,6 +30,7 @@ from stac_fastapi.core.database_logic import (
|
|
|
32
30
|
mk_item_id,
|
|
33
31
|
)
|
|
34
32
|
from stac_fastapi.core.extensions import filter
|
|
33
|
+
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
|
|
35
34
|
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
|
|
36
35
|
from stac_fastapi.opensearch.config import (
|
|
37
36
|
AsyncOpensearchSettings as AsyncSearchSettings,
|
|
@@ -80,24 +79,21 @@ async def create_collection_index() -> None:
|
|
|
80
79
|
"""
|
|
81
80
|
client = AsyncSearchSettings().create_client
|
|
82
81
|
|
|
83
|
-
search_body: Dict[str, Any] = {
|
|
84
|
-
"aliases": {COLLECTIONS_INDEX: {}},
|
|
85
|
-
}
|
|
86
|
-
|
|
87
82
|
index = f"{COLLECTIONS_INDEX}-000001"
|
|
88
83
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
84
|
+
exists = await client.indices.exists(index=index)
|
|
85
|
+
if not exists:
|
|
86
|
+
await client.indices.create(
|
|
87
|
+
index=index,
|
|
88
|
+
body={
|
|
89
|
+
"aliases": {COLLECTIONS_INDEX: {}},
|
|
90
|
+
"mappings": ES_COLLECTIONS_MAPPINGS,
|
|
91
|
+
},
|
|
92
|
+
)
|
|
97
93
|
await client.close()
|
|
98
94
|
|
|
99
95
|
|
|
100
|
-
async def create_item_index(collection_id: str):
|
|
96
|
+
async def create_item_index(collection_id: str) -> None:
|
|
101
97
|
"""
|
|
102
98
|
Create the index for Items. The settings of the index template will be used implicitly.
|
|
103
99
|
|
|
@@ -109,24 +105,22 @@ async def create_item_index(collection_id: str):
|
|
|
109
105
|
|
|
110
106
|
"""
|
|
111
107
|
client = AsyncSearchSettings().create_client
|
|
112
|
-
search_body: Dict[str, Any] = {
|
|
113
|
-
"aliases": {index_alias_by_collection_id(collection_id): {}},
|
|
114
|
-
}
|
|
115
108
|
|
|
116
|
-
|
|
109
|
+
index_name = f"{index_by_collection_id(collection_id)}-000001"
|
|
110
|
+
exists = await client.indices.exists(index=index_name)
|
|
111
|
+
if not exists:
|
|
117
112
|
await client.indices.create(
|
|
118
|
-
index=
|
|
113
|
+
index=index_name,
|
|
114
|
+
body={
|
|
115
|
+
"aliases": {index_alias_by_collection_id(collection_id): {}},
|
|
116
|
+
"mappings": ES_ITEMS_MAPPINGS,
|
|
117
|
+
"settings": ES_ITEMS_SETTINGS,
|
|
118
|
+
},
|
|
119
119
|
)
|
|
120
|
-
except TransportError as e:
|
|
121
|
-
if e.status_code == 400:
|
|
122
|
-
pass # Ignore 400 status codes
|
|
123
|
-
else:
|
|
124
|
-
raise e
|
|
125
|
-
|
|
126
120
|
await client.close()
|
|
127
121
|
|
|
128
122
|
|
|
129
|
-
async def delete_item_index(collection_id: str):
|
|
123
|
+
async def delete_item_index(collection_id: str) -> None:
|
|
130
124
|
"""Delete the index for items in a collection.
|
|
131
125
|
|
|
132
126
|
Args:
|
|
@@ -149,14 +143,20 @@ async def delete_item_index(collection_id: str):
|
|
|
149
143
|
class DatabaseLogic(BaseDatabaseLogic):
|
|
150
144
|
"""Database logic."""
|
|
151
145
|
|
|
152
|
-
|
|
153
|
-
|
|
146
|
+
async_settings: AsyncSearchSettings = attr.ib(factory=AsyncSearchSettings)
|
|
147
|
+
sync_settings: SyncSearchSettings = attr.ib(factory=SyncSearchSettings)
|
|
154
148
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
149
|
+
client = attr.ib(init=False)
|
|
150
|
+
sync_client = attr.ib(init=False)
|
|
151
|
+
|
|
152
|
+
def __attrs_post_init__(self):
|
|
153
|
+
"""Initialize clients after the class is instantiated."""
|
|
154
|
+
self.client = self.async_settings.create_client
|
|
155
|
+
self.sync_client = self.sync_settings.create_client
|
|
156
|
+
|
|
157
|
+
item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
|
|
158
|
+
collection_serializer: Type[CollectionSerializer] = attr.ib(
|
|
159
|
+
default=CollectionSerializer
|
|
160
160
|
)
|
|
161
161
|
|
|
162
162
|
extensions: List[str] = attr.ib(default=attr.Factory(list))
|
|
@@ -335,7 +335,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
335
335
|
|
|
336
336
|
@staticmethod
|
|
337
337
|
def apply_datetime_filter(search: Search, datetime_search):
|
|
338
|
-
"""Apply a filter to search based on datetime field.
|
|
338
|
+
"""Apply a filter to search based on datetime field, start_datetime, and end_datetime fields.
|
|
339
339
|
|
|
340
340
|
Args:
|
|
341
341
|
search (Search): The search object to filter.
|
|
@@ -344,17 +344,109 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
344
344
|
Returns:
|
|
345
345
|
Search: The filtered search object.
|
|
346
346
|
"""
|
|
347
|
+
should = []
|
|
348
|
+
|
|
349
|
+
# If the request is a single datetime return
|
|
350
|
+
# items with datetimes equal to the requested datetime OR
|
|
351
|
+
# the requested datetime is between their start and end datetimes
|
|
347
352
|
if "eq" in datetime_search:
|
|
348
|
-
|
|
349
|
-
|
|
353
|
+
should.extend(
|
|
354
|
+
[
|
|
355
|
+
Q(
|
|
356
|
+
"bool",
|
|
357
|
+
filter=[
|
|
358
|
+
Q(
|
|
359
|
+
"term",
|
|
360
|
+
properties__datetime=datetime_search["eq"],
|
|
361
|
+
),
|
|
362
|
+
],
|
|
363
|
+
),
|
|
364
|
+
Q(
|
|
365
|
+
"bool",
|
|
366
|
+
filter=[
|
|
367
|
+
Q(
|
|
368
|
+
"range",
|
|
369
|
+
properties__start_datetime={
|
|
370
|
+
"lte": datetime_search["eq"],
|
|
371
|
+
},
|
|
372
|
+
),
|
|
373
|
+
Q(
|
|
374
|
+
"range",
|
|
375
|
+
properties__end_datetime={
|
|
376
|
+
"gte": datetime_search["eq"],
|
|
377
|
+
},
|
|
378
|
+
),
|
|
379
|
+
],
|
|
380
|
+
),
|
|
381
|
+
]
|
|
350
382
|
)
|
|
383
|
+
|
|
384
|
+
# If the request is a date range return
|
|
385
|
+
# items with datetimes within the requested date range OR
|
|
386
|
+
# their startdatetime ithin the requested date range OR
|
|
387
|
+
# their enddatetime ithin the requested date range OR
|
|
388
|
+
# the requested daterange within their start and end datetimes
|
|
351
389
|
else:
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
390
|
+
should.extend(
|
|
391
|
+
[
|
|
392
|
+
Q(
|
|
393
|
+
"bool",
|
|
394
|
+
filter=[
|
|
395
|
+
Q(
|
|
396
|
+
"range",
|
|
397
|
+
properties__datetime={
|
|
398
|
+
"gte": datetime_search["gte"],
|
|
399
|
+
"lte": datetime_search["lte"],
|
|
400
|
+
},
|
|
401
|
+
),
|
|
402
|
+
],
|
|
403
|
+
),
|
|
404
|
+
Q(
|
|
405
|
+
"bool",
|
|
406
|
+
filter=[
|
|
407
|
+
Q(
|
|
408
|
+
"range",
|
|
409
|
+
properties__start_datetime={
|
|
410
|
+
"gte": datetime_search["gte"],
|
|
411
|
+
"lte": datetime_search["lte"],
|
|
412
|
+
},
|
|
413
|
+
),
|
|
414
|
+
],
|
|
415
|
+
),
|
|
416
|
+
Q(
|
|
417
|
+
"bool",
|
|
418
|
+
filter=[
|
|
419
|
+
Q(
|
|
420
|
+
"range",
|
|
421
|
+
properties__end_datetime={
|
|
422
|
+
"gte": datetime_search["gte"],
|
|
423
|
+
"lte": datetime_search["lte"],
|
|
424
|
+
},
|
|
425
|
+
),
|
|
426
|
+
],
|
|
427
|
+
),
|
|
428
|
+
Q(
|
|
429
|
+
"bool",
|
|
430
|
+
filter=[
|
|
431
|
+
Q(
|
|
432
|
+
"range",
|
|
433
|
+
properties__start_datetime={
|
|
434
|
+
"lte": datetime_search["gte"]
|
|
435
|
+
},
|
|
436
|
+
),
|
|
437
|
+
Q(
|
|
438
|
+
"range",
|
|
439
|
+
properties__end_datetime={
|
|
440
|
+
"gte": datetime_search["lte"]
|
|
441
|
+
},
|
|
442
|
+
),
|
|
443
|
+
],
|
|
444
|
+
),
|
|
445
|
+
]
|
|
357
446
|
)
|
|
447
|
+
|
|
448
|
+
search = search.query(Q("bool", filter=[Q("bool", should=should)]))
|
|
449
|
+
|
|
358
450
|
return search
|
|
359
451
|
|
|
360
452
|
@staticmethod
|
|
@@ -639,7 +731,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
639
731
|
if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
|
|
640
732
|
raise NotFoundError(f"Collection {collection_id} does not exist")
|
|
641
733
|
|
|
642
|
-
async def
|
|
734
|
+
async def async_prep_create_item(
|
|
643
735
|
self, item: Item, base_url: str, exist_ok: bool = False
|
|
644
736
|
) -> Item:
|
|
645
737
|
"""
|
|
@@ -669,44 +761,113 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
669
761
|
|
|
670
762
|
return self.item_serializer.stac_to_db(item, base_url)
|
|
671
763
|
|
|
672
|
-
def
|
|
764
|
+
async def bulk_async_prep_create_item(
|
|
673
765
|
self, item: Item, base_url: str, exist_ok: bool = False
|
|
674
766
|
) -> Item:
|
|
675
767
|
"""
|
|
676
768
|
Prepare an item for insertion into the database.
|
|
677
769
|
|
|
678
|
-
This method performs pre-insertion preparation on the given `item`,
|
|
679
|
-
|
|
680
|
-
|
|
770
|
+
This method performs pre-insertion preparation on the given `item`, such as:
|
|
771
|
+
- Verifying that the collection the item belongs to exists.
|
|
772
|
+
- Optionally checking if an item with the same ID already exists in the database.
|
|
773
|
+
- Serializing the item into a database-compatible format.
|
|
681
774
|
|
|
682
775
|
Args:
|
|
683
|
-
item (Item): The item to be
|
|
684
|
-
base_url (str): The base URL used
|
|
685
|
-
exist_ok (bool): Indicates whether the item can exist
|
|
776
|
+
item (Item): The item to be prepared for insertion.
|
|
777
|
+
base_url (str): The base URL used to construct the item's self URL.
|
|
778
|
+
exist_ok (bool): Indicates whether the item can already exist in the database.
|
|
779
|
+
If False, a `ConflictError` is raised if the item exists.
|
|
686
780
|
|
|
687
781
|
Returns:
|
|
688
|
-
Item: The item
|
|
782
|
+
Item: The prepared item, serialized into a database-compatible format.
|
|
689
783
|
|
|
690
784
|
Raises:
|
|
691
785
|
NotFoundError: If the collection that the item belongs to does not exist in the database.
|
|
692
|
-
ConflictError: If an item with the same ID already exists in the collection
|
|
786
|
+
ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
|
|
787
|
+
and `RAISE_ON_BULK_ERROR` is set to `true`.
|
|
693
788
|
"""
|
|
694
|
-
|
|
695
|
-
collection_id = item["collection"]
|
|
696
|
-
if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id):
|
|
697
|
-
raise NotFoundError(f"Collection {collection_id} does not exist")
|
|
789
|
+
logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")
|
|
698
790
|
|
|
699
|
-
if
|
|
700
|
-
|
|
701
|
-
|
|
791
|
+
# Check if the collection exists
|
|
792
|
+
await self.check_collection_exists(collection_id=item["collection"])
|
|
793
|
+
|
|
794
|
+
# Check if the item already exists in the database
|
|
795
|
+
if not exist_ok and await self.client.exists(
|
|
796
|
+
index=index_alias_by_collection_id(item["collection"]),
|
|
797
|
+
id=mk_item_id(item["id"], item["collection"]),
|
|
702
798
|
):
|
|
703
|
-
|
|
704
|
-
f"Item {
|
|
799
|
+
error_message = (
|
|
800
|
+
f"Item {item['id']} in collection {item['collection']} already exists."
|
|
705
801
|
)
|
|
802
|
+
if self.async_settings.raise_on_bulk_error:
|
|
803
|
+
raise ConflictError(error_message)
|
|
804
|
+
else:
|
|
805
|
+
logger.warning(
|
|
806
|
+
f"{error_message} Continuing as `RAISE_ON_BULK_ERROR` is set to false."
|
|
807
|
+
)
|
|
808
|
+
# Serialize the item into a database-compatible format
|
|
809
|
+
prepped_item = self.item_serializer.stac_to_db(item, base_url)
|
|
810
|
+
logger.debug(f"Item {item['id']} prepared successfully.")
|
|
811
|
+
return prepped_item
|
|
812
|
+
|
|
813
|
+
def bulk_sync_prep_create_item(
|
|
814
|
+
self, item: Item, base_url: str, exist_ok: bool = False
|
|
815
|
+
) -> Item:
|
|
816
|
+
"""
|
|
817
|
+
Prepare an item for insertion into the database.
|
|
706
818
|
|
|
707
|
-
|
|
819
|
+
This method performs pre-insertion preparation on the given `item`, such as:
|
|
820
|
+
- Verifying that the collection the item belongs to exists.
|
|
821
|
+
- Optionally checking if an item with the same ID already exists in the database.
|
|
822
|
+
- Serializing the item into a database-compatible format.
|
|
708
823
|
|
|
709
|
-
|
|
824
|
+
Args:
|
|
825
|
+
item (Item): The item to be prepared for insertion.
|
|
826
|
+
base_url (str): The base URL used to construct the item's self URL.
|
|
827
|
+
exist_ok (bool): Indicates whether the item can already exist in the database.
|
|
828
|
+
If False, a `ConflictError` is raised if the item exists.
|
|
829
|
+
|
|
830
|
+
Returns:
|
|
831
|
+
Item: The prepared item, serialized into a database-compatible format.
|
|
832
|
+
|
|
833
|
+
Raises:
|
|
834
|
+
NotFoundError: If the collection that the item belongs to does not exist in the database.
|
|
835
|
+
ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
|
|
836
|
+
and `RAISE_ON_BULK_ERROR` is set to `true`.
|
|
837
|
+
"""
|
|
838
|
+
logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")
|
|
839
|
+
|
|
840
|
+
# Check if the collection exists
|
|
841
|
+
if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=item["collection"]):
|
|
842
|
+
raise NotFoundError(f"Collection {item['collection']} does not exist")
|
|
843
|
+
|
|
844
|
+
# Check if the item already exists in the database
|
|
845
|
+
if not exist_ok and self.sync_client.exists(
|
|
846
|
+
index=index_alias_by_collection_id(item["collection"]),
|
|
847
|
+
id=mk_item_id(item["id"], item["collection"]),
|
|
848
|
+
):
|
|
849
|
+
error_message = (
|
|
850
|
+
f"Item {item['id']} in collection {item['collection']} already exists."
|
|
851
|
+
)
|
|
852
|
+
if self.sync_settings.raise_on_bulk_error:
|
|
853
|
+
raise ConflictError(error_message)
|
|
854
|
+
else:
|
|
855
|
+
logger.warning(
|
|
856
|
+
f"{error_message} Continuing as `RAISE_ON_BULK_ERROR` is set to false."
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
# Serialize the item into a database-compatible format
|
|
860
|
+
prepped_item = self.item_serializer.stac_to_db(item, base_url)
|
|
861
|
+
logger.debug(f"Item {item['id']} prepared successfully.")
|
|
862
|
+
return prepped_item
|
|
863
|
+
|
|
864
|
+
async def create_item(
|
|
865
|
+
self,
|
|
866
|
+
item: Item,
|
|
867
|
+
refresh: bool = False,
|
|
868
|
+
base_url: str = "",
|
|
869
|
+
exist_ok: bool = False,
|
|
870
|
+
):
|
|
710
871
|
"""Database logic for creating one item.
|
|
711
872
|
|
|
712
873
|
Args:
|
|
@@ -722,18 +883,16 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
722
883
|
# todo: check if collection exists, but cache
|
|
723
884
|
item_id = item["id"]
|
|
724
885
|
collection_id = item["collection"]
|
|
725
|
-
|
|
886
|
+
item = await self.async_prep_create_item(
|
|
887
|
+
item=item, base_url=base_url, exist_ok=exist_ok
|
|
888
|
+
)
|
|
889
|
+
await self.client.index(
|
|
726
890
|
index=index_alias_by_collection_id(collection_id),
|
|
727
891
|
id=mk_item_id(item_id, collection_id),
|
|
728
892
|
body=item,
|
|
729
893
|
refresh=refresh,
|
|
730
894
|
)
|
|
731
895
|
|
|
732
|
-
if (meta := es_resp.get("meta")) and meta.get("status") == 409:
|
|
733
|
-
raise ConflictError(
|
|
734
|
-
f"Item {item_id} in collection {collection_id} already exists"
|
|
735
|
-
)
|
|
736
|
-
|
|
737
896
|
async def delete_item(
|
|
738
897
|
self, item_id: str, collection_id: str, refresh: bool = False
|
|
739
898
|
):
|
|
@@ -899,52 +1058,72 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
899
1058
|
await delete_item_index(collection_id)
|
|
900
1059
|
|
|
901
1060
|
async def bulk_async(
|
|
902
|
-
self,
|
|
903
|
-
|
|
904
|
-
|
|
1061
|
+
self,
|
|
1062
|
+
collection_id: str,
|
|
1063
|
+
processed_items: List[Item],
|
|
1064
|
+
refresh: bool = False,
|
|
1065
|
+
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1066
|
+
"""
|
|
1067
|
+
Perform a bulk insert of items into the database asynchronously.
|
|
905
1068
|
|
|
906
1069
|
Args:
|
|
907
|
-
self: The instance of the object calling this function.
|
|
908
1070
|
collection_id (str): The ID of the collection to which the items belong.
|
|
909
1071
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
910
1072
|
refresh (bool): Whether to refresh the index after the bulk insert (default: False).
|
|
911
1073
|
|
|
1074
|
+
Returns:
|
|
1075
|
+
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
1076
|
+
- The number of successfully processed actions (`success`).
|
|
1077
|
+
- A list of errors encountered during the bulk operation (`errors`).
|
|
1078
|
+
|
|
912
1079
|
Notes:
|
|
913
|
-
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
914
|
-
insert is performed asynchronously, and the event loop is used to run the operation in a separate executor.
|
|
915
|
-
`mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True,
|
|
916
|
-
index is refreshed after the bulk insert.
|
|
1080
|
+
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1081
|
+
The insert is performed asynchronously, and the event loop is used to run the operation in a separate executor.
|
|
1082
|
+
The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True,
|
|
1083
|
+
the index is refreshed after the bulk insert.
|
|
917
1084
|
"""
|
|
918
|
-
|
|
1085
|
+
raise_on_error = self.async_settings.raise_on_bulk_error
|
|
1086
|
+
success, errors = await helpers.async_bulk(
|
|
919
1087
|
self.client,
|
|
920
1088
|
mk_actions(collection_id, processed_items),
|
|
921
1089
|
refresh=refresh,
|
|
922
|
-
raise_on_error=
|
|
1090
|
+
raise_on_error=raise_on_error,
|
|
923
1091
|
)
|
|
1092
|
+
return success, errors
|
|
924
1093
|
|
|
925
1094
|
def bulk_sync(
|
|
926
|
-
self,
|
|
927
|
-
|
|
928
|
-
|
|
1095
|
+
self,
|
|
1096
|
+
collection_id: str,
|
|
1097
|
+
processed_items: List[Item],
|
|
1098
|
+
refresh: bool = False,
|
|
1099
|
+
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1100
|
+
"""
|
|
1101
|
+
Perform a bulk insert of items into the database synchronously.
|
|
929
1102
|
|
|
930
1103
|
Args:
|
|
931
|
-
self: The instance of the object calling this function.
|
|
932
1104
|
collection_id (str): The ID of the collection to which the items belong.
|
|
933
1105
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
934
1106
|
refresh (bool): Whether to refresh the index after the bulk insert (default: False).
|
|
935
1107
|
|
|
1108
|
+
Returns:
|
|
1109
|
+
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
1110
|
+
- The number of successfully processed actions (`success`).
|
|
1111
|
+
- A list of errors encountered during the bulk operation (`errors`).
|
|
1112
|
+
|
|
936
1113
|
Notes:
|
|
937
|
-
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
938
|
-
insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
1114
|
+
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1115
|
+
The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
939
1116
|
completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to
|
|
940
|
-
True, the index is refreshed after the bulk insert.
|
|
1117
|
+
True, the index is refreshed after the bulk insert.
|
|
941
1118
|
"""
|
|
942
|
-
|
|
1119
|
+
raise_on_error = self.sync_settings.raise_on_bulk_error
|
|
1120
|
+
success, errors = helpers.bulk(
|
|
943
1121
|
self.sync_client,
|
|
944
1122
|
mk_actions(collection_id, processed_items),
|
|
945
1123
|
refresh=refresh,
|
|
946
|
-
raise_on_error=
|
|
1124
|
+
raise_on_error=raise_on_error,
|
|
947
1125
|
)
|
|
1126
|
+
return success, errors
|
|
948
1127
|
|
|
949
1128
|
# DANGER
|
|
950
1129
|
async def delete_items(self) -> None:
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""library version."""
|
|
2
|
-
__version__ = "4.0
|
|
2
|
+
__version__ = "4.1.0"
|
{stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stac-fastapi-opensearch
|
|
3
|
-
Version: 4.0
|
|
3
|
+
Version: 4.1.0
|
|
4
4
|
Summary: Opensearch stac-fastapi backend.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Requires-Python: >=3.9
|
|
17
17
|
Description-Content-Type: text/markdown
|
|
18
|
-
Requires-Dist: stac-fastapi-core==4.0
|
|
18
|
+
Requires-Dist: stac-fastapi-core==4.1.0
|
|
19
19
|
Requires-Dist: opensearch-py~=2.8.0
|
|
20
20
|
Requires-Dist: opensearch-py[async]~=2.8.0
|
|
21
21
|
Requires-Dist: uvicorn~=0.23.0
|
|
@@ -66,8 +66,18 @@ Requires-Dist: uvicorn[standard]~=0.23.0; extra == "server"
|
|
|
66
66
|
- There is [Postman](https://documenter.getpostman.com/view/12888943/2s8ZDSdRHA) documentation here for examples on how to run some of the API routes locally - after starting the elasticsearch backend via the compose.yml file.
|
|
67
67
|
- The `/examples` folder shows an example of running stac-fastapi-elasticsearch from PyPI in docker without needing any code from the repository. There is also a Postman collection here that you can load into Postman for testing the API routes.
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
|
|
70
|
+
### Performance Note
|
|
71
|
+
|
|
72
|
+
The `enable_direct_response` option is provided by the stac-fastapi core library (introduced in stac-fastapi 5.2.0) and is available in this project starting from v4.0.0.
|
|
73
|
+
|
|
74
|
+
**You can now control this setting via the `ENABLE_DIRECT_RESPONSE` environment variable.**
|
|
75
|
+
|
|
76
|
+
When enabled (`ENABLE_DIRECT_RESPONSE=true`), endpoints return Starlette Response objects directly, bypassing FastAPI's default serialization for improved performance. **However, all FastAPI dependencies (including authentication, custom status codes, and validation) are disabled for all routes.**
|
|
77
|
+
|
|
78
|
+
This mode is best suited for public or read-only APIs where authentication and custom logic are not required. Default is `false` for safety.
|
|
79
|
+
|
|
80
|
+
See: [issue #347](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/347)
|
|
71
81
|
|
|
72
82
|
|
|
73
83
|
### To install from PyPI:
|
|
@@ -111,8 +121,9 @@ If you wish to use a different version, put the following in a
|
|
|
111
121
|
file named `.env` in the same directory you run Docker Compose from:
|
|
112
122
|
|
|
113
123
|
```shell
|
|
114
|
-
ELASTICSEARCH_VERSION=
|
|
115
|
-
OPENSEARCH_VERSION=2.11.
|
|
124
|
+
ELASTICSEARCH_VERSION=8.11.0
|
|
125
|
+
OPENSEARCH_VERSION=2.11.1
|
|
126
|
+
ENABLE_DIRECT_RESPONSE=false
|
|
116
127
|
```
|
|
117
128
|
The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
|
|
118
129
|
|
|
@@ -137,8 +148,10 @@ You can customize additional settings in your `.env` file:
|
|
|
137
148
|
| `RELOAD` | Enable auto-reload for development. | `true` | Optional |
|
|
138
149
|
| `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional |
|
|
139
150
|
| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
|
|
140
|
-
| `ELASTICSEARCH_VERSION`
|
|
141
|
-
| `
|
|
151
|
+
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
|
|
152
|
+
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
|
|
153
|
+
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
154
|
+
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
142
155
|
|
|
143
156
|
> [!NOTE]
|
|
144
157
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
stac_fastapi/opensearch/__init__.py,sha256=iJWMUgn7mUvmuPQSO_FlyhJ5eDdbbfmGv1qnFOX5-qk,28
|
|
2
|
+
stac_fastapi/opensearch/app.py,sha256=VL7kL0PvRuMMAHJ-0AsoqvpIov4lFilelXW8ezVNBoU,4376
|
|
3
|
+
stac_fastapi/opensearch/config.py,sha256=T4nFpe3cOfKagUSkslXHIHHB0FHCOwsI1adZaLGi2WY,4243
|
|
4
|
+
stac_fastapi/opensearch/database_logic.py,sha256=qMqDjsfOFFBSXx2RCMZLOPithqfH5qLVY0Goj2c7uYA,43197
|
|
5
|
+
stac_fastapi/opensearch/version.py,sha256=3xU5aBmxgAcPizRlef_YROCW9ULsGOA4flSyd9AQog4,45
|
|
6
|
+
stac_fastapi_opensearch-4.1.0.dist-info/METADATA,sha256=4L2JDka8J23K1jXn8NOfY__spCBwjSPke6DuCiVr8B8,20845
|
|
7
|
+
stac_fastapi_opensearch-4.1.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
8
|
+
stac_fastapi_opensearch-4.1.0.dist-info/entry_points.txt,sha256=zjZ0Xsr9BUNJqMkdPpl6zEIUykv1uFdJtNELFRChp0w,76
|
|
9
|
+
stac_fastapi_opensearch-4.1.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
+
stac_fastapi_opensearch-4.1.0.dist-info/RECORD,,
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
stac_fastapi/opensearch/__init__.py,sha256=iJWMUgn7mUvmuPQSO_FlyhJ5eDdbbfmGv1qnFOX5-qk,28
|
|
2
|
-
stac_fastapi/opensearch/app.py,sha256=NgoH3mkNPhDlrX4wdtNqGTwelzMMnNwIwOHGqUdu9D8,4158
|
|
3
|
-
stac_fastapi/opensearch/config.py,sha256=tr-j9cLib-FYSSDkiOr8qJawICIVeRgcbzA54Z8A-rI,2950
|
|
4
|
-
stac_fastapi/opensearch/database_logic.py,sha256=SQ7ebK5V3EH8scH9PAq9rgFZk_uZ24ElwooANQZTC00,35587
|
|
5
|
-
stac_fastapi/opensearch/version.py,sha256=I6yN3SJF9c8gQmc5k4THOt7Vh8jd_Q50-0Zl7daL29k,47
|
|
6
|
-
stac_fastapi_opensearch-4.0.0a1.dist-info/METADATA,sha256=H_GcFxnJ90DcTQ1gNBY_ptEcYosxBt28PjhcoygiZHo,19416
|
|
7
|
-
stac_fastapi_opensearch-4.0.0a1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
|
8
|
-
stac_fastapi_opensearch-4.0.0a1.dist-info/entry_points.txt,sha256=zjZ0Xsr9BUNJqMkdPpl6zEIUykv1uFdJtNELFRChp0w,76
|
|
9
|
-
stac_fastapi_opensearch-4.0.0a1.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
|
|
10
|
-
stac_fastapi_opensearch-4.0.0a1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{stac_fastapi_opensearch-4.0.0a1.dist-info → stac_fastapi_opensearch-4.1.0.dist-info}/top_level.txt
RENAMED
|
File without changes
|