stac-fastapi-opensearch 4.0.0a2__tar.gz → 4.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/PKG-INFO +3 -2
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/README.md +2 -1
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/setup.py +1 -1
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/app.py +15 -9
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/config.py +2 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/database_logic.py +247 -62
- stac_fastapi_opensearch-4.1.0/stac_fastapi/opensearch/version.py +2 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/PKG-INFO +3 -2
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/requires.txt +1 -1
- stac_fastapi_opensearch-4.0.0a2/stac_fastapi/opensearch/version.py +0 -2
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/setup.cfg +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/__init__.py +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/SOURCES.txt +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/dependency_links.txt +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/entry_points.txt +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/not-zip-safe +0 -0
- {stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi_opensearch.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stac_fastapi_opensearch
|
|
3
|
-
Version: 4.0
|
|
3
|
+
Version: 4.1.0
|
|
4
4
|
Summary: Opensearch stac-fastapi backend.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -134,7 +134,8 @@ You can customize additional settings in your `.env` file:
|
|
|
134
134
|
| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
|
|
135
135
|
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
|
|
136
136
|
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
|
|
137
|
-
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
137
|
+
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
138
|
+
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
138
139
|
|
|
139
140
|
> [!NOTE]
|
|
140
141
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
@@ -113,7 +113,8 @@ You can customize additional settings in your `.env` file:
|
|
|
113
113
|
| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
|
|
114
114
|
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
|
|
115
115
|
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
|
|
116
|
-
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
116
|
+
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
117
|
+
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
117
118
|
|
|
118
119
|
> [!NOTE]
|
|
119
120
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
{stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/app.py
RENAMED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
"""FastAPI application."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
|
|
6
|
+
from fastapi import FastAPI
|
|
4
7
|
|
|
5
8
|
from stac_fastapi.api.app import StacApi
|
|
6
9
|
from stac_fastapi.api.models import create_get_request_model, create_post_request_model
|
|
@@ -87,7 +90,7 @@ post_request_model = create_post_request_model(search_extensions)
|
|
|
87
90
|
api = StacApi(
|
|
88
91
|
title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"),
|
|
89
92
|
description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"),
|
|
90
|
-
api_version=os.getenv("STAC_FASTAPI_VERSION", "4.0
|
|
93
|
+
api_version=os.getenv("STAC_FASTAPI_VERSION", "4.1.0"),
|
|
91
94
|
settings=settings,
|
|
92
95
|
extensions=extensions,
|
|
93
96
|
client=CoreClient(
|
|
@@ -97,18 +100,21 @@ api = StacApi(
|
|
|
97
100
|
search_post_request_model=post_request_model,
|
|
98
101
|
route_dependencies=get_route_dependencies(),
|
|
99
102
|
)
|
|
100
|
-
app = api.app
|
|
101
|
-
app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "")
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# Add rate limit
|
|
105
|
-
setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT"))
|
|
106
103
|
|
|
107
104
|
|
|
108
|
-
@
|
|
109
|
-
async def
|
|
105
|
+
@asynccontextmanager
|
|
106
|
+
async def lifespan(app: FastAPI):
|
|
107
|
+
"""Lifespan handler for FastAPI app. Initializes index templates and collections at startup."""
|
|
110
108
|
await create_index_templates()
|
|
111
109
|
await create_collection_index()
|
|
110
|
+
yield
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
app = api.app
|
|
114
|
+
app.router.lifespan_context = lifespan
|
|
115
|
+
app.root_path = os.getenv("STAC_FASTAPI_ROOT_PATH", "")
|
|
116
|
+
# Add rate limit
|
|
117
|
+
setup_rate_limit(app, rate_limit=os.getenv("STAC_FASTAPI_RATE_LIMIT"))
|
|
112
118
|
|
|
113
119
|
|
|
114
120
|
def run() -> None:
|
{stac_fastapi_opensearch-4.0.0a2 → stac_fastapi_opensearch-4.1.0}/stac_fastapi/opensearch/config.py
RENAMED
|
@@ -83,6 +83,7 @@ class OpensearchSettings(ApiSettings, ApiBaseSettings):
|
|
|
83
83
|
indexed_fields: Set[str] = {"datetime"}
|
|
84
84
|
enable_response_models: bool = False
|
|
85
85
|
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
86
|
+
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
86
87
|
|
|
87
88
|
@property
|
|
88
89
|
def create_client(self):
|
|
@@ -103,6 +104,7 @@ class AsyncOpensearchSettings(ApiSettings, ApiBaseSettings):
|
|
|
103
104
|
indexed_fields: Set[str] = {"datetime"}
|
|
104
105
|
enable_response_models: bool = False
|
|
105
106
|
enable_direct_response: bool = get_bool_env("ENABLE_DIRECT_RESPONSE", default=False)
|
|
107
|
+
raise_on_bulk_error: bool = get_bool_env("RAISE_ON_BULK_ERROR", default=False)
|
|
106
108
|
|
|
107
109
|
@property
|
|
108
110
|
def create_client(self):
|
|
@@ -13,7 +13,6 @@ from opensearchpy.helpers.query import Q
|
|
|
13
13
|
from opensearchpy.helpers.search import Search
|
|
14
14
|
from starlette.requests import Request
|
|
15
15
|
|
|
16
|
-
from stac_fastapi.core import serializers
|
|
17
16
|
from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
|
|
18
17
|
from stac_fastapi.core.database_logic import (
|
|
19
18
|
COLLECTIONS_INDEX,
|
|
@@ -31,6 +30,7 @@ from stac_fastapi.core.database_logic import (
|
|
|
31
30
|
mk_item_id,
|
|
32
31
|
)
|
|
33
32
|
from stac_fastapi.core.extensions import filter
|
|
33
|
+
from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
|
|
34
34
|
from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon
|
|
35
35
|
from stac_fastapi.opensearch.config import (
|
|
36
36
|
AsyncOpensearchSettings as AsyncSearchSettings,
|
|
@@ -143,14 +143,20 @@ async def delete_item_index(collection_id: str) -> None:
|
|
|
143
143
|
class DatabaseLogic(BaseDatabaseLogic):
|
|
144
144
|
"""Database logic."""
|
|
145
145
|
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
async_settings: AsyncSearchSettings = attr.ib(factory=AsyncSearchSettings)
|
|
147
|
+
sync_settings: SyncSearchSettings = attr.ib(factory=SyncSearchSettings)
|
|
148
148
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
149
|
+
client = attr.ib(init=False)
|
|
150
|
+
sync_client = attr.ib(init=False)
|
|
151
|
+
|
|
152
|
+
def __attrs_post_init__(self):
|
|
153
|
+
"""Initialize clients after the class is instantiated."""
|
|
154
|
+
self.client = self.async_settings.create_client
|
|
155
|
+
self.sync_client = self.sync_settings.create_client
|
|
156
|
+
|
|
157
|
+
item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer)
|
|
158
|
+
collection_serializer: Type[CollectionSerializer] = attr.ib(
|
|
159
|
+
default=CollectionSerializer
|
|
154
160
|
)
|
|
155
161
|
|
|
156
162
|
extensions: List[str] = attr.ib(default=attr.Factory(list))
|
|
@@ -329,7 +335,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
329
335
|
|
|
330
336
|
@staticmethod
|
|
331
337
|
def apply_datetime_filter(search: Search, datetime_search):
|
|
332
|
-
"""Apply a filter to search based on datetime field.
|
|
338
|
+
"""Apply a filter to search based on datetime field, start_datetime, and end_datetime fields.
|
|
333
339
|
|
|
334
340
|
Args:
|
|
335
341
|
search (Search): The search object to filter.
|
|
@@ -338,17 +344,109 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
338
344
|
Returns:
|
|
339
345
|
Search: The filtered search object.
|
|
340
346
|
"""
|
|
347
|
+
should = []
|
|
348
|
+
|
|
349
|
+
# If the request is a single datetime return
|
|
350
|
+
# items with datetimes equal to the requested datetime OR
|
|
351
|
+
# the requested datetime is between their start and end datetimes
|
|
341
352
|
if "eq" in datetime_search:
|
|
342
|
-
|
|
343
|
-
|
|
353
|
+
should.extend(
|
|
354
|
+
[
|
|
355
|
+
Q(
|
|
356
|
+
"bool",
|
|
357
|
+
filter=[
|
|
358
|
+
Q(
|
|
359
|
+
"term",
|
|
360
|
+
properties__datetime=datetime_search["eq"],
|
|
361
|
+
),
|
|
362
|
+
],
|
|
363
|
+
),
|
|
364
|
+
Q(
|
|
365
|
+
"bool",
|
|
366
|
+
filter=[
|
|
367
|
+
Q(
|
|
368
|
+
"range",
|
|
369
|
+
properties__start_datetime={
|
|
370
|
+
"lte": datetime_search["eq"],
|
|
371
|
+
},
|
|
372
|
+
),
|
|
373
|
+
Q(
|
|
374
|
+
"range",
|
|
375
|
+
properties__end_datetime={
|
|
376
|
+
"gte": datetime_search["eq"],
|
|
377
|
+
},
|
|
378
|
+
),
|
|
379
|
+
],
|
|
380
|
+
),
|
|
381
|
+
]
|
|
344
382
|
)
|
|
383
|
+
|
|
384
|
+
# If the request is a date range return
|
|
385
|
+
# items with datetimes within the requested date range OR
|
|
386
|
+
# their startdatetime ithin the requested date range OR
|
|
387
|
+
# their enddatetime ithin the requested date range OR
|
|
388
|
+
# the requested daterange within their start and end datetimes
|
|
345
389
|
else:
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
390
|
+
should.extend(
|
|
391
|
+
[
|
|
392
|
+
Q(
|
|
393
|
+
"bool",
|
|
394
|
+
filter=[
|
|
395
|
+
Q(
|
|
396
|
+
"range",
|
|
397
|
+
properties__datetime={
|
|
398
|
+
"gte": datetime_search["gte"],
|
|
399
|
+
"lte": datetime_search["lte"],
|
|
400
|
+
},
|
|
401
|
+
),
|
|
402
|
+
],
|
|
403
|
+
),
|
|
404
|
+
Q(
|
|
405
|
+
"bool",
|
|
406
|
+
filter=[
|
|
407
|
+
Q(
|
|
408
|
+
"range",
|
|
409
|
+
properties__start_datetime={
|
|
410
|
+
"gte": datetime_search["gte"],
|
|
411
|
+
"lte": datetime_search["lte"],
|
|
412
|
+
},
|
|
413
|
+
),
|
|
414
|
+
],
|
|
415
|
+
),
|
|
416
|
+
Q(
|
|
417
|
+
"bool",
|
|
418
|
+
filter=[
|
|
419
|
+
Q(
|
|
420
|
+
"range",
|
|
421
|
+
properties__end_datetime={
|
|
422
|
+
"gte": datetime_search["gte"],
|
|
423
|
+
"lte": datetime_search["lte"],
|
|
424
|
+
},
|
|
425
|
+
),
|
|
426
|
+
],
|
|
427
|
+
),
|
|
428
|
+
Q(
|
|
429
|
+
"bool",
|
|
430
|
+
filter=[
|
|
431
|
+
Q(
|
|
432
|
+
"range",
|
|
433
|
+
properties__start_datetime={
|
|
434
|
+
"lte": datetime_search["gte"]
|
|
435
|
+
},
|
|
436
|
+
),
|
|
437
|
+
Q(
|
|
438
|
+
"range",
|
|
439
|
+
properties__end_datetime={
|
|
440
|
+
"gte": datetime_search["lte"]
|
|
441
|
+
},
|
|
442
|
+
),
|
|
443
|
+
],
|
|
444
|
+
),
|
|
445
|
+
]
|
|
351
446
|
)
|
|
447
|
+
|
|
448
|
+
search = search.query(Q("bool", filter=[Q("bool", should=should)]))
|
|
449
|
+
|
|
352
450
|
return search
|
|
353
451
|
|
|
354
452
|
@staticmethod
|
|
@@ -633,7 +731,7 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
633
731
|
if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id):
|
|
634
732
|
raise NotFoundError(f"Collection {collection_id} does not exist")
|
|
635
733
|
|
|
636
|
-
async def
|
|
734
|
+
async def async_prep_create_item(
|
|
637
735
|
self, item: Item, base_url: str, exist_ok: bool = False
|
|
638
736
|
) -> Item:
|
|
639
737
|
"""
|
|
@@ -663,44 +761,113 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
663
761
|
|
|
664
762
|
return self.item_serializer.stac_to_db(item, base_url)
|
|
665
763
|
|
|
666
|
-
def
|
|
764
|
+
async def bulk_async_prep_create_item(
|
|
667
765
|
self, item: Item, base_url: str, exist_ok: bool = False
|
|
668
766
|
) -> Item:
|
|
669
767
|
"""
|
|
670
768
|
Prepare an item for insertion into the database.
|
|
671
769
|
|
|
672
|
-
This method performs pre-insertion preparation on the given `item`,
|
|
673
|
-
|
|
674
|
-
|
|
770
|
+
This method performs pre-insertion preparation on the given `item`, such as:
|
|
771
|
+
- Verifying that the collection the item belongs to exists.
|
|
772
|
+
- Optionally checking if an item with the same ID already exists in the database.
|
|
773
|
+
- Serializing the item into a database-compatible format.
|
|
675
774
|
|
|
676
775
|
Args:
|
|
677
|
-
item (Item): The item to be
|
|
678
|
-
base_url (str): The base URL used
|
|
679
|
-
exist_ok (bool): Indicates whether the item can exist
|
|
776
|
+
item (Item): The item to be prepared for insertion.
|
|
777
|
+
base_url (str): The base URL used to construct the item's self URL.
|
|
778
|
+
exist_ok (bool): Indicates whether the item can already exist in the database.
|
|
779
|
+
If False, a `ConflictError` is raised if the item exists.
|
|
680
780
|
|
|
681
781
|
Returns:
|
|
682
|
-
Item: The item
|
|
782
|
+
Item: The prepared item, serialized into a database-compatible format.
|
|
683
783
|
|
|
684
784
|
Raises:
|
|
685
785
|
NotFoundError: If the collection that the item belongs to does not exist in the database.
|
|
686
|
-
ConflictError: If an item with the same ID already exists in the collection
|
|
786
|
+
ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
|
|
787
|
+
and `RAISE_ON_BULK_ERROR` is set to `true`.
|
|
687
788
|
"""
|
|
688
|
-
|
|
689
|
-
collection_id = item["collection"]
|
|
690
|
-
if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id):
|
|
691
|
-
raise NotFoundError(f"Collection {collection_id} does not exist")
|
|
789
|
+
logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")
|
|
692
790
|
|
|
693
|
-
if
|
|
694
|
-
|
|
695
|
-
|
|
791
|
+
# Check if the collection exists
|
|
792
|
+
await self.check_collection_exists(collection_id=item["collection"])
|
|
793
|
+
|
|
794
|
+
# Check if the item already exists in the database
|
|
795
|
+
if not exist_ok and await self.client.exists(
|
|
796
|
+
index=index_alias_by_collection_id(item["collection"]),
|
|
797
|
+
id=mk_item_id(item["id"], item["collection"]),
|
|
696
798
|
):
|
|
697
|
-
|
|
698
|
-
f"Item {
|
|
799
|
+
error_message = (
|
|
800
|
+
f"Item {item['id']} in collection {item['collection']} already exists."
|
|
699
801
|
)
|
|
802
|
+
if self.async_settings.raise_on_bulk_error:
|
|
803
|
+
raise ConflictError(error_message)
|
|
804
|
+
else:
|
|
805
|
+
logger.warning(
|
|
806
|
+
f"{error_message} Continuing as `RAISE_ON_BULK_ERROR` is set to false."
|
|
807
|
+
)
|
|
808
|
+
# Serialize the item into a database-compatible format
|
|
809
|
+
prepped_item = self.item_serializer.stac_to_db(item, base_url)
|
|
810
|
+
logger.debug(f"Item {item['id']} prepared successfully.")
|
|
811
|
+
return prepped_item
|
|
812
|
+
|
|
813
|
+
def bulk_sync_prep_create_item(
|
|
814
|
+
self, item: Item, base_url: str, exist_ok: bool = False
|
|
815
|
+
) -> Item:
|
|
816
|
+
"""
|
|
817
|
+
Prepare an item for insertion into the database.
|
|
700
818
|
|
|
701
|
-
|
|
819
|
+
This method performs pre-insertion preparation on the given `item`, such as:
|
|
820
|
+
- Verifying that the collection the item belongs to exists.
|
|
821
|
+
- Optionally checking if an item with the same ID already exists in the database.
|
|
822
|
+
- Serializing the item into a database-compatible format.
|
|
823
|
+
|
|
824
|
+
Args:
|
|
825
|
+
item (Item): The item to be prepared for insertion.
|
|
826
|
+
base_url (str): The base URL used to construct the item's self URL.
|
|
827
|
+
exist_ok (bool): Indicates whether the item can already exist in the database.
|
|
828
|
+
If False, a `ConflictError` is raised if the item exists.
|
|
702
829
|
|
|
703
|
-
|
|
830
|
+
Returns:
|
|
831
|
+
Item: The prepared item, serialized into a database-compatible format.
|
|
832
|
+
|
|
833
|
+
Raises:
|
|
834
|
+
NotFoundError: If the collection that the item belongs to does not exist in the database.
|
|
835
|
+
ConflictError: If an item with the same ID already exists in the collection and `exist_ok` is False,
|
|
836
|
+
and `RAISE_ON_BULK_ERROR` is set to `true`.
|
|
837
|
+
"""
|
|
838
|
+
logger.debug(f"Preparing item {item['id']} in collection {item['collection']}.")
|
|
839
|
+
|
|
840
|
+
# Check if the collection exists
|
|
841
|
+
if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=item["collection"]):
|
|
842
|
+
raise NotFoundError(f"Collection {item['collection']} does not exist")
|
|
843
|
+
|
|
844
|
+
# Check if the item already exists in the database
|
|
845
|
+
if not exist_ok and self.sync_client.exists(
|
|
846
|
+
index=index_alias_by_collection_id(item["collection"]),
|
|
847
|
+
id=mk_item_id(item["id"], item["collection"]),
|
|
848
|
+
):
|
|
849
|
+
error_message = (
|
|
850
|
+
f"Item {item['id']} in collection {item['collection']} already exists."
|
|
851
|
+
)
|
|
852
|
+
if self.sync_settings.raise_on_bulk_error:
|
|
853
|
+
raise ConflictError(error_message)
|
|
854
|
+
else:
|
|
855
|
+
logger.warning(
|
|
856
|
+
f"{error_message} Continuing as `RAISE_ON_BULK_ERROR` is set to false."
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
# Serialize the item into a database-compatible format
|
|
860
|
+
prepped_item = self.item_serializer.stac_to_db(item, base_url)
|
|
861
|
+
logger.debug(f"Item {item['id']} prepared successfully.")
|
|
862
|
+
return prepped_item
|
|
863
|
+
|
|
864
|
+
async def create_item(
|
|
865
|
+
self,
|
|
866
|
+
item: Item,
|
|
867
|
+
refresh: bool = False,
|
|
868
|
+
base_url: str = "",
|
|
869
|
+
exist_ok: bool = False,
|
|
870
|
+
):
|
|
704
871
|
"""Database logic for creating one item.
|
|
705
872
|
|
|
706
873
|
Args:
|
|
@@ -716,18 +883,16 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
716
883
|
# todo: check if collection exists, but cache
|
|
717
884
|
item_id = item["id"]
|
|
718
885
|
collection_id = item["collection"]
|
|
719
|
-
|
|
886
|
+
item = await self.async_prep_create_item(
|
|
887
|
+
item=item, base_url=base_url, exist_ok=exist_ok
|
|
888
|
+
)
|
|
889
|
+
await self.client.index(
|
|
720
890
|
index=index_alias_by_collection_id(collection_id),
|
|
721
891
|
id=mk_item_id(item_id, collection_id),
|
|
722
892
|
body=item,
|
|
723
893
|
refresh=refresh,
|
|
724
894
|
)
|
|
725
895
|
|
|
726
|
-
if (meta := es_resp.get("meta")) and meta.get("status") == 409:
|
|
727
|
-
raise ConflictError(
|
|
728
|
-
f"Item {item_id} in collection {collection_id} already exists"
|
|
729
|
-
)
|
|
730
|
-
|
|
731
896
|
async def delete_item(
|
|
732
897
|
self, item_id: str, collection_id: str, refresh: bool = False
|
|
733
898
|
):
|
|
@@ -893,52 +1058,72 @@ class DatabaseLogic(BaseDatabaseLogic):
|
|
|
893
1058
|
await delete_item_index(collection_id)
|
|
894
1059
|
|
|
895
1060
|
async def bulk_async(
|
|
896
|
-
self,
|
|
897
|
-
|
|
898
|
-
|
|
1061
|
+
self,
|
|
1062
|
+
collection_id: str,
|
|
1063
|
+
processed_items: List[Item],
|
|
1064
|
+
refresh: bool = False,
|
|
1065
|
+
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1066
|
+
"""
|
|
1067
|
+
Perform a bulk insert of items into the database asynchronously.
|
|
899
1068
|
|
|
900
1069
|
Args:
|
|
901
|
-
self: The instance of the object calling this function.
|
|
902
1070
|
collection_id (str): The ID of the collection to which the items belong.
|
|
903
1071
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
904
1072
|
refresh (bool): Whether to refresh the index after the bulk insert (default: False).
|
|
905
1073
|
|
|
1074
|
+
Returns:
|
|
1075
|
+
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
1076
|
+
- The number of successfully processed actions (`success`).
|
|
1077
|
+
- A list of errors encountered during the bulk operation (`errors`).
|
|
1078
|
+
|
|
906
1079
|
Notes:
|
|
907
|
-
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
908
|
-
insert is performed asynchronously, and the event loop is used to run the operation in a separate executor.
|
|
909
|
-
`mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True,
|
|
910
|
-
index is refreshed after the bulk insert.
|
|
1080
|
+
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1081
|
+
The insert is performed asynchronously, and the event loop is used to run the operation in a separate executor.
|
|
1082
|
+
The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True,
|
|
1083
|
+
the index is refreshed after the bulk insert.
|
|
911
1084
|
"""
|
|
912
|
-
|
|
1085
|
+
raise_on_error = self.async_settings.raise_on_bulk_error
|
|
1086
|
+
success, errors = await helpers.async_bulk(
|
|
913
1087
|
self.client,
|
|
914
1088
|
mk_actions(collection_id, processed_items),
|
|
915
1089
|
refresh=refresh,
|
|
916
|
-
raise_on_error=
|
|
1090
|
+
raise_on_error=raise_on_error,
|
|
917
1091
|
)
|
|
1092
|
+
return success, errors
|
|
918
1093
|
|
|
919
1094
|
def bulk_sync(
|
|
920
|
-
self,
|
|
921
|
-
|
|
922
|
-
|
|
1095
|
+
self,
|
|
1096
|
+
collection_id: str,
|
|
1097
|
+
processed_items: List[Item],
|
|
1098
|
+
refresh: bool = False,
|
|
1099
|
+
) -> Tuple[int, List[Dict[str, Any]]]:
|
|
1100
|
+
"""
|
|
1101
|
+
Perform a bulk insert of items into the database synchronously.
|
|
923
1102
|
|
|
924
1103
|
Args:
|
|
925
|
-
self: The instance of the object calling this function.
|
|
926
1104
|
collection_id (str): The ID of the collection to which the items belong.
|
|
927
1105
|
processed_items (List[Item]): A list of `Item` objects to be inserted into the database.
|
|
928
1106
|
refresh (bool): Whether to refresh the index after the bulk insert (default: False).
|
|
929
1107
|
|
|
1108
|
+
Returns:
|
|
1109
|
+
Tuple[int, List[Dict[str, Any]]]: A tuple containing:
|
|
1110
|
+
- The number of successfully processed actions (`success`).
|
|
1111
|
+
- A list of errors encountered during the bulk operation (`errors`).
|
|
1112
|
+
|
|
930
1113
|
Notes:
|
|
931
|
-
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
932
|
-
insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
1114
|
+
This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`.
|
|
1115
|
+
The insert is performed synchronously and blocking, meaning that the function does not return until the insert has
|
|
933
1116
|
completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to
|
|
934
|
-
True, the index is refreshed after the bulk insert.
|
|
1117
|
+
True, the index is refreshed after the bulk insert.
|
|
935
1118
|
"""
|
|
936
|
-
|
|
1119
|
+
raise_on_error = self.sync_settings.raise_on_bulk_error
|
|
1120
|
+
success, errors = helpers.bulk(
|
|
937
1121
|
self.sync_client,
|
|
938
1122
|
mk_actions(collection_id, processed_items),
|
|
939
1123
|
refresh=refresh,
|
|
940
|
-
raise_on_error=
|
|
1124
|
+
raise_on_error=raise_on_error,
|
|
941
1125
|
)
|
|
1126
|
+
return success, errors
|
|
942
1127
|
|
|
943
1128
|
# DANGER
|
|
944
1129
|
async def delete_items(self) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: stac-fastapi-opensearch
|
|
3
|
-
Version: 4.0
|
|
3
|
+
Version: 4.1.0
|
|
4
4
|
Summary: Opensearch stac-fastapi backend.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -134,7 +134,8 @@ You can customize additional settings in your `.env` file:
|
|
|
134
134
|
| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional |
|
|
135
135
|
| `ELASTICSEARCH_VERSION` | Version of Elasticsearch to use. | `8.11.0` | Optional |
|
|
136
136
|
| `ENABLE_DIRECT_RESPONSE` | Enable direct response for maximum performance (disables all FastAPI dependencies, including authentication, custom status codes, and validation) | `false` | Optional |
|
|
137
|
-
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
137
|
+
| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.1` | Optional
|
|
138
|
+
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
138
139
|
|
|
139
140
|
> [!NOTE]
|
|
140
141
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|