stac-fastapi-core 6.1.0__tar.gz → 6.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stac_fastapi_core-6.1.0/stac_fastapi_core.egg-info → stac_fastapi_core-6.2.1}/PKG-INFO +101 -4
- stac_fastapi_core-6.1.0/PKG-INFO → stac_fastapi_core-6.2.1/README.md +77 -19
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/base_database_logic.py +2 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/core.py +25 -10
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/datetime_utils.py +1 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/serializers.py +1 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/session.py +1 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/version.py +1 -1
- stac_fastapi_core-6.1.0/README.md → stac_fastapi_core-6.2.1/stac_fastapi_core.egg-info/PKG-INFO +116 -1
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/top_level.txt +1 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/setup.cfg +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/setup.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/__init__.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/base_settings.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/basic_auth.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/__init__.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/aggregation.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/fields.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/filter.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/query.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/models/__init__.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/models/links.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/models/search.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/rate_limit.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/route_dependencies.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/utilities.py +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/SOURCES.txt +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/dependency_links.txt +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/not-zip-safe +0 -0
- {stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/requires.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
2
|
-
Name:
|
|
3
|
-
Version: 6.1
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stac_fastapi_core
|
|
3
|
+
Version: 6.2.1
|
|
4
4
|
Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -15,6 +15,27 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: License :: OSI Approved :: MIT License
|
|
16
16
|
Requires-Python: >=3.9
|
|
17
17
|
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: fastapi~=0.109.0
|
|
19
|
+
Requires-Dist: attrs>=23.2.0
|
|
20
|
+
Requires-Dist: pydantic<3.0.0,>=2.4.1
|
|
21
|
+
Requires-Dist: stac_pydantic~=3.3.0
|
|
22
|
+
Requires-Dist: stac-fastapi.types==6.0.0
|
|
23
|
+
Requires-Dist: stac-fastapi.api==6.0.0
|
|
24
|
+
Requires-Dist: stac-fastapi.extensions==6.0.0
|
|
25
|
+
Requires-Dist: orjson~=3.9.0
|
|
26
|
+
Requires-Dist: overrides~=7.4.0
|
|
27
|
+
Requires-Dist: geojson-pydantic~=1.0.0
|
|
28
|
+
Requires-Dist: pygeofilter~=0.3.1
|
|
29
|
+
Requires-Dist: jsonschema~=4.0.0
|
|
30
|
+
Requires-Dist: slowapi~=0.1.9
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: requires-python
|
|
38
|
+
Dynamic: summary
|
|
18
39
|
|
|
19
40
|
# stac-fastapi-elasticsearch-opensearch
|
|
20
41
|
|
|
@@ -103,6 +124,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
103
124
|
- [Auth](#auth)
|
|
104
125
|
- [Aggregation](#aggregation)
|
|
105
126
|
- [Rate Limiting](#rate-limiting)
|
|
127
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
106
128
|
|
|
107
129
|
## Documentation & Resources
|
|
108
130
|
|
|
@@ -244,10 +266,86 @@ You can customize additional settings in your `.env` file:
|
|
|
244
266
|
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
245
267
|
| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
|
|
246
268
|
| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
|
|
269
|
+
| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
|
|
247
270
|
|
|
248
271
|
> [!NOTE]
|
|
249
272
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
250
273
|
|
|
274
|
+
## Datetime-Based Index Management
|
|
275
|
+
|
|
276
|
+
### Overview
|
|
277
|
+
|
|
278
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
279
|
+
|
|
280
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
281
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
282
|
+
|
|
283
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
284
|
+
|
|
285
|
+
### When to Use
|
|
286
|
+
|
|
287
|
+
**Recommended for:**
|
|
288
|
+
- Systems with large collections containing millions of items
|
|
289
|
+
- Systems requiring high-performance temporal searching
|
|
290
|
+
|
|
291
|
+
**Pros:**
|
|
292
|
+
- Multiple times faster queries with datetime filter
|
|
293
|
+
- Reduced database load - only relevant indexes are searched
|
|
294
|
+
|
|
295
|
+
**Cons:**
|
|
296
|
+
- Slightly longer item indexing time (automatic index management)
|
|
297
|
+
- Greater management complexity
|
|
298
|
+
|
|
299
|
+
### Configuration
|
|
300
|
+
|
|
301
|
+
#### Enabling Datetime-Based Indexing
|
|
302
|
+
|
|
303
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### Related Configuration Variables
|
|
310
|
+
|
|
311
|
+
| Variable | Description | Default | Example |
|
|
312
|
+
|----------|-------------|---------|---------|
|
|
313
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
314
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
315
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
316
|
+
|
|
317
|
+
## How Datetime-Based Indexing Works
|
|
318
|
+
|
|
319
|
+
### Index and Alias Naming Convention
|
|
320
|
+
|
|
321
|
+
The system uses a precise naming convention:
|
|
322
|
+
|
|
323
|
+
**Physical indexes:**
|
|
324
|
+
```
|
|
325
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
**Aliases:**
|
|
329
|
+
```
|
|
330
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
331
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
332
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
**Example:**
|
|
336
|
+
|
|
337
|
+
*Physical indexes:*
|
|
338
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
339
|
+
|
|
340
|
+
*Aliases:*
|
|
341
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
342
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
343
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
344
|
+
|
|
345
|
+
### Index Size Management
|
|
346
|
+
|
|
347
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
348
|
+
|
|
251
349
|
## Interacting with the API
|
|
252
350
|
|
|
253
351
|
- **Creating a Collection**:
|
|
@@ -556,4 +654,3 @@ You can customize additional settings in your `.env` file:
|
|
|
556
654
|
- Ensures fair resource allocation among all clients
|
|
557
655
|
|
|
558
656
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
559
|
-
|
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: stac_fastapi_core
|
|
3
|
-
Version: 6.1.0
|
|
4
|
-
Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
|
|
5
|
-
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
|
-
License: MIT
|
|
7
|
-
Classifier: Intended Audience :: Developers
|
|
8
|
-
Classifier: Intended Audience :: Information Technology
|
|
9
|
-
Classifier: Intended Audience :: Science/Research
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
-
Requires-Python: >=3.9
|
|
17
|
-
Description-Content-Type: text/markdown
|
|
18
|
-
|
|
19
1
|
# stac-fastapi-elasticsearch-opensearch
|
|
20
2
|
|
|
21
3
|
<!-- markdownlint-disable MD033 MD041 -->
|
|
@@ -103,6 +85,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
103
85
|
- [Auth](#auth)
|
|
104
86
|
- [Aggregation](#aggregation)
|
|
105
87
|
- [Rate Limiting](#rate-limiting)
|
|
88
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
106
89
|
|
|
107
90
|
## Documentation & Resources
|
|
108
91
|
|
|
@@ -244,10 +227,86 @@ You can customize additional settings in your `.env` file:
|
|
|
244
227
|
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
245
228
|
| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
|
|
246
229
|
| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
|
|
230
|
+
| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
|
|
247
231
|
|
|
248
232
|
> [!NOTE]
|
|
249
233
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
250
234
|
|
|
235
|
+
## Datetime-Based Index Management
|
|
236
|
+
|
|
237
|
+
### Overview
|
|
238
|
+
|
|
239
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
240
|
+
|
|
241
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
242
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
243
|
+
|
|
244
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
245
|
+
|
|
246
|
+
### When to Use
|
|
247
|
+
|
|
248
|
+
**Recommended for:**
|
|
249
|
+
- Systems with large collections containing millions of items
|
|
250
|
+
- Systems requiring high-performance temporal searching
|
|
251
|
+
|
|
252
|
+
**Pros:**
|
|
253
|
+
- Multiple times faster queries with datetime filter
|
|
254
|
+
- Reduced database load - only relevant indexes are searched
|
|
255
|
+
|
|
256
|
+
**Cons:**
|
|
257
|
+
- Slightly longer item indexing time (automatic index management)
|
|
258
|
+
- Greater management complexity
|
|
259
|
+
|
|
260
|
+
### Configuration
|
|
261
|
+
|
|
262
|
+
#### Enabling Datetime-Based Indexing
|
|
263
|
+
|
|
264
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
### Related Configuration Variables
|
|
271
|
+
|
|
272
|
+
| Variable | Description | Default | Example |
|
|
273
|
+
|----------|-------------|---------|---------|
|
|
274
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
275
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
276
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
277
|
+
|
|
278
|
+
## How Datetime-Based Indexing Works
|
|
279
|
+
|
|
280
|
+
### Index and Alias Naming Convention
|
|
281
|
+
|
|
282
|
+
The system uses a precise naming convention:
|
|
283
|
+
|
|
284
|
+
**Physical indexes:**
|
|
285
|
+
```
|
|
286
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
**Aliases:**
|
|
290
|
+
```
|
|
291
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
292
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
293
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
**Example:**
|
|
297
|
+
|
|
298
|
+
*Physical indexes:*
|
|
299
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
300
|
+
|
|
301
|
+
*Aliases:*
|
|
302
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
303
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
304
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
305
|
+
|
|
306
|
+
### Index Size Management
|
|
307
|
+
|
|
308
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
309
|
+
|
|
251
310
|
## Interacting with the API
|
|
252
311
|
|
|
253
312
|
- **Creating a Collection**:
|
|
@@ -556,4 +615,3 @@ You can customize additional settings in your `.env` file:
|
|
|
556
615
|
- Ensures fair resource allocation among all clients
|
|
557
616
|
|
|
558
617
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
559
|
-
|
{stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/base_database_logic.py
RENAMED
|
@@ -48,6 +48,7 @@ class BaseDatabaseLogic(abc.ABC):
|
|
|
48
48
|
item_id: str,
|
|
49
49
|
operations: List,
|
|
50
50
|
base_url: str,
|
|
51
|
+
create_nest: bool = False,
|
|
51
52
|
refresh: bool = True,
|
|
52
53
|
) -> Dict:
|
|
53
54
|
"""Patch a item in the database follows RF6902."""
|
|
@@ -94,6 +95,7 @@ class BaseDatabaseLogic(abc.ABC):
|
|
|
94
95
|
collection_id: str,
|
|
95
96
|
operations: List,
|
|
96
97
|
base_url: str,
|
|
98
|
+
create_nest: bool = False,
|
|
97
99
|
refresh: bool = True,
|
|
98
100
|
) -> Dict:
|
|
99
101
|
"""Patch a collection in the database follows RF6902."""
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Core client."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
import os
|
|
4
5
|
from datetime import datetime as datetime_type
|
|
5
6
|
from datetime import timezone
|
|
6
7
|
from enum import Enum
|
|
@@ -234,7 +235,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
234
235
|
"""
|
|
235
236
|
request = kwargs["request"]
|
|
236
237
|
base_url = str(request.base_url)
|
|
237
|
-
limit = int(request.query_params.get("limit", 10))
|
|
238
|
+
limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
|
|
238
239
|
token = request.query_params.get("token")
|
|
239
240
|
|
|
240
241
|
collections, next_token = await self.database.get_all_collections(
|
|
@@ -285,7 +286,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
285
286
|
collection_id: str,
|
|
286
287
|
bbox: Optional[BBox] = None,
|
|
287
288
|
datetime: Optional[str] = None,
|
|
288
|
-
limit: Optional[int] =
|
|
289
|
+
limit: Optional[int] = None,
|
|
289
290
|
token: Optional[str] = None,
|
|
290
291
|
**kwargs,
|
|
291
292
|
) -> stac_types.ItemCollection:
|
|
@@ -295,7 +296,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
295
296
|
collection_id (str): The identifier of the collection to read items from.
|
|
296
297
|
bbox (Optional[BBox]): The bounding box to filter items by.
|
|
297
298
|
datetime (Optional[str]): The datetime range to filter items by.
|
|
298
|
-
limit (int): The maximum number of items to return.
|
|
299
|
+
limit (int): The maximum number of items to return.
|
|
299
300
|
token (str): A token used for pagination.
|
|
300
301
|
request (Request): The incoming request.
|
|
301
302
|
|
|
@@ -324,10 +325,15 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
324
325
|
search=search, collection_ids=[collection_id]
|
|
325
326
|
)
|
|
326
327
|
|
|
327
|
-
|
|
328
|
-
search = self.database.apply_datetime_filter(
|
|
329
|
-
search=search,
|
|
328
|
+
try:
|
|
329
|
+
search, datetime_search = self.database.apply_datetime_filter(
|
|
330
|
+
search=search, datetime=datetime
|
|
330
331
|
)
|
|
332
|
+
except (ValueError, TypeError) as e:
|
|
333
|
+
# Handle invalid interval formats if return_date fails
|
|
334
|
+
msg = f"Invalid interval format: {datetime}, error: {e}"
|
|
335
|
+
logger.error(msg)
|
|
336
|
+
raise HTTPException(status_code=400, detail=msg)
|
|
331
337
|
|
|
332
338
|
if bbox:
|
|
333
339
|
bbox = [float(x) for x in bbox]
|
|
@@ -336,12 +342,14 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
336
342
|
|
|
337
343
|
search = self.database.apply_bbox_filter(search=search, bbox=bbox)
|
|
338
344
|
|
|
345
|
+
limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
|
|
339
346
|
items, maybe_count, next_token = await self.database.execute_search(
|
|
340
347
|
search=search,
|
|
341
348
|
limit=limit,
|
|
342
349
|
sort=None,
|
|
343
350
|
token=token,
|
|
344
351
|
collection_ids=[collection_id],
|
|
352
|
+
datetime_search=datetime_search,
|
|
345
353
|
)
|
|
346
354
|
|
|
347
355
|
items = [
|
|
@@ -387,7 +395,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
387
395
|
ids: Optional[List[str]] = None,
|
|
388
396
|
bbox: Optional[BBox] = None,
|
|
389
397
|
datetime: Optional[str] = None,
|
|
390
|
-
limit: Optional[int] =
|
|
398
|
+
limit: Optional[int] = None,
|
|
391
399
|
query: Optional[str] = None,
|
|
392
400
|
token: Optional[str] = None,
|
|
393
401
|
fields: Optional[List[str]] = None,
|
|
@@ -420,6 +428,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
420
428
|
Raises:
|
|
421
429
|
HTTPException: If any error occurs while searching the catalog.
|
|
422
430
|
"""
|
|
431
|
+
limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10)))
|
|
423
432
|
base_args = {
|
|
424
433
|
"collections": collections,
|
|
425
434
|
"ids": ids,
|
|
@@ -500,10 +509,15 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
500
509
|
search=search, collection_ids=search_request.collections
|
|
501
510
|
)
|
|
502
511
|
|
|
503
|
-
|
|
504
|
-
search = self.database.apply_datetime_filter(
|
|
505
|
-
search=search,
|
|
512
|
+
try:
|
|
513
|
+
search, datetime_search = self.database.apply_datetime_filter(
|
|
514
|
+
search=search, datetime=search_request.datetime
|
|
506
515
|
)
|
|
516
|
+
except (ValueError, TypeError) as e:
|
|
517
|
+
# Handle invalid interval formats if return_date fails
|
|
518
|
+
msg = f"Invalid interval format: {search_request.datetime}, error: {e}"
|
|
519
|
+
logger.error(msg)
|
|
520
|
+
raise HTTPException(status_code=400, detail=msg)
|
|
507
521
|
|
|
508
522
|
if search_request.bbox:
|
|
509
523
|
bbox = search_request.bbox
|
|
@@ -560,6 +574,7 @@ class CoreClient(AsyncBaseCoreClient):
|
|
|
560
574
|
token=search_request.token,
|
|
561
575
|
sort=sort,
|
|
562
576
|
collection_ids=search_request.collections,
|
|
577
|
+
datetime_search=datetime_search,
|
|
563
578
|
)
|
|
564
579
|
|
|
565
580
|
fields = (
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""library version."""
|
|
2
|
-
__version__ = "6.1
|
|
2
|
+
__version__ = "6.2.1"
|
stac_fastapi_core-6.1.0/README.md → stac_fastapi_core-6.2.1/stac_fastapi_core.egg-info/PKG-INFO
RENAMED
|
@@ -1,3 +1,42 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: stac_fastapi_core
|
|
3
|
+
Version: 6.2.1
|
|
4
|
+
Summary: Core library for the Elasticsearch and Opensearch stac-fastapi backends.
|
|
5
|
+
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Intended Audience :: Developers
|
|
8
|
+
Classifier: Intended Audience :: Information Technology
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: fastapi~=0.109.0
|
|
19
|
+
Requires-Dist: attrs>=23.2.0
|
|
20
|
+
Requires-Dist: pydantic<3.0.0,>=2.4.1
|
|
21
|
+
Requires-Dist: stac_pydantic~=3.3.0
|
|
22
|
+
Requires-Dist: stac-fastapi.types==6.0.0
|
|
23
|
+
Requires-Dist: stac-fastapi.api==6.0.0
|
|
24
|
+
Requires-Dist: stac-fastapi.extensions==6.0.0
|
|
25
|
+
Requires-Dist: orjson~=3.9.0
|
|
26
|
+
Requires-Dist: overrides~=7.4.0
|
|
27
|
+
Requires-Dist: geojson-pydantic~=1.0.0
|
|
28
|
+
Requires-Dist: pygeofilter~=0.3.1
|
|
29
|
+
Requires-Dist: jsonschema~=4.0.0
|
|
30
|
+
Requires-Dist: slowapi~=0.1.9
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: requires-python
|
|
38
|
+
Dynamic: summary
|
|
39
|
+
|
|
1
40
|
# stac-fastapi-elasticsearch-opensearch
|
|
2
41
|
|
|
3
42
|
<!-- markdownlint-disable MD033 MD041 -->
|
|
@@ -85,6 +124,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
85
124
|
- [Auth](#auth)
|
|
86
125
|
- [Aggregation](#aggregation)
|
|
87
126
|
- [Rate Limiting](#rate-limiting)
|
|
127
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
88
128
|
|
|
89
129
|
## Documentation & Resources
|
|
90
130
|
|
|
@@ -226,10 +266,86 @@ You can customize additional settings in your `.env` file:
|
|
|
226
266
|
| `RAISE_ON_BULK_ERROR` | Controls whether bulk insert operations raise exceptions on errors. If set to `true`, the operation will stop and raise an exception when an error occurs. If set to `false`, errors will be logged, and the operation will continue. **Note:** STAC Item and ItemCollection validation errors will always raise, regardless of this flag. | `false` | Optional |
|
|
227
267
|
| `DATABASE_REFRESH` | Controls whether database operations refresh the index immediately after changes. If set to `true`, changes will be immediately searchable. If set to `false`, changes may not be immediately visible but can improve performance for bulk operations. If set to `wait_for`, changes will wait for the next refresh cycle to become visible. | `false` | Optional |
|
|
228
268
|
| `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. If set to `false`, the POST `/collections` route and related transaction endpoints (including bulk transaction operations) will be unavailable in the API. This is useful for deployments where mutating the catalog via the API should be prevented. | `true` | Optional |
|
|
269
|
+
| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional |
|
|
229
270
|
|
|
230
271
|
> [!NOTE]
|
|
231
272
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
232
273
|
|
|
274
|
+
## Datetime-Based Index Management
|
|
275
|
+
|
|
276
|
+
### Overview
|
|
277
|
+
|
|
278
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
279
|
+
|
|
280
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
281
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
282
|
+
|
|
283
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
284
|
+
|
|
285
|
+
### When to Use
|
|
286
|
+
|
|
287
|
+
**Recommended for:**
|
|
288
|
+
- Systems with large collections containing millions of items
|
|
289
|
+
- Systems requiring high-performance temporal searching
|
|
290
|
+
|
|
291
|
+
**Pros:**
|
|
292
|
+
- Multiple times faster queries with datetime filter
|
|
293
|
+
- Reduced database load - only relevant indexes are searched
|
|
294
|
+
|
|
295
|
+
**Cons:**
|
|
296
|
+
- Slightly longer item indexing time (automatic index management)
|
|
297
|
+
- Greater management complexity
|
|
298
|
+
|
|
299
|
+
### Configuration
|
|
300
|
+
|
|
301
|
+
#### Enabling Datetime-Based Indexing
|
|
302
|
+
|
|
303
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### Related Configuration Variables
|
|
310
|
+
|
|
311
|
+
| Variable | Description | Default | Example |
|
|
312
|
+
|----------|-------------|---------|---------|
|
|
313
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
314
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
315
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
316
|
+
|
|
317
|
+
## How Datetime-Based Indexing Works
|
|
318
|
+
|
|
319
|
+
### Index and Alias Naming Convention
|
|
320
|
+
|
|
321
|
+
The system uses a precise naming convention:
|
|
322
|
+
|
|
323
|
+
**Physical indexes:**
|
|
324
|
+
```
|
|
325
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
**Aliases:**
|
|
329
|
+
```
|
|
330
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
331
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
332
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
**Example:**
|
|
336
|
+
|
|
337
|
+
*Physical indexes:*
|
|
338
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
339
|
+
|
|
340
|
+
*Aliases:*
|
|
341
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
342
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
343
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
344
|
+
|
|
345
|
+
### Index Size Management
|
|
346
|
+
|
|
347
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
348
|
+
|
|
233
349
|
## Interacting with the API
|
|
234
350
|
|
|
235
351
|
- **Creating a Collection**:
|
|
@@ -538,4 +654,3 @@ You can customize additional settings in your `.env` file:
|
|
|
538
654
|
- Ensures fair resource allocation among all clients
|
|
539
655
|
|
|
540
656
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
541
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/__init__.py
RENAMED
|
File without changes
|
{stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi/core/extensions/aggregation.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{stac_fastapi_core-6.1.0 → stac_fastapi_core-6.2.1}/stac_fastapi_core.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|