sfeos-helpers 6.1.0__tar.gz → 6.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/PKG-INFO +77 -2
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/README.md +76 -1
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/setup.py +1 -1
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/PKG-INFO +77 -2
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/SOURCES.txt +12 -1
- sfeos_helpers-6.2.0/sfeos_helpers.egg-info/requires.txt +1 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/client.py +5 -2
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/__init__.py +5 -1
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/datetime.py +64 -3
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/index.py +59 -2
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/__init__.py +27 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/base.py +51 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/factory.py +36 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +167 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/inserters.py +309 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/managers.py +198 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/__init__.py +15 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/base.py +30 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +127 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/factory.py +37 -0
- sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +129 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/version.py +1 -1
- sfeos_helpers-6.1.0/sfeos_helpers.egg-info/requires.txt +0 -1
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/setup.cfg +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/dependency_links.txt +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/not-zip-safe +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/top_level.txt +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/__init__.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/format.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/document.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/mapping.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/query.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/utils.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/__init__.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/client.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/cql2.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/transform.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/mappings.py +0 -0
- {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/models/patch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sfeos_helpers
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.2.0
|
|
4
4
|
Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -103,6 +103,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
103
103
|
- [Auth](#auth)
|
|
104
104
|
- [Aggregation](#aggregation)
|
|
105
105
|
- [Rate Limiting](#rate-limiting)
|
|
106
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
106
107
|
|
|
107
108
|
## Documentation & Resources
|
|
108
109
|
|
|
@@ -248,6 +249,81 @@ You can customize additional settings in your `.env` file:
|
|
|
248
249
|
> [!NOTE]
|
|
249
250
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
250
251
|
|
|
252
|
+
## Datetime-Based Index Management
|
|
253
|
+
|
|
254
|
+
### Overview
|
|
255
|
+
|
|
256
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
257
|
+
|
|
258
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
259
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
260
|
+
|
|
261
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
262
|
+
|
|
263
|
+
### When to Use
|
|
264
|
+
|
|
265
|
+
**Recommended for:**
|
|
266
|
+
- Systems with large collections containing millions of items
|
|
267
|
+
- Systems requiring high-performance temporal searching
|
|
268
|
+
|
|
269
|
+
**Pros:**
|
|
270
|
+
- Multiple times faster queries with datetime filter
|
|
271
|
+
- Reduced database load - only relevant indexes are searched
|
|
272
|
+
|
|
273
|
+
**Cons:**
|
|
274
|
+
- Slightly longer item indexing time (automatic index management)
|
|
275
|
+
- Greater management complexity
|
|
276
|
+
|
|
277
|
+
### Configuration
|
|
278
|
+
|
|
279
|
+
#### Enabling Datetime-Based Indexing
|
|
280
|
+
|
|
281
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
282
|
+
|
|
283
|
+
```bash
|
|
284
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
### Related Configuration Variables
|
|
288
|
+
|
|
289
|
+
| Variable | Description | Default | Example |
|
|
290
|
+
|----------|-------------|---------|---------|
|
|
291
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
292
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
293
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
294
|
+
|
|
295
|
+
## How Datetime-Based Indexing Works
|
|
296
|
+
|
|
297
|
+
### Index and Alias Naming Convention
|
|
298
|
+
|
|
299
|
+
The system uses a precise naming convention:
|
|
300
|
+
|
|
301
|
+
**Physical indexes:**
|
|
302
|
+
```
|
|
303
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
**Aliases:**
|
|
307
|
+
```
|
|
308
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
309
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
310
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
**Example:**
|
|
314
|
+
|
|
315
|
+
*Physical indexes:*
|
|
316
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
317
|
+
|
|
318
|
+
*Aliases:*
|
|
319
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
320
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
321
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
322
|
+
|
|
323
|
+
### Index Size Management
|
|
324
|
+
|
|
325
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
326
|
+
|
|
251
327
|
## Interacting with the API
|
|
252
328
|
|
|
253
329
|
- **Creating a Collection**:
|
|
@@ -556,4 +632,3 @@ You can customize additional settings in your `.env` file:
|
|
|
556
632
|
- Ensures fair resource allocation among all clients
|
|
557
633
|
|
|
558
634
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
559
|
-
|
|
@@ -85,6 +85,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
85
85
|
- [Auth](#auth)
|
|
86
86
|
- [Aggregation](#aggregation)
|
|
87
87
|
- [Rate Limiting](#rate-limiting)
|
|
88
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
88
89
|
|
|
89
90
|
## Documentation & Resources
|
|
90
91
|
|
|
@@ -230,6 +231,81 @@ You can customize additional settings in your `.env` file:
|
|
|
230
231
|
> [!NOTE]
|
|
231
232
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
232
233
|
|
|
234
|
+
## Datetime-Based Index Management
|
|
235
|
+
|
|
236
|
+
### Overview
|
|
237
|
+
|
|
238
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
239
|
+
|
|
240
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
241
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
242
|
+
|
|
243
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
244
|
+
|
|
245
|
+
### When to Use
|
|
246
|
+
|
|
247
|
+
**Recommended for:**
|
|
248
|
+
- Systems with large collections containing millions of items
|
|
249
|
+
- Systems requiring high-performance temporal searching
|
|
250
|
+
|
|
251
|
+
**Pros:**
|
|
252
|
+
- Multiple times faster queries with datetime filter
|
|
253
|
+
- Reduced database load - only relevant indexes are searched
|
|
254
|
+
|
|
255
|
+
**Cons:**
|
|
256
|
+
- Slightly longer item indexing time (automatic index management)
|
|
257
|
+
- Greater management complexity
|
|
258
|
+
|
|
259
|
+
### Configuration
|
|
260
|
+
|
|
261
|
+
#### Enabling Datetime-Based Indexing
|
|
262
|
+
|
|
263
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
### Related Configuration Variables
|
|
270
|
+
|
|
271
|
+
| Variable | Description | Default | Example |
|
|
272
|
+
|----------|-------------|---------|---------|
|
|
273
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
274
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
275
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
276
|
+
|
|
277
|
+
## How Datetime-Based Indexing Works
|
|
278
|
+
|
|
279
|
+
### Index and Alias Naming Convention
|
|
280
|
+
|
|
281
|
+
The system uses a precise naming convention:
|
|
282
|
+
|
|
283
|
+
**Physical indexes:**
|
|
284
|
+
```
|
|
285
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
**Aliases:**
|
|
289
|
+
```
|
|
290
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
291
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
292
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
**Example:**
|
|
296
|
+
|
|
297
|
+
*Physical indexes:*
|
|
298
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
299
|
+
|
|
300
|
+
*Aliases:*
|
|
301
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
302
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
303
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
304
|
+
|
|
305
|
+
### Index Size Management
|
|
306
|
+
|
|
307
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
308
|
+
|
|
233
309
|
## Interacting with the API
|
|
234
310
|
|
|
235
311
|
- **Creating a Collection**:
|
|
@@ -538,4 +614,3 @@ You can customize additional settings in your `.env` file:
|
|
|
538
614
|
- Ensures fair resource allocation among all clients
|
|
539
615
|
|
|
540
616
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
541
|
-
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sfeos-helpers
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.2.0
|
|
4
4
|
Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
|
|
5
5
|
Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
|
|
6
6
|
License: MIT
|
|
@@ -103,6 +103,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
|
|
|
103
103
|
- [Auth](#auth)
|
|
104
104
|
- [Aggregation](#aggregation)
|
|
105
105
|
- [Rate Limiting](#rate-limiting)
|
|
106
|
+
- [Datetime-Based Index Management](#datetime-based-index-management)
|
|
106
107
|
|
|
107
108
|
## Documentation & Resources
|
|
108
109
|
|
|
@@ -248,6 +249,81 @@ You can customize additional settings in your `.env` file:
|
|
|
248
249
|
> [!NOTE]
|
|
249
250
|
> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
|
|
250
251
|
|
|
252
|
+
## Datetime-Based Index Management
|
|
253
|
+
|
|
254
|
+
### Overview
|
|
255
|
+
|
|
256
|
+
SFEOS supports two indexing strategies for managing STAC items:
|
|
257
|
+
|
|
258
|
+
1. **Simple Indexing** (default) - One index per collection
|
|
259
|
+
2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
|
|
260
|
+
|
|
261
|
+
The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
|
|
262
|
+
|
|
263
|
+
### When to Use
|
|
264
|
+
|
|
265
|
+
**Recommended for:**
|
|
266
|
+
- Systems with large collections containing millions of items
|
|
267
|
+
- Systems requiring high-performance temporal searching
|
|
268
|
+
|
|
269
|
+
**Pros:**
|
|
270
|
+
- Multiple times faster queries with datetime filter
|
|
271
|
+
- Reduced database load - only relevant indexes are searched
|
|
272
|
+
|
|
273
|
+
**Cons:**
|
|
274
|
+
- Slightly longer item indexing time (automatic index management)
|
|
275
|
+
- Greater management complexity
|
|
276
|
+
|
|
277
|
+
### Configuration
|
|
278
|
+
|
|
279
|
+
#### Enabling Datetime-Based Indexing
|
|
280
|
+
|
|
281
|
+
Enable datetime-based indexing by setting the following environment variable:
|
|
282
|
+
|
|
283
|
+
```bash
|
|
284
|
+
ENABLE_DATETIME_INDEX_FILTERING=true
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
### Related Configuration Variables
|
|
288
|
+
|
|
289
|
+
| Variable | Description | Default | Example |
|
|
290
|
+
|----------|-------------|---------|---------|
|
|
291
|
+
| `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
|
|
292
|
+
| `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
|
|
293
|
+
| `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
|
|
294
|
+
|
|
295
|
+
## How Datetime-Based Indexing Works
|
|
296
|
+
|
|
297
|
+
### Index and Alias Naming Convention
|
|
298
|
+
|
|
299
|
+
The system uses a precise naming convention:
|
|
300
|
+
|
|
301
|
+
**Physical indexes:**
|
|
302
|
+
```
|
|
303
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
**Aliases:**
|
|
307
|
+
```
|
|
308
|
+
{ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
|
|
309
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
|
|
310
|
+
{ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
**Example:**
|
|
314
|
+
|
|
315
|
+
*Physical indexes:*
|
|
316
|
+
- `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
|
|
317
|
+
|
|
318
|
+
*Aliases:*
|
|
319
|
+
- `items_sentinel-2-l2a` - main collection alias
|
|
320
|
+
- `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
|
|
321
|
+
- `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
|
|
322
|
+
|
|
323
|
+
### Index Size Management
|
|
324
|
+
|
|
325
|
+
**Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
|
|
326
|
+
|
|
251
327
|
## Interacting with the API
|
|
252
328
|
|
|
253
329
|
- **Creating a Collection**:
|
|
@@ -556,4 +632,3 @@ You can customize additional settings in your `.env` file:
|
|
|
556
632
|
- Ensures fair resource allocation among all clients
|
|
557
633
|
|
|
558
634
|
- **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
|
|
559
|
-
|
|
@@ -23,4 +23,15 @@ stac_fastapi/sfeos_helpers/filter/__init__.py
|
|
|
23
23
|
stac_fastapi/sfeos_helpers/filter/client.py
|
|
24
24
|
stac_fastapi/sfeos_helpers/filter/cql2.py
|
|
25
25
|
stac_fastapi/sfeos_helpers/filter/transform.py
|
|
26
|
-
stac_fastapi/sfeos_helpers/models/patch.py
|
|
26
|
+
stac_fastapi/sfeos_helpers/models/patch.py
|
|
27
|
+
stac_fastapi/sfeos_helpers/search_engine/__init__.py
|
|
28
|
+
stac_fastapi/sfeos_helpers/search_engine/base.py
|
|
29
|
+
stac_fastapi/sfeos_helpers/search_engine/factory.py
|
|
30
|
+
stac_fastapi/sfeos_helpers/search_engine/index_operations.py
|
|
31
|
+
stac_fastapi/sfeos_helpers/search_engine/inserters.py
|
|
32
|
+
stac_fastapi/sfeos_helpers/search_engine/managers.py
|
|
33
|
+
stac_fastapi/sfeos_helpers/search_engine/selection/__init__.py
|
|
34
|
+
stac_fastapi/sfeos_helpers/search_engine/selection/base.py
|
|
35
|
+
stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py
|
|
36
|
+
stac_fastapi/sfeos_helpers/search_engine/selection/factory.py
|
|
37
|
+
stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
stac-fastapi.core==6.2.0
|
{sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/client.py
RENAMED
|
@@ -313,9 +313,11 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
|
|
|
313
313
|
)
|
|
314
314
|
|
|
315
315
|
if aggregate_request.datetime:
|
|
316
|
-
search = self.database.apply_datetime_filter(
|
|
317
|
-
search=search,
|
|
316
|
+
search, datetime_search = self.database.apply_datetime_filter(
|
|
317
|
+
search=search, datetime=aggregate_request.datetime
|
|
318
318
|
)
|
|
319
|
+
else:
|
|
320
|
+
datetime_search = {"gte": None, "lte": None}
|
|
319
321
|
|
|
320
322
|
if aggregate_request.bbox:
|
|
321
323
|
bbox = aggregate_request.bbox
|
|
@@ -414,6 +416,7 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
|
|
|
414
416
|
geometry_geohash_grid_precision,
|
|
415
417
|
geometry_geotile_grid_precision,
|
|
416
418
|
datetime_frequency_interval,
|
|
419
|
+
datetime_search,
|
|
417
420
|
)
|
|
418
421
|
except Exception as error:
|
|
419
422
|
if not isinstance(error, IndexError):
|
|
@@ -30,11 +30,12 @@ Function Naming Conventions:
|
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
32
|
# Re-export all functions for backward compatibility
|
|
33
|
-
from .datetime import return_date
|
|
33
|
+
from .datetime import extract_date, extract_first_date_from_index, return_date
|
|
34
34
|
from .document import mk_actions, mk_item_id
|
|
35
35
|
from .index import (
|
|
36
36
|
create_index_templates_shared,
|
|
37
37
|
delete_item_index_shared,
|
|
38
|
+
filter_indexes_by_datetime,
|
|
38
39
|
index_alias_by_collection_id,
|
|
39
40
|
index_by_collection_id,
|
|
40
41
|
indices,
|
|
@@ -53,6 +54,7 @@ __all__ = [
|
|
|
53
54
|
"delete_item_index_shared",
|
|
54
55
|
"index_alias_by_collection_id",
|
|
55
56
|
"index_by_collection_id",
|
|
57
|
+
"filter_indexes_by_datetime",
|
|
56
58
|
"indices",
|
|
57
59
|
# Query operations
|
|
58
60
|
"apply_free_text_filter_shared",
|
|
@@ -68,4 +70,6 @@ __all__ = [
|
|
|
68
70
|
"get_bool_env",
|
|
69
71
|
# Datetime utilities
|
|
70
72
|
"return_date",
|
|
73
|
+
"extract_date",
|
|
74
|
+
"extract_first_date_from_index",
|
|
71
75
|
]
|
|
@@ -4,14 +4,19 @@ This module provides datetime utility functions specifically designed for
|
|
|
4
4
|
Elasticsearch and OpenSearch query formatting.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
+
import logging
|
|
8
|
+
import re
|
|
9
|
+
from datetime import date
|
|
7
10
|
from datetime import datetime as datetime_type
|
|
8
11
|
from typing import Dict, Optional, Union
|
|
9
12
|
|
|
10
13
|
from stac_fastapi.types.rfc3339 import DateTimeType
|
|
11
14
|
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
12
17
|
|
|
13
18
|
def return_date(
|
|
14
|
-
interval: Optional[Union[DateTimeType, str]]
|
|
19
|
+
interval: Optional[Union[DateTimeType, str]],
|
|
15
20
|
) -> Dict[str, Optional[str]]:
|
|
16
21
|
"""
|
|
17
22
|
Convert a date interval to an Elasticsearch/OpenSearch query format.
|
|
@@ -39,8 +44,14 @@ def return_date(
|
|
|
39
44
|
if isinstance(interval, str):
|
|
40
45
|
if "/" in interval:
|
|
41
46
|
parts = interval.split("/")
|
|
42
|
-
result["gte"] =
|
|
43
|
-
|
|
47
|
+
result["gte"] = (
|
|
48
|
+
parts[0] if parts[0] != ".." else datetime_type.min.isoformat() + "Z"
|
|
49
|
+
)
|
|
50
|
+
result["lte"] = (
|
|
51
|
+
parts[1]
|
|
52
|
+
if len(parts) > 1 and parts[1] != ".."
|
|
53
|
+
else datetime_type.max.isoformat() + "Z"
|
|
54
|
+
)
|
|
44
55
|
else:
|
|
45
56
|
converted_time = interval if interval != ".." else None
|
|
46
57
|
result["gte"] = result["lte"] = converted_time
|
|
@@ -58,3 +69,53 @@ def return_date(
|
|
|
58
69
|
result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
59
70
|
|
|
60
71
|
return result
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def extract_date(date_str: str) -> date:
|
|
75
|
+
"""Extract date from ISO format string.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
date_str: ISO format date string
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
A date object extracted from the input string.
|
|
82
|
+
"""
|
|
83
|
+
date_str = date_str.replace("Z", "+00:00")
|
|
84
|
+
return datetime_type.fromisoformat(date_str).date()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def extract_first_date_from_index(index_name: str) -> date:
|
|
88
|
+
"""Extract the first date from an index name containing date patterns.
|
|
89
|
+
|
|
90
|
+
Searches for date patterns (YYYY-MM-DD) within the index name string
|
|
91
|
+
and returns the first found date as a date object.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
index_name: Index name containing date patterns.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
A date object extracted from the first date pattern found in the index name.
|
|
98
|
+
|
|
99
|
+
"""
|
|
100
|
+
date_pattern = r"\d{4}-\d{2}-\d{2}"
|
|
101
|
+
match = re.search(date_pattern, index_name)
|
|
102
|
+
|
|
103
|
+
if not match:
|
|
104
|
+
logger.error(f"No date pattern found in index name: '{index_name}'")
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"No date pattern (YYYY-MM-DD) found in index name: '{index_name}'"
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
date_string = match.group(0)
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
extracted_date = datetime_type.strptime(date_string, "%Y-%m-%d").date()
|
|
113
|
+
return extracted_date
|
|
114
|
+
except ValueError as e:
|
|
115
|
+
logger.error(
|
|
116
|
+
f"Invalid date format found in index name '{index_name}': "
|
|
117
|
+
f"'{date_string}' - {str(e)}"
|
|
118
|
+
)
|
|
119
|
+
raise ValueError(
|
|
120
|
+
f"Invalid date format in index name '{index_name}': '{date_string}'"
|
|
121
|
+
) from e
|
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
This module provides functions for creating and managing indices in Elasticsearch/OpenSearch.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
+
import re
|
|
7
|
+
from datetime import datetime
|
|
6
8
|
from functools import lru_cache
|
|
7
9
|
from typing import Any, List, Optional
|
|
8
10
|
|
|
11
|
+
from dateutil.parser import parse # type: ignore[import]
|
|
12
|
+
|
|
9
13
|
from stac_fastapi.sfeos_helpers.mappings import (
|
|
10
14
|
_ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
|
|
11
15
|
COLLECTIONS_INDEX,
|
|
@@ -66,6 +70,59 @@ def indices(collection_ids: Optional[List[str]]) -> str:
|
|
|
66
70
|
)
|
|
67
71
|
|
|
68
72
|
|
|
73
|
+
def filter_indexes_by_datetime(
|
|
74
|
+
indexes: List[str], gte: Optional[str], lte: Optional[str]
|
|
75
|
+
) -> List[str]:
|
|
76
|
+
"""Filter indexes based on datetime range extracted from index names.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
indexes: List of index names containing dates
|
|
80
|
+
gte: Greater than or equal date filter (ISO format, optional 'Z' suffix)
|
|
81
|
+
lte: Less than or equal date filter (ISO format, optional 'Z' suffix)
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
List of filtered index names
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def parse_datetime(dt_str: str) -> datetime:
|
|
88
|
+
"""Parse datetime string, handling both with and without 'Z' suffix."""
|
|
89
|
+
return parse(dt_str).replace(tzinfo=None)
|
|
90
|
+
|
|
91
|
+
def extract_date_range_from_index(index_name: str) -> tuple:
|
|
92
|
+
"""Extract start and end dates from index name."""
|
|
93
|
+
date_pattern = r"(\d{4}-\d{2}-\d{2})"
|
|
94
|
+
dates = re.findall(date_pattern, index_name)
|
|
95
|
+
|
|
96
|
+
if len(dates) == 1:
|
|
97
|
+
start_date = datetime.strptime(dates[0], "%Y-%m-%d")
|
|
98
|
+
max_date = datetime.max.replace(microsecond=0)
|
|
99
|
+
return start_date, max_date
|
|
100
|
+
else:
|
|
101
|
+
start_date = datetime.strptime(dates[0], "%Y-%m-%d")
|
|
102
|
+
end_date = datetime.strptime(dates[1], "%Y-%m-%d")
|
|
103
|
+
return start_date, end_date
|
|
104
|
+
|
|
105
|
+
def is_index_in_range(
|
|
106
|
+
start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime
|
|
107
|
+
) -> bool:
|
|
108
|
+
"""Check if index date range overlaps with filter range."""
|
|
109
|
+
return not (
|
|
110
|
+
end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date()
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0)
|
|
114
|
+
lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0)
|
|
115
|
+
|
|
116
|
+
filtered_indexes = []
|
|
117
|
+
|
|
118
|
+
for index in indexes:
|
|
119
|
+
start_date, end_date = extract_date_range_from_index(index)
|
|
120
|
+
if is_index_in_range(start_date, end_date, gte_dt, lte_dt):
|
|
121
|
+
filtered_indexes.append(index)
|
|
122
|
+
|
|
123
|
+
return filtered_indexes
|
|
124
|
+
|
|
125
|
+
|
|
69
126
|
async def create_index_templates_shared(settings: Any) -> None:
|
|
70
127
|
"""Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
|
|
71
128
|
|
|
@@ -120,11 +177,11 @@ async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
|
|
|
120
177
|
client = settings.create_client
|
|
121
178
|
|
|
122
179
|
name = index_alias_by_collection_id(collection_id)
|
|
123
|
-
resolved = await client.indices.resolve_index(name=name)
|
|
180
|
+
resolved = await client.indices.resolve_index(name=name, ignore=[404])
|
|
124
181
|
if "aliases" in resolved and resolved["aliases"]:
|
|
125
182
|
[alias] = resolved["aliases"]
|
|
126
183
|
await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
|
|
127
184
|
await client.indices.delete(index=alias["indices"])
|
|
128
185
|
else:
|
|
129
|
-
await client.indices.delete(index=name)
|
|
186
|
+
await client.indices.delete(index=name, ignore=[404])
|
|
130
187
|
await client.close()
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Search engine index management package."""
|
|
2
|
+
|
|
3
|
+
from .base import BaseIndexInserter
|
|
4
|
+
from .factory import IndexInsertionFactory
|
|
5
|
+
from .index_operations import IndexOperations
|
|
6
|
+
from .inserters import DatetimeIndexInserter, SimpleIndexInserter
|
|
7
|
+
from .managers import DatetimeIndexManager, IndexSizeManager
|
|
8
|
+
from .selection import (
|
|
9
|
+
BaseIndexSelector,
|
|
10
|
+
DatetimeBasedIndexSelector,
|
|
11
|
+
IndexSelectorFactory,
|
|
12
|
+
UnfilteredIndexSelector,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"BaseIndexInserter",
|
|
17
|
+
"BaseIndexSelector",
|
|
18
|
+
"IndexOperations",
|
|
19
|
+
"IndexSizeManager",
|
|
20
|
+
"DatetimeIndexManager",
|
|
21
|
+
"DatetimeIndexInserter",
|
|
22
|
+
"SimpleIndexInserter",
|
|
23
|
+
"IndexInsertionFactory",
|
|
24
|
+
"DatetimeBasedIndexSelector",
|
|
25
|
+
"UnfilteredIndexSelector",
|
|
26
|
+
"IndexSelectorFactory",
|
|
27
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Base classes for index inserters."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import Any, Dict, List
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BaseIndexInserter(ABC):
|
|
8
|
+
"""Base async index inserter with common async methods."""
|
|
9
|
+
|
|
10
|
+
@abstractmethod
|
|
11
|
+
async def get_target_index(
|
|
12
|
+
self, collection_id: str, product: Dict[str, Any]
|
|
13
|
+
) -> str:
|
|
14
|
+
"""Get target index for a product asynchronously.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
collection_id (str): Collection identifier.
|
|
18
|
+
product (Dict[str, Any]): Product data.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
str: Target index name.
|
|
22
|
+
"""
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
async def prepare_bulk_actions(
|
|
27
|
+
self, collection_id: str, items: List[Dict[str, Any]]
|
|
28
|
+
) -> List[Dict[str, Any]]:
|
|
29
|
+
"""Prepare bulk actions for multiple items asynchronously.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
collection_id (str): Collection identifier.
|
|
33
|
+
items (List[Dict[str, Any]]): List of items to process.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
List[Dict[str, Any]]: List of bulk actions.
|
|
37
|
+
"""
|
|
38
|
+
pass
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
async def create_simple_index(self, client: Any, collection_id: str) -> str:
|
|
42
|
+
"""Create a simple index asynchronously.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
client: Search engine client instance.
|
|
46
|
+
collection_id (str): Collection identifier.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
str: Created index name.
|
|
50
|
+
"""
|
|
51
|
+
pass
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Factory for creating index insertion strategies."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from stac_fastapi.core.utilities import get_bool_env
|
|
6
|
+
|
|
7
|
+
from .base import BaseIndexInserter
|
|
8
|
+
from .index_operations import IndexOperations
|
|
9
|
+
from .inserters import DatetimeIndexInserter, SimpleIndexInserter
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class IndexInsertionFactory:
|
|
13
|
+
"""Factory for creating index insertion strategies."""
|
|
14
|
+
|
|
15
|
+
@staticmethod
|
|
16
|
+
def create_insertion_strategy(
|
|
17
|
+
client: Any,
|
|
18
|
+
) -> BaseIndexInserter:
|
|
19
|
+
"""Create async insertion strategy based on configuration.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
client: Async search engine client instance.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
BaseIndexInserter: Configured async insertion strategy.
|
|
26
|
+
"""
|
|
27
|
+
index_operations = IndexOperations()
|
|
28
|
+
|
|
29
|
+
use_datetime_partitioning = get_bool_env(
|
|
30
|
+
"ENABLE_DATETIME_INDEX_FILTERING", default="false"
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
if use_datetime_partitioning:
|
|
34
|
+
return DatetimeIndexInserter(client, index_operations)
|
|
35
|
+
else:
|
|
36
|
+
return SimpleIndexInserter(index_operations, client)
|