sfeos-helpers 6.1.0__tar.gz → 6.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/PKG-INFO +77 -2
  2. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/README.md +76 -1
  3. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/setup.py +1 -1
  4. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/PKG-INFO +77 -2
  5. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/SOURCES.txt +12 -1
  6. sfeos_helpers-6.2.0/sfeos_helpers.egg-info/requires.txt +1 -0
  7. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/client.py +5 -2
  8. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/__init__.py +5 -1
  9. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/datetime.py +64 -3
  10. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/index.py +59 -2
  11. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/__init__.py +27 -0
  12. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/base.py +51 -0
  13. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/factory.py +36 -0
  14. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/index_operations.py +167 -0
  15. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/inserters.py +309 -0
  16. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/managers.py +198 -0
  17. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/__init__.py +15 -0
  18. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/base.py +30 -0
  19. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py +127 -0
  20. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/factory.py +37 -0
  21. sfeos_helpers-6.2.0/stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py +129 -0
  22. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/version.py +1 -1
  23. sfeos_helpers-6.1.0/sfeos_helpers.egg-info/requires.txt +0 -1
  24. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/setup.cfg +0 -0
  25. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/dependency_links.txt +0 -0
  26. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/not-zip-safe +0 -0
  27. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/sfeos_helpers.egg-info/top_level.txt +0 -0
  28. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/__init__.py +0 -0
  29. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/aggregation/format.py +0 -0
  30. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/document.py +0 -0
  31. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/mapping.py +0 -0
  32. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/query.py +0 -0
  33. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/database/utils.py +0 -0
  34. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/__init__.py +0 -0
  35. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/client.py +0 -0
  36. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/cql2.py +0 -0
  37. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/filter/transform.py +0 -0
  38. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/mappings.py +0 -0
  39. {sfeos_helpers-6.1.0 → sfeos_helpers-6.2.0}/stac_fastapi/sfeos_helpers/models/patch.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sfeos_helpers
3
- Version: 6.1.0
3
+ Version: 6.2.0
4
4
  Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
5
5
  Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
6
  License: MIT
@@ -103,6 +103,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
103
103
  - [Auth](#auth)
104
104
  - [Aggregation](#aggregation)
105
105
  - [Rate Limiting](#rate-limiting)
106
+ - [Datetime-Based Index Management](#datetime-based-index-management)
106
107
 
107
108
  ## Documentation & Resources
108
109
 
@@ -248,6 +249,81 @@ You can customize additional settings in your `.env` file:
248
249
  > [!NOTE]
249
250
  > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
250
251
 
252
+ ## Datetime-Based Index Management
253
+
254
+ ### Overview
255
+
256
+ SFEOS supports two indexing strategies for managing STAC items:
257
+
258
+ 1. **Simple Indexing** (default) - One index per collection
259
+ 2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
260
+
261
+ The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
262
+
263
+ ### When to Use
264
+
265
+ **Recommended for:**
266
+ - Systems with large collections containing millions of items
267
+ - Systems requiring high-performance temporal searching
268
+
269
+ **Pros:**
270
+ - Multiple times faster queries with datetime filter
271
+ - Reduced database load - only relevant indexes are searched
272
+
273
+ **Cons:**
274
+ - Slightly longer item indexing time (automatic index management)
275
+ - Greater management complexity
276
+
277
+ ### Configuration
278
+
279
+ #### Enabling Datetime-Based Indexing
280
+
281
+ Enable datetime-based indexing by setting the following environment variable:
282
+
283
+ ```bash
284
+ ENABLE_DATETIME_INDEX_FILTERING=true
285
+ ```
286
+
287
+ ### Related Configuration Variables
288
+
289
+ | Variable | Description | Default | Example |
290
+ |----------|-------------|---------|---------|
291
+ | `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
292
+ | `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
293
+ | `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
294
+
295
+ ## How Datetime-Based Indexing Works
296
+
297
+ ### Index and Alias Naming Convention
298
+
299
+ The system uses a precise naming convention:
300
+
301
+ **Physical indexes:**
302
+ ```
303
+ {ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
304
+ ```
305
+
306
+ **Aliases:**
307
+ ```
308
+ {ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
309
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
310
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
311
+ ```
312
+
313
+ **Example:**
314
+
315
+ *Physical indexes:*
316
+ - `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
317
+
318
+ *Aliases:*
319
+ - `items_sentinel-2-l2a` - main collection alias
320
+ - `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
321
+ - `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
322
+
323
+ ### Index Size Management
324
+
325
+ **Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
326
+
251
327
  ## Interacting with the API
252
328
 
253
329
  - **Creating a Collection**:
@@ -556,4 +632,3 @@ You can customize additional settings in your `.env` file:
556
632
  - Ensures fair resource allocation among all clients
557
633
 
558
634
  - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
559
-
@@ -85,6 +85,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
85
85
  - [Auth](#auth)
86
86
  - [Aggregation](#aggregation)
87
87
  - [Rate Limiting](#rate-limiting)
88
+ - [Datetime-Based Index Management](#datetime-based-index-management)
88
89
 
89
90
  ## Documentation & Resources
90
91
 
@@ -230,6 +231,81 @@ You can customize additional settings in your `.env` file:
230
231
  > [!NOTE]
231
232
  > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
232
233
 
234
+ ## Datetime-Based Index Management
235
+
236
+ ### Overview
237
+
238
+ SFEOS supports two indexing strategies for managing STAC items:
239
+
240
+ 1. **Simple Indexing** (default) - One index per collection
241
+ 2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
242
+
243
+ The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
244
+
245
+ ### When to Use
246
+
247
+ **Recommended for:**
248
+ - Systems with large collections containing millions of items
249
+ - Systems requiring high-performance temporal searching
250
+
251
+ **Pros:**
252
+ - Multiple times faster queries with datetime filter
253
+ - Reduced database load - only relevant indexes are searched
254
+
255
+ **Cons:**
256
+ - Slightly longer item indexing time (automatic index management)
257
+ - Greater management complexity
258
+
259
+ ### Configuration
260
+
261
+ #### Enabling Datetime-Based Indexing
262
+
263
+ Enable datetime-based indexing by setting the following environment variable:
264
+
265
+ ```bash
266
+ ENABLE_DATETIME_INDEX_FILTERING=true
267
+ ```
268
+
269
+ ### Related Configuration Variables
270
+
271
+ | Variable | Description | Default | Example |
272
+ |----------|-------------|---------|---------|
273
+ | `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
274
+ | `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
275
+ | `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
276
+
277
+ ## How Datetime-Based Indexing Works
278
+
279
+ ### Index and Alias Naming Convention
280
+
281
+ The system uses a precise naming convention:
282
+
283
+ **Physical indexes:**
284
+ ```
285
+ {ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
286
+ ```
287
+
288
+ **Aliases:**
289
+ ```
290
+ {ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
291
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
292
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
293
+ ```
294
+
295
+ **Example:**
296
+
297
+ *Physical indexes:*
298
+ - `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
299
+
300
+ *Aliases:*
301
+ - `items_sentinel-2-l2a` - main collection alias
302
+ - `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
303
+ - `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
304
+
305
+ ### Index Size Management
306
+
307
+ **Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
308
+
233
309
  ## Interacting with the API
234
310
 
235
311
  - **Creating a Collection**:
@@ -538,4 +614,3 @@ You can customize additional settings in your `.env` file:
538
614
  - Ensures fair resource allocation among all clients
539
615
 
540
616
  - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
541
-
@@ -6,7 +6,7 @@ with open("README.md") as f:
6
6
  desc = f.read()
7
7
 
8
8
  install_requires = [
9
- "stac-fastapi.core==6.1.0",
9
+ "stac-fastapi.core==6.2.0",
10
10
  ]
11
11
 
12
12
  setup(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sfeos-helpers
3
- Version: 6.1.0
3
+ Version: 6.2.0
4
4
  Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
5
5
  Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
6
6
  License: MIT
@@ -103,6 +103,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI
103
103
  - [Auth](#auth)
104
104
  - [Aggregation](#aggregation)
105
105
  - [Rate Limiting](#rate-limiting)
106
+ - [Datetime-Based Index Management](#datetime-based-index-management)
106
107
 
107
108
  ## Documentation & Resources
108
109
 
@@ -248,6 +249,81 @@ You can customize additional settings in your `.env` file:
248
249
  > [!NOTE]
249
250
  > The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, `ES_VERIFY_CERTS` and `ES_TIMEOUT` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch.
250
251
 
252
+ ## Datetime-Based Index Management
253
+
254
+ ### Overview
255
+
256
+ SFEOS supports two indexing strategies for managing STAC items:
257
+
258
+ 1. **Simple Indexing** (default) - One index per collection
259
+ 2. **Datetime-Based Indexing** - Time-partitioned indexes with automatic management
260
+
261
+ The datetime-based indexing strategy is particularly useful for large temporal datasets. When a user provides a datetime parameter in a query, the system knows exactly which index to search, providing **multiple times faster searches** and significantly **reducing database load**.
262
+
263
+ ### When to Use
264
+
265
+ **Recommended for:**
266
+ - Systems with large collections containing millions of items
267
+ - Systems requiring high-performance temporal searching
268
+
269
+ **Pros:**
270
+ - Multiple times faster queries with datetime filter
271
+ - Reduced database load - only relevant indexes are searched
272
+
273
+ **Cons:**
274
+ - Slightly longer item indexing time (automatic index management)
275
+ - Greater management complexity
276
+
277
+ ### Configuration
278
+
279
+ #### Enabling Datetime-Based Indexing
280
+
281
+ Enable datetime-based indexing by setting the following environment variable:
282
+
283
+ ```bash
284
+ ENABLE_DATETIME_INDEX_FILTERING=true
285
+ ```
286
+
287
+ ### Related Configuration Variables
288
+
289
+ | Variable | Description | Default | Example |
290
+ |----------|-------------|---------|---------|
291
+ | `ENABLE_DATETIME_INDEX_FILTERING` | Enables time-based index partitioning | `false` | `true` |
292
+ | `DATETIME_INDEX_MAX_SIZE_GB` | Maximum size limit for datetime indexes (GB) - note: add +20% to target size due to ES/OS compression | `25` | `50` |
293
+ | `STAC_ITEMS_INDEX_PREFIX` | Prefix for item indexes | `items_` | `stac_items_` |
294
+
295
+ ## How Datetime-Based Indexing Works
296
+
297
+ ### Index and Alias Naming Convention
298
+
299
+ The system uses a precise naming convention:
300
+
301
+ **Physical indexes:**
302
+ ```
303
+ {ITEMS_INDEX_PREFIX}{collection-id}_{uuid4}
304
+ ```
305
+
306
+ **Aliases:**
307
+ ```
308
+ {ITEMS_INDEX_PREFIX}{collection-id} # Main collection alias
309
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime} # Temporal alias
310
+ {ITEMS_INDEX_PREFIX}{collection-id}_{start-datetime}_{end-datetime} # Closed index alias
311
+ ```
312
+
313
+ **Example:**
314
+
315
+ *Physical indexes:*
316
+ - `items_sentinel-2-l2a_a1b2c3d4-e5f6-7890-abcd-ef1234567890`
317
+
318
+ *Aliases:*
319
+ - `items_sentinel-2-l2a` - main collection alias
320
+ - `items_sentinel-2-l2a_2024-01-01` - active alias from January 1, 2024
321
+ - `items_sentinel-2-l2a_2024-01-01_2024-03-15` - closed index alias (reached size limit)
322
+
323
+ ### Index Size Management
324
+
325
+ **Important - Data Compression:** Elasticsearch and OpenSearch automatically compress data. The configured `DATETIME_INDEX_MAX_SIZE_GB` limit refers to the compressed size on disk. It is recommended to add +20% to the target size to account for compression overhead and metadata.
326
+
251
327
  ## Interacting with the API
252
328
 
253
329
  - **Creating a Collection**:
@@ -556,4 +632,3 @@ You can customize additional settings in your `.env` file:
556
632
  - Ensures fair resource allocation among all clients
557
633
 
558
634
  - **Examples**: Implementation examples are available in the [examples/rate_limit](examples/rate_limit) directory.
559
-
@@ -23,4 +23,15 @@ stac_fastapi/sfeos_helpers/filter/__init__.py
23
23
  stac_fastapi/sfeos_helpers/filter/client.py
24
24
  stac_fastapi/sfeos_helpers/filter/cql2.py
25
25
  stac_fastapi/sfeos_helpers/filter/transform.py
26
- stac_fastapi/sfeos_helpers/models/patch.py
26
+ stac_fastapi/sfeos_helpers/models/patch.py
27
+ stac_fastapi/sfeos_helpers/search_engine/__init__.py
28
+ stac_fastapi/sfeos_helpers/search_engine/base.py
29
+ stac_fastapi/sfeos_helpers/search_engine/factory.py
30
+ stac_fastapi/sfeos_helpers/search_engine/index_operations.py
31
+ stac_fastapi/sfeos_helpers/search_engine/inserters.py
32
+ stac_fastapi/sfeos_helpers/search_engine/managers.py
33
+ stac_fastapi/sfeos_helpers/search_engine/selection/__init__.py
34
+ stac_fastapi/sfeos_helpers/search_engine/selection/base.py
35
+ stac_fastapi/sfeos_helpers/search_engine/selection/cache_manager.py
36
+ stac_fastapi/sfeos_helpers/search_engine/selection/factory.py
37
+ stac_fastapi/sfeos_helpers/search_engine/selection/selectors.py
@@ -0,0 +1 @@
1
+ stac-fastapi.core==6.2.0
@@ -313,9 +313,11 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
313
313
  )
314
314
 
315
315
  if aggregate_request.datetime:
316
- search = self.database.apply_datetime_filter(
317
- search=search, interval=aggregate_request.datetime
316
+ search, datetime_search = self.database.apply_datetime_filter(
317
+ search=search, datetime=aggregate_request.datetime
318
318
  )
319
+ else:
320
+ datetime_search = {"gte": None, "lte": None}
319
321
 
320
322
  if aggregate_request.bbox:
321
323
  bbox = aggregate_request.bbox
@@ -414,6 +416,7 @@ class EsAsyncBaseAggregationClient(AsyncBaseAggregationClient):
414
416
  geometry_geohash_grid_precision,
415
417
  geometry_geotile_grid_precision,
416
418
  datetime_frequency_interval,
419
+ datetime_search,
417
420
  )
418
421
  except Exception as error:
419
422
  if not isinstance(error, IndexError):
@@ -30,11 +30,12 @@ Function Naming Conventions:
30
30
  """
31
31
 
32
32
  # Re-export all functions for backward compatibility
33
- from .datetime import return_date
33
+ from .datetime import extract_date, extract_first_date_from_index, return_date
34
34
  from .document import mk_actions, mk_item_id
35
35
  from .index import (
36
36
  create_index_templates_shared,
37
37
  delete_item_index_shared,
38
+ filter_indexes_by_datetime,
38
39
  index_alias_by_collection_id,
39
40
  index_by_collection_id,
40
41
  indices,
@@ -53,6 +54,7 @@ __all__ = [
53
54
  "delete_item_index_shared",
54
55
  "index_alias_by_collection_id",
55
56
  "index_by_collection_id",
57
+ "filter_indexes_by_datetime",
56
58
  "indices",
57
59
  # Query operations
58
60
  "apply_free_text_filter_shared",
@@ -68,4 +70,6 @@ __all__ = [
68
70
  "get_bool_env",
69
71
  # Datetime utilities
70
72
  "return_date",
73
+ "extract_date",
74
+ "extract_first_date_from_index",
71
75
  ]
@@ -4,14 +4,19 @@ This module provides datetime utility functions specifically designed for
4
4
  Elasticsearch and OpenSearch query formatting.
5
5
  """
6
6
 
7
+ import logging
8
+ import re
9
+ from datetime import date
7
10
  from datetime import datetime as datetime_type
8
11
  from typing import Dict, Optional, Union
9
12
 
10
13
  from stac_fastapi.types.rfc3339 import DateTimeType
11
14
 
15
+ logger = logging.getLogger(__name__)
16
+
12
17
 
13
18
  def return_date(
14
- interval: Optional[Union[DateTimeType, str]]
19
+ interval: Optional[Union[DateTimeType, str]],
15
20
  ) -> Dict[str, Optional[str]]:
16
21
  """
17
22
  Convert a date interval to an Elasticsearch/OpenSearch query format.
@@ -39,8 +44,14 @@ def return_date(
39
44
  if isinstance(interval, str):
40
45
  if "/" in interval:
41
46
  parts = interval.split("/")
42
- result["gte"] = parts[0] if parts[0] != ".." else None
43
- result["lte"] = parts[1] if len(parts) > 1 and parts[1] != ".." else None
47
+ result["gte"] = (
48
+ parts[0] if parts[0] != ".." else datetime_type.min.isoformat() + "Z"
49
+ )
50
+ result["lte"] = (
51
+ parts[1]
52
+ if len(parts) > 1 and parts[1] != ".."
53
+ else datetime_type.max.isoformat() + "Z"
54
+ )
44
55
  else:
45
56
  converted_time = interval if interval != ".." else None
46
57
  result["gte"] = result["lte"] = converted_time
@@ -58,3 +69,53 @@ def return_date(
58
69
  result["lte"] = end.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
59
70
 
60
71
  return result
72
+
73
+
74
+ def extract_date(date_str: str) -> date:
75
+ """Extract date from ISO format string.
76
+
77
+ Args:
78
+ date_str: ISO format date string
79
+
80
+ Returns:
81
+ A date object extracted from the input string.
82
+ """
83
+ date_str = date_str.replace("Z", "+00:00")
84
+ return datetime_type.fromisoformat(date_str).date()
85
+
86
+
87
+ def extract_first_date_from_index(index_name: str) -> date:
88
+ """Extract the first date from an index name containing date patterns.
89
+
90
+ Searches for date patterns (YYYY-MM-DD) within the index name string
91
+ and returns the first found date as a date object.
92
+
93
+ Args:
94
+ index_name: Index name containing date patterns.
95
+
96
+ Returns:
97
+ A date object extracted from the first date pattern found in the index name.
98
+
99
+ """
100
+ date_pattern = r"\d{4}-\d{2}-\d{2}"
101
+ match = re.search(date_pattern, index_name)
102
+
103
+ if not match:
104
+ logger.error(f"No date pattern found in index name: '{index_name}'")
105
+ raise ValueError(
106
+ f"No date pattern (YYYY-MM-DD) found in index name: '{index_name}'"
107
+ )
108
+
109
+ date_string = match.group(0)
110
+
111
+ try:
112
+ extracted_date = datetime_type.strptime(date_string, "%Y-%m-%d").date()
113
+ return extracted_date
114
+ except ValueError as e:
115
+ logger.error(
116
+ f"Invalid date format found in index name '{index_name}': "
117
+ f"'{date_string}' - {str(e)}"
118
+ )
119
+ raise ValueError(
120
+ f"Invalid date format in index name '{index_name}': '{date_string}'"
121
+ ) from e
@@ -3,9 +3,13 @@
3
3
  This module provides functions for creating and managing indices in Elasticsearch/OpenSearch.
4
4
  """
5
5
 
6
+ import re
7
+ from datetime import datetime
6
8
  from functools import lru_cache
7
9
  from typing import Any, List, Optional
8
10
 
11
+ from dateutil.parser import parse # type: ignore[import]
12
+
9
13
  from stac_fastapi.sfeos_helpers.mappings import (
10
14
  _ES_INDEX_NAME_UNSUPPORTED_CHARS_TABLE,
11
15
  COLLECTIONS_INDEX,
@@ -66,6 +70,59 @@ def indices(collection_ids: Optional[List[str]]) -> str:
66
70
  )
67
71
 
68
72
 
73
+ def filter_indexes_by_datetime(
74
+ indexes: List[str], gte: Optional[str], lte: Optional[str]
75
+ ) -> List[str]:
76
+ """Filter indexes based on datetime range extracted from index names.
77
+
78
+ Args:
79
+ indexes: List of index names containing dates
80
+ gte: Greater than or equal date filter (ISO format, optional 'Z' suffix)
81
+ lte: Less than or equal date filter (ISO format, optional 'Z' suffix)
82
+
83
+ Returns:
84
+ List of filtered index names
85
+ """
86
+
87
+ def parse_datetime(dt_str: str) -> datetime:
88
+ """Parse datetime string, handling both with and without 'Z' suffix."""
89
+ return parse(dt_str).replace(tzinfo=None)
90
+
91
+ def extract_date_range_from_index(index_name: str) -> tuple:
92
+ """Extract start and end dates from index name."""
93
+ date_pattern = r"(\d{4}-\d{2}-\d{2})"
94
+ dates = re.findall(date_pattern, index_name)
95
+
96
+ if len(dates) == 1:
97
+ start_date = datetime.strptime(dates[0], "%Y-%m-%d")
98
+ max_date = datetime.max.replace(microsecond=0)
99
+ return start_date, max_date
100
+ else:
101
+ start_date = datetime.strptime(dates[0], "%Y-%m-%d")
102
+ end_date = datetime.strptime(dates[1], "%Y-%m-%d")
103
+ return start_date, end_date
104
+
105
+ def is_index_in_range(
106
+ start_date: datetime, end_date: datetime, gte_dt: datetime, lte_dt: datetime
107
+ ) -> bool:
108
+ """Check if index date range overlaps with filter range."""
109
+ return not (
110
+ end_date.date() < gte_dt.date() or start_date.date() > lte_dt.date()
111
+ )
112
+
113
+ gte_dt = parse_datetime(gte) if gte else datetime.min.replace(microsecond=0)
114
+ lte_dt = parse_datetime(lte) if lte else datetime.max.replace(microsecond=0)
115
+
116
+ filtered_indexes = []
117
+
118
+ for index in indexes:
119
+ start_date, end_date = extract_date_range_from_index(index)
120
+ if is_index_in_range(start_date, end_date, gte_dt, lte_dt):
121
+ filtered_indexes.append(index)
122
+
123
+ return filtered_indexes
124
+
125
+
69
126
  async def create_index_templates_shared(settings: Any) -> None:
70
127
  """Create index templates for Elasticsearch/OpenSearch Collection and Item indices.
71
128
 
@@ -120,11 +177,11 @@ async def delete_item_index_shared(settings: Any, collection_id: str) -> None:
120
177
  client = settings.create_client
121
178
 
122
179
  name = index_alias_by_collection_id(collection_id)
123
- resolved = await client.indices.resolve_index(name=name)
180
+ resolved = await client.indices.resolve_index(name=name, ignore=[404])
124
181
  if "aliases" in resolved and resolved["aliases"]:
125
182
  [alias] = resolved["aliases"]
126
183
  await client.indices.delete_alias(index=alias["indices"], name=alias["name"])
127
184
  await client.indices.delete(index=alias["indices"])
128
185
  else:
129
- await client.indices.delete(index=name)
186
+ await client.indices.delete(index=name, ignore=[404])
130
187
  await client.close()
@@ -0,0 +1,27 @@
1
+ """Search engine index management package."""
2
+
3
+ from .base import BaseIndexInserter
4
+ from .factory import IndexInsertionFactory
5
+ from .index_operations import IndexOperations
6
+ from .inserters import DatetimeIndexInserter, SimpleIndexInserter
7
+ from .managers import DatetimeIndexManager, IndexSizeManager
8
+ from .selection import (
9
+ BaseIndexSelector,
10
+ DatetimeBasedIndexSelector,
11
+ IndexSelectorFactory,
12
+ UnfilteredIndexSelector,
13
+ )
14
+
15
+ __all__ = [
16
+ "BaseIndexInserter",
17
+ "BaseIndexSelector",
18
+ "IndexOperations",
19
+ "IndexSizeManager",
20
+ "DatetimeIndexManager",
21
+ "DatetimeIndexInserter",
22
+ "SimpleIndexInserter",
23
+ "IndexInsertionFactory",
24
+ "DatetimeBasedIndexSelector",
25
+ "UnfilteredIndexSelector",
26
+ "IndexSelectorFactory",
27
+ ]
@@ -0,0 +1,51 @@
1
+ """Base classes for index inserters."""
2
+
3
+ from abc import ABC, abstractmethod
4
+ from typing import Any, Dict, List
5
+
6
+
7
+ class BaseIndexInserter(ABC):
8
+ """Base async index inserter with common async methods."""
9
+
10
+ @abstractmethod
11
+ async def get_target_index(
12
+ self, collection_id: str, product: Dict[str, Any]
13
+ ) -> str:
14
+ """Get target index for a product asynchronously.
15
+
16
+ Args:
17
+ collection_id (str): Collection identifier.
18
+ product (Dict[str, Any]): Product data.
19
+
20
+ Returns:
21
+ str: Target index name.
22
+ """
23
+ pass
24
+
25
+ @abstractmethod
26
+ async def prepare_bulk_actions(
27
+ self, collection_id: str, items: List[Dict[str, Any]]
28
+ ) -> List[Dict[str, Any]]:
29
+ """Prepare bulk actions for multiple items asynchronously.
30
+
31
+ Args:
32
+ collection_id (str): Collection identifier.
33
+ items (List[Dict[str, Any]]): List of items to process.
34
+
35
+ Returns:
36
+ List[Dict[str, Any]]: List of bulk actions.
37
+ """
38
+ pass
39
+
40
+ @abstractmethod
41
+ async def create_simple_index(self, client: Any, collection_id: str) -> str:
42
+ """Create a simple index asynchronously.
43
+
44
+ Args:
45
+ client: Search engine client instance.
46
+ collection_id (str): Collection identifier.
47
+
48
+ Returns:
49
+ str: Created index name.
50
+ """
51
+ pass
@@ -0,0 +1,36 @@
1
+ """Factory for creating index insertion strategies."""
2
+
3
+ from typing import Any
4
+
5
+ from stac_fastapi.core.utilities import get_bool_env
6
+
7
+ from .base import BaseIndexInserter
8
+ from .index_operations import IndexOperations
9
+ from .inserters import DatetimeIndexInserter, SimpleIndexInserter
10
+
11
+
12
+ class IndexInsertionFactory:
13
+ """Factory for creating index insertion strategies."""
14
+
15
+ @staticmethod
16
+ def create_insertion_strategy(
17
+ client: Any,
18
+ ) -> BaseIndexInserter:
19
+ """Create async insertion strategy based on configuration.
20
+
21
+ Args:
22
+ client: Async search engine client instance.
23
+
24
+ Returns:
25
+ BaseIndexInserter: Configured async insertion strategy.
26
+ """
27
+ index_operations = IndexOperations()
28
+
29
+ use_datetime_partitioning = get_bool_env(
30
+ "ENABLE_DATETIME_INDEX_FILTERING", default="false"
31
+ )
32
+
33
+ if use_datetime_partitioning:
34
+ return DatetimeIndexInserter(client, index_operations)
35
+ else:
36
+ return SimpleIndexInserter(index_operations, client)