sfeos-tools 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sfeos-tools
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments
5
5
  Author: CloudFerro S.A.
6
6
  Author-email: Jonathan Healy <jon@healy-hypersaptial.dev>
@@ -35,11 +35,12 @@ Classifier: Development Status :: 4 - Beta
35
35
  Classifier: Intended Audience :: Developers
36
36
  Classifier: Intended Audience :: Science/Research
37
37
  Classifier: License :: OSI Approved :: MIT License
38
- Classifier: Programming Language :: Python :: 3.8
39
38
  Classifier: Programming Language :: Python :: 3.9
40
39
  Classifier: Programming Language :: Python :: 3.10
41
40
  Classifier: Programming Language :: Python :: 3.11
42
41
  Classifier: Programming Language :: Python :: 3.12
42
+ Classifier: Programming Language :: Python :: 3.13
43
+ Classifier: Programming Language :: Python :: 3.14
43
44
  Classifier: Topic :: Scientific/Engineering
44
45
  Classifier: Topic :: Scientific/Engineering :: GIS
45
46
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -47,14 +48,17 @@ Requires-Python: >=3.8
47
48
  Description-Content-Type: text/markdown
48
49
  License-File: LICENSE
49
50
  Requires-Dist: click>=8.0.0
51
+ Requires-Dist: httpx>=0.24.0
52
+ Requires-Dist: orjson>=3.0.0
50
53
  Provides-Extra: elasticsearch
51
- Requires-Dist: stac-fastapi-elasticsearch; extra == "elasticsearch"
54
+ Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "elasticsearch"
52
55
  Provides-Extra: opensearch
53
- Requires-Dist: stac-fastapi-opensearch; extra == "opensearch"
56
+ Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "opensearch"
54
57
  Provides-Extra: dev
55
- Requires-Dist: stac-fastapi-elasticsearch; extra == "dev"
56
- Requires-Dist: stac-fastapi-opensearch; extra == "dev"
58
+ Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "dev"
59
+ Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "dev"
57
60
  Requires-Dist: pytest>=6.0; extra == "dev"
61
+ Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
58
62
  Requires-Dist: pytest-cov>=2.0; extra == "dev"
59
63
  Requires-Dist: black>=21.0; extra == "dev"
60
64
  Requires-Dist: isort>=5.0; extra == "dev"
@@ -66,6 +70,22 @@ Dynamic: license-file
66
70
 
67
71
  CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
68
72
 
73
+ <!-- markdownlint-disable MD033 MD041 -->
74
+
75
+
76
+ <p align="left">
77
+ <img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
78
+ </p>
79
+
80
+ <!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
81
+
82
+ [![Downloads](https://static.pepy.tech/badge/sfeos-tools?color=blue)](https://pepy.tech/project/sfeos-tools)
83
+ [![GitHub contributors](https://img.shields.io/github/contributors/healy-hyperspatial/sfeos-tools?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
84
+ [![GitHub stars](https://img.shields.io/github/stars/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
85
+ [![GitHub forks](https://img.shields.io/github/forks/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
86
+ [![PyPI version](https://img.shields.io/pypi/v/sfeos-tools.svg?color=blue)](https://pypi.org/project/sfeos-tools/)
87
+ [![STAC](https://img.shields.io/badge/STAC-1.1.0-blue.svg)](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
88
+
69
89
  ## Table of Contents
70
90
 
71
91
  - [Installation](#installation)
@@ -76,6 +96,7 @@ CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.co
76
96
  - [Commands](#commands)
77
97
  - [add-bbox-shape](#add-bbox-shape)
78
98
  - [reindex](#reindex)
99
+ - [load-data](#load-data)
79
100
  - [Development](#development)
80
101
  - [License](#license)
81
102
 
@@ -165,62 +186,55 @@ Options:
165
186
  - `--password`: Database password (default: ES_PASS env var)
166
187
  - `--yes`: Skip confirmation prompt
167
188
 
168
- Example:
189
+ Examples:
169
190
  ```bash
170
191
  # Reindex Elasticsearch with custom host and no SSL
171
- sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl
192
+ sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl --yes
172
193
 
173
194
  # Reindex OpenSearch with default settings
174
- sfeos-tools reindex --backend opensearch
195
+ sfeos-tools reindex --backend opensearch --yes
175
196
  ```
176
197
 
177
- # Get help for a specific command
178
- sfeos-tools add-bbox-shape --help
179
- ```
180
-
181
- ## Commands
182
-
183
- ### add-bbox-shape
184
-
185
- Add `bbox_shape` field to existing collections for spatial search support.
198
+ ### load-data
186
199
 
187
- **Basic usage:**
200
+ Load STAC collections and items from local JSON files into a STAC API instance. This command is useful for:
201
+ - Populating a new STAC API deployment with test data
202
+ - Migrating data between STAC API instances
203
+ - Bulk loading STAC collections and items
188
204
 
189
205
  ```bash
190
- # Elasticsearch
191
- sfeos-tools add-bbox-shape --backend elasticsearch
192
-
193
- # OpenSearch
194
- sfeos-tools add-bbox-shape --backend opensearch
206
+ sfeos-tools load-data --base-url <stac-api-url> [options]
195
207
  ```
196
208
 
197
- **Connection options:**
209
+ Options:
210
+ - `--base-url`: Base URL of the STAC API (required)
211
+ - `--collection-id`: ID of the collection to create/update (default: test-collection)
212
+ - `--data-dir`: Directory containing collection.json and feature collection files (default: sample_data/)
213
+ - `--use-bulk`: Use bulk insert method for items (faster for large datasets)
198
214
 
199
- ```bash
200
- # Local Docker Compose (no SSL)
201
- sfeos-tools add-bbox-shape --backend elasticsearch --no-ssl
202
-
203
- # Remote server with SSL
204
- sfeos-tools add-bbox-shape \
205
- --backend elasticsearch \
206
- --host db.example.com \
207
- --port 9200 \
208
- --user admin \
209
- --password secret
210
-
211
- # Using environment variables
212
- ES_HOST=my-cluster.cloud.com ES_PORT=9243 ES_USER=elastic ES_PASS=changeme \
213
- sfeos-tools add-bbox-shape --backend elasticsearch
214
- ```
215
+ **Data Directory Structure:**
215
216
 
216
- **Available options:**
217
+ Your data directory should contain:
218
+ - `collection.json`: STAC collection definition
219
+ - One or more `.json` files: Feature collections with STAC items
217
220
 
218
- - `--backend`: Database backend (elasticsearch or opensearch) - **required**
219
- - `--host`: Database host (default: localhost or ES_HOST env var)
220
- - `--port`: Database port (default: 9200 or ES_PORT env var)
221
- - `--use-ssl / --no-ssl`: Use SSL connection (default: true or ES_USE_SSL env var)
222
- - `--user`: Database username (default: ES_USER env var)
223
- - `--password`: Database password (default: ES_PASS env var)
221
+ Examples:
222
+ ```bash
223
+ # Load data from default directory
224
+ sfeos-tools load-data --base-url http://localhost:8080
225
+
226
+ # Load with custom collection ID and bulk insert
227
+ sfeos-tools load-data \
228
+ --base-url http://localhost:8080 \
229
+ --collection-id my-collection \
230
+ --use-bulk
231
+
232
+ # Load from custom directory
233
+ sfeos-tools load-data \
234
+ --base-url http://localhost:8080 \
235
+ --data-dir /path/to/stac/data \
236
+ --collection-id production-data
237
+ ```
224
238
 
225
239
  ## Development
226
240
 
@@ -241,3 +255,6 @@ pre-commit install
241
255
  pre-commit run --all-files
242
256
  ```
243
257
 
258
+ ## License
259
+
260
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -2,6 +2,22 @@
2
2
 
3
3
  CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
4
4
 
5
+ <!-- markdownlint-disable MD033 MD041 -->
6
+
7
+
8
+ <p align="left">
9
+ <img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
10
+ </p>
11
+
12
+ <!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
13
+
14
+ [![Downloads](https://static.pepy.tech/badge/sfeos-tools?color=blue)](https://pepy.tech/project/sfeos-tools)
15
+ [![GitHub contributors](https://img.shields.io/github/contributors/healy-hyperspatial/sfeos-tools?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
16
+ [![GitHub stars](https://img.shields.io/github/stars/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
17
+ [![GitHub forks](https://img.shields.io/github/forks/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
18
+ [![PyPI version](https://img.shields.io/pypi/v/sfeos-tools.svg?color=blue)](https://pypi.org/project/sfeos-tools/)
19
+ [![STAC](https://img.shields.io/badge/STAC-1.1.0-blue.svg)](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
20
+
5
21
  ## Table of Contents
6
22
 
7
23
  - [Installation](#installation)
@@ -12,6 +28,7 @@ CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.co
12
28
  - [Commands](#commands)
13
29
  - [add-bbox-shape](#add-bbox-shape)
14
30
  - [reindex](#reindex)
31
+ - [load-data](#load-data)
15
32
  - [Development](#development)
16
33
  - [License](#license)
17
34
 
@@ -101,62 +118,55 @@ Options:
101
118
  - `--password`: Database password (default: ES_PASS env var)
102
119
  - `--yes`: Skip confirmation prompt
103
120
 
104
- Example:
121
+ Examples:
105
122
  ```bash
106
123
  # Reindex Elasticsearch with custom host and no SSL
107
- sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl
124
+ sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl --yes
108
125
 
109
126
  # Reindex OpenSearch with default settings
110
- sfeos-tools reindex --backend opensearch
127
+ sfeos-tools reindex --backend opensearch --yes
111
128
  ```
112
129
 
113
- # Get help for a specific command
114
- sfeos-tools add-bbox-shape --help
115
- ```
116
-
117
- ## Commands
118
-
119
- ### add-bbox-shape
120
-
121
- Add `bbox_shape` field to existing collections for spatial search support.
130
+ ### load-data
122
131
 
123
- **Basic usage:**
132
+ Load STAC collections and items from local JSON files into a STAC API instance. This command is useful for:
133
+ - Populating a new STAC API deployment with test data
134
+ - Migrating data between STAC API instances
135
+ - Bulk loading STAC collections and items
124
136
 
125
137
  ```bash
126
- # Elasticsearch
127
- sfeos-tools add-bbox-shape --backend elasticsearch
128
-
129
- # OpenSearch
130
- sfeos-tools add-bbox-shape --backend opensearch
138
+ sfeos-tools load-data --base-url <stac-api-url> [options]
131
139
  ```
132
140
 
133
- **Connection options:**
141
+ Options:
142
+ - `--base-url`: Base URL of the STAC API (required)
143
+ - `--collection-id`: ID of the collection to create/update (default: test-collection)
144
+ - `--data-dir`: Directory containing collection.json and feature collection files (default: sample_data/)
145
+ - `--use-bulk`: Use bulk insert method for items (faster for large datasets)
134
146
 
135
- ```bash
136
- # Local Docker Compose (no SSL)
137
- sfeos-tools add-bbox-shape --backend elasticsearch --no-ssl
138
-
139
- # Remote server with SSL
140
- sfeos-tools add-bbox-shape \
141
- --backend elasticsearch \
142
- --host db.example.com \
143
- --port 9200 \
144
- --user admin \
145
- --password secret
146
-
147
- # Using environment variables
148
- ES_HOST=my-cluster.cloud.com ES_PORT=9243 ES_USER=elastic ES_PASS=changeme \
149
- sfeos-tools add-bbox-shape --backend elasticsearch
150
- ```
147
+ **Data Directory Structure:**
151
148
 
152
- **Available options:**
149
+ Your data directory should contain:
150
+ - `collection.json`: STAC collection definition
151
+ - One or more `.json` files: Feature collections with STAC items
153
152
 
154
- - `--backend`: Database backend (elasticsearch or opensearch) - **required**
155
- - `--host`: Database host (default: localhost or ES_HOST env var)
156
- - `--port`: Database port (default: 9200 or ES_PORT env var)
157
- - `--use-ssl / --no-ssl`: Use SSL connection (default: true or ES_USE_SSL env var)
158
- - `--user`: Database username (default: ES_USER env var)
159
- - `--password`: Database password (default: ES_PASS env var)
153
+ Examples:
154
+ ```bash
155
+ # Load data from default directory
156
+ sfeos-tools load-data --base-url http://localhost:8080
157
+
158
+ # Load with custom collection ID and bulk insert
159
+ sfeos-tools load-data \
160
+ --base-url http://localhost:8080 \
161
+ --collection-id my-collection \
162
+ --use-bulk
163
+
164
+ # Load from custom directory
165
+ sfeos-tools load-data \
166
+ --base-url http://localhost:8080 \
167
+ --data-dir /path/to/stac/data \
168
+ --collection-id production-data
169
+ ```
160
170
 
161
171
  ## Development
162
172
 
@@ -177,3 +187,6 @@ pre-commit install
177
187
  pre-commit run --all-files
178
188
  ```
179
189
 
190
+ ## License
191
+
192
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "sfeos-tools"
7
- version = "0.1.0"
7
+ version = "0.2.0"
8
8
  description = "CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments"
9
9
  readme = "README.md"
10
10
  authors = [
@@ -21,17 +21,20 @@ classifiers = [
21
21
  "Intended Audience :: Developers",
22
22
  "Intended Audience :: Science/Research",
23
23
  "License :: OSI Approved :: MIT License",
24
- "Programming Language :: Python :: 3.8",
25
24
  "Programming Language :: Python :: 3.9",
26
25
  "Programming Language :: Python :: 3.10",
27
26
  "Programming Language :: Python :: 3.11",
28
27
  "Programming Language :: Python :: 3.12",
28
+ "Programming Language :: Python :: 3.13",
29
+ "Programming Language :: Python :: 3.14",
29
30
  "Topic :: Scientific/Engineering",
30
31
  "Topic :: Scientific/Engineering :: GIS",
31
32
  "Topic :: Software Development :: Libraries :: Python Modules"
32
33
  ]
33
34
  dependencies = [
34
35
  "click>=8.0.0",
36
+ "httpx>=0.24.0",
37
+ "orjson>=3.0.0",
35
38
  ]
36
39
 
37
40
  [project.urls]
@@ -42,16 +45,17 @@ Source = "https://github.com/Healy-Hyperspatial/sfeos-tools"
42
45
 
43
46
  [project.optional-dependencies]
44
47
  elasticsearch = [
45
- "stac-fastapi-elasticsearch",
48
+ "stac-fastapi-elasticsearch>=6.6.0",
46
49
  ]
47
50
  opensearch = [
48
- "stac-fastapi-opensearch",
51
+ "stac-fastapi-opensearch>=6.6.0",
49
52
  ]
50
53
  dev = [
51
- "stac-fastapi-elasticsearch",
52
- "stac-fastapi-opensearch",
54
+ "stac-fastapi-elasticsearch>=6.6.0",
55
+ "stac-fastapi-opensearch>=6.6.0",
53
56
  # Development tools
54
57
  "pytest>=6.0",
58
+ "pytest-asyncio>=0.21.0",
55
59
  "pytest-cov>=2.0",
56
60
  "black>=21.0",
57
61
  "isort>=5.0",
@@ -79,7 +83,13 @@ line_length = 88
79
83
  [tool.pytest.ini_options]
80
84
  testpaths = ["tests"]
81
85
  python_files = "test_*.py"
82
- addopts = "-v --cov=sfeos_tool --cov-report=term-missing"
86
+ addopts = "-v --cov=sfeos_tools --cov-report=term-missing"
87
+ asyncio_mode = "auto"
88
+ filterwarnings = [
89
+ "ignore::DeprecationWarning:pydantic._internal._config",
90
+ "ignore::pydantic.warnings.PydanticDeprecatedSince20",
91
+ "ignore:coroutine.*was never awaited:RuntimeWarning",
92
+ ]
83
93
 
84
94
  [tool.coverage.report]
85
95
  exclude_lines = [
@@ -0,0 +1,114 @@
1
+ """Bbox shape migration utilities for SFEOS collections."""
2
+
3
+ import logging
4
+
5
+ from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
6
+ from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+
11
+ async def process_collection_bbox_shape(client, collection_doc, backend):
12
+ """Process a single collection document to add bbox_shape field.
13
+
14
+ Args:
15
+ client: Elasticsearch/OpenSearch client
16
+ collection_doc: Collection document from database
17
+ backend: Backend type ('elasticsearch' or 'opensearch')
18
+
19
+ Returns:
20
+ bool: True if collection was updated, False if no update was needed
21
+ """
22
+ collection = collection_doc["_source"]
23
+ collection_id = collection.get("id", collection_doc["_id"])
24
+
25
+ # Use the shared function to add bbox_shape
26
+ was_added = add_bbox_shape_to_collection(collection)
27
+
28
+ if not was_added:
29
+ return False
30
+
31
+ # Update the collection in the database
32
+ if backend == "elasticsearch":
33
+ await client.index(
34
+ index=COLLECTIONS_INDEX,
35
+ id=collection_id,
36
+ document=collection,
37
+ refresh=True,
38
+ )
39
+ else: # opensearch
40
+ await client.index(
41
+ index=COLLECTIONS_INDEX,
42
+ id=collection_id,
43
+ body=collection,
44
+ refresh=True,
45
+ )
46
+
47
+ logger.info(f"Collection '{collection_id}': Added bbox_shape field")
48
+ return True
49
+
50
+
51
+ async def run_add_bbox_shape(backend):
52
+ """Add bbox_shape field to all existing collections.
53
+
54
+ Args:
55
+ backend: Backend type ('elasticsearch' or 'opensearch')
56
+ """
57
+ import os
58
+
59
+ logger.info(
60
+ f"Starting migration: Adding bbox_shape to existing collections ({backend})"
61
+ )
62
+
63
+ # Log connection info (showing what will be used by the client)
64
+ es_host = os.getenv("ES_HOST", "localhost")
65
+ es_port = os.getenv(
66
+ "ES_PORT", "9200"
67
+ ) # Both backends default to 9200 in their config
68
+ es_use_ssl = os.getenv("ES_USE_SSL", "true")
69
+ logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
70
+
71
+ # Create client based on backend
72
+ if backend == "elasticsearch":
73
+ from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
74
+
75
+ settings = AsyncElasticsearchSettings()
76
+ else: # opensearch
77
+ from stac_fastapi.opensearch.config import AsyncOpensearchSettings
78
+
79
+ settings = AsyncOpensearchSettings()
80
+
81
+ client = settings.create_client
82
+
83
+ try:
84
+ # Get all collections
85
+ response = await client.search(
86
+ index=COLLECTIONS_INDEX,
87
+ body={
88
+ "query": {"match_all": {}},
89
+ "size": 10000,
90
+ }, # Adjust size if you have more collections
91
+ )
92
+
93
+ total_collections = response["hits"]["total"]["value"]
94
+ logger.info(f"Found {total_collections} collections to process")
95
+
96
+ updated_count = 0
97
+ skipped_count = 0
98
+
99
+ for hit in response["hits"]["hits"]:
100
+ was_updated = await process_collection_bbox_shape(client, hit, backend)
101
+ if was_updated:
102
+ updated_count += 1
103
+ else:
104
+ skipped_count += 1
105
+
106
+ logger.info(
107
+ f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
108
+ )
109
+
110
+ except Exception as e:
111
+ logger.error(f"Migration failed with error: {e}")
112
+ raise
113
+ finally:
114
+ await client.close()
@@ -13,123 +13,24 @@ import logging
13
13
  import sys
14
14
 
15
15
  import click
16
- from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
17
- from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
18
16
 
17
+ try:
18
+ from importlib.metadata import version as _get_version
19
+ except ImportError:
20
+ from importlib_metadata import version as _get_version # type: ignore[no-redef]
21
+
22
+ __version__ = _get_version("sfeos-tools")
23
+
24
+ from .bbox_shape import run_add_bbox_shape
25
+ from .data_loader import load_items
19
26
  from .reindex import run as unified_reindex_run
20
27
 
21
28
  logging.basicConfig(level=logging.INFO)
22
29
  logger = logging.getLogger(__name__)
23
30
 
24
31
 
25
- async def process_collection_bbox_shape(client, collection_doc, backend):
26
- """Process a single collection document to add bbox_shape field.
27
-
28
- Args:
29
- client: Elasticsearch/OpenSearch client
30
- collection_doc: Collection document from database
31
- backend: Backend type ('elasticsearch' or 'opensearch')
32
-
33
- Returns:
34
- bool: True if collection was updated, False if no update was needed
35
- """
36
- collection = collection_doc["_source"]
37
- collection_id = collection.get("id", collection_doc["_id"])
38
-
39
- # Use the shared function to add bbox_shape
40
- was_added = add_bbox_shape_to_collection(collection)
41
-
42
- if not was_added:
43
- return False
44
-
45
- # Update the collection in the database
46
- if backend == "elasticsearch":
47
- await client.index(
48
- index=COLLECTIONS_INDEX,
49
- id=collection_id,
50
- document=collection,
51
- refresh=True,
52
- )
53
- else: # opensearch
54
- await client.index(
55
- index=COLLECTIONS_INDEX,
56
- id=collection_id,
57
- body=collection,
58
- refresh=True,
59
- )
60
-
61
- logger.info(f"Collection '{collection_id}': Added bbox_shape field")
62
- return True
63
-
64
-
65
- async def run_add_bbox_shape(backend):
66
- """Add bbox_shape field to all existing collections.
67
-
68
- Args:
69
- backend: Backend type ('elasticsearch' or 'opensearch')
70
- """
71
- import os
72
-
73
- logger.info(
74
- f"Starting migration: Adding bbox_shape to existing collections ({backend})"
75
- )
76
-
77
- # Log connection info (showing what will be used by the client)
78
- es_host = os.getenv("ES_HOST", "localhost")
79
- es_port = os.getenv(
80
- "ES_PORT", "9200"
81
- ) # Both backends default to 9200 in their config
82
- es_use_ssl = os.getenv("ES_USE_SSL", "true")
83
- logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
84
-
85
- # Create client based on backend
86
- if backend == "elasticsearch":
87
- from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
88
-
89
- settings = AsyncElasticsearchSettings()
90
- else: # opensearch
91
- from stac_fastapi.opensearch.config import AsyncOpensearchSettings
92
-
93
- settings = AsyncOpensearchSettings()
94
-
95
- client = settings.create_client
96
-
97
- try:
98
- # Get all collections
99
- response = await client.search(
100
- index=COLLECTIONS_INDEX,
101
- body={
102
- "query": {"match_all": {}},
103
- "size": 10000,
104
- }, # Adjust size if you have more collections
105
- )
106
-
107
- total_collections = response["hits"]["total"]["value"]
108
- logger.info(f"Found {total_collections} collections to process")
109
-
110
- updated_count = 0
111
- skipped_count = 0
112
-
113
- for hit in response["hits"]["hits"]:
114
- was_updated = await process_collection_bbox_shape(client, hit, backend)
115
- if was_updated:
116
- updated_count += 1
117
- else:
118
- skipped_count += 1
119
-
120
- logger.info(
121
- f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
122
- )
123
-
124
- except Exception as e:
125
- logger.error(f"Migration failed with error: {e}")
126
- raise
127
- finally:
128
- await client.close()
129
-
130
-
131
32
  @click.group()
132
- @click.version_option(version="0.1.0", prog_name="sfeos-tools")
33
+ @click.version_option(version=__version__, prog_name="sfeos-tools")
133
34
  def cli():
134
35
  """SFEOS Tools - Utilities for managing stac-fastapi-elasticsearch-opensearch deployments."""
135
36
  pass
@@ -332,5 +233,47 @@ def reindex(backend, host, port, use_ssl, user, password, yes):
332
233
  sys.exit(1)
333
234
 
334
235
 
236
+ @cli.command("load-data")
237
+ @click.option("--base-url", required=True, help="Base URL of the STAC API")
238
+ @click.option(
239
+ "--collection-id",
240
+ default="test-collection",
241
+ help="ID of the collection to which items are added",
242
+ )
243
+ @click.option("--use-bulk", is_flag=True, help="Use bulk insert method for items")
244
+ @click.option(
245
+ "--data-dir",
246
+ type=click.Path(exists=True),
247
+ default="sample_data/",
248
+ help="Directory containing collection.json and feature collection file",
249
+ )
250
+ def load_data(base_url: str, collection_id: str, use_bulk: bool, data_dir: str) -> None:
251
+ """Load STAC items into the database via STAC API.
252
+
253
+ This command loads a STAC collection and its items from local JSON files
254
+ into a STAC API instance. It expects a directory containing:
255
+ - collection.json: The STAC collection definition
256
+ - One or more feature collection JSON files with STAC items
257
+
258
+ Examples:
259
+ sfeos-tools load-data --base-url http://localhost:8080
260
+ sfeos-tools load-data --base-url http://localhost:8080 --collection-id my-collection --use-bulk
261
+ sfeos-tools load-data --base-url http://localhost:8080 --data-dir /path/to/data
262
+ """
263
+ from httpx import Client
264
+
265
+ try:
266
+ with Client(base_url=base_url) as client:
267
+ load_items(client, collection_id, use_bulk, data_dir)
268
+ click.echo(click.style("✓ Data loading completed successfully", fg="green"))
269
+ except KeyboardInterrupt:
270
+ click.echo(click.style("\n✗ Data loading interrupted by user", fg="yellow"))
271
+ sys.exit(1)
272
+ except Exception as e:
273
+ error_msg = str(e)
274
+ click.echo(click.style(f"✗ Data loading failed: {error_msg}", fg="red"))
275
+ sys.exit(1)
276
+
277
+
335
278
  if __name__ == "__main__":
336
279
  cli()