sfeos-tools 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sfeos_tools-0.1.0/sfeos_tools.egg-info → sfeos_tools-0.2.0}/PKG-INFO +65 -48
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/README.md +55 -42
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/pyproject.toml +17 -7
- sfeos_tools-0.2.0/sfeos_tools/bbox_shape.py +114 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools/cli.py +52 -109
- sfeos_tools-0.2.0/sfeos_tools/data_loader.py +102 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0/sfeos_tools.egg-info}/PKG-INFO +65 -48
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools.egg-info/SOURCES.txt +6 -1
- sfeos_tools-0.2.0/sfeos_tools.egg-info/requires.txt +20 -0
- sfeos_tools-0.2.0/tests/test_bbox_shape.py +525 -0
- sfeos_tools-0.2.0/tests/test_data_loader.py +386 -0
- sfeos_tools-0.2.0/tests/test_reindex.py +566 -0
- sfeos_tools-0.1.0/sfeos_tools.egg-info/requires.txt +0 -17
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/LICENSE +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/setup.cfg +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools/__init__.py +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools/reindex.py +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools.egg-info/dependency_links.txt +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools.egg-info/entry_points.txt +0 -0
- {sfeos_tools-0.1.0 → sfeos_tools-0.2.0}/sfeos_tools.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: sfeos-tools
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.2.0
|
4
4
|
Summary: CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments
|
5
5
|
Author: CloudFerro S.A.
|
6
6
|
Author-email: Jonathan Healy <jon@healy-hypersaptial.dev>
|
@@ -35,11 +35,12 @@ Classifier: Development Status :: 4 - Beta
|
|
35
35
|
Classifier: Intended Audience :: Developers
|
36
36
|
Classifier: Intended Audience :: Science/Research
|
37
37
|
Classifier: License :: OSI Approved :: MIT License
|
38
|
-
Classifier: Programming Language :: Python :: 3.8
|
39
38
|
Classifier: Programming Language :: Python :: 3.9
|
40
39
|
Classifier: Programming Language :: Python :: 3.10
|
41
40
|
Classifier: Programming Language :: Python :: 3.11
|
42
41
|
Classifier: Programming Language :: Python :: 3.12
|
42
|
+
Classifier: Programming Language :: Python :: 3.13
|
43
|
+
Classifier: Programming Language :: Python :: 3.14
|
43
44
|
Classifier: Topic :: Scientific/Engineering
|
44
45
|
Classifier: Topic :: Scientific/Engineering :: GIS
|
45
46
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
@@ -47,14 +48,17 @@ Requires-Python: >=3.8
|
|
47
48
|
Description-Content-Type: text/markdown
|
48
49
|
License-File: LICENSE
|
49
50
|
Requires-Dist: click>=8.0.0
|
51
|
+
Requires-Dist: httpx>=0.24.0
|
52
|
+
Requires-Dist: orjson>=3.0.0
|
50
53
|
Provides-Extra: elasticsearch
|
51
|
-
Requires-Dist: stac-fastapi-elasticsearch; extra == "elasticsearch"
|
54
|
+
Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "elasticsearch"
|
52
55
|
Provides-Extra: opensearch
|
53
|
-
Requires-Dist: stac-fastapi-opensearch; extra == "opensearch"
|
56
|
+
Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "opensearch"
|
54
57
|
Provides-Extra: dev
|
55
|
-
Requires-Dist: stac-fastapi-elasticsearch; extra == "dev"
|
56
|
-
Requires-Dist: stac-fastapi-opensearch; extra == "dev"
|
58
|
+
Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "dev"
|
59
|
+
Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "dev"
|
57
60
|
Requires-Dist: pytest>=6.0; extra == "dev"
|
61
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
58
62
|
Requires-Dist: pytest-cov>=2.0; extra == "dev"
|
59
63
|
Requires-Dist: black>=21.0; extra == "dev"
|
60
64
|
Requires-Dist: isort>=5.0; extra == "dev"
|
@@ -66,6 +70,22 @@ Dynamic: license-file
|
|
66
70
|
|
67
71
|
CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
|
68
72
|
|
73
|
+
<!-- markdownlint-disable MD033 MD041 -->
|
74
|
+
|
75
|
+
|
76
|
+
<p align="left">
|
77
|
+
<img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
|
78
|
+
</p>
|
79
|
+
|
80
|
+
<!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
|
81
|
+
|
82
|
+
[](https://pepy.tech/project/sfeos-tools)
|
83
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
|
84
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
|
85
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
|
86
|
+
[](https://pypi.org/project/sfeos-tools/)
|
87
|
+
[](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
|
88
|
+
|
69
89
|
## Table of Contents
|
70
90
|
|
71
91
|
- [Installation](#installation)
|
@@ -76,6 +96,7 @@ CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.co
|
|
76
96
|
- [Commands](#commands)
|
77
97
|
- [add-bbox-shape](#add-bbox-shape)
|
78
98
|
- [reindex](#reindex)
|
99
|
+
- [load-data](#load-data)
|
79
100
|
- [Development](#development)
|
80
101
|
- [License](#license)
|
81
102
|
|
@@ -165,62 +186,55 @@ Options:
|
|
165
186
|
- `--password`: Database password (default: ES_PASS env var)
|
166
187
|
- `--yes`: Skip confirmation prompt
|
167
188
|
|
168
|
-
|
189
|
+
Examples:
|
169
190
|
```bash
|
170
191
|
# Reindex Elasticsearch with custom host and no SSL
|
171
|
-
sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl
|
192
|
+
sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl --yes
|
172
193
|
|
173
194
|
# Reindex OpenSearch with default settings
|
174
|
-
sfeos-tools reindex --backend opensearch
|
195
|
+
sfeos-tools reindex --backend opensearch --yes
|
175
196
|
```
|
176
197
|
|
177
|
-
|
178
|
-
sfeos-tools add-bbox-shape --help
|
179
|
-
```
|
180
|
-
|
181
|
-
## Commands
|
182
|
-
|
183
|
-
### add-bbox-shape
|
184
|
-
|
185
|
-
Add `bbox_shape` field to existing collections for spatial search support.
|
198
|
+
### load-data
|
186
199
|
|
187
|
-
|
200
|
+
Load STAC collections and items from local JSON files into a STAC API instance. This command is useful for:
|
201
|
+
- Populating a new STAC API deployment with test data
|
202
|
+
- Migrating data between STAC API instances
|
203
|
+
- Bulk loading STAC collections and items
|
188
204
|
|
189
205
|
```bash
|
190
|
-
|
191
|
-
sfeos-tools add-bbox-shape --backend elasticsearch
|
192
|
-
|
193
|
-
# OpenSearch
|
194
|
-
sfeos-tools add-bbox-shape --backend opensearch
|
206
|
+
sfeos-tools load-data --base-url <stac-api-url> [options]
|
195
207
|
```
|
196
208
|
|
197
|
-
|
209
|
+
Options:
|
210
|
+
- `--base-url`: Base URL of the STAC API (required)
|
211
|
+
- `--collection-id`: ID of the collection to create/update (default: test-collection)
|
212
|
+
- `--data-dir`: Directory containing collection.json and feature collection files (default: sample_data/)
|
213
|
+
- `--use-bulk`: Use bulk insert method for items (faster for large datasets)
|
198
214
|
|
199
|
-
|
200
|
-
# Local Docker Compose (no SSL)
|
201
|
-
sfeos-tools add-bbox-shape --backend elasticsearch --no-ssl
|
202
|
-
|
203
|
-
# Remote server with SSL
|
204
|
-
sfeos-tools add-bbox-shape \
|
205
|
-
--backend elasticsearch \
|
206
|
-
--host db.example.com \
|
207
|
-
--port 9200 \
|
208
|
-
--user admin \
|
209
|
-
--password secret
|
210
|
-
|
211
|
-
# Using environment variables
|
212
|
-
ES_HOST=my-cluster.cloud.com ES_PORT=9243 ES_USER=elastic ES_PASS=changeme \
|
213
|
-
sfeos-tools add-bbox-shape --backend elasticsearch
|
214
|
-
```
|
215
|
+
**Data Directory Structure:**
|
215
216
|
|
216
|
-
|
217
|
+
Your data directory should contain:
|
218
|
+
- `collection.json`: STAC collection definition
|
219
|
+
- One or more `.json` files: Feature collections with STAC items
|
217
220
|
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
-
|
222
|
-
|
223
|
-
|
221
|
+
Examples:
|
222
|
+
```bash
|
223
|
+
# Load data from default directory
|
224
|
+
sfeos-tools load-data --base-url http://localhost:8080
|
225
|
+
|
226
|
+
# Load with custom collection ID and bulk insert
|
227
|
+
sfeos-tools load-data \
|
228
|
+
--base-url http://localhost:8080 \
|
229
|
+
--collection-id my-collection \
|
230
|
+
--use-bulk
|
231
|
+
|
232
|
+
# Load from custom directory
|
233
|
+
sfeos-tools load-data \
|
234
|
+
--base-url http://localhost:8080 \
|
235
|
+
--data-dir /path/to/stac/data \
|
236
|
+
--collection-id production-data
|
237
|
+
```
|
224
238
|
|
225
239
|
## Development
|
226
240
|
|
@@ -241,3 +255,6 @@ pre-commit install
|
|
241
255
|
pre-commit run --all-files
|
242
256
|
```
|
243
257
|
|
258
|
+
## License
|
259
|
+
|
260
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
@@ -2,6 +2,22 @@
|
|
2
2
|
|
3
3
|
CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
|
4
4
|
|
5
|
+
<!-- markdownlint-disable MD033 MD041 -->
|
6
|
+
|
7
|
+
|
8
|
+
<p align="left">
|
9
|
+
<img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
|
10
|
+
</p>
|
11
|
+
|
12
|
+
<!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
|
13
|
+
|
14
|
+
[](https://pepy.tech/project/sfeos-tools)
|
15
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
|
16
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
|
17
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
|
18
|
+
[](https://pypi.org/project/sfeos-tools/)
|
19
|
+
[](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
|
20
|
+
|
5
21
|
## Table of Contents
|
6
22
|
|
7
23
|
- [Installation](#installation)
|
@@ -12,6 +28,7 @@ CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.co
|
|
12
28
|
- [Commands](#commands)
|
13
29
|
- [add-bbox-shape](#add-bbox-shape)
|
14
30
|
- [reindex](#reindex)
|
31
|
+
- [load-data](#load-data)
|
15
32
|
- [Development](#development)
|
16
33
|
- [License](#license)
|
17
34
|
|
@@ -101,62 +118,55 @@ Options:
|
|
101
118
|
- `--password`: Database password (default: ES_PASS env var)
|
102
119
|
- `--yes`: Skip confirmation prompt
|
103
120
|
|
104
|
-
|
121
|
+
Examples:
|
105
122
|
```bash
|
106
123
|
# Reindex Elasticsearch with custom host and no SSL
|
107
|
-
sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl
|
124
|
+
sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl --yes
|
108
125
|
|
109
126
|
# Reindex OpenSearch with default settings
|
110
|
-
sfeos-tools reindex --backend opensearch
|
127
|
+
sfeos-tools reindex --backend opensearch --yes
|
111
128
|
```
|
112
129
|
|
113
|
-
|
114
|
-
sfeos-tools add-bbox-shape --help
|
115
|
-
```
|
116
|
-
|
117
|
-
## Commands
|
118
|
-
|
119
|
-
### add-bbox-shape
|
120
|
-
|
121
|
-
Add `bbox_shape` field to existing collections for spatial search support.
|
130
|
+
### load-data
|
122
131
|
|
123
|
-
|
132
|
+
Load STAC collections and items from local JSON files into a STAC API instance. This command is useful for:
|
133
|
+
- Populating a new STAC API deployment with test data
|
134
|
+
- Migrating data between STAC API instances
|
135
|
+
- Bulk loading STAC collections and items
|
124
136
|
|
125
137
|
```bash
|
126
|
-
|
127
|
-
sfeos-tools add-bbox-shape --backend elasticsearch
|
128
|
-
|
129
|
-
# OpenSearch
|
130
|
-
sfeos-tools add-bbox-shape --backend opensearch
|
138
|
+
sfeos-tools load-data --base-url <stac-api-url> [options]
|
131
139
|
```
|
132
140
|
|
133
|
-
|
141
|
+
Options:
|
142
|
+
- `--base-url`: Base URL of the STAC API (required)
|
143
|
+
- `--collection-id`: ID of the collection to create/update (default: test-collection)
|
144
|
+
- `--data-dir`: Directory containing collection.json and feature collection files (default: sample_data/)
|
145
|
+
- `--use-bulk`: Use bulk insert method for items (faster for large datasets)
|
134
146
|
|
135
|
-
|
136
|
-
# Local Docker Compose (no SSL)
|
137
|
-
sfeos-tools add-bbox-shape --backend elasticsearch --no-ssl
|
138
|
-
|
139
|
-
# Remote server with SSL
|
140
|
-
sfeos-tools add-bbox-shape \
|
141
|
-
--backend elasticsearch \
|
142
|
-
--host db.example.com \
|
143
|
-
--port 9200 \
|
144
|
-
--user admin \
|
145
|
-
--password secret
|
146
|
-
|
147
|
-
# Using environment variables
|
148
|
-
ES_HOST=my-cluster.cloud.com ES_PORT=9243 ES_USER=elastic ES_PASS=changeme \
|
149
|
-
sfeos-tools add-bbox-shape --backend elasticsearch
|
150
|
-
```
|
147
|
+
**Data Directory Structure:**
|
151
148
|
|
152
|
-
|
149
|
+
Your data directory should contain:
|
150
|
+
- `collection.json`: STAC collection definition
|
151
|
+
- One or more `.json` files: Feature collections with STAC items
|
153
152
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
-
|
158
|
-
|
159
|
-
|
153
|
+
Examples:
|
154
|
+
```bash
|
155
|
+
# Load data from default directory
|
156
|
+
sfeos-tools load-data --base-url http://localhost:8080
|
157
|
+
|
158
|
+
# Load with custom collection ID and bulk insert
|
159
|
+
sfeos-tools load-data \
|
160
|
+
--base-url http://localhost:8080 \
|
161
|
+
--collection-id my-collection \
|
162
|
+
--use-bulk
|
163
|
+
|
164
|
+
# Load from custom directory
|
165
|
+
sfeos-tools load-data \
|
166
|
+
--base-url http://localhost:8080 \
|
167
|
+
--data-dir /path/to/stac/data \
|
168
|
+
--collection-id production-data
|
169
|
+
```
|
160
170
|
|
161
171
|
## Development
|
162
172
|
|
@@ -177,3 +187,6 @@ pre-commit install
|
|
177
187
|
pre-commit run --all-files
|
178
188
|
```
|
179
189
|
|
190
|
+
## License
|
191
|
+
|
192
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
4
4
|
|
5
5
|
[project]
|
6
6
|
name = "sfeos-tools"
|
7
|
-
version = "0.
|
7
|
+
version = "0.2.0"
|
8
8
|
description = "CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments"
|
9
9
|
readme = "README.md"
|
10
10
|
authors = [
|
@@ -21,17 +21,20 @@ classifiers = [
|
|
21
21
|
"Intended Audience :: Developers",
|
22
22
|
"Intended Audience :: Science/Research",
|
23
23
|
"License :: OSI Approved :: MIT License",
|
24
|
-
"Programming Language :: Python :: 3.8",
|
25
24
|
"Programming Language :: Python :: 3.9",
|
26
25
|
"Programming Language :: Python :: 3.10",
|
27
26
|
"Programming Language :: Python :: 3.11",
|
28
27
|
"Programming Language :: Python :: 3.12",
|
28
|
+
"Programming Language :: Python :: 3.13",
|
29
|
+
"Programming Language :: Python :: 3.14",
|
29
30
|
"Topic :: Scientific/Engineering",
|
30
31
|
"Topic :: Scientific/Engineering :: GIS",
|
31
32
|
"Topic :: Software Development :: Libraries :: Python Modules"
|
32
33
|
]
|
33
34
|
dependencies = [
|
34
35
|
"click>=8.0.0",
|
36
|
+
"httpx>=0.24.0",
|
37
|
+
"orjson>=3.0.0",
|
35
38
|
]
|
36
39
|
|
37
40
|
[project.urls]
|
@@ -42,16 +45,17 @@ Source = "https://github.com/Healy-Hyperspatial/sfeos-tools"
|
|
42
45
|
|
43
46
|
[project.optional-dependencies]
|
44
47
|
elasticsearch = [
|
45
|
-
"stac-fastapi-elasticsearch",
|
48
|
+
"stac-fastapi-elasticsearch>=6.6.0",
|
46
49
|
]
|
47
50
|
opensearch = [
|
48
|
-
"stac-fastapi-opensearch",
|
51
|
+
"stac-fastapi-opensearch>=6.6.0",
|
49
52
|
]
|
50
53
|
dev = [
|
51
|
-
"stac-fastapi-elasticsearch",
|
52
|
-
"stac-fastapi-opensearch",
|
54
|
+
"stac-fastapi-elasticsearch>=6.6.0",
|
55
|
+
"stac-fastapi-opensearch>=6.6.0",
|
53
56
|
# Development tools
|
54
57
|
"pytest>=6.0",
|
58
|
+
"pytest-asyncio>=0.21.0",
|
55
59
|
"pytest-cov>=2.0",
|
56
60
|
"black>=21.0",
|
57
61
|
"isort>=5.0",
|
@@ -79,7 +83,13 @@ line_length = 88
|
|
79
83
|
[tool.pytest.ini_options]
|
80
84
|
testpaths = ["tests"]
|
81
85
|
python_files = "test_*.py"
|
82
|
-
addopts = "-v --cov=
|
86
|
+
addopts = "-v --cov=sfeos_tools --cov-report=term-missing"
|
87
|
+
asyncio_mode = "auto"
|
88
|
+
filterwarnings = [
|
89
|
+
"ignore::DeprecationWarning:pydantic._internal._config",
|
90
|
+
"ignore::pydantic.warnings.PydanticDeprecatedSince20",
|
91
|
+
"ignore:coroutine.*was never awaited:RuntimeWarning",
|
92
|
+
]
|
83
93
|
|
84
94
|
[tool.coverage.report]
|
85
95
|
exclude_lines = [
|
@@ -0,0 +1,114 @@
|
|
1
|
+
"""Bbox shape migration utilities for SFEOS collections."""
|
2
|
+
|
3
|
+
import logging
|
4
|
+
|
5
|
+
from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
|
6
|
+
from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
|
7
|
+
|
8
|
+
logger = logging.getLogger(__name__)
|
9
|
+
|
10
|
+
|
11
|
+
async def process_collection_bbox_shape(client, collection_doc, backend):
|
12
|
+
"""Process a single collection document to add bbox_shape field.
|
13
|
+
|
14
|
+
Args:
|
15
|
+
client: Elasticsearch/OpenSearch client
|
16
|
+
collection_doc: Collection document from database
|
17
|
+
backend: Backend type ('elasticsearch' or 'opensearch')
|
18
|
+
|
19
|
+
Returns:
|
20
|
+
bool: True if collection was updated, False if no update was needed
|
21
|
+
"""
|
22
|
+
collection = collection_doc["_source"]
|
23
|
+
collection_id = collection.get("id", collection_doc["_id"])
|
24
|
+
|
25
|
+
# Use the shared function to add bbox_shape
|
26
|
+
was_added = add_bbox_shape_to_collection(collection)
|
27
|
+
|
28
|
+
if not was_added:
|
29
|
+
return False
|
30
|
+
|
31
|
+
# Update the collection in the database
|
32
|
+
if backend == "elasticsearch":
|
33
|
+
await client.index(
|
34
|
+
index=COLLECTIONS_INDEX,
|
35
|
+
id=collection_id,
|
36
|
+
document=collection,
|
37
|
+
refresh=True,
|
38
|
+
)
|
39
|
+
else: # opensearch
|
40
|
+
await client.index(
|
41
|
+
index=COLLECTIONS_INDEX,
|
42
|
+
id=collection_id,
|
43
|
+
body=collection,
|
44
|
+
refresh=True,
|
45
|
+
)
|
46
|
+
|
47
|
+
logger.info(f"Collection '{collection_id}': Added bbox_shape field")
|
48
|
+
return True
|
49
|
+
|
50
|
+
|
51
|
+
async def run_add_bbox_shape(backend):
|
52
|
+
"""Add bbox_shape field to all existing collections.
|
53
|
+
|
54
|
+
Args:
|
55
|
+
backend: Backend type ('elasticsearch' or 'opensearch')
|
56
|
+
"""
|
57
|
+
import os
|
58
|
+
|
59
|
+
logger.info(
|
60
|
+
f"Starting migration: Adding bbox_shape to existing collections ({backend})"
|
61
|
+
)
|
62
|
+
|
63
|
+
# Log connection info (showing what will be used by the client)
|
64
|
+
es_host = os.getenv("ES_HOST", "localhost")
|
65
|
+
es_port = os.getenv(
|
66
|
+
"ES_PORT", "9200"
|
67
|
+
) # Both backends default to 9200 in their config
|
68
|
+
es_use_ssl = os.getenv("ES_USE_SSL", "true")
|
69
|
+
logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
|
70
|
+
|
71
|
+
# Create client based on backend
|
72
|
+
if backend == "elasticsearch":
|
73
|
+
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
|
74
|
+
|
75
|
+
settings = AsyncElasticsearchSettings()
|
76
|
+
else: # opensearch
|
77
|
+
from stac_fastapi.opensearch.config import AsyncOpensearchSettings
|
78
|
+
|
79
|
+
settings = AsyncOpensearchSettings()
|
80
|
+
|
81
|
+
client = settings.create_client
|
82
|
+
|
83
|
+
try:
|
84
|
+
# Get all collections
|
85
|
+
response = await client.search(
|
86
|
+
index=COLLECTIONS_INDEX,
|
87
|
+
body={
|
88
|
+
"query": {"match_all": {}},
|
89
|
+
"size": 10000,
|
90
|
+
}, # Adjust size if you have more collections
|
91
|
+
)
|
92
|
+
|
93
|
+
total_collections = response["hits"]["total"]["value"]
|
94
|
+
logger.info(f"Found {total_collections} collections to process")
|
95
|
+
|
96
|
+
updated_count = 0
|
97
|
+
skipped_count = 0
|
98
|
+
|
99
|
+
for hit in response["hits"]["hits"]:
|
100
|
+
was_updated = await process_collection_bbox_shape(client, hit, backend)
|
101
|
+
if was_updated:
|
102
|
+
updated_count += 1
|
103
|
+
else:
|
104
|
+
skipped_count += 1
|
105
|
+
|
106
|
+
logger.info(
|
107
|
+
f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
|
108
|
+
)
|
109
|
+
|
110
|
+
except Exception as e:
|
111
|
+
logger.error(f"Migration failed with error: {e}")
|
112
|
+
raise
|
113
|
+
finally:
|
114
|
+
await client.close()
|
@@ -13,123 +13,24 @@ import logging
|
|
13
13
|
import sys
|
14
14
|
|
15
15
|
import click
|
16
|
-
from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
|
17
|
-
from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
|
18
16
|
|
17
|
+
try:
|
18
|
+
from importlib.metadata import version as _get_version
|
19
|
+
except ImportError:
|
20
|
+
from importlib_metadata import version as _get_version # type: ignore[no-redef]
|
21
|
+
|
22
|
+
__version__ = _get_version("sfeos-tools")
|
23
|
+
|
24
|
+
from .bbox_shape import run_add_bbox_shape
|
25
|
+
from .data_loader import load_items
|
19
26
|
from .reindex import run as unified_reindex_run
|
20
27
|
|
21
28
|
logging.basicConfig(level=logging.INFO)
|
22
29
|
logger = logging.getLogger(__name__)
|
23
30
|
|
24
31
|
|
25
|
-
async def process_collection_bbox_shape(client, collection_doc, backend):
|
26
|
-
"""Process a single collection document to add bbox_shape field.
|
27
|
-
|
28
|
-
Args:
|
29
|
-
client: Elasticsearch/OpenSearch client
|
30
|
-
collection_doc: Collection document from database
|
31
|
-
backend: Backend type ('elasticsearch' or 'opensearch')
|
32
|
-
|
33
|
-
Returns:
|
34
|
-
bool: True if collection was updated, False if no update was needed
|
35
|
-
"""
|
36
|
-
collection = collection_doc["_source"]
|
37
|
-
collection_id = collection.get("id", collection_doc["_id"])
|
38
|
-
|
39
|
-
# Use the shared function to add bbox_shape
|
40
|
-
was_added = add_bbox_shape_to_collection(collection)
|
41
|
-
|
42
|
-
if not was_added:
|
43
|
-
return False
|
44
|
-
|
45
|
-
# Update the collection in the database
|
46
|
-
if backend == "elasticsearch":
|
47
|
-
await client.index(
|
48
|
-
index=COLLECTIONS_INDEX,
|
49
|
-
id=collection_id,
|
50
|
-
document=collection,
|
51
|
-
refresh=True,
|
52
|
-
)
|
53
|
-
else: # opensearch
|
54
|
-
await client.index(
|
55
|
-
index=COLLECTIONS_INDEX,
|
56
|
-
id=collection_id,
|
57
|
-
body=collection,
|
58
|
-
refresh=True,
|
59
|
-
)
|
60
|
-
|
61
|
-
logger.info(f"Collection '{collection_id}': Added bbox_shape field")
|
62
|
-
return True
|
63
|
-
|
64
|
-
|
65
|
-
async def run_add_bbox_shape(backend):
|
66
|
-
"""Add bbox_shape field to all existing collections.
|
67
|
-
|
68
|
-
Args:
|
69
|
-
backend: Backend type ('elasticsearch' or 'opensearch')
|
70
|
-
"""
|
71
|
-
import os
|
72
|
-
|
73
|
-
logger.info(
|
74
|
-
f"Starting migration: Adding bbox_shape to existing collections ({backend})"
|
75
|
-
)
|
76
|
-
|
77
|
-
# Log connection info (showing what will be used by the client)
|
78
|
-
es_host = os.getenv("ES_HOST", "localhost")
|
79
|
-
es_port = os.getenv(
|
80
|
-
"ES_PORT", "9200"
|
81
|
-
) # Both backends default to 9200 in their config
|
82
|
-
es_use_ssl = os.getenv("ES_USE_SSL", "true")
|
83
|
-
logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
|
84
|
-
|
85
|
-
# Create client based on backend
|
86
|
-
if backend == "elasticsearch":
|
87
|
-
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
|
88
|
-
|
89
|
-
settings = AsyncElasticsearchSettings()
|
90
|
-
else: # opensearch
|
91
|
-
from stac_fastapi.opensearch.config import AsyncOpensearchSettings
|
92
|
-
|
93
|
-
settings = AsyncOpensearchSettings()
|
94
|
-
|
95
|
-
client = settings.create_client
|
96
|
-
|
97
|
-
try:
|
98
|
-
# Get all collections
|
99
|
-
response = await client.search(
|
100
|
-
index=COLLECTIONS_INDEX,
|
101
|
-
body={
|
102
|
-
"query": {"match_all": {}},
|
103
|
-
"size": 10000,
|
104
|
-
}, # Adjust size if you have more collections
|
105
|
-
)
|
106
|
-
|
107
|
-
total_collections = response["hits"]["total"]["value"]
|
108
|
-
logger.info(f"Found {total_collections} collections to process")
|
109
|
-
|
110
|
-
updated_count = 0
|
111
|
-
skipped_count = 0
|
112
|
-
|
113
|
-
for hit in response["hits"]["hits"]:
|
114
|
-
was_updated = await process_collection_bbox_shape(client, hit, backend)
|
115
|
-
if was_updated:
|
116
|
-
updated_count += 1
|
117
|
-
else:
|
118
|
-
skipped_count += 1
|
119
|
-
|
120
|
-
logger.info(
|
121
|
-
f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
|
122
|
-
)
|
123
|
-
|
124
|
-
except Exception as e:
|
125
|
-
logger.error(f"Migration failed with error: {e}")
|
126
|
-
raise
|
127
|
-
finally:
|
128
|
-
await client.close()
|
129
|
-
|
130
|
-
|
131
32
|
@click.group()
|
132
|
-
@click.version_option(version=
|
33
|
+
@click.version_option(version=__version__, prog_name="sfeos-tools")
|
133
34
|
def cli():
|
134
35
|
"""SFEOS Tools - Utilities for managing stac-fastapi-elasticsearch-opensearch deployments."""
|
135
36
|
pass
|
@@ -332,5 +233,47 @@ def reindex(backend, host, port, use_ssl, user, password, yes):
|
|
332
233
|
sys.exit(1)
|
333
234
|
|
334
235
|
|
236
|
+
@cli.command("load-data")
|
237
|
+
@click.option("--base-url", required=True, help="Base URL of the STAC API")
|
238
|
+
@click.option(
|
239
|
+
"--collection-id",
|
240
|
+
default="test-collection",
|
241
|
+
help="ID of the collection to which items are added",
|
242
|
+
)
|
243
|
+
@click.option("--use-bulk", is_flag=True, help="Use bulk insert method for items")
|
244
|
+
@click.option(
|
245
|
+
"--data-dir",
|
246
|
+
type=click.Path(exists=True),
|
247
|
+
default="sample_data/",
|
248
|
+
help="Directory containing collection.json and feature collection file",
|
249
|
+
)
|
250
|
+
def load_data(base_url: str, collection_id: str, use_bulk: bool, data_dir: str) -> None:
|
251
|
+
"""Load STAC items into the database via STAC API.
|
252
|
+
|
253
|
+
This command loads a STAC collection and its items from local JSON files
|
254
|
+
into a STAC API instance. It expects a directory containing:
|
255
|
+
- collection.json: The STAC collection definition
|
256
|
+
- One or more feature collection JSON files with STAC items
|
257
|
+
|
258
|
+
Examples:
|
259
|
+
sfeos-tools load-data --base-url http://localhost:8080
|
260
|
+
sfeos-tools load-data --base-url http://localhost:8080 --collection-id my-collection --use-bulk
|
261
|
+
sfeos-tools load-data --base-url http://localhost:8080 --data-dir /path/to/data
|
262
|
+
"""
|
263
|
+
from httpx import Client
|
264
|
+
|
265
|
+
try:
|
266
|
+
with Client(base_url=base_url) as client:
|
267
|
+
load_items(client, collection_id, use_bulk, data_dir)
|
268
|
+
click.echo(click.style("✓ Data loading completed successfully", fg="green"))
|
269
|
+
except KeyboardInterrupt:
|
270
|
+
click.echo(click.style("\n✗ Data loading interrupted by user", fg="yellow"))
|
271
|
+
sys.exit(1)
|
272
|
+
except Exception as e:
|
273
|
+
error_msg = str(e)
|
274
|
+
click.echo(click.style(f"✗ Data loading failed: {error_msg}", fg="red"))
|
275
|
+
sys.exit(1)
|
276
|
+
|
277
|
+
|
335
278
|
if __name__ == "__main__":
|
336
279
|
cli()
|