sfeos-tools 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sfeos_tools/bbox_shape.py +114 -0
- sfeos_tools/cli.py +1 -108
- {sfeos_tools-0.1.0.dist-info → sfeos_tools-0.1.1.dist-info}/METADATA +24 -6
- sfeos_tools-0.1.1.dist-info/RECORD +10 -0
- sfeos_tools-0.1.0.dist-info/RECORD +0 -9
- {sfeos_tools-0.1.0.dist-info → sfeos_tools-0.1.1.dist-info}/WHEEL +0 -0
- {sfeos_tools-0.1.0.dist-info → sfeos_tools-0.1.1.dist-info}/entry_points.txt +0 -0
- {sfeos_tools-0.1.0.dist-info → sfeos_tools-0.1.1.dist-info}/licenses/LICENSE +0 -0
- {sfeos_tools-0.1.0.dist-info → sfeos_tools-0.1.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Bbox shape migration utilities for SFEOS collections."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
|
|
6
|
+
from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
async def process_collection_bbox_shape(client, collection_doc, backend):
|
|
12
|
+
"""Process a single collection document to add bbox_shape field.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
client: Elasticsearch/OpenSearch client
|
|
16
|
+
collection_doc: Collection document from database
|
|
17
|
+
backend: Backend type ('elasticsearch' or 'opensearch')
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
bool: True if collection was updated, False if no update was needed
|
|
21
|
+
"""
|
|
22
|
+
collection = collection_doc["_source"]
|
|
23
|
+
collection_id = collection.get("id", collection_doc["_id"])
|
|
24
|
+
|
|
25
|
+
# Use the shared function to add bbox_shape
|
|
26
|
+
was_added = add_bbox_shape_to_collection(collection)
|
|
27
|
+
|
|
28
|
+
if not was_added:
|
|
29
|
+
return False
|
|
30
|
+
|
|
31
|
+
# Update the collection in the database
|
|
32
|
+
if backend == "elasticsearch":
|
|
33
|
+
await client.index(
|
|
34
|
+
index=COLLECTIONS_INDEX,
|
|
35
|
+
id=collection_id,
|
|
36
|
+
document=collection,
|
|
37
|
+
refresh=True,
|
|
38
|
+
)
|
|
39
|
+
else: # opensearch
|
|
40
|
+
await client.index(
|
|
41
|
+
index=COLLECTIONS_INDEX,
|
|
42
|
+
id=collection_id,
|
|
43
|
+
body=collection,
|
|
44
|
+
refresh=True,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
logger.info(f"Collection '{collection_id}': Added bbox_shape field")
|
|
48
|
+
return True
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def run_add_bbox_shape(backend):
|
|
52
|
+
"""Add bbox_shape field to all existing collections.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
backend: Backend type ('elasticsearch' or 'opensearch')
|
|
56
|
+
"""
|
|
57
|
+
import os
|
|
58
|
+
|
|
59
|
+
logger.info(
|
|
60
|
+
f"Starting migration: Adding bbox_shape to existing collections ({backend})"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Log connection info (showing what will be used by the client)
|
|
64
|
+
es_host = os.getenv("ES_HOST", "localhost")
|
|
65
|
+
es_port = os.getenv(
|
|
66
|
+
"ES_PORT", "9200"
|
|
67
|
+
) # Both backends default to 9200 in their config
|
|
68
|
+
es_use_ssl = os.getenv("ES_USE_SSL", "true")
|
|
69
|
+
logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
|
|
70
|
+
|
|
71
|
+
# Create client based on backend
|
|
72
|
+
if backend == "elasticsearch":
|
|
73
|
+
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
|
|
74
|
+
|
|
75
|
+
settings = AsyncElasticsearchSettings()
|
|
76
|
+
else: # opensearch
|
|
77
|
+
from stac_fastapi.opensearch.config import AsyncOpensearchSettings
|
|
78
|
+
|
|
79
|
+
settings = AsyncOpensearchSettings()
|
|
80
|
+
|
|
81
|
+
client = settings.create_client
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
# Get all collections
|
|
85
|
+
response = await client.search(
|
|
86
|
+
index=COLLECTIONS_INDEX,
|
|
87
|
+
body={
|
|
88
|
+
"query": {"match_all": {}},
|
|
89
|
+
"size": 10000,
|
|
90
|
+
}, # Adjust size if you have more collections
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
total_collections = response["hits"]["total"]["value"]
|
|
94
|
+
logger.info(f"Found {total_collections} collections to process")
|
|
95
|
+
|
|
96
|
+
updated_count = 0
|
|
97
|
+
skipped_count = 0
|
|
98
|
+
|
|
99
|
+
for hit in response["hits"]["hits"]:
|
|
100
|
+
was_updated = await process_collection_bbox_shape(client, hit, backend)
|
|
101
|
+
if was_updated:
|
|
102
|
+
updated_count += 1
|
|
103
|
+
else:
|
|
104
|
+
skipped_count += 1
|
|
105
|
+
|
|
106
|
+
logger.info(
|
|
107
|
+
f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
except Exception as e:
|
|
111
|
+
logger.error(f"Migration failed with error: {e}")
|
|
112
|
+
raise
|
|
113
|
+
finally:
|
|
114
|
+
await client.close()
|
sfeos_tools/cli.py
CHANGED
|
@@ -13,121 +13,14 @@ import logging
|
|
|
13
13
|
import sys
|
|
14
14
|
|
|
15
15
|
import click
|
|
16
|
-
from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
|
|
17
|
-
from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
|
|
18
16
|
|
|
17
|
+
from .bbox_shape import run_add_bbox_shape
|
|
19
18
|
from .reindex import run as unified_reindex_run
|
|
20
19
|
|
|
21
20
|
logging.basicConfig(level=logging.INFO)
|
|
22
21
|
logger = logging.getLogger(__name__)
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
async def process_collection_bbox_shape(client, collection_doc, backend):
|
|
26
|
-
"""Process a single collection document to add bbox_shape field.
|
|
27
|
-
|
|
28
|
-
Args:
|
|
29
|
-
client: Elasticsearch/OpenSearch client
|
|
30
|
-
collection_doc: Collection document from database
|
|
31
|
-
backend: Backend type ('elasticsearch' or 'opensearch')
|
|
32
|
-
|
|
33
|
-
Returns:
|
|
34
|
-
bool: True if collection was updated, False if no update was needed
|
|
35
|
-
"""
|
|
36
|
-
collection = collection_doc["_source"]
|
|
37
|
-
collection_id = collection.get("id", collection_doc["_id"])
|
|
38
|
-
|
|
39
|
-
# Use the shared function to add bbox_shape
|
|
40
|
-
was_added = add_bbox_shape_to_collection(collection)
|
|
41
|
-
|
|
42
|
-
if not was_added:
|
|
43
|
-
return False
|
|
44
|
-
|
|
45
|
-
# Update the collection in the database
|
|
46
|
-
if backend == "elasticsearch":
|
|
47
|
-
await client.index(
|
|
48
|
-
index=COLLECTIONS_INDEX,
|
|
49
|
-
id=collection_id,
|
|
50
|
-
document=collection,
|
|
51
|
-
refresh=True,
|
|
52
|
-
)
|
|
53
|
-
else: # opensearch
|
|
54
|
-
await client.index(
|
|
55
|
-
index=COLLECTIONS_INDEX,
|
|
56
|
-
id=collection_id,
|
|
57
|
-
body=collection,
|
|
58
|
-
refresh=True,
|
|
59
|
-
)
|
|
60
|
-
|
|
61
|
-
logger.info(f"Collection '{collection_id}': Added bbox_shape field")
|
|
62
|
-
return True
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
async def run_add_bbox_shape(backend):
|
|
66
|
-
"""Add bbox_shape field to all existing collections.
|
|
67
|
-
|
|
68
|
-
Args:
|
|
69
|
-
backend: Backend type ('elasticsearch' or 'opensearch')
|
|
70
|
-
"""
|
|
71
|
-
import os
|
|
72
|
-
|
|
73
|
-
logger.info(
|
|
74
|
-
f"Starting migration: Adding bbox_shape to existing collections ({backend})"
|
|
75
|
-
)
|
|
76
|
-
|
|
77
|
-
# Log connection info (showing what will be used by the client)
|
|
78
|
-
es_host = os.getenv("ES_HOST", "localhost")
|
|
79
|
-
es_port = os.getenv(
|
|
80
|
-
"ES_PORT", "9200"
|
|
81
|
-
) # Both backends default to 9200 in their config
|
|
82
|
-
es_use_ssl = os.getenv("ES_USE_SSL", "true")
|
|
83
|
-
logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
|
|
84
|
-
|
|
85
|
-
# Create client based on backend
|
|
86
|
-
if backend == "elasticsearch":
|
|
87
|
-
from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
|
|
88
|
-
|
|
89
|
-
settings = AsyncElasticsearchSettings()
|
|
90
|
-
else: # opensearch
|
|
91
|
-
from stac_fastapi.opensearch.config import AsyncOpensearchSettings
|
|
92
|
-
|
|
93
|
-
settings = AsyncOpensearchSettings()
|
|
94
|
-
|
|
95
|
-
client = settings.create_client
|
|
96
|
-
|
|
97
|
-
try:
|
|
98
|
-
# Get all collections
|
|
99
|
-
response = await client.search(
|
|
100
|
-
index=COLLECTIONS_INDEX,
|
|
101
|
-
body={
|
|
102
|
-
"query": {"match_all": {}},
|
|
103
|
-
"size": 10000,
|
|
104
|
-
}, # Adjust size if you have more collections
|
|
105
|
-
)
|
|
106
|
-
|
|
107
|
-
total_collections = response["hits"]["total"]["value"]
|
|
108
|
-
logger.info(f"Found {total_collections} collections to process")
|
|
109
|
-
|
|
110
|
-
updated_count = 0
|
|
111
|
-
skipped_count = 0
|
|
112
|
-
|
|
113
|
-
for hit in response["hits"]["hits"]:
|
|
114
|
-
was_updated = await process_collection_bbox_shape(client, hit, backend)
|
|
115
|
-
if was_updated:
|
|
116
|
-
updated_count += 1
|
|
117
|
-
else:
|
|
118
|
-
skipped_count += 1
|
|
119
|
-
|
|
120
|
-
logger.info(
|
|
121
|
-
f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
except Exception as e:
|
|
125
|
-
logger.error(f"Migration failed with error: {e}")
|
|
126
|
-
raise
|
|
127
|
-
finally:
|
|
128
|
-
await client.close()
|
|
129
|
-
|
|
130
|
-
|
|
131
24
|
@click.group()
|
|
132
25
|
@click.version_option(version="0.1.0", prog_name="sfeos-tools")
|
|
133
26
|
def cli():
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sfeos-tools
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments
|
|
5
5
|
Author: CloudFerro S.A.
|
|
6
6
|
Author-email: Jonathan Healy <jon@healy-hypersaptial.dev>
|
|
@@ -35,11 +35,12 @@ Classifier: Development Status :: 4 - Beta
|
|
|
35
35
|
Classifier: Intended Audience :: Developers
|
|
36
36
|
Classifier: Intended Audience :: Science/Research
|
|
37
37
|
Classifier: License :: OSI Approved :: MIT License
|
|
38
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
39
38
|
Classifier: Programming Language :: Python :: 3.9
|
|
40
39
|
Classifier: Programming Language :: Python :: 3.10
|
|
41
40
|
Classifier: Programming Language :: Python :: 3.11
|
|
42
41
|
Classifier: Programming Language :: Python :: 3.12
|
|
42
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
43
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
43
44
|
Classifier: Topic :: Scientific/Engineering
|
|
44
45
|
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
45
46
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
@@ -48,13 +49,14 @@ Description-Content-Type: text/markdown
|
|
|
48
49
|
License-File: LICENSE
|
|
49
50
|
Requires-Dist: click>=8.0.0
|
|
50
51
|
Provides-Extra: elasticsearch
|
|
51
|
-
Requires-Dist: stac-fastapi-elasticsearch; extra == "elasticsearch"
|
|
52
|
+
Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "elasticsearch"
|
|
52
53
|
Provides-Extra: opensearch
|
|
53
|
-
Requires-Dist: stac-fastapi-opensearch; extra == "opensearch"
|
|
54
|
+
Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "opensearch"
|
|
54
55
|
Provides-Extra: dev
|
|
55
|
-
Requires-Dist: stac-fastapi-elasticsearch; extra == "dev"
|
|
56
|
-
Requires-Dist: stac-fastapi-opensearch; extra == "dev"
|
|
56
|
+
Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "dev"
|
|
57
|
+
Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "dev"
|
|
57
58
|
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
59
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
58
60
|
Requires-Dist: pytest-cov>=2.0; extra == "dev"
|
|
59
61
|
Requires-Dist: black>=21.0; extra == "dev"
|
|
60
62
|
Requires-Dist: isort>=5.0; extra == "dev"
|
|
@@ -66,6 +68,22 @@ Dynamic: license-file
|
|
|
66
68
|
|
|
67
69
|
CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
|
|
68
70
|
|
|
71
|
+
<!-- markdownlint-disable MD033 MD041 -->
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
<p align="left">
|
|
75
|
+
<img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
|
|
76
|
+
</p>
|
|
77
|
+
|
|
78
|
+
<!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
|
|
79
|
+
|
|
80
|
+
[](https://pepy.tech/project/sfeos-tools)
|
|
81
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
|
|
82
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
|
|
83
|
+
[](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
|
|
84
|
+
[](https://pypi.org/project/sfeos-tools/)
|
|
85
|
+
[](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
|
|
86
|
+
|
|
69
87
|
## Table of Contents
|
|
70
88
|
|
|
71
89
|
- [Installation](#installation)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
sfeos_tools/__init__.py,sha256=E6dzjiV5qnPvtjRAEfXy3lS6yqgS6olUV9gW3rnKo2Y,117
|
|
2
|
+
sfeos_tools/bbox_shape.py,sha256=lL7vq-lD36XKcZ4ScZF6tDs-KRW9DHCiDquXaE9gL9A,3491
|
|
3
|
+
sfeos_tools/cli.py,sha256=s6fK1uyqTXY1avaFAodBDnAOU8NUKqYQvqavQX83Kgk,6876
|
|
4
|
+
sfeos_tools/reindex.py,sha256=JIglWrRxvmrSwsTSqKrhqcTJgNKP_5blcNxujFesbgw,4277
|
|
5
|
+
sfeos_tools-0.1.1.dist-info/licenses/LICENSE,sha256=euoUdQTBFz_vxdUnYkUnTUeAtrzfmJnBmGrO3NceeO8,1111
|
|
6
|
+
sfeos_tools-0.1.1.dist-info/METADATA,sha256=NARvC1EiJj68s2K45U6KDKzRa-04tfpPzrh_mmQR4Qs,9137
|
|
7
|
+
sfeos_tools-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
8
|
+
sfeos_tools-0.1.1.dist-info/entry_points.txt,sha256=fgwnuQndU6x7RNOw4twi0ZkqDFgb5HOijB-wRuo1cp8,52
|
|
9
|
+
sfeos_tools-0.1.1.dist-info/top_level.txt,sha256=wBLaEvh7OmhjMhGluVm2vwE0ufSuc90eHrlTph-51kM,12
|
|
10
|
+
sfeos_tools-0.1.1.dist-info/RECORD,,
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
sfeos_tools/__init__.py,sha256=E6dzjiV5qnPvtjRAEfXy3lS6yqgS6olUV9gW3rnKo2Y,117
|
|
2
|
-
sfeos_tools/cli.py,sha256=eUauJIRjuQZfrZPTfXAMDm_cn4oFNNi1eJWTkcZh-0U,10209
|
|
3
|
-
sfeos_tools/reindex.py,sha256=JIglWrRxvmrSwsTSqKrhqcTJgNKP_5blcNxujFesbgw,4277
|
|
4
|
-
sfeos_tools-0.1.0.dist-info/licenses/LICENSE,sha256=euoUdQTBFz_vxdUnYkUnTUeAtrzfmJnBmGrO3NceeO8,1111
|
|
5
|
-
sfeos_tools-0.1.0.dist-info/METADATA,sha256=1aHsyPTxW28AjEyOSAt-YkeBhWWsbQoKxwh27P3TIjU,7772
|
|
6
|
-
sfeos_tools-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
7
|
-
sfeos_tools-0.1.0.dist-info/entry_points.txt,sha256=fgwnuQndU6x7RNOw4twi0ZkqDFgb5HOijB-wRuo1cp8,52
|
|
8
|
-
sfeos_tools-0.1.0.dist-info/top_level.txt,sha256=wBLaEvh7OmhjMhGluVm2vwE0ufSuc90eHrlTph-51kM,12
|
|
9
|
-
sfeos_tools-0.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|