sfeos-tools 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

sfeos_tools/bbox_shape.py

@@ -0,0 +1,114 @@
+"""Bbox shape migration utilities for SFEOS collections."""
+
+import logging
+
+from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
+from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
+
+logger = logging.getLogger(__name__)
+
+
+async def process_collection_bbox_shape(client, collection_doc, backend):
+    """Process a single collection document to add bbox_shape field.
+
+    Args:
+        client: Elasticsearch/OpenSearch client
+        collection_doc: Collection document from database
+        backend: Backend type ('elasticsearch' or 'opensearch')
+
+    Returns:
+        bool: True if collection was updated, False if no update was needed
+    """
+    collection = collection_doc["_source"]
+    collection_id = collection.get("id", collection_doc["_id"])
+
+    # Use the shared function to add bbox_shape
+    was_added = add_bbox_shape_to_collection(collection)
+
+    if not was_added:
+        return False
+
+    # Update the collection in the database
+    if backend == "elasticsearch":
+        await client.index(
+            index=COLLECTIONS_INDEX,
+            id=collection_id,
+            document=collection,
+            refresh=True,
+        )
+    else:  # opensearch
+        await client.index(
+            index=COLLECTIONS_INDEX,
+            id=collection_id,
+            body=collection,
+            refresh=True,
+        )
+
+    logger.info(f"Collection '{collection_id}': Added bbox_shape field")
+    return True
+
+
+async def run_add_bbox_shape(backend):
+    """Add bbox_shape field to all existing collections.
+
+    Args:
+        backend: Backend type ('elasticsearch' or 'opensearch')
+    """
+    import os
+
+    logger.info(
+        f"Starting migration: Adding bbox_shape to existing collections ({backend})"
+    )
+
+    # Log connection info (showing what will be used by the client)
+    es_host = os.getenv("ES_HOST", "localhost")
+    es_port = os.getenv(
+        "ES_PORT", "9200"
+    )  # Both backends default to 9200 in their config
+    es_use_ssl = os.getenv("ES_USE_SSL", "true")
+    logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
+
+    # Create client based on backend
+    if backend == "elasticsearch":
+        from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
+
+        settings = AsyncElasticsearchSettings()
+    else:  # opensearch
+        from stac_fastapi.opensearch.config import AsyncOpensearchSettings
+
+        settings = AsyncOpensearchSettings()
+
+    client = settings.create_client
+
+    try:
+        # Get all collections
+        response = await client.search(
+            index=COLLECTIONS_INDEX,
+            body={
+                "query": {"match_all": {}},
+                "size": 10000,
+            },  # Adjust size if you have more collections
+        )
+
+        total_collections = response["hits"]["total"]["value"]
+        logger.info(f"Found {total_collections} collections to process")
+
+        updated_count = 0
+        skipped_count = 0
+
+        for hit in response["hits"]["hits"]:
+            was_updated = await process_collection_bbox_shape(client, hit, backend)
+            if was_updated:
+                updated_count += 1
+            else:
+                skipped_count += 1
+
+        logger.info(
+            f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
+        )
+
+    except Exception as e:
+        logger.error(f"Migration failed with error: {e}")
+        raise
+    finally:
+        await client.close()

sfeos_tools/cli.py

@@ -13,123 +13,24 @@ import logging
 import sys
 
 import click
-from stac_fastapi.sfeos_helpers.database import add_bbox_shape_to_collection
-from stac_fastapi.sfeos_helpers.mappings import COLLECTIONS_INDEX
 
+try:
+    from importlib.metadata import version as _get_version
+except ImportError:
+    from importlib_metadata import version as _get_version  # type: ignore[no-redef]
+
+__version__ = _get_version("sfeos-tools")
+
+from .bbox_shape import run_add_bbox_shape
+from .data_loader import load_items
 from .reindex import run as unified_reindex_run
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
-async def process_collection_bbox_shape(client, collection_doc, backend):
-    """Process a single collection document to add bbox_shape field.
-
-    Args:
-        client: Elasticsearch/OpenSearch client
-        collection_doc: Collection document from database
-        backend: Backend type ('elasticsearch' or 'opensearch')
-
-    Returns:
-        bool: True if collection was updated, False if no update was needed
-    """
-    collection = collection_doc["_source"]
-    collection_id = collection.get("id", collection_doc["_id"])
-
-    # Use the shared function to add bbox_shape
-    was_added = add_bbox_shape_to_collection(collection)
-
-    if not was_added:
-        return False
-
-    # Update the collection in the database
-    if backend == "elasticsearch":
-        await client.index(
-            index=COLLECTIONS_INDEX,
-            id=collection_id,
-            document=collection,
-            refresh=True,
-        )
-    else:  # opensearch
-        await client.index(
-            index=COLLECTIONS_INDEX,
-            id=collection_id,
-            body=collection,
-            refresh=True,
-        )
-
-    logger.info(f"Collection '{collection_id}': Added bbox_shape field")
-    return True
-
-
-async def run_add_bbox_shape(backend):
-    """Add bbox_shape field to all existing collections.
-
-    Args:
-        backend: Backend type ('elasticsearch' or 'opensearch')
-    """
-    import os
-
-    logger.info(
-        f"Starting migration: Adding bbox_shape to existing collections ({backend})"
-    )
-
-    # Log connection info (showing what will be used by the client)
-    es_host = os.getenv("ES_HOST", "localhost")
-    es_port = os.getenv(
-        "ES_PORT", "9200"
-    )  # Both backends default to 9200 in their config
-    es_use_ssl = os.getenv("ES_USE_SSL", "true")
-    logger.info(f"Connecting to {backend} at {es_host}:{es_port} (SSL: {es_use_ssl})")
-
-    # Create client based on backend
-    if backend == "elasticsearch":
-        from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings
-
-        settings = AsyncElasticsearchSettings()
-    else:  # opensearch
-        from stac_fastapi.opensearch.config import AsyncOpensearchSettings
-
-        settings = AsyncOpensearchSettings()
-
-    client = settings.create_client
-
-    try:
-        # Get all collections
-        response = await client.search(
-            index=COLLECTIONS_INDEX,
-            body={
-                "query": {"match_all": {}},
-                "size": 10000,
-            },  # Adjust size if you have more collections
-        )
-
-        total_collections = response["hits"]["total"]["value"]
-        logger.info(f"Found {total_collections} collections to process")
-
-        updated_count = 0
-        skipped_count = 0
-
-        for hit in response["hits"]["hits"]:
-            was_updated = await process_collection_bbox_shape(client, hit, backend)
-            if was_updated:
-                updated_count += 1
-            else:
-                skipped_count += 1
-
-        logger.info(
-            f"Migration complete: {updated_count} collections updated, {skipped_count} skipped"
-        )
-
-    except Exception as e:
-        logger.error(f"Migration failed with error: {e}")
-        raise
-    finally:
-        await client.close()
-
-
 @click.group()
-@click.version_option(version="0.1.0", prog_name="sfeos-tools")
+@click.version_option(version=__version__, prog_name="sfeos-tools")
 def cli():
     """SFEOS Tools - Utilities for managing stac-fastapi-elasticsearch-opensearch deployments."""
     pass
@@ -332,5 +233,47 @@ def reindex(backend, host, port, use_ssl, user, password, yes):
         sys.exit(1)
 
 
+@cli.command("load-data")
+@click.option("--base-url", required=True, help="Base URL of the STAC API")
+@click.option(
+    "--collection-id",
+    default="test-collection",
+    help="ID of the collection to which items are added",
+)
+@click.option("--use-bulk", is_flag=True, help="Use bulk insert method for items")
+@click.option(
+    "--data-dir",
+    type=click.Path(exists=True),
+    default="sample_data/",
+    help="Directory containing collection.json and feature collection file",
+)
+def load_data(base_url: str, collection_id: str, use_bulk: bool, data_dir: str) -> None:
+    """Load STAC items into the database via STAC API.
+
+    This command loads a STAC collection and its items from local JSON files
+    into a STAC API instance. It expects a directory containing:
+    - collection.json: The STAC collection definition
+    - One or more feature collection JSON files with STAC items
+
+    Examples:
+        sfeos-tools load-data --base-url http://localhost:8080
+        sfeos-tools load-data --base-url http://localhost:8080 --collection-id my-collection --use-bulk
+        sfeos-tools load-data --base-url http://localhost:8080 --data-dir /path/to/data
+    """
+    from httpx import Client
+
+    try:
+        with Client(base_url=base_url) as client:
+            load_items(client, collection_id, use_bulk, data_dir)
+        click.echo(click.style("✓ Data loading completed successfully", fg="green"))
+    except KeyboardInterrupt:
+        click.echo(click.style("\n✗ Data loading interrupted by user", fg="yellow"))
+        sys.exit(1)
+    except Exception as e:
+        error_msg = str(e)
+        click.echo(click.style(f"✗ Data loading failed: {error_msg}", fg="red"))
+        sys.exit(1)
+
+
 if __name__ == "__main__":
     cli()

sfeos_tools/data_loader.py

@@ -0,0 +1,102 @@
+"""Data Loader CLI STAC_API Ingestion Tool."""
+
+import os
+from typing import Any
+
+import click
+import orjson
+from httpx import Client
+
+
+def load_data(filepath: str) -> dict[str, Any]:
+    """Load json data from a file within the specified data directory."""
+    try:
+        with open(filepath, "rb") as file:
+            return orjson.loads(file.read())
+    except FileNotFoundError as e:
+        click.secho(f"File not found: {filepath}", fg="red", err=True)
+        raise click.Abort() from e
+
+
+def load_collection(client: Client, collection_id: str, data_dir: str) -> None:
+    """Load a STAC collection into the database."""
+    collection = load_data(os.path.join(data_dir, "collection.json"))
+    collection["id"] = collection_id
+    resp = client.post("/collections", json=collection)
+    if resp.status_code == 200 or resp.status_code == 201:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo(f"Added collection: {collection['id']}")
+    elif resp.status_code == 409:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo(f"Collection: {collection['id']} already exists")
+    else:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo(f"Error writing {collection['id']} collection. Message: {resp.text}")
+
+
+def load_items(
+    client: Client, collection_id: str, use_bulk: bool, data_dir: str
+) -> None:
+    """Load STAC items into the database based on the method selected."""
+    with os.scandir(data_dir) as entries:
+        # Attempt to dynamically find a suitable feature collection file
+        # Use the first found feature collection file
+        feature_file = next(
+            (
+                entry.path
+                for entry in entries
+                if entry.is_file()
+                and entry.name.endswith(".json")
+                and entry.name != "collection.json"
+            ),
+            None,
+        )
+
+    if feature_file is None:
+        click.secho(
+            "No feature collection files found in the specified directory.",
+            fg="red",
+            err=True,
+        )
+        raise click.Abort()
+
+    feature_collection = load_data(feature_file)
+
+    load_collection(client, collection_id, data_dir)
+    if use_bulk:
+        load_items_bulk_insert(client, collection_id, feature_collection)
+    else:
+        load_items_one_by_one(client, collection_id, feature_collection)
+
+
+def load_items_one_by_one(
+    client: Client, collection_id: str, feature_collection: dict[str, Any]
+) -> None:
+    """Load STAC items into the database one by one."""
+    for feature in feature_collection["features"]:
+        feature["collection"] = collection_id
+        resp = client.post(f"/collections/{collection_id}/items", json=feature)
+        if resp.status_code == 200:
+            click.echo(f"Status code: {resp.status_code}")
+            click.echo(f"Added item: {feature['id']}")
+        elif resp.status_code == 409:
+            click.echo(f"Status code: {resp.status_code}")
+            click.echo(f"Item: {feature['id']} already exists")
+
+
+def load_items_bulk_insert(
+    client: Client, collection_id: str, feature_collection: dict[str, Any]
+) -> None:
+    """Load STAC items into the database via bulk insert."""
+    for feature in feature_collection["features"]:
+        feature["collection"] = collection_id
+    resp = client.post(f"/collections/{collection_id}/items", json=feature_collection)
+    if resp.status_code == 200:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo("Bulk inserted items successfully.")
+    elif resp.status_code == 204:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo("Bulk update successful, no content returned.")
+    elif resp.status_code == 409:
+        click.echo(f"Status code: {resp.status_code}")
+        click.echo("Conflict detected, some items might already exist.")

{sfeos_tools-0.1.0.dist-info → sfeos_tools-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sfeos-tools
-Version: 0.1.0
+Version: 0.2.0
 Summary: CLI tools for managing stac-fastapi-elasticsearch-opensearch deployments
 Author: CloudFerro S.A.
 Author-email: Jonathan Healy <jon@healy-hypersaptial.dev>
@@ -35,11 +35,12 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: GIS
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -47,14 +48,17 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: click>=8.0.0
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: orjson>=3.0.0
 Provides-Extra: elasticsearch
-Requires-Dist: stac-fastapi-elasticsearch; extra == "elasticsearch"
+Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "elasticsearch"
 Provides-Extra: opensearch
-Requires-Dist: stac-fastapi-opensearch; extra == "opensearch"
+Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "opensearch"
 Provides-Extra: dev
-Requires-Dist: stac-fastapi-elasticsearch; extra == "dev"
-Requires-Dist: stac-fastapi-opensearch; extra == "dev"
+Requires-Dist: stac-fastapi-elasticsearch>=6.6.0; extra == "dev"
+Requires-Dist: stac-fastapi-opensearch>=6.6.0; extra == "dev"
 Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
 Requires-Dist: pytest-cov>=2.0; extra == "dev"
 Requires-Dist: black>=21.0; extra == "dev"
 Requires-Dist: isort>=5.0; extra == "dev"
@@ -66,6 +70,22 @@ Dynamic: license-file
 
 CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch) deployments.
 
+<!-- markdownlint-disable MD033 MD041 -->
+
+
+<p align="left">
+  <img src="https://raw.githubusercontent.com/stac-utils/stac-fastapi-elasticsearch-opensearch/refs/heads/main/assets/sfeos.png" width=1000>
+</p>
+
+<!-- **Jump to:** [Project Introduction](#project-introduction---what-is-sfeos) | [Quick Start](#quick-start) | [Table of Contents](#table-of-contents) -->
+
+  [![Downloads](https://static.pepy.tech/badge/sfeos-tools?color=blue)](https://pepy.tech/project/sfeos-tools)
+  [![GitHub contributors](https://img.shields.io/github/contributors/healy-hyperspatial/sfeos-tools?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/graphs/contributors)
+  [![GitHub stars](https://img.shields.io/github/stars/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/stargazers)
+  [![GitHub forks](https://img.shields.io/github/forks/healy-hyperspatial/sfeos-tools.svg?color=blue)](https://github.com/healy-hyperspatial/sfeos-tools/network/members)
+   [![PyPI version](https://img.shields.io/pypi/v/sfeos-tools.svg?color=blue)](https://pypi.org/project/sfeos-tools/)
+  [![STAC](https://img.shields.io/badge/STAC-1.1.0-blue.svg)](https://github.com/radiantearth/stac-spec/tree/v1.1.0)
+
 ## Table of Contents
 
 - [Installation](#installation)
@@ -76,6 +96,7 @@ CLI tools for managing [stac-fastapi-elasticsearch-opensearch](https://github.co
 - [Commands](#commands)
   - [add-bbox-shape](#add-bbox-shape)
   - [reindex](#reindex)
+  - [load-data](#load-data)
 - [Development](#development)
 - [License](#license)
 
@@ -165,62 +186,55 @@ Options:
 - `--password`: Database password (default: ES_PASS env var)
 - `--yes`: Skip confirmation prompt
 
-Example:
+Examples:
 ```bash
 # Reindex Elasticsearch with custom host and no SSL
-sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl
+sfeos-tools reindex --backend elasticsearch --host localhost --port 9200 --no-ssl --yes
 
 # Reindex OpenSearch with default settings
-sfeos-tools reindex --backend opensearch
+sfeos-tools reindex --backend opensearch --yes
 ```
 
-# Get help for a specific command
-sfeos-tools add-bbox-shape --help
-```
-
-## Commands
-
-### add-bbox-shape
-
-Add `bbox_shape` field to existing collections for spatial search support.
+### load-data
 
-**Basic usage:**
+Load STAC collections and items from local JSON files into a STAC API instance. This command is useful for:
+- Populating a new STAC API deployment with test data
+- Migrating data between STAC API instances
+- Bulk loading STAC collections and items
 
 ```bash
-# Elasticsearch
-sfeos-tools add-bbox-shape --backend elasticsearch
-
-# OpenSearch
-sfeos-tools add-bbox-shape --backend opensearch
+sfeos-tools load-data --base-url <stac-api-url> [options]
 ```
 
-**Connection options:**
+Options:
+- `--base-url`: Base URL of the STAC API (required)
+- `--collection-id`: ID of the collection to create/update (default: test-collection)
+- `--data-dir`: Directory containing collection.json and feature collection files (default: sample_data/)
+- `--use-bulk`: Use bulk insert method for items (faster for large datasets)
 
-```bash
-# Local Docker Compose (no SSL)
-sfeos-tools add-bbox-shape --backend elasticsearch --no-ssl
-
-# Remote server with SSL
-sfeos-tools add-bbox-shape \
-  --backend elasticsearch \
-  --host db.example.com \
-  --port 9200 \
-  --user admin \
-  --password secret
-
-# Using environment variables
-ES_HOST=my-cluster.cloud.com ES_PORT=9243 ES_USER=elastic ES_PASS=changeme \
-  sfeos-tools add-bbox-shape --backend elasticsearch
-```
+**Data Directory Structure:**
 
-**Available options:**
+Your data directory should contain:
+- `collection.json`: STAC collection definition
+- One or more `.json` files: Feature collections with STAC items
 
-- `--backend`: Database backend (elasticsearch or opensearch) - **required**
-- `--host`: Database host (default: localhost or ES_HOST env var)
-- `--port`: Database port (default: 9200 or ES_PORT env var)
-- `--use-ssl / --no-ssl`: Use SSL connection (default: true or ES_USE_SSL env var)
-- `--user`: Database username (default: ES_USER env var)
-- `--password`: Database password (default: ES_PASS env var)
+Examples:
+```bash
+# Load data from default directory
+sfeos-tools load-data --base-url http://localhost:8080
+
+# Load with custom collection ID and bulk insert
+sfeos-tools load-data \
+  --base-url http://localhost:8080 \
+  --collection-id my-collection \
+  --use-bulk
+
+# Load from custom directory
+sfeos-tools load-data \
+  --base-url http://localhost:8080 \
+  --data-dir /path/to/stac/data \
+  --collection-id production-data
+```
 
 ## Development
 
@@ -241,3 +255,6 @@ pre-commit install
 pre-commit run --all-files
 ```
 
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

sfeos_tools-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+sfeos_tools/__init__.py,sha256=E6dzjiV5qnPvtjRAEfXy3lS6yqgS6olUV9gW3rnKo2Y,117
+sfeos_tools/bbox_shape.py,sha256=lL7vq-lD36XKcZ4ScZF6tDs-KRW9DHCiDquXaE9gL9A,3491
+sfeos_tools/cli.py,sha256=eTm7cfyuYU8x2KQ3ORBMmbw2AX1Ys3ct2tkjbzOBwQc,8838
+sfeos_tools/data_loader.py,sha256=dh_JL7qxIYajVS4Dhz2hJRSJ-L1k-c_6K4V8VXgumcs,3876
+sfeos_tools/reindex.py,sha256=JIglWrRxvmrSwsTSqKrhqcTJgNKP_5blcNxujFesbgw,4277
+sfeos_tools-0.2.0.dist-info/licenses/LICENSE,sha256=euoUdQTBFz_vxdUnYkUnTUeAtrzfmJnBmGrO3NceeO8,1111
+sfeos_tools-0.2.0.dist-info/METADATA,sha256=4I8ebe3FciZ6WsYMC-Q3ZSg_fFgivMJ6G_WmM-7mCV8,9394
+sfeos_tools-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sfeos_tools-0.2.0.dist-info/entry_points.txt,sha256=fgwnuQndU6x7RNOw4twi0ZkqDFgb5HOijB-wRuo1cp8,52
+sfeos_tools-0.2.0.dist-info/top_level.txt,sha256=wBLaEvh7OmhjMhGluVm2vwE0ufSuc90eHrlTph-51kM,12
+sfeos_tools-0.2.0.dist-info/RECORD,,

sfeos_tools-0.1.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-sfeos_tools/__init__.py,sha256=E6dzjiV5qnPvtjRAEfXy3lS6yqgS6olUV9gW3rnKo2Y,117
-sfeos_tools/cli.py,sha256=eUauJIRjuQZfrZPTfXAMDm_cn4oFNNi1eJWTkcZh-0U,10209
-sfeos_tools/reindex.py,sha256=JIglWrRxvmrSwsTSqKrhqcTJgNKP_5blcNxujFesbgw,4277
-sfeos_tools-0.1.0.dist-info/licenses/LICENSE,sha256=euoUdQTBFz_vxdUnYkUnTUeAtrzfmJnBmGrO3NceeO8,1111
-sfeos_tools-0.1.0.dist-info/METADATA,sha256=1aHsyPTxW28AjEyOSAt-YkeBhWWsbQoKxwh27P3TIjU,7772
-sfeos_tools-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sfeos_tools-0.1.0.dist-info/entry_points.txt,sha256=fgwnuQndU6x7RNOw4twi0ZkqDFgb5HOijB-wRuo1cp8,52
-sfeos_tools-0.1.0.dist-info/top_level.txt,sha256=wBLaEvh7OmhjMhGluVm2vwE0ufSuc90eHrlTph-51kM,12
-sfeos_tools-0.1.0.dist-info/RECORD,,