sfeos-helpers 5.0.0a0__tar.gz → 6.0.0__tar.gz

{sfeos_helpers-5.0.0a0 → sfeos_helpers-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sfeos_helpers
-Version: 5.0.0a0
+Version: 6.0.0
 Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT

{sfeos_helpers-5.0.0a0 → sfeos_helpers-6.0.0}/setup.py

@@ -6,7 +6,7 @@ with open("README.md") as f:
     desc = f.read()
 
 install_requires = [
-    "stac-fastapi.core==5.0.0a0",
+    "stac-fastapi.core==6.0.0",
 ]
 
 setup(

{sfeos_helpers-5.0.0a0 → sfeos_helpers-6.0.0}/sfeos_helpers.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sfeos-helpers
-Version: 5.0.0a0
+Version: 6.0.0
 Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT

{sfeos_helpers-5.0.0a0 → sfeos_helpers-6.0.0}/sfeos_helpers.egg-info/SOURCES.txt

@@ -22,4 +22,5 @@ stac_fastapi/sfeos_helpers/database/utils.py
 stac_fastapi/sfeos_helpers/filter/__init__.py
 stac_fastapi/sfeos_helpers/filter/client.py
 stac_fastapi/sfeos_helpers/filter/cql2.py
-stac_fastapi/sfeos_helpers/filter/transform.py
+stac_fastapi/sfeos_helpers/filter/transform.py
+stac_fastapi/sfeos_helpers/models/patch.py

sfeos_helpers-6.0.0/sfeos_helpers.egg-info/requires.txt

@@ -0,0 +1 @@
+stac-fastapi.core==6.0.0

sfeos_helpers-6.0.0/stac_fastapi/sfeos_helpers/database/utils.py

@@ -0,0 +1,248 @@
+"""Utility functions for database operations in Elasticsearch/OpenSearch.
+
+This module provides utility functions for working with database operations
+in Elasticsearch/OpenSearch, such as parameter validation.
+"""
+
+import logging
+from typing import Dict, List, Union
+
+from stac_fastapi.core.utilities import get_bool_env
+from stac_fastapi.extensions.core.transaction.request import (
+    PatchAddReplaceTest,
+    PatchOperation,
+    PatchRemove,
+)
+from stac_fastapi.sfeos_helpers.models.patch import ElasticPath, ESCommandSet
+
+
+def validate_refresh(value: Union[str, bool]) -> str:
+    """
+    Validate the `refresh` parameter value.
+
+    Args:
+        value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
+
+    Returns:
+        str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
+    """
+    logger = logging.getLogger(__name__)
+
+    # Handle boolean-like values using get_bool_env
+    if isinstance(value, bool) or value in {
+        "true",
+        "false",
+        "1",
+        "0",
+        "yes",
+        "no",
+        "y",
+        "n",
+    }:
+        is_true = get_bool_env("DATABASE_REFRESH", default=value)
+        return "true" if is_true else "false"
+
+    # Normalize to lowercase for case-insensitivity
+    value = value.lower()
+
+    # Handle "wait_for" explicitly
+    if value == "wait_for":
+        return "wait_for"
+
+    # Log a warning for invalid values and default to "false"
+    logger.warning(
+        f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
+    )
+    return "false"
+
+
+def merge_to_operations(data: Dict) -> List:
+    """Convert merge operation to list of RF6902 operations.
+
+    Args:
+        data: dictionary to convert.
+
+    Returns:
+        List: list of RF6902 operations.
+    """
+    operations = []
+
+    for key, value in data.copy().items():
+
+        if value is None:
+            operations.append(PatchRemove(op="remove", path=key))
+
+        elif isinstance(value, dict):
+            nested_operations = merge_to_operations(value)
+
+            for nested_operation in nested_operations:
+                nested_operation.path = f"{key}.{nested_operation.path}"
+                operations.append(nested_operation)
+
+        else:
+            operations.append(PatchAddReplaceTest(op="add", path=key, value=value))
+
+    return operations
+
+
+def check_commands(
+    commands: ESCommandSet,
+    op: str,
+    path: ElasticPath,
+    from_path: bool = False,
+) -> None:
+    """Add Elasticsearch checks to operation.
+
+    Args:
+        commands (List[str]): current commands
+        op (str): the operation of script
+        path (Dict): path of variable to run operation on
+        from_path (bool): True if path is a from path
+
+    """
+    if path.nest:
+        commands.add(
+            f"if (!ctx._source.containsKey('{path.nest}'))"
+            f"{{Debug.explain('{path.nest} does not exist');}}"
+        )
+
+    if path.index or op in ["remove", "replace", "test"] or from_path:
+        commands.add(
+            f"if (!ctx._source{path.es_nest}.containsKey('{path.key}'))"
+            f"{{Debug.explain('{path.key}  does not exist in {path.nest}');}}"
+        )
+
+    if from_path and path.index is not None:
+        commands.add(
+            f"if ((ctx._source{path.es_location} instanceof ArrayList"
+            f" && ctx._source{path.es_location}.size() < {path.index})"
+            f" || (!(ctx._source{path.es_location} instanceof ArrayList)"
+            f" && !ctx._source{path.es_location}.containsKey('{path.index}')))"
+            f"{{Debug.explain('{path.path} does not exist');}}"
+        )
+
+
+def remove_commands(commands: ESCommandSet, path: ElasticPath) -> None:
+    """Remove value at path.
+
+    Args:
+        commands (List[str]): current commands
+        path (ElasticPath): Path to value to be removed
+
+    """
+    if path.index is not None:
+        commands.add(
+            f"def {path.variable_name} = ctx._source{path.es_location}.remove({path.index});"
+        )
+
+    else:
+        commands.add(
+            f"def {path.variable_name} = ctx._source{path.es_nest}.remove('{path.key}');"
+        )
+
+
+def add_commands(
+    commands: ESCommandSet,
+    operation: PatchOperation,
+    path: ElasticPath,
+    from_path: ElasticPath,
+    params: Dict,
+) -> None:
+    """Add value at path.
+
+    Args:
+        commands (List[str]): current commands
+        operation (PatchOperation): operation to run
+        path (ElasticPath): path for value to be added
+
+    """
+    if from_path is not None:
+        value = (
+            from_path.variable_name
+            if operation.op == "move"
+            else f"ctx._source.{from_path.es_path}"
+        )
+    else:
+        value = f"params.{path.param_key}"
+        params[path.param_key] = operation.value
+
+    if path.index is not None:
+        commands.add(
+            f"if (ctx._source{path.es_location} instanceof ArrayList)"
+            f"{{ctx._source{path.es_location}.{'add' if operation.op in ['add', 'move'] else 'set'}({path.index}, {value})}}"
+            f"else{{ctx._source.{path.es_path} = {value}}}"
+        )
+
+    else:
+        commands.add(f"ctx._source.{path.es_path} = {value};")
+
+
+def test_commands(
+    commands: ESCommandSet, operation: PatchOperation, path: ElasticPath, params: Dict
+) -> None:
+    """Test value at path.
+
+    Args:
+        commands (List[str]): current commands
+        operation (PatchOperation): operation to run
+        path (ElasticPath): path for value to be tested
+    """
+    value = f"params.{path.param_key}"
+    params[path.param_key] = operation.value
+
+    commands.add(
+        f"if (ctx._source.{path.es_path} != {value})"
+        f"{{Debug.explain('Test failed `{path.path}` | "
+        f"{operation.json_value} != ' + ctx._source.{path.es_path});}}"
+    )
+
+
+def operations_to_script(operations: List) -> Dict:
+    """Convert list of operation to painless script.
+
+    Args:
+        operations: List of RF6902 operations.
+
+    Returns:
+        Dict: elasticsearch update script.
+    """
+    commands: ESCommandSet = ESCommandSet()
+    params: Dict = {}
+
+    for operation in operations:
+        path = ElasticPath(path=operation.path)
+        from_path = (
+            ElasticPath(path=operation.from_) if hasattr(operation, "from_") else None
+        )
+
+        check_commands(commands=commands, op=operation.op, path=path)
+        if from_path is not None:
+            check_commands(
+                commands=commands, op=operation.op, path=from_path, from_path=True
+            )
+
+        if operation.op in ["remove", "move"]:
+            remove_path = from_path if from_path else path
+            remove_commands(commands=commands, path=remove_path)
+
+        if operation.op in ["add", "replace", "copy", "move"]:
+            add_commands(
+                commands=commands,
+                operation=operation,
+                path=path,
+                from_path=from_path,
+                params=params,
+            )
+
+        if operation.op == "test":
+            test_commands(
+                commands=commands, operation=operation, path=path, params=params
+            )
+
+        source = "".join(commands)
+
+    return {
+        "source": source,
+        "lang": "painless",
+        "params": params,
+    }

{sfeos_helpers-5.0.0a0 → sfeos_helpers-6.0.0}/stac_fastapi/sfeos_helpers/filter/client.py

@@ -1,12 +1,12 @@
 """Filter client implementation for Elasticsearch/OpenSearch."""
 
 from collections import deque
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 
 import attr
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES
+from stac_fastapi.core.extensions.filter import ALL_QUERYABLES, DEFAULT_QUERYABLES
 from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
 from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON
 
@@ -59,31 +59,31 @@ class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
 
         mapping_data = await self.database.get_items_mapping(collection_id)
         mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
-        stack = deque(mapping_properties.items())
+        stack: deque[Tuple[str, Dict[str, Any]]] = deque(mapping_properties.items())
+        enum_fields: Dict[str, Dict[str, Any]] = {}
 
         while stack:
-            field_name, field_def = stack.popleft()
+            field_fqn, field_def = stack.popleft()
 
             # Iterate over nested fields
             field_properties = field_def.get("properties")
             if field_properties:
-                # Fields in Item Properties should be exposed with their un-prefixed names,
-                # and not require expressions to prefix them with properties,
-                # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
-                if field_name == "properties":
-                    stack.extend(field_properties.items())
-                else:
-                    stack.extend(
-                        (f"{field_name}.{k}", v) for k, v in field_properties.items()
-                    )
+                stack.extend(
+                    (f"{field_fqn}.{k}", v) for k, v in field_properties.items()
+                )
 
             # Skip non-indexed or disabled fields
             field_type = field_def.get("type")
             if not field_type or not field_def.get("enabled", True):
                 continue
 
+            # Fields in Item Properties should be exposed with their un-prefixed names,
+            # and not require expressions to prefix them with properties,
+            # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
+            field_name = field_fqn.removeprefix("properties.")
+
             # Generate field properties
-            field_result = DEFAULT_QUERYABLES.get(field_name, {})
+            field_result = ALL_QUERYABLES.get(field_name, {})
             properties[field_name] = field_result
 
             field_name_human = field_name.replace("_", " ").title()
@@ -95,4 +95,13 @@ class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
             if field_type in {"date", "date_nanos"}:
                 field_result.setdefault("format", "date-time")
 
+            if field_result.pop("$enum", False):
+                enum_fields[field_fqn] = field_result
+
+        if enum_fields:
+            for field_fqn, unique_values in (
+                await self.database.get_items_unique_values(collection_id, enum_fields)
+            ).items():
+                enum_fields[field_fqn]["enum"] = unique_values
+
         return queryables

sfeos_helpers-6.0.0/stac_fastapi/sfeos_helpers/models/patch.py

@@ -0,0 +1,166 @@
+"""patch helpers."""
+
+import re
+from typing import Any, Dict, Optional, Union
+
+from pydantic import BaseModel, computed_field, model_validator
+
+regex = re.compile(r"([^.' ]*:[^.'[ ]*)\.?")
+replacements = str.maketrans({"/": "", ".": "", ":": "", "[": "", "]": ""})
+
+
+class ESCommandSet:
+    """Uses dictionary keys to behaviour of ordered set.
+
+    Yields:
+        str: Elasticsearch commands
+    """
+
+    dict_: Dict[str, None] = {}
+
+    def __init__(self):
+        """Initialise ESCommandSet instance."""
+        self.dict_ = {}
+
+    def add(self, value: str):
+        """Add command.
+
+        Args:
+            value (str): value to be added
+        """
+        self.dict_[value] = None
+
+    def remove(self, value: str):
+        """Remove command.
+
+        Args:
+            value (str): value to be removed
+        """
+        del self.dict_[value]
+
+    def __iter__(self):
+        """Iterate Elasticsearch commands.
+
+        Yields:
+            str: Elasticsearch command
+        """
+        yield from self.dict_.keys()
+
+
+def to_es(string: str):
+    """Convert patch operation key to Elasticsearch key.
+
+    Args:
+        string (str): string to be converted
+
+    Returns:
+        _type_: converted string
+    """
+    if matches := regex.findall(string):
+        for match in set(matches):
+            string = re.sub(rf"\.?{match}", f"['{match}']", string)
+
+    return string
+
+
+class ElasticPath(BaseModel):
+    """Converts a JSON path to an Elasticsearch path.
+
+    Args:
+        path (str): JSON path to be converted.
+
+    """
+
+    path: str
+    nest: Optional[str] = None
+    partition: Optional[str] = None
+    key: Optional[str] = None
+
+    es_path: Optional[str] = None
+    es_nest: Optional[str] = None
+    es_key: Optional[str] = None
+
+    index_: Optional[int] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_model(cls, data: Any):
+        """Set optional fields from JSON path.
+
+        Args:
+            data (Any): input data
+        """
+        data["path"] = data["path"].lstrip("/").replace("/", ".")
+        data["nest"], data["partition"], data["key"] = data["path"].rpartition(".")
+
+        if data["key"].lstrip("-").isdigit() or data["key"] == "-":
+            data["index_"] = -1 if data["key"] == "-" else int(data["key"])
+            data["path"] = f"{data['nest']}[{data['index_']}]"
+            data["nest"], data["partition"], data["key"] = data["nest"].rpartition(".")
+
+        data["es_path"] = to_es(data["path"])
+        data["es_nest"] = f".{to_es(data['nest'])}" if data["nest"] else ""
+        data["es_key"] = to_es(data["key"])
+
+        return data
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def index(self) -> Union[int, str, None]:
+        """Compute location of path.
+
+        Returns:
+            str: path index
+        """
+        if self.index_ and self.index_ < 0:
+
+            return f"ctx._source.{self.location}.size() - {-self.index_}"
+
+        return self.index_
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def location(self) -> str:
+        """Compute location of path.
+
+        Returns:
+            str: path location
+        """
+        return self.nest + self.partition + self.key
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def es_location(self) -> str:
+        """Compute location of path.
+
+        Returns:
+            str: path location
+        """
+        if self.es_key and ":" in self.es_key:
+            return self.es_nest + self.es_key
+        return self.es_nest + self.partition + self.es_key
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def variable_name(self) -> str:
+        """Variable name for scripting.
+
+        Returns:
+            str: variable name
+        """
+        if self.index is not None:
+            return f"{self.location.replace('.','_').replace(':','_')}_{self.index}"
+
+        return (
+            f"{self.nest.replace('.','_').replace(':','_')}_{self.key.replace(':','_')}"
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def param_key(self) -> str:
+        """Param key for scripting.
+
+        Returns:
+            str: param key
+        """
+        return self.path.translate(replacements)

sfeos_helpers-6.0.0/stac_fastapi/sfeos_helpers/version.py

@@ -0,0 +1,2 @@
+"""library version."""
+__version__ = "6.0.0"

sfeos_helpers-5.0.0a0/sfeos_helpers.egg-info/requires.txt

@@ -1 +0,0 @@
-stac-fastapi.core==5.0.0a0

sfeos_helpers-5.0.0a0/stac_fastapi/sfeos_helpers/database/utils.py

@@ -1,50 +0,0 @@
-"""Utility functions for database operations in Elasticsearch/OpenSearch.
-
-This module provides utility functions for working with database operations
-in Elasticsearch/OpenSearch, such as parameter validation.
-"""
-
-import logging
-from typing import Union
-
-from stac_fastapi.core.utilities import get_bool_env
-
-
-def validate_refresh(value: Union[str, bool]) -> str:
-    """
-    Validate the `refresh` parameter value.
-
-    Args:
-        value (Union[str, bool]): The `refresh` parameter value, which can be a string or a boolean.
-
-    Returns:
-        str: The validated value of the `refresh` parameter, which can be "true", "false", or "wait_for".
-    """
-    logger = logging.getLogger(__name__)
-
-    # Handle boolean-like values using get_bool_env
-    if isinstance(value, bool) or value in {
-        "true",
-        "false",
-        "1",
-        "0",
-        "yes",
-        "no",
-        "y",
-        "n",
-    }:
-        is_true = get_bool_env("DATABASE_REFRESH", default=value)
-        return "true" if is_true else "false"
-
-    # Normalize to lowercase for case-insensitivity
-    value = value.lower()
-
-    # Handle "wait_for" explicitly
-    if value == "wait_for":
-        return "wait_for"
-
-    # Log a warning for invalid values and default to "false"
-    logger.warning(
-        f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
-    )
-    return "false"

sfeos_helpers-5.0.0a0/stac_fastapi/sfeos_helpers/version.py

@@ -1,2 +0,0 @@
-"""library version."""
-__version__ = "5.0.0a0"