sfeos-helpers 5.0.0a0__py3-none-any.whl → 6.0.0__py3-none-any.whl

{sfeos_helpers-5.0.0a0.dist-info → sfeos_helpers-6.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sfeos-helpers
-Version: 5.0.0a0
+Version: 6.0.0
 Summary: Helper library for the Elasticsearch and Opensearch stac-fastapi backends.
 Home-page: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch
 License: MIT
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: stac-fastapi.core==5.0.0a0
+Requires-Dist: stac-fastapi.core==6.0.0
 
 # stac-fastapi-elasticsearch-opensearch
 

{sfeos_helpers-5.0.0a0.dist-info → sfeos_helpers-6.0.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 stac_fastapi/sfeos_helpers/mappings.py,sha256=z6GJFJUE7bRKF9ODc8_ddkb7JCOokMtj4p2LeaQqrQQ,8237
-stac_fastapi/sfeos_helpers/version.py,sha256=c2gZCryeGGpiwcZ3c6UrJnUyJqZkYSBi5EhoLK_wRU0,47
+stac_fastapi/sfeos_helpers/version.py,sha256=Fo5UFEQVxJZ3nywa3IY-enu5UQBE0X45nrQaRBe8c9o,45
 stac_fastapi/sfeos_helpers/aggregation/__init__.py,sha256=Mym17lFh90by1GnoQgMyIKAqRNJnvCgVSXDYzjBiPQk,1210
 stac_fastapi/sfeos_helpers/aggregation/client.py,sha256=JCUVBXsUXHdUXn59WaytGxB-R6OLhC_LApPm-pzQNTI,17818
 stac_fastapi/sfeos_helpers/aggregation/format.py,sha256=qUW1jjh2EEjy-V7riliFR77grpi-AgsTmP76z60K5Lo,2011
@@ -9,12 +9,13 @@ stac_fastapi/sfeos_helpers/database/document.py,sha256=LtjX15gvaOuZC_k2t_oQhys_c
 stac_fastapi/sfeos_helpers/database/index.py,sha256=7xFNzOQzFxBAtQUKUmhKcqz5j3_HRPuwIecdEEWb0i0,4476
 stac_fastapi/sfeos_helpers/database/mapping.py,sha256=4-MSd4xH5wg7yoC4aPjzYMDSEvP026bw4k2TfffMT5E,1387
 stac_fastapi/sfeos_helpers/database/query.py,sha256=3aFby56ggvTxtBg1vH1AXw2P2_CloPIV-cwL260P47E,2972
-stac_fastapi/sfeos_helpers/database/utils.py,sha256=UTUNQWOqpY_TpwXorBQtUFqkG06Ciew7-s5QZVxD-NE,1421
+stac_fastapi/sfeos_helpers/database/utils.py,sha256=nPTWqP-pdSZumeYRx92XTN4xmmufRSqCDHaiSLdg-3o,7311
 stac_fastapi/sfeos_helpers/filter/__init__.py,sha256=n3zL_MhEGOoxMz1KeijyK_UKiZ0MKPl90zHtYI5RAy8,1557
-stac_fastapi/sfeos_helpers/filter/client.py,sha256=q7we754Ms04EcRNNKMMiaP4p_H9Zv6khIdInaJek2Lw,3864
+stac_fastapi/sfeos_helpers/filter/client.py,sha256=QwjYWXkevoVS7HPtoXfeSzDy-_GJnFhPJtJM49D14oU,4229
 stac_fastapi/sfeos_helpers/filter/cql2.py,sha256=Cg9kRYD9CVkVSyRqOyB5oVXmlyteSn2bw88sqklGpUM,955
 stac_fastapi/sfeos_helpers/filter/transform.py,sha256=1GEWQSp-rbq7_1nDVv1ApDbWxt8DswJWxwaxzV85gj4,4644
-sfeos_helpers-5.0.0a0.dist-info/METADATA,sha256=LmNQqszPNw1ba2SRDwhw9MWzc-5ObOAK7NOospWIB6I,31177
-sfeos_helpers-5.0.0a0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-sfeos_helpers-5.0.0a0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
-sfeos_helpers-5.0.0a0.dist-info/RECORD,,
+stac_fastapi/sfeos_helpers/models/patch.py,sha256=akzfF7vXI6AyGDrxgP8KqVwBvBDQayXYNNf4pm6Q8qM,4209
+sfeos_helpers-6.0.0.dist-info/METADATA,sha256=huD7geSPsXMjM-KvE7KsaM6gTuGHRO7QPDBlpyTBayc,31173
+sfeos_helpers-6.0.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+sfeos_helpers-6.0.0.dist-info/top_level.txt,sha256=vqn-D9-HsRPTTxy0Vk_KkDmTiMES4owwBQ3ydSZYb2s,13
+sfeos_helpers-6.0.0.dist-info/RECORD,,

stac_fastapi/sfeos_helpers/database/utils.py

@@ -5,9 +5,15 @@ in Elasticsearch/OpenSearch, such as parameter validation.
 """
 
 import logging
-from typing import Union
+from typing import Dict, List, Union
 
 from stac_fastapi.core.utilities import get_bool_env
+from stac_fastapi.extensions.core.transaction.request import (
+    PatchAddReplaceTest,
+    PatchOperation,
+    PatchRemove,
+)
+from stac_fastapi.sfeos_helpers.models.patch import ElasticPath, ESCommandSet
 
 
 def validate_refresh(value: Union[str, bool]) -> str:
@@ -48,3 +54,195 @@ def validate_refresh(value: Union[str, bool]) -> str:
         f"Invalid value for `refresh`: '{value}'. Expected 'true', 'false', or 'wait_for'. Defaulting to 'false'."
     )
     return "false"
+
+
+def merge_to_operations(data: Dict) -> List:
+    """Convert merge operation to list of RF6902 operations.
+
+    Args:
+        data: dictionary to convert.
+
+    Returns:
+        List: list of RF6902 operations.
+    """
+    operations = []
+
+    for key, value in data.copy().items():
+
+        if value is None:
+            operations.append(PatchRemove(op="remove", path=key))
+
+        elif isinstance(value, dict):
+            nested_operations = merge_to_operations(value)
+
+            for nested_operation in nested_operations:
+                nested_operation.path = f"{key}.{nested_operation.path}"
+                operations.append(nested_operation)
+
+        else:
+            operations.append(PatchAddReplaceTest(op="add", path=key, value=value))
+
+    return operations
+
+
+def check_commands(
+    commands: ESCommandSet,
+    op: str,
+    path: ElasticPath,
+    from_path: bool = False,
+) -> None:
+    """Add Elasticsearch checks to operation.
+
+    Args:
+        commands (List[str]): current commands
+        op (str): the operation of script
+        path (Dict): path of variable to run operation on
+        from_path (bool): True if path is a from path
+
+    """
+    if path.nest:
+        commands.add(
+            f"if (!ctx._source.containsKey('{path.nest}'))"
+            f"{{Debug.explain('{path.nest} does not exist');}}"
+        )
+
+    if path.index or op in ["remove", "replace", "test"] or from_path:
+        commands.add(
+            f"if (!ctx._source{path.es_nest}.containsKey('{path.key}'))"
+            f"{{Debug.explain('{path.key}  does not exist in {path.nest}');}}"
+        )
+
+    if from_path and path.index is not None:
+        commands.add(
+            f"if ((ctx._source{path.es_location} instanceof ArrayList"
+            f" && ctx._source{path.es_location}.size() < {path.index})"
+            f" || (!(ctx._source{path.es_location} instanceof ArrayList)"
+            f" && !ctx._source{path.es_location}.containsKey('{path.index}')))"
+            f"{{Debug.explain('{path.path} does not exist');}}"
+        )
+
+
+def remove_commands(commands: ESCommandSet, path: ElasticPath) -> None:
+    """Remove value at path.
+
+    Args:
+        commands (List[str]): current commands
+        path (ElasticPath): Path to value to be removed
+
+    """
+    if path.index is not None:
+        commands.add(
+            f"def {path.variable_name} = ctx._source{path.es_location}.remove({path.index});"
+        )
+
+    else:
+        commands.add(
+            f"def {path.variable_name} = ctx._source{path.es_nest}.remove('{path.key}');"
+        )
+
+
+def add_commands(
+    commands: ESCommandSet,
+    operation: PatchOperation,
+    path: ElasticPath,
+    from_path: ElasticPath,
+    params: Dict,
+) -> None:
+    """Add value at path.
+
+    Args:
+        commands (List[str]): current commands
+        operation (PatchOperation): operation to run
+        path (ElasticPath): path for value to be added
+
+    """
+    if from_path is not None:
+        value = (
+            from_path.variable_name
+            if operation.op == "move"
+            else f"ctx._source.{from_path.es_path}"
+        )
+    else:
+        value = f"params.{path.param_key}"
+        params[path.param_key] = operation.value
+
+    if path.index is not None:
+        commands.add(
+            f"if (ctx._source{path.es_location} instanceof ArrayList)"
+            f"{{ctx._source{path.es_location}.{'add' if operation.op in ['add', 'move'] else 'set'}({path.index}, {value})}}"
+            f"else{{ctx._source.{path.es_path} = {value}}}"
+        )
+
+    else:
+        commands.add(f"ctx._source.{path.es_path} = {value};")
+
+
+def test_commands(
+    commands: ESCommandSet, operation: PatchOperation, path: ElasticPath, params: Dict
+) -> None:
+    """Test value at path.
+
+    Args:
+        commands (List[str]): current commands
+        operation (PatchOperation): operation to run
+        path (ElasticPath): path for value to be tested
+    """
+    value = f"params.{path.param_key}"
+    params[path.param_key] = operation.value
+
+    commands.add(
+        f"if (ctx._source.{path.es_path} != {value})"
+        f"{{Debug.explain('Test failed `{path.path}` | "
+        f"{operation.json_value} != ' + ctx._source.{path.es_path});}}"
+    )
+
+
+def operations_to_script(operations: List) -> Dict:
+    """Convert list of operation to painless script.
+
+    Args:
+        operations: List of RF6902 operations.
+
+    Returns:
+        Dict: elasticsearch update script.
+    """
+    commands: ESCommandSet = ESCommandSet()
+    params: Dict = {}
+
+    for operation in operations:
+        path = ElasticPath(path=operation.path)
+        from_path = (
+            ElasticPath(path=operation.from_) if hasattr(operation, "from_") else None
+        )
+
+        check_commands(commands=commands, op=operation.op, path=path)
+        if from_path is not None:
+            check_commands(
+                commands=commands, op=operation.op, path=from_path, from_path=True
+            )
+
+        if operation.op in ["remove", "move"]:
+            remove_path = from_path if from_path else path
+            remove_commands(commands=commands, path=remove_path)
+
+        if operation.op in ["add", "replace", "copy", "move"]:
+            add_commands(
+                commands=commands,
+                operation=operation,
+                path=path,
+                from_path=from_path,
+                params=params,
+            )
+
+        if operation.op == "test":
+            test_commands(
+                commands=commands, operation=operation, path=path, params=params
+            )
+
+        source = "".join(commands)
+
+    return {
+        "source": source,
+        "lang": "painless",
+        "params": params,
+    }

stac_fastapi/sfeos_helpers/filter/client.py

@@ -1,12 +1,12 @@
 """Filter client implementation for Elasticsearch/OpenSearch."""
 
 from collections import deque
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 
 import attr
 
 from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES
+from stac_fastapi.core.extensions.filter import ALL_QUERYABLES, DEFAULT_QUERYABLES
 from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient
 from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON
 
@@ -59,31 +59,31 @@ class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
 
         mapping_data = await self.database.get_items_mapping(collection_id)
         mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"]
-        stack = deque(mapping_properties.items())
+        stack: deque[Tuple[str, Dict[str, Any]]] = deque(mapping_properties.items())
+        enum_fields: Dict[str, Dict[str, Any]] = {}
 
         while stack:
-            field_name, field_def = stack.popleft()
+            field_fqn, field_def = stack.popleft()
 
             # Iterate over nested fields
             field_properties = field_def.get("properties")
             if field_properties:
-                # Fields in Item Properties should be exposed with their un-prefixed names,
-                # and not require expressions to prefix them with properties,
-                # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
-                if field_name == "properties":
-                    stack.extend(field_properties.items())
-                else:
-                    stack.extend(
-                        (f"{field_name}.{k}", v) for k, v in field_properties.items()
-                    )
+                stack.extend(
+                    (f"{field_fqn}.{k}", v) for k, v in field_properties.items()
+                )
 
             # Skip non-indexed or disabled fields
             field_type = field_def.get("type")
             if not field_type or not field_def.get("enabled", True):
                 continue
 
+            # Fields in Item Properties should be exposed with their un-prefixed names,
+            # and not require expressions to prefix them with properties,
+            # e.g., eo:cloud_cover instead of properties.eo:cloud_cover.
+            field_name = field_fqn.removeprefix("properties.")
+
             # Generate field properties
-            field_result = DEFAULT_QUERYABLES.get(field_name, {})
+            field_result = ALL_QUERYABLES.get(field_name, {})
             properties[field_name] = field_result
 
             field_name_human = field_name.replace("_", " ").title()
@@ -95,4 +95,13 @@ class EsAsyncBaseFiltersClient(AsyncBaseFiltersClient):
             if field_type in {"date", "date_nanos"}:
                 field_result.setdefault("format", "date-time")
 
+            if field_result.pop("$enum", False):
+                enum_fields[field_fqn] = field_result
+
+        if enum_fields:
+            for field_fqn, unique_values in (
+                await self.database.get_items_unique_values(collection_id, enum_fields)
+            ).items():
+                enum_fields[field_fqn]["enum"] = unique_values
+
         return queryables

stac_fastapi/sfeos_helpers/models/patch.py

@@ -0,0 +1,166 @@
+"""patch helpers."""
+
+import re
+from typing import Any, Dict, Optional, Union
+
+from pydantic import BaseModel, computed_field, model_validator
+
+regex = re.compile(r"([^.' ]*:[^.'[ ]*)\.?")
+replacements = str.maketrans({"/": "", ".": "", ":": "", "[": "", "]": ""})
+
+
+class ESCommandSet:
+    """Uses dictionary keys to behaviour of ordered set.
+
+    Yields:
+        str: Elasticsearch commands
+    """
+
+    dict_: Dict[str, None] = {}
+
+    def __init__(self):
+        """Initialise ESCommandSet instance."""
+        self.dict_ = {}
+
+    def add(self, value: str):
+        """Add command.
+
+        Args:
+            value (str): value to be added
+        """
+        self.dict_[value] = None
+
+    def remove(self, value: str):
+        """Remove command.
+
+        Args:
+            value (str): value to be removed
+        """
+        del self.dict_[value]
+
+    def __iter__(self):
+        """Iterate Elasticsearch commands.
+
+        Yields:
+            str: Elasticsearch command
+        """
+        yield from self.dict_.keys()
+
+
+def to_es(string: str):
+    """Convert patch operation key to Elasticsearch key.
+
+    Args:
+        string (str): string to be converted
+
+    Returns:
+        _type_: converted string
+    """
+    if matches := regex.findall(string):
+        for match in set(matches):
+            string = re.sub(rf"\.?{match}", f"['{match}']", string)
+
+    return string
+
+
+class ElasticPath(BaseModel):
+    """Converts a JSON path to an Elasticsearch path.
+
+    Args:
+        path (str): JSON path to be converted.
+
+    """
+
+    path: str
+    nest: Optional[str] = None
+    partition: Optional[str] = None
+    key: Optional[str] = None
+
+    es_path: Optional[str] = None
+    es_nest: Optional[str] = None
+    es_key: Optional[str] = None
+
+    index_: Optional[int] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_model(cls, data: Any):
+        """Set optional fields from JSON path.
+
+        Args:
+            data (Any): input data
+        """
+        data["path"] = data["path"].lstrip("/").replace("/", ".")
+        data["nest"], data["partition"], data["key"] = data["path"].rpartition(".")
+
+        if data["key"].lstrip("-").isdigit() or data["key"] == "-":
+            data["index_"] = -1 if data["key"] == "-" else int(data["key"])
+            data["path"] = f"{data['nest']}[{data['index_']}]"
+            data["nest"], data["partition"], data["key"] = data["nest"].rpartition(".")
+
+        data["es_path"] = to_es(data["path"])
+        data["es_nest"] = f".{to_es(data['nest'])}" if data["nest"] else ""
+        data["es_key"] = to_es(data["key"])
+
+        return data
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def index(self) -> Union[int, str, None]:
+        """Compute location of path.
+
+        Returns:
+            str: path index
+        """
+        if self.index_ and self.index_ < 0:
+
+            return f"ctx._source.{self.location}.size() - {-self.index_}"
+
+        return self.index_
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def location(self) -> str:
+        """Compute location of path.
+
+        Returns:
+            str: path location
+        """
+        return self.nest + self.partition + self.key
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def es_location(self) -> str:
+        """Compute location of path.
+
+        Returns:
+            str: path location
+        """
+        if self.es_key and ":" in self.es_key:
+            return self.es_nest + self.es_key
+        return self.es_nest + self.partition + self.es_key
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def variable_name(self) -> str:
+        """Variable name for scripting.
+
+        Returns:
+            str: variable name
+        """
+        if self.index is not None:
+            return f"{self.location.replace('.','_').replace(':','_')}_{self.index}"
+
+        return (
+            f"{self.nest.replace('.','_').replace(':','_')}_{self.key.replace(':','_')}"
+        )
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def param_key(self) -> str:
+        """Param key for scripting.
+
+        Returns:
+            str: param key
+        """
+        return self.path.translate(replacements)

stac_fastapi/sfeos_helpers/version.py

@@ -1,2 +1,2 @@
 """library version."""
-__version__ = "5.0.0a0"
+__version__ = "6.0.0"