muscles-data-elasticsearch 0.1.0__tar.gz

muscles_data_elasticsearch-0.1.0/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: muscles-data-elasticsearch
+Version: 0.1.0
+Summary: Elasticsearch SearchIndexPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: elasticsearch<9.0,>=8.0
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-elasticsearch
+
+Elasticsearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Elasticsearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_elasticsearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_elasticsearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_elasticsearch import ElasticsearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(ElasticsearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.elastic", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.elastic:
+      type: elasticsearch
+      url_env: ELASTICSEARCH_URL
+      api_key: ${ELASTICSEARCH_API_KEY}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the Elasticsearch client and index lazily on search, index/delete,
+explicit native access or `data.doctor`. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+Missing indexes are created with a default mapping for `text`, `title` and
+`metadata`; `mapping` and `settings` resource options can customize it.
+`upsert_documents()` uses Elasticsearch bulk operations.
+
+See `muscular-example/example_data_elasticsearch_1` for an executable example.

muscles_data_elasticsearch-0.1.0/README.md

@@ -0,0 +1,68 @@
+# muscles-data-elasticsearch
+
+Elasticsearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Elasticsearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_elasticsearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_elasticsearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_elasticsearch import ElasticsearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(ElasticsearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.elastic", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.elastic:
+      type: elasticsearch
+      url_env: ELASTICSEARCH_URL
+      api_key: ${ELASTICSEARCH_API_KEY}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the Elasticsearch client and index lazily on search, index/delete,
+explicit native access or `data.doctor`. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+Missing indexes are created with a default mapping for `text`, `title` and
+`metadata`; `mapping` and `settings` resource options can customize it.
+`upsert_documents()` uses Elasticsearch bulk operations.
+
+See `muscular-example/example_data_elasticsearch_1` for an executable example.

muscles_data_elasticsearch-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "muscles-data-elasticsearch"
+version = "0.1.0"
+description = "Elasticsearch SearchIndexPort adapter for muscles-data"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Denis B.", email = "denis@butko.info" }]
+dependencies = [
+  "muscles-data>=0.1.0,<1.0.0",
+  "elasticsearch>=8.0,<9.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0,<9.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+  "integration: tests that require the local backend integration stack",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

muscles_data_elasticsearch-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch/__init__.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from .adapter import (
+    ElasticsearchAdapterError,
+    ElasticsearchClientMissingError,
+    ElasticsearchConfigError,
+    ElasticsearchConnectionError,
+    ElasticsearchFilterError,
+    ElasticsearchResourceMissingError,
+    ElasticsearchSchemaError,
+    ElasticsearchSearchAdapter,
+    ElasticsearchSearchFactory,
+    elasticsearch_filter_from_mapping,
+)
+
+
+__all__ = [
+    "ElasticsearchAdapterError",
+    "ElasticsearchClientMissingError",
+    "ElasticsearchConfigError",
+    "ElasticsearchConnectionError",
+    "ElasticsearchFilterError",
+    "ElasticsearchResourceMissingError",
+    "ElasticsearchSchemaError",
+    "ElasticsearchSearchAdapter",
+    "ElasticsearchSearchFactory",
+    "elasticsearch_filter_from_mapping",
+]

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch/adapter.py

@@ -0,0 +1,541 @@
+from __future__ import annotations
+
+import importlib
+import threading
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any, Mapping
+from urllib.parse import urlsplit
+
+from muscles_data.config import DataResourceConfig
+from muscles_data.errors import (
+    DataConfigurationError,
+    DataConnectionError,
+    DataResourceMissingError,
+    DataSchemaMismatchError,
+    DataUnsupportedOperationError,
+    DataError,
+)
+from muscles_data.models import DataCapability, HealthResult, InspectResult, SearchHit, WriteResult
+
+
+_CLIENT_UNSET = object()
+_RANGE_OPERATORS = {"gt", "gte", "lt", "lte"}
+_ALLOWED_OPTIONS = {
+    "url",
+    "url_env",
+    "api_key",
+    "username",
+    "password",
+    "basic_auth",
+    "index",
+    "timeout",
+    "verify_certs",
+    "native_client",
+    "text_field",
+    "metadata_field",
+    "mapping",
+    "settings",
+}
+
+DEFAULT_INDEX_MAPPING = {
+    "properties": {
+        "text": {"type": "text"},
+        "title": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "metadata": {"type": "object", "dynamic": True},
+    }
+}
+
+
+class ElasticsearchAdapterError(DataError):
+    """Base error for Elasticsearch search adapter failures."""
+
+
+class ElasticsearchConfigError(ValueError, DataConfigurationError, ElasticsearchAdapterError):
+    """Raised when an Elasticsearch resource config cannot be mapped safely."""
+
+
+class ElasticsearchClientMissingError(DataConfigurationError, ElasticsearchAdapterError):
+    """Raised when Elasticsearch adapter is used without an available client."""
+
+
+class ElasticsearchConnectionError(DataConnectionError, ElasticsearchAdapterError):
+    """Raised when an Elasticsearch operation cannot reach or use the backend."""
+
+
+class ElasticsearchFilterError(DataUnsupportedOperationError, ElasticsearchAdapterError):
+    """Raised when a data filter cannot be translated to Elasticsearch."""
+
+
+class ElasticsearchSchemaError(DataSchemaMismatchError, ElasticsearchAdapterError):
+    """Raised when the configured index mapping cannot be prepared."""
+
+
+class ElasticsearchResourceMissingError(DataResourceMissingError, ElasticsearchAdapterError):
+    """Raised when an index is missing and cannot be created."""
+
+
+class ElasticsearchSearchAdapter:
+    resource_type = "elasticsearch"
+
+    def __init__(
+        self,
+        config: DataResourceConfig,
+        *,
+        client_factory: Callable[[DataResourceConfig], Any] | None = None,
+    ) -> None:
+        self.config = config
+        self._client_factory = client_factory
+        self._client: Any = _CLIENT_UNSET
+        self._lock = threading.RLock()
+        self._index_ready = False
+        self.closed = False
+
+    def search_text(
+        self,
+        query: str,
+        filters: Mapping[str, Any] | None = None,
+        limit: int = 10,
+        options: Mapping[str, Any] | None = None,
+    ) -> list[SearchHit]:
+        options = dict(options or {})
+        kwargs: dict[str, Any] = {
+            "index": self.index_name(),
+            "query": self._query(query, filters),
+            "size": max(0, int(limit)),
+        }
+        if options.pop("highlight", False):
+            fields = options.pop("highlight_fields", [self.text_field()])
+            kwargs["highlight"] = {"fields": {str(field): {} for field in fields}}
+        for key in ("from_", "sort", "track_total_hits", "routing", "preference", "search_after"):
+            if key in options:
+                kwargs[key] = options[key]
+        try:
+            response = self._client_instance().search(**kwargs)
+        except (ElasticsearchClientMissingError, ElasticsearchConfigError, ElasticsearchFilterError):
+            raise
+        except Exception as exc:
+            raise ElasticsearchConnectionError(self._safe_error(exc)) from exc
+        return [
+            _hit_from_response(hit, text_field=self.text_field(), metadata_field=self.metadata_field())
+            for hit in _hits(response)
+        ]
+
+    def upsert_documents(self, items: list[Mapping[str, Any]], options: Mapping[str, Any] | None = None) -> WriteResult:
+        options = dict(options or {})
+        client = self._client_instance()
+        try:
+            operations: list[dict[str, Any]] = []
+            for item in items:
+                document_id = str(item["id"])
+                operations.extend(
+                    [
+                        {"index": {"_index": self.index_name(), "_id": document_id}},
+                        self._document_from_item(item),
+                    ]
+                )
+            bulk = getattr(client, "bulk", None)
+            if callable(bulk):
+                kwargs: dict[str, Any] = {"operations": operations}
+                for key in ("refresh", "routing", "pipeline"):
+                    if key in options:
+                        kwargs[key] = options[key]
+                response = bulk(**kwargs)
+                errors = _bulk_error_messages(response)
+                if errors:
+                    written = max(0, len(items) - len(errors))
+                    return WriteResult(
+                        status="partial" if written else "failed",
+                        written=written,
+                        matched=written,
+                        errors=errors,
+                    )
+            else:
+                for item in items:
+                    client.index(
+                        index=self.index_name(),
+                        id=str(item["id"]),
+                        document=self._document_from_item(item),
+                    )
+        except KeyError as exc:
+            raise ElasticsearchConfigError(f"Elasticsearch document item requires {exc.args[0]}") from exc
+        except (ElasticsearchClientMissingError, ElasticsearchConfigError, ElasticsearchFilterError):
+            raise
+        except Exception as exc:
+            raise ElasticsearchConnectionError(self._safe_error(exc)) from exc
+        return WriteResult(written=len(items), matched=len(items))
+
+    def delete_documents(
+        self,
+        ids: list[str] | None = None,
+        filters: Mapping[str, Any] | None = None,
+        options: Mapping[str, Any] | None = None,
+    ) -> WriteResult:
+        options = dict(options or {})
+        if ids is None and not filters:
+            return WriteResult()
+        client = self._client_instance()
+        try:
+            if ids is not None:
+                deleted = 0
+                for document_id in ids:
+                    kwargs: dict[str, Any] = {"index": self.index_name(), "id": str(document_id)}
+                    for key in ("refresh", "routing"):
+                        if key in options:
+                            kwargs[key] = options[key]
+                    client.delete(**kwargs)
+                    deleted += 1
+                return WriteResult(deleted=deleted, matched=deleted)
+            query = self._filter_query(filters)
+            kwargs = {"index": self.index_name(), "query": query}
+            for key in ("refresh", "routing", "conflicts"):
+                if key in options:
+                    kwargs[key] = options[key]
+            response = client.delete_by_query(**kwargs)
+        except (ElasticsearchClientMissingError, ElasticsearchConfigError, ElasticsearchFilterError):
+            raise
+        except Exception as exc:
+            raise ElasticsearchConnectionError(self._safe_error(exc)) from exc
+        deleted = int(_mapping_get(response, "deleted", 0) or 0)
+        return WriteResult(deleted=deleted, matched=deleted)
+
+    def inspect(self) -> dict[str, Any]:
+        return asdict(
+            InspectResult(
+                name=self.config.name,
+                type=self.config.type,
+                capabilities=[],
+                initialized=self._client is not _CLIENT_UNSET,
+                status="ok",
+                options=self.config.safe_options(),
+                details={"backend": "elasticsearch", "index": self.index_name(), "mapping": self.mapping()},
+            )
+        )
+
+    def health(self) -> HealthResult:
+        try:
+            client = self._client_instance()
+            ping = getattr(client, "ping", None)
+            if callable(ping) and not bool(ping()):
+                return HealthResult(
+                    status="failed",
+                    code="connection_failed",
+                    message="Elasticsearch ping failed",
+                    details={"index": self.index_name()},
+                )
+            indices = getattr(client, "indices", None)
+            exists = self._index_ready
+            if not exists and indices is not None and hasattr(indices, "exists"):
+                exists = bool(indices.exists(index=self.index_name()))
+        except ElasticsearchResourceMissingError as exc:
+            return HealthResult(status="failed", code="resource_missing", message=str(exc), details={"index": self.index_name()})
+        except ElasticsearchSchemaError as exc:
+            return HealthResult(status="failed", code="schema_mismatch", message=str(exc), details={"index": self.index_name()})
+        except DataConfigurationError as exc:
+            return HealthResult(status="failed", code="configuration_error", message=str(exc), details={"index": self.index_name()})
+        except Exception as exc:
+            return HealthResult(status="failed", code="connection_failed", message=self._safe_error(exc), details={"index": self.index_name()})
+        if not exists:
+            return HealthResult(
+                status="failed",
+                code="resource_missing",
+                message=f"Elasticsearch index '{self.index_name()}' is not available",
+                details={"index": self.index_name()},
+            )
+        return HealthResult(status="ok", code="ok", message="Elasticsearch index is available", details={"index": self.index_name()})
+
+    def close(self) -> None:
+        if self._client is _CLIENT_UNSET:
+            self.closed = True
+            return
+        close = getattr(self._client, "close", None)
+        if callable(close):
+            close()
+        self.closed = True
+
+    def native_client(self):
+        return self._client_instance()
+
+    def index_name(self) -> str:
+        try:
+            return str(self.config.options["index"])
+        except KeyError as exc:
+            raise ElasticsearchConfigError("Elasticsearch resource requires index") from exc
+
+    def mapping(self) -> dict[str, Any]:
+        mapping = self.config.options.get("mapping")
+        if mapping is None:
+            return dict(DEFAULT_INDEX_MAPPING)
+        if not isinstance(mapping, Mapping):
+            raise ElasticsearchConfigError("Elasticsearch mapping must be a mapping")
+        return dict(mapping)
+
+    def text_field(self) -> str:
+        return str(self.config.options.get("text_field", "text"))
+
+    def metadata_field(self) -> str:
+        return str(self.config.options.get("metadata_field", "metadata"))
+
+    def _query(self, query: str, filters: Mapping[str, Any] | None) -> dict[str, Any]:
+        clauses: dict[str, Any] = {}
+        if str(query).strip():
+            clauses["must"] = [{"match": {self.text_field(): {"query": str(query)}}}]
+        else:
+            clauses["must"] = [{"match_all": {}}]
+        filter_clauses = elasticsearch_filter_from_mapping(filters or {}, field_prefix=self.metadata_field())
+        if filter_clauses:
+            clauses["filter"] = filter_clauses
+        return {"bool": clauses}
+
+    def _filter_query(self, filters: Mapping[str, Any] | None) -> dict[str, Any]:
+        filter_clauses = elasticsearch_filter_from_mapping(filters or {}, field_prefix=self.metadata_field())
+        return {"bool": {"filter": filter_clauses}} if filter_clauses else {"match_all": {}}
+
+    def _document_from_item(self, item: Mapping[str, Any]) -> dict[str, Any]:
+        metadata = dict(item.get("payload", {}) or {})
+        metadata.update(dict(item.get("metadata", {}) or {}))
+        document = {
+            self.text_field(): str(item.get("text", "")),
+            self.metadata_field(): metadata,
+        }
+        if item.get("title") is not None:
+            document["title"] = str(item["title"])
+        fields = item.get("fields")
+        if isinstance(fields, Mapping):
+            document.update(dict(fields))
+        return document
+
+    def _client_instance(self):
+        if self._client is _CLIENT_UNSET:
+            with self._lock:
+                if self._client is _CLIENT_UNSET:
+                    self._validate_options()
+                    client = (
+                        self._client_factory(self.config)
+                        if self._client_factory
+                        else _default_elasticsearch_client(self.config)
+                    )
+                    if client is None:
+                        raise ElasticsearchClientMissingError("Elasticsearch client is not available")
+                    self._client = client
+                    try:
+                        self._ensure_index(client)
+                    except Exception:
+                        self._client = _CLIENT_UNSET
+                        close = getattr(client, "close", None)
+                        if callable(close):
+                            close()
+                        raise
+        return self._client
+
+    def _validate_options(self) -> None:
+        unknown = sorted(set(self.config.options) - _ALLOWED_OPTIONS)
+        if unknown:
+            names = ", ".join(unknown)
+            raise ElasticsearchConfigError(f"Unsupported Elasticsearch resource options: {names}")
+        if not self.config.options.get("url") and not self.config.options.get("url_env"):
+            raise ElasticsearchConfigError("Elasticsearch resource requires url_env")
+        basic_auth = self.config.options.get("basic_auth")
+        if basic_auth is not None and (not isinstance(basic_auth, (list, tuple)) or len(basic_auth) != 2):
+            raise ElasticsearchConfigError("Elasticsearch basic_auth must contain username and password")
+
+    def _ensure_index(self, client: Any) -> None:
+        indices = getattr(client, "indices", None)
+        exists = getattr(indices, "exists", None)
+        create = getattr(indices, "create", None)
+        if not callable(exists) or not callable(create):
+            return
+        try:
+            if bool(exists(index=self.index_name())):
+                self._index_ready = True
+                return
+            kwargs: dict[str, Any] = {"index": self.index_name(), "mappings": self.mapping()}
+            settings = self.config.options.get("settings")
+            if settings is not None:
+                kwargs["settings"] = settings
+            create(**kwargs)
+            self._index_ready = True
+        except Exception as exc:
+            message = self._safe_error(exc)
+            if "already exists" in message.lower() or "resource_already_exists" in message.lower():
+                return
+            if "mapping" in message.lower() or "schema" in message.lower():
+                raise ElasticsearchSchemaError(message) from exc
+            raise ElasticsearchResourceMissingError(message) from exc
+
+    def _safe_error(self, exc: Exception) -> str:
+        message = str(exc)
+        try:
+            options = self.config.resolved_options()
+        except DataConfigurationError:
+            options = self.config.options
+        for key, value in options.items():
+            if key in {"url", "api_key", "password", "basic_auth"} and value:
+                if isinstance(value, (list, tuple)):
+                    for item in value:
+                        message = message.replace(str(item), "***")
+                else:
+                    message = message.replace(str(value), "***")
+                    if key == "url":
+                        parsed = urlsplit(str(value))
+                        for item in (parsed.username, parsed.password):
+                            if item:
+                                message = message.replace(str(item), "***")
+        return message
+
+
+class ElasticsearchSearchFactory:
+    resource_type = "elasticsearch"
+
+    def __init__(self, *, client_factory: Callable[[DataResourceConfig], Any] | None = None) -> None:
+        self._client_factory = client_factory
+
+    def capabilities(self, config: DataResourceConfig) -> set[DataCapability]:
+        native = {DataCapability.NATIVE_CLIENT} if bool(config.options.get("native_client")) else set()
+        return {DataCapability.KEYWORD_SEARCH, DataCapability.DOCUMENT_INDEX, DataCapability.HEALTHCHECK} | native
+
+    def create(self, config: DataResourceConfig) -> ElasticsearchSearchAdapter:
+        return ElasticsearchSearchAdapter(config, client_factory=self._client_factory)
+
+
+def elasticsearch_filter_from_mapping(
+    filters: Mapping[str, Any],
+    *,
+    field_prefix: str | None = "metadata",
+) -> list[dict[str, Any]]:
+    clauses: list[dict[str, Any]] = []
+    for key in sorted(filters, key=_filter_sort_key):
+        value = filters[key]
+        if key == "$and":
+            clauses.append({"bool": {"filter": _logical_conditions(value, field_prefix=field_prefix)}})
+        elif key == "$or":
+            clauses.append(
+                {
+                    "bool": {
+                        "should": _logical_conditions(value, field_prefix=field_prefix),
+                        "minimum_should_match": 1,
+                    }
+                }
+            )
+        elif key == "$not":
+            clauses.append({"bool": {"must_not": _logical_conditions(value, field_prefix=field_prefix)}})
+        else:
+            clauses.append(_field_condition(str(key), value, field_prefix=field_prefix))
+    return clauses
+
+
+def _logical_conditions(value: Any, *, field_prefix: str | None) -> list[dict[str, Any]]:
+    if isinstance(value, Mapping):
+        return elasticsearch_filter_from_mapping(value, field_prefix=field_prefix)
+    if isinstance(value, list):
+        conditions: list[dict[str, Any]] = []
+        for item in value:
+            if not isinstance(item, Mapping):
+                raise ElasticsearchFilterError("Elasticsearch logical filter items must be mappings")
+            conditions.extend(elasticsearch_filter_from_mapping(item, field_prefix=field_prefix))
+        return conditions
+    raise ElasticsearchFilterError("Elasticsearch logical filters must be mappings or lists of mappings")
+
+
+def _field_condition(key: str, value: Any, *, field_prefix: str | None) -> dict[str, Any]:
+    field = _field_name(key, field_prefix)
+    if isinstance(value, Mapping):
+        operators = set(value)
+        if operators <= _RANGE_OPERATORS:
+            return {"range": {field: dict(value)}}
+        if operators == {"eq"}:
+            return {"term": {field: value["eq"]}}
+        if operators == {"any"}:
+            return {"terms": {field: list(value["any"])}}
+        unsupported = ", ".join(sorted(str(operator) for operator in operators))
+        raise ElasticsearchFilterError(f"Unsupported Elasticsearch filter operator for '{key}': {unsupported}")
+    if isinstance(value, (list, tuple, set)):
+        return {"terms": {field: list(value)}}
+    return {"term": {field: value}}
+
+
+def _field_name(key: str, field_prefix: str | None) -> str:
+    if not field_prefix or "." in key:
+        return key
+    return f"{field_prefix}.{key}"
+
+
+def _filter_sort_key(key: Any) -> tuple[int, str]:
+    order = {"$and": 0, "$or": 1, "$not": 2}
+    key_text = str(key)
+    return (order.get(key_text, 10), key_text)
+
+
+def _hit_from_response(hit: Mapping[str, Any], *, text_field: str, metadata_field: str) -> SearchHit:
+    source = dict(hit.get("_source", {}) or {})
+    metadata = dict(source.get(metadata_field, {}) or {})
+    highlights = {
+        str(key): [str(item) for item in value]
+        for key, value in dict(hit.get("highlight", {}) or {}).items()
+    }
+    return SearchHit(
+        id=str(hit.get("_id", "")),
+        score=float(hit.get("_score", 0.0) or 0.0),
+        text=source.get(text_field),
+        title=source.get("title"),
+        metadata=metadata,
+        highlights=highlights,
+    )
+
+
+def _hits(response: Any) -> list[Mapping[str, Any]]:
+    return list(_mapping_get(_mapping_get(response, "hits", {}), "hits", []) or [])
+
+
+def _mapping_get(value: Any, key: str, default: Any = None) -> Any:
+    if isinstance(value, Mapping):
+        return value.get(key, default)
+    try:
+        return value[key]
+    except Exception:
+        return default
+
+
+def _bulk_error_messages(response: Any) -> list[str]:
+    """Extract actionable per-document errors from an Elasticsearch bulk reply."""
+    if not bool(_mapping_get(response, "errors", False)):
+        return []
+    messages: list[str] = []
+    items = _mapping_get(response, "items", []) or []
+    for item in items:
+        if not isinstance(item, Mapping) or not item:
+            continue
+        operation, result = next(iter(item.items()))
+        if not isinstance(result, Mapping):
+            continue
+        error = result.get("error")
+        status = int(result.get("status", 0) or 0)
+        if error or status >= 300:
+            detail = error if error is not None else f"status={status}"
+            messages.append(f"{operation}: {detail}")
+    return messages or ["Elasticsearch bulk upsert reported item errors"]
+
+
+def _default_elasticsearch_client(config: DataResourceConfig):
+    try:
+        client_module = importlib.import_module("elasticsearch")
+    except Exception as exc:
+        raise ElasticsearchClientMissingError("Install elasticsearch or muscles-data-elasticsearch to use type=elasticsearch") from exc
+
+    options = config.resolved_options()
+    kwargs: dict[str, Any] = {}
+    if options.get("api_key"):
+        kwargs["api_key"] = options["api_key"]
+    if options.get("basic_auth"):
+        kwargs["basic_auth"] = tuple(options["basic_auth"])
+    elif options.get("username") or options.get("password"):
+        kwargs["basic_auth"] = (options.get("username"), options.get("password"))
+    if options.get("timeout") is not None:
+        kwargs["request_timeout"] = options["timeout"]
+    if options.get("verify_certs") is not None:
+        kwargs["verify_certs"] = bool(options["verify_certs"])
+
+    try:
+        return client_module.Elasticsearch(options["url"], **kwargs)
+    except Exception as exc:
+        raise ElasticsearchConnectionError(str(exc)) from exc

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch.egg-info/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: muscles-data-elasticsearch
+Version: 0.1.0
+Summary: Elasticsearch SearchIndexPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: elasticsearch<9.0,>=8.0
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-elasticsearch
+
+Elasticsearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Elasticsearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_elasticsearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_elasticsearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_elasticsearch import ElasticsearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(ElasticsearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.elastic", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.elastic:
+      type: elasticsearch
+      url_env: ELASTICSEARCH_URL
+      api_key: ${ELASTICSEARCH_API_KEY}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the Elasticsearch client and index lazily on search, index/delete,
+explicit native access or `data.doctor`. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+Missing indexes are created with a default mapping for `text`, `title` and
+`metadata`; `mapping` and `settings` resource options can customize it.
+`upsert_documents()` uses Elasticsearch bulk operations.
+
+See `muscular-example/example_data_elasticsearch_1` for an executable example.

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/muscles_data_elasticsearch/__init__.py
+src/muscles_data_elasticsearch/adapter.py
+src/muscles_data_elasticsearch.egg-info/PKG-INFO
+src/muscles_data_elasticsearch.egg-info/SOURCES.txt
+src/muscles_data_elasticsearch.egg-info/dependency_links.txt
+src/muscles_data_elasticsearch.egg-info/requires.txt
+src/muscles_data_elasticsearch.egg-info/top_level.txt
+tests/test_elasticsearch_adapter.py
+tests/test_integration.py

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch.egg-info/requires.txt

@@ -0,0 +1,5 @@
+muscles-data<1.0.0,>=0.1.0
+elasticsearch<9.0,>=8.0
+
+[dev]
+pytest<9.0,>=8.0

muscles_data_elasticsearch-0.1.0/src/muscles_data_elasticsearch.egg-info/top_level.txt

@@ -0,0 +1 @@
+muscles_data_elasticsearch

muscles_data_elasticsearch-0.1.0/tests/test_elasticsearch_adapter.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_elasticsearch import (
+    ElasticsearchClientMissingError,
+    ElasticsearchConnectionError,
+    ElasticsearchFilterError,
+    ElasticsearchSearchFactory,
+    elasticsearch_filter_from_mapping,
+)
+
+
+class FakeIndices:
+    def __init__(self, client: "FakeElasticsearchClient") -> None:
+        self.client = client
+
+    def exists(self, *, index: str) -> bool:
+        self.client.index_checks.append(index)
+        if self.client.fail_health:
+            raise TimeoutError("elastic password=secret timed out")
+        return index == "docs"
+
+    def create(self, *, index: str, mappings: dict, settings: dict | None = None):
+        self.client.created_indices.append({"index": index, "mappings": mappings, "settings": settings})
+        return {"acknowledged": True}
+
+
+class FakeElasticsearchClient:
+    def __init__(self, *, fail_health: bool = False) -> None:
+        self.fail_health = fail_health
+        self.searches: list[dict] = []
+        self.indexes: list[dict] = []
+        self.bulks: list[dict] = []
+        self.created_indices: list[dict] = []
+        self.deletes: list[dict] = []
+        self.delete_queries: list[dict] = []
+        self.index_checks: list[str] = []
+        self.closed = False
+        self.indices = FakeIndices(self)
+
+    def search(self, **kwargs):
+        self.searches.append(kwargs)
+        return {
+            "hits": {
+                "hits": [
+                    {
+                        "_id": "doc-1",
+                        "_score": 4.2,
+                        "_source": {"text": "Muscles data ports", "metadata": {"section": "docs"}},
+                        "highlight": {"text": ["<em>Muscles</em> data ports"]},
+                    }
+                ]
+            }
+        }
+
+    def index(self, **kwargs):
+        self.indexes.append(kwargs)
+        return {"result": "created"}
+
+    def bulk(self, **kwargs):
+        self.bulks.append(kwargs)
+        return {"errors": False, "items": []}
+
+    def delete(self, **kwargs):
+        self.deletes.append(kwargs)
+        return {"result": "deleted"}
+
+    def delete_by_query(self, **kwargs):
+        self.delete_queries.append(kwargs)
+        return {"deleted": 3}
+
+    def ping(self) -> bool:
+        if self.fail_health:
+            raise TimeoutError("elastic password=secret timed out")
+        return True
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def _config(url: str = "https://elastic.example") -> dict:
+    return {
+        "data": {
+            "resources": {
+                "search.elastic": {
+                    "type": "elasticsearch",
+                    "url": url,
+                    "api_key": "elastic-secret",
+                    "index": "docs",
+                    "native_client": True,
+                }
+            }
+        }
+    }
+
+
+def _runtime(client: FakeElasticsearchClient | None, url: str = "https://elastic.example") -> DataRuntime:
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(ElasticsearchSearchFactory(client_factory=lambda _config: client))
+    return DataRuntime(config=DataConfig.from_raw(_config(url)), catalog=catalog)
+
+
+def test_elasticsearch_external_adapter_maps_search_index_delete_and_native_access():
+    client = FakeElasticsearchClient()
+    runtime = _runtime(client)
+
+    listed = runtime.list_resources()[0]
+    assert listed["type"] == "elasticsearch"
+    assert {"keyword_search", "document_index"} <= set(listed["capabilities"])
+    assert listed["initialized"] is False
+
+    search = runtime.require_port("search.elastic", SearchIndexPort)
+    hits = search.search_text("muscles", filters={"section": "docs"}, limit=2, options={"highlight": True})
+    write = search.upsert_documents([{"id": "doc-1", "text": "Muscles data ports", "metadata": {"section": "docs"}}])
+    deleted = search.delete_documents(filters={"section": ["docs", "notes"]})
+
+    assert [hit.id for hit in hits] == ["doc-1"]
+    assert hits[0].highlights["text"] == ["<em>Muscles</em> data ports"]
+    assert client.searches[0]["query"]["bool"]["filter"] == [{"term": {"metadata.section": "docs"}}]
+    assert write.written == 1
+    assert client.bulks[0]["operations"][1]["metadata"] == {"section": "docs"}
+    assert deleted.deleted == 3
+    assert client.delete_queries[0]["query"]["bool"]["filter"][0] == {"terms": {"metadata.section": ["docs", "notes"]}}
+    assert runtime.require_resource("search.elastic", DataCapability.NATIVE_CLIENT).native_client() is client
+    assert runtime.doctor()["status"] == "ok"
+    assert client.index_checks == ["docs"]
+    assert runtime.close()["status"] == "ok"
+    assert client.closed is True
+
+
+def test_elasticsearch_bulk_upsert_returns_actionable_partial_result():
+    client = FakeElasticsearchClient()
+    client.bulk = lambda **_kwargs: {
+        "errors": True,
+        "items": [
+            {"index": {"status": 400, "error": {"type": "mapper_parsing_exception", "reason": "bad field"}}},
+            {"index": {"status": 201}},
+        ],
+    }
+    search = _runtime(client).require_port("search.elastic", SearchIndexPort)
+
+    result = search.upsert_documents([{"id": "bad"}, {"id": "good"}])
+
+    assert result.status == "partial"
+    assert result.written == 1
+    assert result.errors == ["index: {'type': 'mapper_parsing_exception', 'reason': 'bad field'}"]
+
+
+def test_elasticsearch_external_adapter_filters_and_safe_failures():
+    translated = elasticsearch_filter_from_mapping({"score": {"gte": 0.5}, "$not": {"archived": True}})
+    assert translated[0] == {"bool": {"must_not": [{"term": {"metadata.archived": True}}]}}
+    assert translated[1] == {"range": {"metadata.score": {"gte": 0.5}}}
+    with pytest.raises(ElasticsearchFilterError):
+        elasticsearch_filter_from_mapping({"score": {"near": 1.0}})
+
+    with pytest.raises(ElasticsearchClientMissingError):
+        _runtime(None).require_port("search.elastic", SearchIndexPort).search_text("x")
+
+    failing = _runtime(FakeElasticsearchClient(fail_health=True), "https://user:secret@elastic.example").doctor()
+    assert failing["status"] == "failed"
+    assert "secret" not in repr(failing)
+
+    bad_client = FakeElasticsearchClient()
+    bad_client.search = lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("network unavailable"))
+    with pytest.raises(ElasticsearchConnectionError):
+        _runtime(bad_client).require_port("search.elastic", SearchIndexPort).search_text("x")
+
+
+def test_elasticsearch_creates_missing_index_with_mapping_and_resolves_url_env(monkeypatch):
+    monkeypatch.setenv("ELASTICSEARCH_URL", "https://elastic.example")
+    client = FakeElasticsearchClient()
+    client.indices.exists = lambda *, index: False
+    captured: list[dict] = []
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(
+        ElasticsearchSearchFactory(
+            client_factory=lambda config: captured.append(config.resolved_options()) or client
+        )
+    )
+    runtime = DataRuntime(
+        config=DataConfig.from_raw(
+            {
+                "data": {
+                    "resources": {
+                        "search.elastic": {
+                            "type": "elasticsearch",
+                            "url_env": "ELASTICSEARCH_URL",
+                            "index": "assetforge-rag",
+                        }
+                    }
+                }
+            }
+        ),
+        catalog=catalog,
+    )
+
+    runtime.require_port("search.elastic", SearchIndexPort).upsert_documents(
+        [{"id": "doc-1", "title": "Title", "text": "content", "metadata": {"status": "ready"}}]
+    )
+
+    assert captured == [{"url_env": "ELASTICSEARCH_URL", "index": "assetforge-rag", "url": "https://elastic.example"}]
+    assert client.created_indices[0]["index"] == "assetforge-rag"
+    assert client.created_indices[0]["mappings"]["properties"]["text"]["type"] == "text"

muscles_data_elasticsearch-0.1.0/tests/test_integration.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import os
+from uuid import uuid4
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_elasticsearch import ElasticsearchSearchFactory
+
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(not os.getenv("MUSCLES_DATA_INTEGRATION"), reason="backend integration is disabled"),
+]
+
+
+def test_elasticsearch_real_search_index_lifecycle():
+    index = f"muscles-data-it-{uuid4().hex[:12]}"
+    config = DataConfig.from_raw(
+        {
+            "data": {
+                "resources": {
+                    "search.elastic": {
+                        "type": "elasticsearch",
+                        "url_env": "ELASTICSEARCH_URL",
+                        "index": index,
+                        "native_client": True,
+                    }
+                }
+            }
+        }
+    )
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(ElasticsearchSearchFactory())
+    runtime = DataRuntime(config=config, catalog=catalog)
+
+    client = None
+    try:
+        search = runtime.require_port("search.elastic", SearchIndexPort)
+        assert search.upsert_documents(
+            [
+                {"id": "alpha", "title": "Alpha", "text": "alpha document", "metadata": {"status": "ready"}},
+                {"id": "beta", "title": "Beta", "text": "beta document", "metadata": {"status": "draft"}},
+            ],
+            options={"refresh": "wait_for"},
+        ).written == 2
+        hits = search.search_text("alpha", filters={"status": "ready"}, options={"highlight": True})
+        assert [hit.id for hit in hits] == ["alpha"]
+        assert hits[0].highlights
+        assert runtime.doctor()["status"] == "ok"
+        assert search.delete_documents(ids=["alpha"], options={"refresh": "wait_for"}).deleted == 1
+        assert search.search_text("alpha") == []
+        contracts = pytest.importorskip("muscles_data.contracts")
+        contract = getattr(contracts, "assert_search_index_contract", None)
+        if contract is not None:
+            contract(lambda: search)
+    finally:
+        try:
+            if client is None:
+                try:
+                    client = runtime.require_resource("search.elastic", DataCapability.NATIVE_CLIENT).native_client()
+                except Exception:
+                    client = None
+            if client is not None:
+                client.options(ignore_status=404).indices.delete(index=index)
+        finally:
+            runtime.close()