muscles-data-opensearch 0.1.0__tar.gz

muscles_data_opensearch-0.1.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: muscles-data-opensearch
+Version: 0.1.0
+Summary: OpenSearch SearchIndexPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: opensearch-py<3.0,>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-opensearch
+
+OpenSearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+OpenSearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_opensearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_opensearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_opensearch import OpenSearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(OpenSearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.public", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.public:
+      type: opensearch
+      url: ${OPENSEARCH_URL}
+      username: ${OPENSEARCH_USER}
+      password: ${OPENSEARCH_PASSWORD}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the OpenSearch client and a missing index lazily on search,
+index/delete, explicit native access or `data.doctor`. New indexes receive a
+default mapping for `text`, `title` and `metadata`; `mapping` and `settings`
+resource options customize it. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+See `muscular-example/example_data_opensearch_1` for an executable example.

muscles_data_opensearch-0.1.0/README.md

@@ -0,0 +1,67 @@
+# muscles-data-opensearch
+
+OpenSearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+OpenSearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_opensearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_opensearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_opensearch import OpenSearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(OpenSearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.public", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.public:
+      type: opensearch
+      url: ${OPENSEARCH_URL}
+      username: ${OPENSEARCH_USER}
+      password: ${OPENSEARCH_PASSWORD}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the OpenSearch client and a missing index lazily on search,
+index/delete, explicit native access or `data.doctor`. New indexes receive a
+default mapping for `text`, `title` and `metadata`; `mapping` and `settings`
+resource options customize it. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+See `muscular-example/example_data_opensearch_1` for an executable example.

muscles_data_opensearch-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "muscles-data-opensearch"
+version = "0.1.0"
+description = "OpenSearch SearchIndexPort adapter for muscles-data"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Denis B.", email = "denis@butko.info" }]
+dependencies = [
+  "muscles-data>=0.1.0,<1.0.0",
+  "opensearch-py>=2.0,<3.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0,<9.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+  "integration: tests that require the local backend integration stack",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

muscles_data_opensearch-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch/__init__.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from .adapter import (
+    OpenSearchAdapterError,
+    OpenSearchClientMissingError,
+    OpenSearchConfigError,
+    OpenSearchConnectionError,
+    OpenSearchFilterError,
+    OpenSearchSearchAdapter,
+    OpenSearchSearchFactory,
+    opensearch_filter_from_mapping,
+)
+
+
+__all__ = [
+    "OpenSearchAdapterError",
+    "OpenSearchClientMissingError",
+    "OpenSearchConfigError",
+    "OpenSearchConnectionError",
+    "OpenSearchFilterError",
+    "OpenSearchSearchAdapter",
+    "OpenSearchSearchFactory",
+    "opensearch_filter_from_mapping",
+]

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch/adapter.py

@@ -0,0 +1,488 @@
+from __future__ import annotations
+
+import importlib
+import threading
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any, Mapping
+from urllib.parse import urlsplit
+
+from muscles_data.config import DataResourceConfig
+from muscles_data.errors import DataError
+from muscles_data.models import DataCapability, HealthResult, InspectResult, SearchHit, WriteResult
+
+
+_CLIENT_UNSET = object()
+_RANGE_OPERATORS = {"gt", "gte", "lt", "lte"}
+DEFAULT_INDEX_MAPPING = {
+    "properties": {
+        "text": {"type": "text"},
+        "title": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "metadata": {"type": "object", "dynamic": True},
+    }
+}
+_ALLOWED_OPTIONS = {
+    "url",
+    "url_env",
+    "username",
+    "password",
+    "http_auth",
+    "index",
+    "timeout",
+    "verify_certs",
+    "use_ssl",
+    "ssl_assert_hostname",
+    "ssl_show_warn",
+    "ca_certs",
+    "native_client",
+    "text_field",
+    "metadata_field",
+    "mapping",
+    "settings",
+}
+
+
+class OpenSearchAdapterError(DataError):
+    """Base error for OpenSearch search adapter failures."""
+
+
+class OpenSearchConfigError(ValueError, OpenSearchAdapterError):
+    """Raised when an OpenSearch resource config cannot be mapped safely."""
+
+
+class OpenSearchClientMissingError(OpenSearchAdapterError):
+    """Raised when OpenSearch adapter is used without an available client."""
+
+
+class OpenSearchConnectionError(OpenSearchAdapterError):
+    """Raised when an OpenSearch operation cannot reach or use the backend."""
+
+
+class OpenSearchFilterError(OpenSearchAdapterError):
+    """Raised when a data filter cannot be translated to OpenSearch."""
+
+
+class OpenSearchSearchAdapter:
+    resource_type = "opensearch"
+
+    def __init__(
+        self,
+        config: DataResourceConfig,
+        *,
+        client_factory: Callable[[DataResourceConfig], Any] | None = None,
+    ) -> None:
+        self.config = config
+        self._client_factory = client_factory
+        self._client: Any = _CLIENT_UNSET
+        self._lock = threading.RLock()
+        self._index_ready = False
+        self.closed = False
+
+    def search_text(
+        self,
+        query: str,
+        filters: Mapping[str, Any] | None = None,
+        limit: int = 10,
+        options: Mapping[str, Any] | None = None,
+    ) -> list[SearchHit]:
+        options = dict(options or {})
+        body: dict[str, Any] = {
+            "query": self._query(query, filters),
+            "size": max(0, int(limit)),
+        }
+        if options.pop("highlight", False):
+            fields = options.pop("highlight_fields", [self.text_field()])
+            body["highlight"] = {"fields": {str(field): {} for field in fields}}
+        kwargs: dict[str, Any] = {"index": self.index_name(), "body": body}
+        for key in ("from_", "sort", "track_total_hits", "routing", "preference", "search_after"):
+            if key in options:
+                kwargs[key] = options[key]
+        try:
+            response = self._client_instance().search(**kwargs)
+        except (OpenSearchClientMissingError, OpenSearchConfigError, OpenSearchFilterError):
+            raise
+        except Exception as exc:
+            raise OpenSearchConnectionError(self._safe_error(exc)) from exc
+        return [
+            _hit_from_response(hit, text_field=self.text_field(), metadata_field=self.metadata_field())
+            for hit in _hits(response)
+        ]
+
+    def upsert_documents(self, items: list[Mapping[str, Any]], options: Mapping[str, Any] | None = None) -> WriteResult:
+        options = dict(options or {})
+        client = self._client_instance()
+        try:
+            for item in items:
+                kwargs: dict[str, Any] = {
+                    "index": self.index_name(),
+                    "id": str(item["id"]),
+                    "body": self._document_from_item(item),
+                }
+                for key in ("refresh", "routing", "pipeline"):
+                    if key in options:
+                        kwargs[key] = options[key]
+                client.index(**kwargs)
+        except KeyError as exc:
+            raise OpenSearchConfigError(f"OpenSearch document item requires {exc.args[0]}") from exc
+        except (OpenSearchClientMissingError, OpenSearchConfigError, OpenSearchFilterError):
+            raise
+        except Exception as exc:
+            raise OpenSearchConnectionError(self._safe_error(exc)) from exc
+        return WriteResult(written=len(items), matched=len(items))
+
+    def delete_documents(
+        self,
+        ids: list[str] | None = None,
+        filters: Mapping[str, Any] | None = None,
+        options: Mapping[str, Any] | None = None,
+    ) -> WriteResult:
+        options = dict(options or {})
+        if ids is None and not filters:
+            return WriteResult()
+        client = self._client_instance()
+        try:
+            if ids is not None:
+                deleted = 0
+                for document_id in ids:
+                    kwargs: dict[str, Any] = {"index": self.index_name(), "id": str(document_id)}
+                    for key in ("refresh", "routing"):
+                        if key in options:
+                            kwargs[key] = options[key]
+                    client.delete(**kwargs)
+                    deleted += 1
+                return WriteResult(deleted=deleted, matched=deleted)
+            kwargs = {
+                "index": self.index_name(),
+                "body": {"query": self._filter_query(filters)},
+            }
+            for key in ("refresh", "routing", "conflicts"):
+                if key in options:
+                    kwargs[key] = options[key]
+            response = client.delete_by_query(**kwargs)
+        except (OpenSearchClientMissingError, OpenSearchConfigError, OpenSearchFilterError):
+            raise
+        except Exception as exc:
+            raise OpenSearchConnectionError(self._safe_error(exc)) from exc
+        deleted = int(_mapping_get(response, "deleted", 0) or 0)
+        return WriteResult(deleted=deleted, matched=deleted)
+
+    def inspect(self) -> dict[str, Any]:
+        return asdict(
+            InspectResult(
+                name=self.config.name,
+                type=self.config.type,
+                capabilities=[],
+                initialized=self._client is not _CLIENT_UNSET,
+                status="ok",
+                options=self.config.safe_options(),
+                details={"backend": "opensearch", "index": self.index_name()},
+            )
+        )
+
+    def health(self) -> HealthResult:
+        try:
+            client = self._client_instance()
+            ping = getattr(client, "ping", None)
+            if callable(ping) and not bool(ping()):
+                return HealthResult(
+                    status="failed",
+                    message="OpenSearch ping failed",
+                    details={"index": self.index_name()},
+                )
+            indices = getattr(client, "indices", None)
+            exists = self._index_ready
+            if not exists and indices is not None and hasattr(indices, "exists"):
+                exists = bool(indices.exists(index=self.index_name()))
+        except Exception as exc:
+            return HealthResult(status="failed", message=self._safe_error(exc), details={"index": self.index_name()})
+        if not exists:
+            return HealthResult(
+                status="failed",
+                message=f"OpenSearch index '{self.index_name()}' is not available",
+                details={"index": self.index_name()},
+            )
+        return HealthResult(status="ok", message="OpenSearch index is available", details={"index": self.index_name()})
+
+    def close(self) -> None:
+        if self._client is _CLIENT_UNSET:
+            self.closed = True
+            return
+        close = getattr(self._client, "close", None)
+        if callable(close):
+            close()
+        self.closed = True
+
+    def native_client(self):
+        return self._client_instance()
+
+    def index_name(self) -> str:
+        return str(self.config.options["index"])
+
+    def text_field(self) -> str:
+        return str(self.config.options.get("text_field", "text"))
+
+    def metadata_field(self) -> str:
+        return str(self.config.options.get("metadata_field", "metadata"))
+
+    def _query(self, query: str, filters: Mapping[str, Any] | None) -> dict[str, Any]:
+        clauses: dict[str, Any] = {}
+        if str(query).strip():
+            clauses["must"] = [{"match": {self.text_field(): {"query": str(query)}}}]
+        else:
+            clauses["must"] = [{"match_all": {}}]
+        filter_clauses = opensearch_filter_from_mapping(filters or {}, field_prefix=self.metadata_field())
+        if filter_clauses:
+            clauses["filter"] = filter_clauses
+        return {"bool": clauses}
+
+    def _filter_query(self, filters: Mapping[str, Any] | None) -> dict[str, Any]:
+        filter_clauses = opensearch_filter_from_mapping(filters or {}, field_prefix=self.metadata_field())
+        return {"bool": {"filter": filter_clauses}} if filter_clauses else {"match_all": {}}
+
+    def _document_from_item(self, item: Mapping[str, Any]) -> dict[str, Any]:
+        metadata = dict(item.get("payload", {}) or {})
+        metadata.update(dict(item.get("metadata", {}) or {}))
+        document = {
+            self.text_field(): str(item.get("text", "")),
+            self.metadata_field(): metadata,
+        }
+        if item.get("title") is not None:
+            document["title"] = str(item["title"])
+        fields = item.get("fields")
+        if isinstance(fields, Mapping):
+            document.update(dict(fields))
+        return document
+
+    def _client_instance(self):
+        if self._client is _CLIENT_UNSET:
+            with self._lock:
+                if self._client is _CLIENT_UNSET:
+                    self._validate_options()
+                    client = (
+                        self._client_factory(self.config)
+                        if self._client_factory
+                        else _default_opensearch_client(self.config)
+                    )
+                    if client is None:
+                        raise OpenSearchClientMissingError("OpenSearch client is not available")
+                    self._client = client
+                    try:
+                        self._ensure_index(client)
+                    except Exception:
+                        self._client = _CLIENT_UNSET
+                        close = getattr(client, "close", None)
+                        if callable(close):
+                            close()
+                        raise
+        return self._client
+
+    def _validate_options(self) -> None:
+        unknown = sorted(set(self.config.options) - _ALLOWED_OPTIONS)
+        if unknown:
+            names = ", ".join(unknown)
+            raise OpenSearchConfigError(f"Unsupported OpenSearch resource options: {names}")
+        http_auth = self.config.options.get("http_auth")
+        if http_auth is not None and (not isinstance(http_auth, (list, tuple)) or len(http_auth) != 2):
+            raise OpenSearchConfigError("OpenSearch http_auth must contain username and password")
+        if not self.config.options.get("url") and not self.config.options.get("url_env"):
+            raise OpenSearchConfigError("OpenSearch resource requires url or url_env")
+        self.index_name()
+
+    def mapping(self) -> dict[str, Any]:
+        mapping = self.config.options.get("mapping")
+        if mapping is None:
+            return dict(DEFAULT_INDEX_MAPPING)
+        if not isinstance(mapping, Mapping):
+            raise OpenSearchConfigError("OpenSearch mapping must be a mapping")
+        return dict(mapping)
+
+    def _ensure_index(self, client: Any) -> None:
+        indices = getattr(client, "indices", None)
+        exists = getattr(indices, "exists", None)
+        create = getattr(indices, "create", None)
+        if not callable(exists) or not callable(create):
+            return
+        try:
+            if bool(exists(index=self.index_name())):
+                self._index_ready = True
+                return
+            body: dict[str, Any] = {"mappings": self.mapping()}
+            settings = self.config.options.get("settings")
+            if settings is not None:
+                if not isinstance(settings, Mapping):
+                    raise OpenSearchConfigError("OpenSearch settings must be a mapping")
+                body["settings"] = dict(settings)
+            create(index=self.index_name(), body=body)
+            self._index_ready = True
+        except OpenSearchConfigError:
+            raise
+        except Exception as exc:
+            message = self._safe_error(exc)
+            if "already exists" in message.lower() or "resource_already_exists" in message.lower():
+                self._index_ready = True
+                return
+            raise OpenSearchConnectionError(message) from exc
+
+    def _safe_error(self, exc: Exception) -> str:
+        message = str(exc)
+        try:
+            options = self.config.resolved_options()
+        except Exception:
+            options = self.config.options
+        for key, value in options.items():
+            if key in {"url", "username", "password", "http_auth"} and value:
+                if isinstance(value, (list, tuple)):
+                    for item in value:
+                        message = message.replace(str(item), "***")
+                else:
+                    message = message.replace(str(value), "***")
+                    if key == "url":
+                        parsed = urlsplit(str(value))
+                        for item in (parsed.username, parsed.password):
+                            if item:
+                                message = message.replace(str(item), "***")
+        return message
+
+
+class OpenSearchSearchFactory:
+    resource_type = "opensearch"
+
+    def __init__(self, *, client_factory: Callable[[DataResourceConfig], Any] | None = None) -> None:
+        self._client_factory = client_factory
+
+    def capabilities(self, config: DataResourceConfig) -> set[DataCapability]:
+        native = {DataCapability.NATIVE_CLIENT} if bool(config.options.get("native_client")) else set()
+        return {DataCapability.KEYWORD_SEARCH, DataCapability.DOCUMENT_INDEX, DataCapability.HEALTHCHECK} | native
+
+    def create(self, config: DataResourceConfig) -> OpenSearchSearchAdapter:
+        return OpenSearchSearchAdapter(config, client_factory=self._client_factory)
+
+
+def opensearch_filter_from_mapping(
+    filters: Mapping[str, Any],
+    *,
+    field_prefix: str | None = "metadata",
+) -> list[dict[str, Any]]:
+    clauses: list[dict[str, Any]] = []
+    for key in sorted(filters, key=_filter_sort_key):
+        value = filters[key]
+        if key == "$and":
+            clauses.append({"bool": {"filter": _logical_conditions(value, field_prefix=field_prefix)}})
+        elif key == "$or":
+            clauses.append(
+                {
+                    "bool": {
+                        "should": _logical_conditions(value, field_prefix=field_prefix),
+                        "minimum_should_match": 1,
+                    }
+                }
+            )
+        elif key == "$not":
+            clauses.append({"bool": {"must_not": _logical_conditions(value, field_prefix=field_prefix)}})
+        else:
+            clauses.append(_field_condition(str(key), value, field_prefix=field_prefix))
+    return clauses
+
+
+def _logical_conditions(value: Any, *, field_prefix: str | None) -> list[dict[str, Any]]:
+    if isinstance(value, Mapping):
+        return opensearch_filter_from_mapping(value, field_prefix=field_prefix)
+    if isinstance(value, list):
+        conditions: list[dict[str, Any]] = []
+        for item in value:
+            if not isinstance(item, Mapping):
+                raise OpenSearchFilterError("OpenSearch logical filter items must be mappings")
+            conditions.extend(opensearch_filter_from_mapping(item, field_prefix=field_prefix))
+        return conditions
+    raise OpenSearchFilterError("OpenSearch logical filters must be mappings or lists of mappings")
+
+
+def _field_condition(key: str, value: Any, *, field_prefix: str | None) -> dict[str, Any]:
+    field = _field_name(key, field_prefix)
+    if isinstance(value, Mapping):
+        operators = set(value)
+        if operators <= _RANGE_OPERATORS:
+            return {"range": {field: dict(value)}}
+        if operators == {"eq"}:
+            return {"term": {field: value["eq"]}}
+        if operators == {"any"}:
+            return {"terms": {field: list(value["any"])}}
+        unsupported = ", ".join(sorted(str(operator) for operator in operators))
+        raise OpenSearchFilterError(f"Unsupported OpenSearch filter operator for '{key}': {unsupported}")
+    if isinstance(value, (list, tuple, set)):
+        return {"terms": {field: list(value)}}
+    return {"term": {field: value}}
+
+
+def _field_name(key: str, field_prefix: str | None) -> str:
+    if not field_prefix or "." in key:
+        return key
+    return f"{field_prefix}.{key}"
+
+
+def _filter_sort_key(key: Any) -> tuple[int, str]:
+    order = {"$and": 0, "$or": 1, "$not": 2}
+    key_text = str(key)
+    return (order.get(key_text, 10), key_text)
+
+
+def _hit_from_response(hit: Mapping[str, Any], *, text_field: str, metadata_field: str) -> SearchHit:
+    source = dict(hit.get("_source", {}) or {})
+    metadata = dict(source.get(metadata_field, {}) or {})
+    highlights = {
+        str(key): [str(item) for item in value]
+        for key, value in dict(hit.get("highlight", {}) or {}).items()
+    }
+    return SearchHit(
+        id=str(hit.get("_id", "")),
+        score=float(hit.get("_score", 0.0) or 0.0),
+        text=source.get(text_field),
+        metadata=metadata,
+        highlights=highlights,
+        title=source.get("title"),
+    )
+
+
+def _hits(response: Any) -> list[Mapping[str, Any]]:
+    return list(_mapping_get(_mapping_get(response, "hits", {}), "hits", []) or [])
+
+
+def _mapping_get(value: Any, key: str, default: Any = None) -> Any:
+    if isinstance(value, Mapping):
+        return value.get(key, default)
+    try:
+        return value[key]
+    except Exception:
+        return default
+
+
+def _default_opensearch_client(config: DataResourceConfig):
+    try:
+        client_module = importlib.import_module("opensearchpy")
+    except Exception as exc:
+        raise OpenSearchClientMissingError("Install opensearch-py or muscles-data-opensearch to use type=opensearch") from exc
+
+    try:
+        options = config.resolved_options()
+    except Exception as exc:
+        raise OpenSearchConfigError(str(exc)) from exc
+    url = str(options["url"])
+    parsed = urlsplit(url)
+    kwargs: dict[str, Any] = {"hosts": [url]}
+    if options.get("http_auth"):
+        kwargs["http_auth"] = tuple(options["http_auth"])
+    elif options.get("username") or options.get("password"):
+        kwargs["http_auth"] = (options.get("username"), options.get("password"))
+    if options.get("timeout") is not None:
+        kwargs["timeout"] = options["timeout"]
+    if options.get("verify_certs") is not None:
+        kwargs["verify_certs"] = bool(options["verify_certs"])
+    kwargs["use_ssl"] = bool(options.get("use_ssl", parsed.scheme == "https"))
+    for key in ("ssl_assert_hostname", "ssl_show_warn", "ca_certs"):
+        if key in options:
+            kwargs[key] = options[key]
+
+    try:
+        return client_module.OpenSearch(**kwargs)
+    except Exception as exc:
+        raise OpenSearchConnectionError(str(exc)) from exc

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch.egg-info/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: muscles-data-opensearch
+Version: 0.1.0
+Summary: OpenSearch SearchIndexPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: opensearch-py<3.0,>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-opensearch
+
+OpenSearch adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+OpenSearch-backed `SearchIndexPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- Qdrant vector adapter:
+  [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_opensearch_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_opensearch_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_opensearch import OpenSearchSearchFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(OpenSearchSearchFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+search = runtime.require_port("search.public", SearchIndexPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    search.public:
+      type: opensearch
+      url: ${OPENSEARCH_URL}
+      username: ${OPENSEARCH_USER}
+      password: ${OPENSEARCH_PASSWORD}
+      index: docs
+      timeout: 3
+      verify_certs: true
+```
+
+The adapter creates the OpenSearch client and a missing index lazily on search,
+index/delete, explicit native access or `data.doctor`. New indexes receive a
+default mapping for `text`, `title` and `metadata`; `mapping` and `settings`
+resource options customize it. Application code should use
+`SearchIndexPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+See `muscular-example/example_data_opensearch_1` for an executable example.

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/muscles_data_opensearch/__init__.py
+src/muscles_data_opensearch/adapter.py
+src/muscles_data_opensearch.egg-info/PKG-INFO
+src/muscles_data_opensearch.egg-info/SOURCES.txt
+src/muscles_data_opensearch.egg-info/dependency_links.txt
+src/muscles_data_opensearch.egg-info/requires.txt
+src/muscles_data_opensearch.egg-info/top_level.txt
+tests/test_integration.py
+tests/test_opensearch_adapter.py

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch.egg-info/requires.txt

@@ -0,0 +1,5 @@
+muscles-data<1.0.0,>=0.1.0
+opensearch-py<3.0,>=2.0
+
+[dev]
+pytest<9.0,>=8.0

muscles_data_opensearch-0.1.0/src/muscles_data_opensearch.egg-info/top_level.txt

@@ -0,0 +1 @@
+muscles_data_opensearch

muscles_data_opensearch-0.1.0/tests/test_integration.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import os
+from uuid import uuid4
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_opensearch import OpenSearchSearchFactory
+
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(not os.getenv("MUSCLES_DATA_INTEGRATION"), reason="backend integration is disabled"),
+]
+
+
+def test_opensearch_real_search_index_lifecycle():
+    index = f"muscles-data-it-{uuid4().hex[:12]}"
+    config = DataConfig.from_raw(
+        {
+            "data": {
+                "resources": {
+                    "search.open": {
+                        "type": "opensearch",
+                        "url_env": "OPENSEARCH_URL",
+                        "index": index,
+                        "verify_certs": False,
+                        "native_client": True,
+                    }
+                }
+            }
+        }
+    )
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(OpenSearchSearchFactory())
+    runtime = DataRuntime(config=config, catalog=catalog)
+
+    client = None
+    try:
+        search = runtime.require_port("search.open", SearchIndexPort)
+        assert search.upsert_documents(
+            [
+                {"id": "alpha", "title": "Alpha", "text": "alpha document", "metadata": {"status": "ready"}},
+                {"id": "beta", "title": "Beta", "text": "beta document", "metadata": {"status": "draft"}},
+            ],
+            options={"refresh": "wait_for"},
+        ).written == 2
+        hits = search.search_text("alpha", filters={"status": "ready"}, options={"highlight": True})
+        assert [hit.id for hit in hits] == ["alpha"]
+        assert hits[0].highlights
+        assert runtime.doctor()["status"] == "ok"
+        assert search.delete_documents(ids=["alpha"], options={"refresh": "wait_for"}).deleted == 1
+        assert search.search_text("alpha") == []
+        contracts = pytest.importorskip("muscles_data.contracts")
+        contract = getattr(contracts, "assert_search_index_contract", None)
+        if contract is not None:
+            contract(lambda: search)
+    finally:
+        try:
+            if client is None:
+                try:
+                    client = runtime.require_resource("search.open", DataCapability.NATIVE_CLIENT).native_client()
+                except Exception:
+                    client = None
+            if client is not None:
+                client.indices.delete(index=index, ignore=[404])
+        finally:
+            runtime.close()

muscles_data_opensearch-0.1.0/tests/test_opensearch_adapter.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import SearchIndexPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_opensearch import (
+    OpenSearchClientMissingError,
+    OpenSearchConnectionError,
+    OpenSearchFilterError,
+    OpenSearchSearchFactory,
+    opensearch_filter_from_mapping,
+)
+
+
+class FakeIndices:
+    def __init__(self, client: "FakeOpenSearchClient") -> None:
+        self.client = client
+        self.created: list[dict] = []
+
+    def exists(self, *, index: str) -> bool:
+        self.client.index_checks.append(index)
+        if self.client.fail_health:
+            raise TimeoutError("opensearch password=secret timed out")
+        return index == "docs" or any(item["index"] == index for item in self.created)
+
+    def create(self, **kwargs):
+        self.created.append(dict(kwargs))
+        return {"acknowledged": True}
+
+
+class FakeOpenSearchClient:
+    def __init__(self, *, fail_health: bool = False) -> None:
+        self.fail_health = fail_health
+        self.searches: list[dict] = []
+        self.indexes: list[dict] = []
+        self.delete_queries: list[dict] = []
+        self.index_checks: list[str] = []
+        self.closed = False
+        self.indices = FakeIndices(self)
+
+    def search(self, **kwargs):
+        self.searches.append(kwargs)
+        return {"hits": {"hits": [{"_id": "doc-1", "_score": 4.2, "_source": {"title": "Ports", "text": "Muscles data ports", "metadata": {"section": "docs"}}}]}}
+
+    def index(self, **kwargs):
+        self.indexes.append(kwargs)
+        return {"result": "created"}
+
+    def delete_by_query(self, **kwargs):
+        self.delete_queries.append(kwargs)
+        return {"deleted": 3}
+
+    def ping(self) -> bool:
+        if self.fail_health:
+            raise TimeoutError("opensearch password=secret timed out")
+        return True
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def _config(url: str = "https://opensearch.example") -> dict:
+    return {
+        "data": {
+            "resources": {
+                "search.open": {
+                    "type": "opensearch",
+                    "url": url,
+                    "username": "admin",
+                    "password": "open-secret",
+                    "index": "docs",
+                    "native_client": True,
+                }
+            }
+        }
+    }
+
+
+def _runtime(client: FakeOpenSearchClient | None, url: str = "https://opensearch.example") -> DataRuntime:
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(OpenSearchSearchFactory(client_factory=lambda _config: client))
+    return DataRuntime(config=DataConfig.from_raw(_config(url)), catalog=catalog)
+
+
+def test_opensearch_external_adapter_maps_search_index_delete_and_native_access():
+    client = FakeOpenSearchClient()
+    runtime = _runtime(client)
+
+    listed = runtime.list_resources()[0]
+    assert listed["type"] == "opensearch"
+    assert {"keyword_search", "document_index"} <= set(listed["capabilities"])
+    assert listed["initialized"] is False
+
+    search = runtime.require_port("search.open", SearchIndexPort)
+    hits = search.search_text("muscles", filters={"section": "docs"}, limit=2)
+    write = search.upsert_documents([{"id": "doc-1", "title": "Ports", "text": "Muscles data ports", "metadata": {"section": "docs"}}])
+    deleted = search.delete_documents(filters={"section": ["docs", "notes"]})
+
+    assert [hit.id for hit in hits] == ["doc-1"]
+    assert hits[0].title == "Ports"
+    assert client.searches[0]["body"]["query"]["bool"]["filter"] == [{"term": {"metadata.section": "docs"}}]
+    assert write.written == 1
+    assert client.indexes[0]["body"]["title"] == "Ports"
+    assert client.indexes[0]["body"]["metadata"] == {"section": "docs"}
+    assert deleted.deleted == 3
+    assert client.delete_queries[0]["body"]["query"]["bool"]["filter"][0] == {"terms": {"metadata.section": ["docs", "notes"]}}
+    assert runtime.require_resource("search.open", DataCapability.NATIVE_CLIENT).native_client() is client
+    assert runtime.doctor()["status"] == "ok"
+    assert client.index_checks == ["docs"]
+    assert runtime.close()["status"] == "ok"
+    assert client.closed is True
+
+
+def test_opensearch_creates_missing_index_with_default_mapping():
+    client = FakeOpenSearchClient()
+    config = DataConfig.from_raw(
+        {
+            "data": {
+                "resources": {
+                    "search.new": {
+                        "type": "opensearch",
+                        "url": "https://opensearch.example",
+                        "index": "new-docs",
+                    }
+                }
+            }
+        }
+    )
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(OpenSearchSearchFactory(client_factory=lambda _config: client))
+    runtime = DataRuntime(config=config, catalog=catalog)
+
+    runtime.require_port("search.new", SearchIndexPort).search_text("first")
+
+    assert client.indices.created[0]["index"] == "new-docs"
+    assert client.indices.created[0]["body"]["mappings"]["properties"]["text"] == {"type": "text"}
+
+
+def test_opensearch_external_adapter_filters_and_safe_failures():
+    translated = opensearch_filter_from_mapping({"score": {"gte": 0.5}, "$not": {"archived": True}})
+    assert translated[0] == {"bool": {"must_not": [{"term": {"metadata.archived": True}}]}}
+    assert translated[1] == {"range": {"metadata.score": {"gte": 0.5}}}
+    with pytest.raises(OpenSearchFilterError):
+        opensearch_filter_from_mapping({"score": {"near": 1.0}})
+
+    with pytest.raises(OpenSearchClientMissingError):
+        _runtime(None).require_port("search.open", SearchIndexPort).search_text("x")
+
+    failing = _runtime(FakeOpenSearchClient(fail_health=True), "https://user:secret@opensearch.example").doctor()
+    assert failing["status"] == "failed"
+    assert "secret" not in repr(failing)
+
+    bad_client = FakeOpenSearchClient()
+    bad_client.search = lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("network unavailable"))
+    with pytest.raises(OpenSearchConnectionError):
+        _runtime(bad_client).require_port("search.open", SearchIndexPort).search_text("x")