muscles-data-qdrant 0.1.0__tar.gz

muscles_data_qdrant-0.1.0/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: muscles-data-qdrant
+Version: 0.1.0
+Summary: Qdrant VectorSearchPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: qdrant-client<2.0,>=1.10
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-qdrant
+
+Qdrant adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Qdrant-backed `VectorSearchPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_qdrant_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_qdrant_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import VectorSearchPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_qdrant import QdrantVectorFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(QdrantVectorFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+vector = runtime.require_port("vector.docs", VectorSearchPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    vector.docs:
+      type: qdrant
+      url_env: QDRANT_URL
+      api_key: ${QDRANT_API_KEY}
+      collection: docs
+      vector_size: 1536
+      distance: cosine
+      timeout: 3
+      prefer_grpc: false
+```
+
+The adapter creates the Qdrant client and collection lazily on vector search/write/delete,
+explicit native access or `data.doctor`. Application code should use
+`VectorSearchPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+`vector_size` is required when a missing collection must be created. Request
+dimensions are validated against the configured/existing collection, and
+keyword payload indexes are created for standard AssetForge filter fields.
+Embeddings are not generated by this adapter.
+
+See `muscular-example/example_data_qdrant_1` for an executable example.

muscles_data_qdrant-0.1.0/README.md

@@ -0,0 +1,71 @@
+# muscles-data-qdrant
+
+Qdrant adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Qdrant-backed `VectorSearchPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_qdrant_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_qdrant_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import VectorSearchPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_qdrant import QdrantVectorFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(QdrantVectorFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+vector = runtime.require_port("vector.docs", VectorSearchPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    vector.docs:
+      type: qdrant
+      url_env: QDRANT_URL
+      api_key: ${QDRANT_API_KEY}
+      collection: docs
+      vector_size: 1536
+      distance: cosine
+      timeout: 3
+      prefer_grpc: false
+```
+
+The adapter creates the Qdrant client and collection lazily on vector search/write/delete,
+explicit native access or `data.doctor`. Application code should use
+`VectorSearchPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+`vector_size` is required when a missing collection must be created. Request
+dimensions are validated against the configured/existing collection, and
+keyword payload indexes are created for standard AssetForge filter fields.
+Embeddings are not generated by this adapter.
+
+See `muscular-example/example_data_qdrant_1` for an executable example.

muscles_data_qdrant-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "muscles-data-qdrant"
+version = "0.1.0"
+description = "Qdrant VectorSearchPort adapter for muscles-data"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Denis B.", email = "denis@butko.info" }]
+dependencies = [
+  "muscles-data>=0.1.0,<1.0.0",
+  "qdrant-client>=1.10,<2.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.0,<9.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+  "integration: tests that require the local backend integration stack",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]

muscles_data_qdrant-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant/__init__.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from .adapter import (
+    QdrantAdapterError,
+    QdrantClientMissingError,
+    QdrantConfigError,
+    QdrantConnectionError,
+    QdrantDimensionError,
+    QdrantFilterError,
+    QdrantVectorAdapter,
+    QdrantVectorFactory,
+    qdrant_filter_from_mapping,
+)
+
+
+__all__ = [
+    "QdrantAdapterError",
+    "QdrantClientMissingError",
+    "QdrantConfigError",
+    "QdrantConnectionError",
+    "QdrantDimensionError",
+    "QdrantFilterError",
+    "QdrantVectorAdapter",
+    "QdrantVectorFactory",
+    "qdrant_filter_from_mapping",
+]

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant/adapter.py

@@ -0,0 +1,509 @@
+from __future__ import annotations
+
+import importlib
+import threading
+import uuid
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any, Mapping
+
+from muscles_data.config import DataResourceConfig
+from muscles_data.errors import (
+    DataConfigurationError,
+    DataConnectionError,
+    DataError,
+    DataSchemaMismatchError,
+    DataUnsupportedOperationError,
+    DataVectorDimensionError,
+)
+from muscles_data.models import DataCapability, HealthResult, InspectResult, VectorHit, WriteResult
+
+
+_RANGE_OPERATORS = {"gt", "gte", "lt", "lte"}
+_CLIENT_UNSET = object()
+_MODELS_UNSET = object()
+_PUBLIC_ID_FIELD = "_muscles_id"
+
+
+class QdrantAdapterError(DataError):
+    """Base error for Qdrant vector adapter failures."""
+
+
+class QdrantConfigError(DataConfigurationError, QdrantAdapterError):
+    """Raised when Qdrant resource configuration is incomplete or invalid."""
+
+
+class QdrantClientMissingError(DataConfigurationError, QdrantAdapterError):
+    """Raised when Qdrant adapter is used without an available client."""
+
+
+class QdrantConnectionError(DataConnectionError, QdrantAdapterError):
+    """Raised when Qdrant operation cannot reach or use the backend."""
+
+
+class QdrantFilterError(DataUnsupportedOperationError, QdrantAdapterError):
+    """Raised when a data filter cannot be translated to Qdrant."""
+
+
+class QdrantDimensionError(DataVectorDimensionError, QdrantAdapterError):
+    """Raised when vectors are empty or incompatible with the Qdrant collection."""
+
+
+class QdrantVectorAdapter:
+    resource_type = "qdrant"
+
+    def __init__(
+        self,
+        config: DataResourceConfig,
+        *,
+        client_factory: Callable[[DataResourceConfig], Any] | None = None,
+        models_provider: Callable[[], Any] | None = None,
+    ) -> None:
+        self.config = config
+        self._client_factory = client_factory
+        self._models_provider = models_provider
+        self._client: Any = _CLIENT_UNSET
+        self._models: Any = _MODELS_UNSET
+        self._lock = threading.RLock()
+        self._collection_initialized = False
+        self.closed = False
+
+    def search_vectors(
+        self,
+        vector: list[float],
+        filters: Mapping[str, Any] | None = None,
+        limit: int = 10,
+        options: Mapping[str, Any] | None = None,
+    ) -> list[VectorHit]:
+        options = dict(options or {})
+        normalized_vector = _normalize_vector(vector)
+        self._validate_dimension(normalized_vector)
+        self._ensure_collection(len(normalized_vector))
+        query_filter = self._filter(filters)
+        kwargs: dict[str, Any] = {
+            "collection_name": self.collection_name(),
+            "query": normalized_vector,
+            "limit": max(0, int(limit)),
+            "with_payload": options.pop("with_payload", True),
+            "with_vectors": options.pop("with_vectors", False),
+        }
+        if query_filter is not None:
+            kwargs["query_filter"] = query_filter
+        for key in ("score_threshold", "using", "search_params", "params", "consistency", "shard_key_selector"):
+            if key in options:
+                kwargs[key] = options[key]
+        try:
+            result = self._client_instance().query_points(**kwargs)
+        except (QdrantClientMissingError, QdrantDimensionError, QdrantFilterError):
+            raise
+        except Exception as exc:
+            _raise_qdrant_operation_error(exc)
+        points = getattr(result, "points", result)
+        return [_hit_from_point(point) for point in points]
+
+    def upsert_vectors(self, items: list[Mapping[str, Any]], options: Mapping[str, Any] | None = None) -> WriteResult:
+        options = dict(options or {})
+        models = self._models_instance()
+        try:
+            normalized_items = []
+            for item in items:
+                vector = _normalize_vector(item["vector"])
+                self._validate_dimension(vector)
+                normalized_items.append((item, vector))
+            points = [
+                models.PointStruct(
+                    id=_qdrant_point_id(str(item["id"])),
+                    vector=vector,
+                    payload=_payload_with_public_id(item),
+                )
+                for item, vector in normalized_items
+            ]
+        except KeyError as exc:
+            raise QdrantDimensionError(f"Vector item requires {exc.args[0]}") from exc
+        if normalized_items:
+            self._ensure_collection(len(normalized_items[0][1]))
+        kwargs: dict[str, Any] = {"collection_name": self.collection_name(), "points": points}
+        if "wait" in options:
+            kwargs["wait"] = bool(options["wait"])
+        try:
+            self._client_instance().upsert(**kwargs)
+        except (QdrantClientMissingError, QdrantDimensionError, QdrantFilterError):
+            raise
+        except Exception as exc:
+            _raise_qdrant_operation_error(exc)
+        return WriteResult(written=len(items), matched=len(items))
+
+    def delete_vectors(
+        self,
+        ids: list[str] | None = None,
+        filters: Mapping[str, Any] | None = None,
+        options: Mapping[str, Any] | None = None,
+    ) -> WriteResult:
+        options = dict(options or {})
+        if ids is None and not filters:
+            return WriteResult()
+        models = self._models_instance()
+        self._ensure_collection()
+        if ids is not None:
+            selector = models.PointIdsList(points=[_qdrant_point_id(str(item_id)) for item_id in ids])
+            deleted = len(ids)
+        else:
+            selector = models.FilterSelector(filter=self._filter(filters))
+            deleted = 0
+        kwargs: dict[str, Any] = {"collection_name": self.collection_name(), "points_selector": selector}
+        if "wait" in options:
+            kwargs["wait"] = bool(options["wait"])
+        try:
+            self._client_instance().delete(**kwargs)
+        except (QdrantClientMissingError, QdrantDimensionError, QdrantFilterError):
+            raise
+        except Exception as exc:
+            _raise_qdrant_operation_error(exc)
+        return WriteResult(deleted=deleted, matched=deleted)
+
+    def inspect(self) -> dict[str, Any]:
+        return asdict(
+            InspectResult(
+                name=self.config.name,
+                type=self.config.type,
+                capabilities=[],
+                initialized=self._client is not _CLIENT_UNSET,
+                status="ok",
+                options=self.config.safe_options(),
+                details={"backend": "qdrant", "collection": self.collection_name()},
+            )
+        )
+
+    def health(self) -> HealthResult:
+        try:
+            self._ensure_collection()
+            client = self._client_instance()
+            collection = self.collection_name()
+            exists = self._collection_initialized
+            if not exists:
+                if hasattr(client, "collection_exists"):
+                    exists = bool(client.collection_exists(collection_name=collection))
+                else:
+                    client.get_collection(collection_name=collection)
+                    exists = True
+        except DataConfigurationError as exc:
+            return HealthResult(status="failed", code="configuration_error", message=str(exc), details={"collection": self.collection_name()})
+        except DataSchemaMismatchError as exc:
+            return HealthResult(status="failed", code="schema_mismatch", message=str(exc), details={"collection": self.collection_name()})
+        except Exception as exc:
+            return HealthResult(status="failed", code="connection_failed", message=self._safe_error(exc), details={"collection": self.collection_name()})
+        if not exists:
+            return HealthResult(
+                status="failed",
+                code="resource_missing",
+                message=f"Qdrant collection '{self.collection_name()}' is not available",
+                details={"collection": self.collection_name()},
+            )
+        return HealthResult(status="ok", code="ok", message="Qdrant collection is available", details={"collection": self.collection_name()})
+
+    def close(self) -> None:
+        if self._client is _CLIENT_UNSET:
+            self.closed = True
+            return
+        close = getattr(self._client, "close", None)
+        if callable(close):
+            close()
+        self.closed = True
+
+    def native_client(self):
+        return self._client_instance()
+
+    def collection_name(self) -> str:
+        try:
+            return str(self.config.options["collection"])
+        except KeyError as exc:
+            raise QdrantConfigError("Qdrant resource requires collection") from exc
+
+    def vector_size(self) -> int | None:
+        value = self.config.options.get("vector_size")
+        if value is None:
+            return None
+        try:
+            normalized = int(value)
+        except (TypeError, ValueError) as exc:
+            raise QdrantConfigError("Qdrant vector_size must be a positive integer") from exc
+        if normalized <= 0:
+            raise QdrantConfigError("Qdrant vector_size must be a positive integer")
+        return normalized
+
+    def distance(self) -> str:
+        value = str(self.config.options.get("distance", "cosine")).lower()
+        if value not in {"cosine", "dot", "euclid"}:
+            raise QdrantConfigError("Qdrant distance must be cosine, dot or euclid")
+        return value
+
+    def payload_indexes(self) -> list[str]:
+        configured = self.config.options.get(
+            "payload_indexes",
+            ["workspace_uid", "resource_type", "section_uid", "category", "status"],
+        )
+        if not isinstance(configured, (list, tuple, set)):
+            raise QdrantConfigError("Qdrant payload_indexes must be a list")
+        return [str(field) for field in configured]
+
+    def _validate_dimension(self, vector: list[float]) -> None:
+        expected = self.vector_size()
+        if expected is not None and len(vector) != expected:
+            raise QdrantDimensionError(
+                f"Qdrant vector dimension mismatch: expected {expected}, got {len(vector)}"
+            )
+
+    def _ensure_collection(self, vector_size: int | None = None) -> None:
+        if self._collection_initialized:
+            return
+        client = self._client_instance()
+        collection = self.collection_name()
+        try:
+            if hasattr(client, "collection_exists"):
+                exists = bool(client.collection_exists(collection_name=collection))
+            else:
+                client.get_collection(collection_name=collection)
+                exists = True
+            if not exists:
+                size = self.vector_size() or vector_size
+                if size is None:
+                    raise QdrantConfigError("Qdrant vector_size is required to create a collection")
+                models = self._models_instance()
+                distance_name = self.distance().upper()
+                distance = getattr(getattr(models, "Distance", None), distance_name, distance_name.title())
+                client.create_collection(
+                    collection_name=collection,
+                    vectors_config=models.VectorParams(size=size, distance=distance),
+                )
+            else:
+                self._validate_existing_collection(client)
+            self._ensure_payload_indexes(client)
+            self._collection_initialized = True
+        except (QdrantConfigError, QdrantDimensionError):
+            raise
+        except Exception as exc:
+            _raise_qdrant_operation_error(exc)
+
+    def _ensure_payload_indexes(self, client: Any) -> None:
+        create = getattr(client, "create_payload_index", None)
+        if not callable(create):
+            return
+        models = self._models_instance()
+        schema_type = getattr(getattr(models, "PayloadSchemaType", None), "KEYWORD", "keyword")
+        for field in self.payload_indexes():
+            try:
+                create(
+                    collection_name=self.collection_name(),
+                    field_name=field,
+                    field_schema=schema_type,
+                )
+            except Exception as exc:
+                if "already exists" not in str(exc).lower():
+                    raise
+
+    def _validate_existing_collection(self, client: Any) -> None:
+        expected = self.vector_size()
+        get_collection = getattr(client, "get_collection", None)
+        if expected is None or not callable(get_collection):
+            return
+        info = get_collection(collection_name=self.collection_name())
+        vectors = getattr(getattr(getattr(info, "config", None), "params", None), "vectors", None)
+        remote_size = getattr(vectors, "size", None)
+        if remote_size is not None and int(remote_size) != expected:
+            raise QdrantDimensionError(
+                f"Qdrant collection dimension mismatch: expected {expected}, got {remote_size}"
+            )
+
+    def _client_instance(self):
+        if self._client is _CLIENT_UNSET:
+            with self._lock:
+                if self._client is _CLIENT_UNSET:
+                    client = self._client_factory(self.config) if self._client_factory else _default_qdrant_client(self.config)
+                    if client is None:
+                        raise QdrantClientMissingError("Qdrant client is not available")
+                    self._client = client
+        return self._client
+
+    def _models_instance(self):
+        if self._models is _MODELS_UNSET:
+            with self._lock:
+                if self._models is _MODELS_UNSET:
+                    self._models = self._models_provider() if self._models_provider else _default_qdrant_models()
+        return self._models
+
+    def _filter(self, filters: Mapping[str, Any] | None):
+        if not filters:
+            return None
+        return qdrant_filter_from_mapping(filters, models=self._models_instance())
+
+    def _safe_error(self, exc: Exception) -> str:
+        message = str(exc)
+        try:
+            options = self.config.resolved_options()
+        except DataConfigurationError:
+            options = self.config.options
+        for key, value in options.items():
+            if key in {"url", "api_key", "token", "password"} and value:
+                message = message.replace(str(value), "***")
+        return message
+
+
+class QdrantVectorFactory:
+    resource_type = "qdrant"
+
+    def __init__(
+        self,
+        *,
+        client_factory: Callable[[DataResourceConfig], Any] | None = None,
+        models_provider: Callable[[], Any] | None = None,
+    ) -> None:
+        self._client_factory = client_factory
+        self._models_provider = models_provider
+
+    def capabilities(self, config: DataResourceConfig) -> set[DataCapability]:
+        native = {DataCapability.NATIVE_CLIENT} if bool(config.options.get("native_client")) else set()
+        return {DataCapability.VECTOR_SEARCH, DataCapability.VECTOR_WRITE, DataCapability.HEALTHCHECK} | native
+
+    def create(self, config: DataResourceConfig) -> QdrantVectorAdapter:
+        return QdrantVectorAdapter(
+            config,
+            client_factory=self._client_factory,
+            models_provider=self._models_provider,
+        )
+
+
+def qdrant_filter_from_mapping(filters: Mapping[str, Any], *, models=None):
+    models = models or _default_qdrant_models()
+    must: list[Any] = []
+    should: list[Any] = []
+    must_not: list[Any] = []
+    for key in sorted(filters, key=str):
+        value = filters[key]
+        if key == "$and":
+            must.extend(_logical_conditions(value, models=models))
+        elif key == "$or":
+            should.extend(_logical_conditions(value, models=models))
+        elif key == "$not":
+            must_not.extend(_logical_conditions(value, models=models))
+        else:
+            must.append(_field_condition(str(key), value, models=models))
+    return models.Filter(must=must, should=should, must_not=must_not)
+
+
+def _logical_conditions(value: Any, *, models) -> list[Any]:
+    if isinstance(value, Mapping):
+        return _conditions_from_mapping(value, models=models)
+    if isinstance(value, list):
+        conditions: list[Any] = []
+        for item in value:
+            if not isinstance(item, Mapping):
+                raise QdrantFilterError("Qdrant logical filter items must be mappings")
+            conditions.extend(_conditions_from_mapping(item, models=models))
+        return conditions
+    raise QdrantFilterError("Qdrant logical filters must be mappings or lists of mappings")
+
+
+def _conditions_from_mapping(filters: Mapping[str, Any], *, models) -> list[Any]:
+    conditions: list[Any] = []
+    for key in sorted(filters, key=str):
+        value = filters[key]
+        if key in {"$and", "$or", "$not"}:
+            conditions.append(qdrant_filter_from_mapping({key: value}, models=models))
+        else:
+            conditions.append(_field_condition(str(key), value, models=models))
+    return conditions
+
+
+def _field_condition(key: str, value: Any, *, models):
+    if isinstance(value, Mapping):
+        operators = set(value)
+        if operators <= _RANGE_OPERATORS:
+            return models.FieldCondition(key=key, range=models.Range(**dict(value)))
+        if operators == {"eq"}:
+            return models.FieldCondition(key=key, match=models.MatchValue(value=value["eq"]))
+        if operators == {"any"}:
+            return models.FieldCondition(key=key, match=models.MatchAny(any=list(value["any"])))
+        unsupported = ", ".join(sorted(str(operator) for operator in operators))
+        raise QdrantFilterError(f"Unsupported Qdrant filter operator for '{key}': {unsupported}")
+    if isinstance(value, (list, tuple, set)):
+        return models.FieldCondition(key=key, match=models.MatchAny(any=list(value)))
+    return models.FieldCondition(key=key, match=models.MatchValue(value=value))
+
+
+def _default_qdrant_client(config: DataResourceConfig):
+    try:
+        client_module = importlib.import_module("qdrant_client")
+    except Exception as exc:
+        raise QdrantClientMissingError("Install qdrant-client or muscles-data-qdrant to use type=qdrant") from exc
+    options = config.resolved_options()
+    kwargs = {
+        "url": options["url"],
+        "api_key": options.get("api_key"),
+        "timeout": options.get("timeout"),
+        "prefer_grpc": options.get("prefer_grpc"),
+    }
+    kwargs = {key: value for key, value in kwargs.items() if value is not None}
+    try:
+        return client_module.QdrantClient(**kwargs)
+    except Exception as exc:
+        raise QdrantConnectionError(str(exc)) from exc
+
+
+def _default_qdrant_models():
+    try:
+        return importlib.import_module("qdrant_client.models")
+    except Exception as exc:
+        raise QdrantClientMissingError("Install qdrant-client or muscles-data-qdrant to use type=qdrant") from exc
+
+
+def _normalize_vector(vector: Any) -> list[float]:
+    try:
+        normalized = [float(value) for value in vector]
+    except TypeError as exc:
+        raise QdrantDimensionError("Vector must be an iterable of numbers") from exc
+    if not normalized:
+        raise QdrantDimensionError("Vector must not be empty")
+    return normalized
+
+
+def _hit_from_point(point: Any) -> VectorHit:
+    payload = dict(getattr(point, "payload", {}) or {})
+    public_id = str(payload.pop(_PUBLIC_ID_FIELD, getattr(point, "id")))
+    metadata = {"backend": "qdrant"}
+    version = getattr(point, "version", None)
+    if version is not None:
+        metadata["version"] = version
+    return VectorHit(
+        id=public_id,
+        score=float(getattr(point, "score", 0.0)),
+        payload=payload,
+        metadata=metadata,
+        text=payload.get("text"),
+        title=payload.get("title"),
+    )
+
+
+def _payload_with_public_id(item: Mapping[str, Any]) -> dict[str, Any]:
+    payload = dict(item.get("payload", {}) or {})
+    item_id = str(item["id"])
+    existing_id = payload.get(_PUBLIC_ID_FIELD)
+    if existing_id is not None and str(existing_id) != item_id:
+        raise QdrantConfigError(f"Qdrant payload field '{_PUBLIC_ID_FIELD}' is reserved")
+    payload[_PUBLIC_ID_FIELD] = item_id
+    return payload
+
+
+def _qdrant_point_id(value: str) -> str | int:
+    """Map the string port id to a Qdrant-supported stable point id."""
+    try:
+        return int(value) if value.isdigit() else str(uuid.UUID(value))
+    except (ValueError, AttributeError):
+        return str(uuid.uuid5(uuid.NAMESPACE_URL, f"muscles-data:qdrant:{value}"))
+
+
+def _raise_qdrant_operation_error(exc: Exception):
+    message = str(exc)
+    lowered = message.lower()
+    if "dimension" in lowered or "vector size" in lowered or "vector dimension" in lowered:
+        raise QdrantDimensionError(message) from exc
+    raise QdrantConnectionError(message) from exc

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant.egg-info/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: muscles-data-qdrant
+Version: 0.1.0
+Summary: Qdrant VectorSearchPort adapter for muscles-data
+Author-email: "Denis B." <denis@butko.info>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: muscles-data<1.0.0,>=0.1.0
+Requires-Dist: qdrant-client<2.0,>=1.10
+Provides-Extra: dev
+Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
+
+# muscles-data-qdrant
+
+Qdrant adapter package for `muscles-data`.
+
+This package is intentionally separate from `muscles-data`: the core package
+owns typed ports, resource runtime and diagnostics, while this package owns the
+Qdrant-backed `VectorSearchPort` implementation.
+
+## Related packages
+
+- Core runtime and port contracts:
+  [`muscles-data`](https://github.com/butkoden/muscles-data)
+- Elasticsearch search adapter:
+  [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
+- OpenSearch search adapter:
+  [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
+- Redis key-value/lock/stream adapter:
+  [`muscles-data-redis`](https://github.com/butkoden/muscles-data-redis)
+- MongoDB document-store adapter:
+  [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
+- S3 object-store adapter:
+  [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
+- SQLAlchemy direct SQL resource adapter:
+  [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
+- Executable example:
+  [`example_data_qdrant_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_qdrant_1)
+
+## Usage
+
+Register the factory in the project composition root:
+
+```python
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.ports import VectorSearchPort
+from muscles_data.runtime import DataRuntime
+from muscles_data_qdrant import QdrantVectorFactory
+
+catalog = DataAdapterCatalog.with_defaults()
+catalog.register(QdrantVectorFactory())
+
+runtime = DataRuntime(config=config, catalog=catalog)
+vector = runtime.require_port("vector.docs", VectorSearchPort)
+```
+
+Resource config stays in the project:
+
+```yaml
+data:
+  resources:
+    vector.docs:
+      type: qdrant
+      url_env: QDRANT_URL
+      api_key: ${QDRANT_API_KEY}
+      collection: docs
+      vector_size: 1536
+      distance: cosine
+      timeout: 3
+      prefer_grpc: false
+```
+
+The adapter creates the Qdrant client and collection lazily on vector search/write/delete,
+explicit native access or `data.doctor`. Application code should use
+`VectorSearchPort`; direct client access is only an advanced escape hatch with
+`native_client: true`.
+
+`vector_size` is required when a missing collection must be created. Request
+dimensions are validated against the configured/existing collection, and
+keyword payload indexes are created for standard AssetForge filter fields.
+Embeddings are not generated by this adapter.
+
+See `muscular-example/example_data_qdrant_1` for an executable example.

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/muscles_data_qdrant/__init__.py
+src/muscles_data_qdrant/adapter.py
+src/muscles_data_qdrant.egg-info/PKG-INFO
+src/muscles_data_qdrant.egg-info/SOURCES.txt
+src/muscles_data_qdrant.egg-info/dependency_links.txt
+src/muscles_data_qdrant.egg-info/requires.txt
+src/muscles_data_qdrant.egg-info/top_level.txt
+tests/test_integration.py
+tests/test_qdrant_adapter.py

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant.egg-info/requires.txt

@@ -0,0 +1,5 @@
+muscles-data<1.0.0,>=0.1.0
+qdrant-client<2.0,>=1.10
+
+[dev]
+pytest<9.0,>=8.0

muscles_data_qdrant-0.1.0/src/muscles_data_qdrant.egg-info/top_level.txt

@@ -0,0 +1 @@
+muscles_data_qdrant

muscles_data_qdrant-0.1.0/tests/test_integration.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import os
+import inspect
+from uuid import uuid4
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import VectorSearchPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_qdrant import QdrantVectorFactory
+
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(not os.getenv("MUSCLES_DATA_INTEGRATION"), reason="backend integration is disabled"),
+]
+
+
+def test_qdrant_real_collection_vector_lifecycle():
+    collection = f"muscles_data_it_{uuid4().hex[:12]}"
+    config = DataConfig.from_raw(
+        {
+            "data": {
+                "resources": {
+                    "vector.qdrant": {
+                        "type": "qdrant",
+                        "url_env": "QDRANT_URL",
+                        "collection": collection,
+                        "vector_size": 3,
+                        "distance": "cosine",
+                        "payload_indexes": ["status"],
+                        "native_client": True,
+                    }
+                }
+            }
+        }
+    )
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(QdrantVectorFactory())
+    runtime = DataRuntime(config=config, catalog=catalog)
+
+    client = None
+    try:
+        vector = runtime.require_port("vector.qdrant", VectorSearchPort)
+        assert vector.upsert_vectors(
+            [
+                {"id": "alpha", "vector": [1.0, 0.0, 0.0], "payload": {"status": "ready"}},
+                {"id": "beta", "vector": [0.0, 1.0, 0.0], "payload": {"status": "draft"}},
+            ],
+            options={"wait": True},
+        ).written == 2
+        hits = vector.search_vectors([1.0, 0.0, 0.0], filters={"status": "ready"}, limit=10)
+        assert [hit.id for hit in hits] == ["alpha"]
+        assert hits[0].payload["status"] == "ready"
+        assert runtime.doctor()["status"] == "ok"
+        assert vector.delete_vectors(ids=["alpha"], options={"wait": True}).deleted == 1
+        contracts = pytest.importorskip("muscles_data.contracts")
+        contract = getattr(contracts, "assert_vector_search_contract", None)
+        if contract is not None:
+            if "dimension" in inspect.signature(contract).parameters:
+                contract(lambda: vector, dimension=3)
+        client = runtime.require_resource("vector.qdrant", DataCapability.NATIVE_CLIENT).native_client()
+        client.delete_collection(collection_name=collection)
+    finally:
+        if client is not None:
+            try:
+                client.delete_collection(collection_name=collection)
+            except Exception:
+                pass
+        runtime.close()

muscles_data_qdrant-0.1.0/tests/test_qdrant_adapter.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import pytest
+from muscles_data.catalog import DataAdapterCatalog
+from muscles_data.config import DataConfig
+from muscles_data.models import DataCapability
+from muscles_data.ports import VectorSearchPort
+from muscles_data.runtime import DataRuntime
+
+from muscles_data_qdrant import (
+    QdrantClientMissingError,
+    QdrantConnectionError,
+    QdrantDimensionError,
+    QdrantFilterError,
+    QdrantVectorFactory,
+    qdrant_filter_from_mapping,
+)
+
+
+class FakePoint:
+    def __init__(self, point_id: str, score: float, payload: dict[str, Any]) -> None:
+        self.id = point_id
+        self.score = score
+        self.payload = payload
+        self.version = 7
+
+
+class FakeQueryResult:
+    def __init__(self, points: list[FakePoint]) -> None:
+        self.points = points
+
+
+class FakeQdrantClient:
+    def __init__(self, *, fail_health: bool = False) -> None:
+        self.fail_health = fail_health
+        self.queries: list[dict[str, Any]] = []
+        self.upserts: list[dict[str, Any]] = []
+        self.deletes: list[dict[str, Any]] = []
+        self.collection_checks: list[str] = []
+        self.collections_created: list[dict[str, Any]] = []
+        self.payload_indexes: list[dict[str, Any]] = []
+        self.closed = False
+
+    def query_points(self, **kwargs):
+        self.queries.append(kwargs)
+        return FakeQueryResult([
+            FakePoint("doc-1", 0.91, {"section": "docs"}),
+            FakePoint("doc-2", 0.42, {"section": "other"}),
+        ])
+
+    def upsert(self, **kwargs):
+        self.upserts.append(kwargs)
+        return SimpleNamespace(status="completed")
+
+    def delete(self, **kwargs):
+        self.deletes.append(kwargs)
+        return SimpleNamespace(status="completed")
+
+    def collection_exists(self, collection_name: str) -> bool:
+        self.collection_checks.append(collection_name)
+        if self.fail_health:
+            raise TimeoutError("qdrant api_key=qdrant-secret timed out")
+        return collection_name == "docs"
+
+    def create_collection(self, **kwargs):
+        self.collections_created.append(kwargs)
+
+    def create_payload_index(self, **kwargs):
+        self.payload_indexes.append(kwargs)
+
+    def close(self) -> None:
+        self.closed = True
+
+
+class FakeModels:
+    class Distance:
+        COSINE = "Cosine"
+        DOT = "Dot"
+        EUCLID = "Euclid"
+
+    class PayloadSchemaType:
+        KEYWORD = "keyword"
+
+    class VectorParams:
+        def __init__(self, *, size: int, distance) -> None:
+            self.size = size
+            self.distance = distance
+
+    class MatchValue:
+        def __init__(self, value) -> None:
+            self.value = value
+
+    class MatchAny:
+        def __init__(self, any) -> None:
+            self.any = any
+
+    class Range:
+        def __init__(self, **kwargs) -> None:
+            self.values = kwargs
+
+    class FieldCondition:
+        def __init__(self, *, key: str, match=None, range=None) -> None:
+            self.key = key
+            self.match = match
+            self.range = range
+
+    class Filter:
+        def __init__(self, *, must=None, should=None, must_not=None) -> None:
+            self.must = must or []
+            self.should = should or []
+            self.must_not = must_not or []
+
+    class PointStruct:
+        def __init__(self, *, id, vector, payload=None) -> None:
+            self.id = id
+            self.vector = vector
+            self.payload = payload or {}
+
+    class PointIdsList:
+        def __init__(self, *, points) -> None:
+            self.points = points
+
+    class FilterSelector:
+        def __init__(self, *, filter) -> None:
+            self.filter = filter
+
+
+def _config() -> dict:
+    return {
+        "data": {
+            "resources": {
+                "vector.qdrant": {
+                    "type": "qdrant",
+                    "url": "https://qdrant.example",
+                    "api_key": "qdrant-secret",
+                    "collection": "docs",
+                    "vector_size": 2,
+                    "distance": "cosine",
+                    "native_client": True,
+                }
+            }
+        }
+    }
+
+
+def _runtime(client: FakeQdrantClient | None):
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(QdrantVectorFactory(client_factory=lambda _config: client, models_provider=lambda: FakeModels))
+    return DataRuntime(config=DataConfig.from_raw(_config()), catalog=catalog)
+
+
+def test_qdrant_external_adapter_maps_vector_operations_and_native_access():
+    client = FakeQdrantClient()
+    runtime = _runtime(client)
+
+    listed = runtime.list_resources()[0]
+    assert listed["type"] == "qdrant"
+    assert {"vector_search", "vector_write"} <= set(listed["capabilities"])
+    assert listed["initialized"] is False
+
+    vector = runtime.require_port("vector.qdrant", VectorSearchPort)
+    hits = vector.search_vectors([0.1, 0.9], filters={"section": "docs"}, limit=2)
+    write = vector.upsert_vectors([{"id": "doc-1", "vector": [0.1, 0.9], "payload": {"section": "docs"}}])
+    deleted = vector.delete_vectors(ids=["doc-1"])
+
+    assert [hit.id for hit in hits] == ["doc-1", "doc-2"]
+    assert client.queries[0]["query_filter"].must[0].key == "section"
+    assert write.written == 1
+    assert client.upserts[0]["points"][0].payload == {"section": "docs", "_muscles_id": "doc-1"}
+    assert deleted.deleted == 1
+    assert client.deletes[0]["points_selector"].points == [str(uuid5(NAMESPACE_URL, "muscles-data:qdrant:doc-1"))]
+    assert runtime.require_resource("vector.qdrant", DataCapability.NATIVE_CLIENT).native_client() is client
+    assert runtime.doctor()["status"] == "ok"
+    assert client.collection_checks == ["docs"]
+    assert runtime.close()["status"] == "ok"
+    assert client.closed is True
+
+
+def test_qdrant_external_adapter_filters_dimensions_and_safe_failures():
+    translated = qdrant_filter_from_mapping({"score": {"gte": 0.5}, "$not": {"archived": True}}, models=FakeModels)
+    assert translated.must[0].key == "score"
+    assert translated.must[0].range.values == {"gte": 0.5}
+    assert translated.must_not[0].key == "archived"
+
+    with pytest.raises(QdrantFilterError):
+        qdrant_filter_from_mapping({"score": {"near": 1.0}}, models=FakeModels)
+    with pytest.raises(QdrantDimensionError):
+        _runtime(FakeQdrantClient()).require_port("vector.qdrant", VectorSearchPort).search_vectors([])
+    with pytest.raises(QdrantDimensionError):
+        _runtime(FakeQdrantClient()).require_port("vector.qdrant", VectorSearchPort).search_vectors([1.0])
+    with pytest.raises(QdrantClientMissingError):
+        _runtime(None).require_port("vector.qdrant", VectorSearchPort).search_vectors([1.0, 0.0])
+
+    failing = _runtime(FakeQdrantClient(fail_health=True)).doctor()
+    assert failing["status"] == "failed"
+    assert "qdrant-secret" not in repr(failing)
+
+    bad_client = FakeQdrantClient()
+    bad_client.query_points = lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("network unavailable"))
+    with pytest.raises(QdrantConnectionError):
+        _runtime(bad_client).require_port("vector.qdrant", VectorSearchPort).search_vectors([1.0, 0.0])
+
+
+def test_qdrant_creates_collection_indexes_payload_and_resolves_url_env(monkeypatch):
+    monkeypatch.setenv("QDRANT_URL", "https://qdrant.example")
+    client = FakeQdrantClient()
+    client.collection_exists = lambda collection_name: False
+    captured: list[dict] = []
+    catalog = DataAdapterCatalog.with_defaults()
+    catalog.register(
+        QdrantVectorFactory(
+            client_factory=lambda config: captured.append(config.resolved_options()) or client,
+            models_provider=lambda: FakeModels,
+        )
+    )
+    runtime = DataRuntime(
+        config=DataConfig.from_raw(
+            {
+                "data": {
+                    "resources": {
+                        "vector.qdrant": {
+                            "type": "qdrant",
+                            "url_env": "QDRANT_URL",
+                            "collection": "assetforge_rag",
+                            "vector_size": 2,
+                            "distance": "cosine",
+                            "payload_indexes": ["workspace_uid", "status"],
+                        }
+                    }
+                }
+            }
+        ),
+        catalog=catalog,
+    )
+
+    runtime.require_port("vector.qdrant", VectorSearchPort).upsert_vectors(
+        [{"id": "doc-1", "vector": [1.0, 0.0], "payload": {"status": "ready"}}]
+    )
+
+    assert captured[0]["url"] == "https://qdrant.example"
+    assert client.collections_created[0]["collection_name"] == "assetforge_rag"
+    assert client.collections_created[0]["vectors_config"].size == 2
+    assert {item["field_name"] for item in client.payload_indexes} == {"workspace_uid", "status"}
+
+
+def test_qdrant_normalizes_arbitrary_string_ids_for_backend_and_keeps_public_id():
+    client = FakeQdrantClient()
+    runtime = _runtime(client)
+    vector = runtime.require_port("vector.qdrant", VectorSearchPort)
+
+    vector.upsert_vectors([{"id": "document-alpha", "vector": [1.0, 0.0], "payload": {"kind": "doc"}}])
+
+    point = client.upserts[-1]["points"][0]
+    UUID(str(point.id))
+    assert point.payload["_muscles_id"] == "document-alpha"
+
+
+def test_qdrant_rejects_configured_dimension_mismatch_with_existing_collection():
+    client = FakeQdrantClient()
+    client.get_collection = lambda **_kwargs: SimpleNamespace(
+        config=SimpleNamespace(params=SimpleNamespace(vectors=SimpleNamespace(size=3)))
+    )
+    with pytest.raises(QdrantDimensionError, match="expected 2, got 3"):
+        _runtime(client).require_port("vector.qdrant", VectorSearchPort).search_vectors([1.0, 0.0])