weaviate-haystack 7.0.0__tar.gz → 7.2.0__tar.gz

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [integrations/weaviate-v7.1.0] - 2026-01-29
+
+### 🚀 Features
+
+- Adding count with filtering operations to `WeaviateDocumentStore` (#2767)
+
+
+## [integrations/weaviate-v7.0.0] - 2026-01-13
+
+### 🐛 Bug Fixes
+
+- `WeaviateDocumentStore `_to_document()` and `to_data_object()` should be static methods (#2669)
+
+### 🧹 Chores
+
+- Make fmt command more forgiving (#2671)
+- [**breaking**] Weaviate - drop Python 3.9 and use X|Y typing (#2733)
+
+
 ## [integrations/weaviate-v6.5.0] - 2026-01-08
 
 ### 🚀 Features

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: weaviate-haystack
-Version: 7.0.0
+Version: 7.2.0
 Summary: An integration of Weaviate vector database with Haystack
 Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/weaviate/README.md
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
-Requires-Dist: haystack-ai>=2.22.0
+Requires-Dist: haystack-ai>=2.24.0
 Requires-Dist: python-dateutil
 Requires-Dist: weaviate-client>=4.9
 Description-Content-Type: text/markdown

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.4'
 services:
   weaviate:
     command:

weaviate_haystack-7.2.0/pydoc/config_docusaurus.yml

@@ -0,0 +1,17 @@
+loaders:
+  - modules:
+      - haystack_integrations.document_stores.weaviate.auth
+      - haystack_integrations.document_stores.weaviate.document_store
+      - haystack_integrations.components.retrievers.weaviate.bm25_retriever
+      - haystack_integrations.components.retrievers.weaviate.embedding_retriever
+      - haystack_integrations.components.retrievers.weaviate.hybrid_retriever
+    search_path: [../src]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  description: Weaviate integration for Haystack
+  id: integrations-weaviate
+  filename: weaviate.md
+  title: Weaviate

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
-  "haystack-ai>=2.22.0",
+  "haystack-ai>=2.24.0",
   "weaviate-client>=4.9",
   "python-dateutil",
 ]
@@ -49,7 +49,7 @@ installer = "uv"
 dependencies = ["haystack-pydoc-tools", "ruff"]
 
 [tool.hatch.envs.default.scripts]
-docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
+docs = ["haystack-pydoc pydoc/config_docusaurus.yml"]
 fmt = "ruff check --fix {args}; ruff format {args}"
 fmt-check = "ruff check {args} && ruff format --check {args}"
 

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/src/haystack_integrations/components/retrievers/weaviate/bm25_retriever.py

@@ -18,8 +18,12 @@ class WeaviateBM25Retriever:
 
     Example usage:
     ```python
-    from haystack_integrations.document_stores.weaviate.document_store import WeaviateDocumentStore
-    from haystack_integrations.components.retrievers.weaviate.bm25_retriever import WeaviateBM25Retriever
+    from haystack_integrations.document_stores.weaviate.document_store import (
+        WeaviateDocumentStore,
+    )
+    from haystack_integrations.components.retrievers.weaviate.bm25_retriever import (
+        WeaviateBM25Retriever,
+    )
 
     document_store = WeaviateDocumentStore(url="http://localhost:8080")
     retriever = WeaviateBM25Retriever(document_store=document_store)

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/src/haystack_integrations/document_stores/weaviate/_filters.py

@@ -26,6 +26,13 @@ def validate_filters(filters: dict[str, Any] | None) -> None:
 def convert_filters(filters: dict[str, Any]) -> FilterReturn:
     """
     Convert filters from Haystack format to Weaviate format.
+
+    Supported comparison operators: ``==``, ``!=``, ``>``, ``>=``, ``<``, ``<=``,
+    ``in``, ``not in``, ``contains``.
+
+    Note: The ``contains`` operator performs substring matching and is
+    **case-sensitive**. For case-insensitive matching, normalize the value
+    (e.g., lowercase) before building the filter.
     """
     if not isinstance(filters, dict):
         msg = "Filters must be a dictionary"
@@ -228,6 +235,19 @@ def _not_in(field: str, value: Any) -> FilterReturn:
     return Filter.all_of(operands)
 
 
+def _contains(field: str, value: Any) -> FilterReturn:
+    """
+    Creates a filter for substring matching using Weaviate's 'like' operator.
+
+    The matching is case-sensitive. For case-insensitive matching, consider
+    normalizing the value before passing it to this function.
+    """
+    if not isinstance(value, str):
+        msg = "Filter value must be a string when using 'contains' comparator"
+        raise FilterError(msg)
+    return weaviate.classes.query.Filter.by_property(field).like(f"*{value}*")
+
+
 COMPARISON_OPERATORS = {
     "==": _equal,
     "!=": _not_equal,
@@ -237,6 +257,7 @@ COMPARISON_OPERATORS = {
     "<=": _less_than_equal,
     "in": _in,
     "not in": _not_in,
+    "contains": _contains,
 }
 
 

{weaviate_haystack-7.0.0 → weaviate_haystack-7.2.0}/src/haystack_integrations/document_stores/weaviate/document_store.py

@@ -15,6 +15,13 @@ from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumen
 from haystack.document_stores.types.policy import DuplicatePolicy
 
 import weaviate
+from weaviate.collections.classes.aggregate import (
+    GroupByAggregate,
+    Metrics,
+    _MetricsDate,
+    _MetricsInteger,
+    _MetricsNumber,
+)
 from weaviate.collections.classes.data import DataObject
 from weaviate.config import AdditionalConfig
 from weaviate.embedded import EmbeddedOptions
@@ -65,7 +72,9 @@ class WeaviateDocumentStore:
     ```python
     import os
     from haystack_integrations.document_stores.weaviate.auth import AuthApiKey
-    from haystack_integrations.document_stores.weaviate.document_store import WeaviateDocumentStore
+    from haystack_integrations.document_stores.weaviate.document_store import (
+        WeaviateDocumentStore,
+    )
 
     os.environ["WEAVIATE_API_KEY"] = "MY_API_KEY"
 
@@ -77,7 +86,9 @@ class WeaviateDocumentStore:
 
     Usage example with self-hosted Weaviate:
     ```python
-    from haystack_integrations.document_stores.weaviate.document_store import WeaviateDocumentStore
+    from haystack_integrations.document_stores.weaviate.document_store import (
+        WeaviateDocumentStore,
+    )
 
     document_store = WeaviateDocumentStore(url="http://localhost:8080")
     ```
@@ -206,8 +217,6 @@ class WeaviateDocumentStore:
 
         self._client.connect()
 
-        # Test connection, it will raise an exception if it fails.
-        self._client.collections.list_all(simple=True)
         if not self._client.collections.exists(self._collection_settings["class"]):
             self._client.collections.create_from_dict(self._collection_settings)
 
@@ -251,8 +260,7 @@ class WeaviateDocumentStore:
             )
 
         await self._async_client.connect()
-        # Test connection, it will raise an exception if it fails.
-        await self._async_client.collections.list_all(simple=True)
+
         if not await self._async_client.collections.exists(self._collection_settings["class"]):
             await self._async_client.collections.create_from_dict(self._collection_settings)
 
@@ -276,6 +284,24 @@ class WeaviateDocumentStore:
         self._async_collection = async_client.collections.get(self._collection_settings["class"])
         return self._async_collection
 
+    def close(self) -> None:
+        """
+        Close the synchronous Weaviate client connection.
+        """
+        if self._client:
+            self._client.close()
+            self._client = None
+            self._collection = None
+
+    async def close_async(self) -> None:
+        """
+        Close the asynchronous Weaviate client connection.
+        """
+        if self._async_client:
+            await self._async_client.close()
+            self._async_client = None
+            self._async_collection = None
+
     def to_dict(self) -> dict[str, Any]:
         """
         Serializes the component to a dictionary.
@@ -326,6 +352,368 @@ class WeaviateDocumentStore:
         total = self.collection.aggregate.over_all(total_count=True).total_count
         return total if total else 0
 
+    def count_documents_by_filter(self, filters: dict[str, Any]) -> int:
+        """
+        Returns the number of documents that match the provided filters.
+
+        :param filters: The filters to apply to count documents.
+            For filter syntax, see
+            [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering).
+        :returns: The number of documents that match the filters.
+        """
+        validate_filters(filters)
+        weaviate_filter = convert_filters(filters)
+        total = self.collection.aggregate.over_all(filters=weaviate_filter, total_count=True).total_count
+        return total if total else 0
+
+    async def count_documents_by_filter_async(self, filters: dict[str, Any]) -> int:
+        """
+        Asynchronously returns the number of documents that match the provided filters.
+
+        :param filters: The filters to apply to count documents.
+            For filter syntax, see
+            [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering).
+        :returns: The number of documents that match the filters.
+        """
+        validate_filters(filters)
+        collection = await self.async_collection
+        weaviate_filter = convert_filters(filters)
+        result = await collection.aggregate.over_all(filters=weaviate_filter, total_count=True)
+        return result.total_count if result.total_count else 0
+
+    def get_metadata_fields_info(self) -> dict[str, dict[str, str]]:
+        """
+        Returns metadata field names and their types, excluding special fields.
+
+        Special fields (content, blob_data, blob_mime_type, _original_id, score) are excluded
+        as they are not user metadata fields.
+
+        :returns: A dictionary where keys are field names and values are dictionaries
+            containing type information, e.g.:
+            ```python
+            {
+                'number': {'type': 'int'},
+                'date': {'type': 'date'},
+                'category': {'type': 'text'},
+                'status': {'type': 'text'}
+            }
+            ```
+        """
+        config = self.collection.config.get()
+        special_fields = {prop["name"] for prop in DOCUMENT_COLLECTION_PROPERTIES}
+        fields_info = {}
+        for prop in config.properties:
+            if prop.name not in special_fields:
+                data_type = str(prop.data_type.value) if hasattr(prop.data_type, "value") else str(prop.data_type)
+                fields_info[prop.name] = {"type": data_type}
+        return fields_info
+
+    async def get_metadata_fields_info_async(self) -> dict[str, dict[str, str]]:
+        """
+        Asynchronously returns metadata field names and their types, excluding special fields.
+
+        Special fields (content, blob_data, blob_mime_type, _original_id, score) are excluded
+        as they are not user metadata fields.
+
+        :returns: A dictionary where keys are field names and values are dictionaries
+            containing type information, e.g.:
+            ```python
+            {
+                'number': {'type': 'int'},
+                'date': {'type': 'date'},
+                'category': {'type': 'text'},
+                'status': {'type': 'text'}
+            }
+            ```
+        """
+        collection = await self.async_collection
+        config = await collection.config.get()
+        special_fields = {prop["name"] for prop in DOCUMENT_COLLECTION_PROPERTIES}
+        fields_info = {}
+        for prop in config.properties:
+            if prop.name not in special_fields:
+                data_type = str(prop.data_type.value) if hasattr(prop.data_type, "value") else str(prop.data_type)
+                fields_info[prop.name] = {"type": data_type}
+        return fields_info
+
+    @staticmethod
+    def _normalize_metadata_field_name(metadata_field: str) -> str:
+        """
+        Removes 'meta.' prefix from field name if present.
+
+        :param metadata_field: The field name, possibly prefixed with 'meta.'.
+        :returns: The field name without the 'meta.' prefix.
+        """
+        return metadata_field[5:] if metadata_field.startswith("meta.") else metadata_field
+
+    def get_metadata_field_min_max(self, metadata_field: str) -> dict[str, Any]:
+        """
+        Returns the minimum and maximum values for a numeric or date metadata field.
+
+        :param metadata_field: The metadata field name to get min/max for.
+            Can be prefixed with 'meta.' (e.g., 'meta.year' or 'year').
+        :returns: A dictionary with 'min' and 'max' keys containing the respective values.
+        :raises ValueError: If the field is not found or doesn't support min/max operations.
+        """
+        field_name = self._normalize_metadata_field_name(metadata_field)
+
+        # Get field type from schema
+        config = self.collection.config.get()
+        field_type = None
+        for prop in config.properties:
+            if prop.name == field_name:
+                field_type = prop.data_type
+                break
+
+        if field_type is None:
+            msg = f"Field '{field_name}' not found in collection schema"
+            raise ValueError(msg)
+
+        data_type_str = str(field_type.value) if hasattr(field_type, "value") else str(field_type)
+
+        # Build metrics based on type.
+        # Explicit type annotation needed for mypy compatibility with Python 3.10,
+        # which infers the type from the first assignment and fails on reassignment.
+        metrics: _MetricsInteger | _MetricsNumber | _MetricsDate
+        if data_type_str.lower() == "int":
+            metrics = Metrics(field_name).integer(minimum=True, maximum=True)
+        elif data_type_str.lower() == "number":
+            metrics = Metrics(field_name).number(minimum=True, maximum=True)
+        elif data_type_str.lower() == "date":
+            metrics = Metrics(field_name).date_(minimum=True, maximum=True)
+        else:
+            msg = f"Field type '{data_type_str}' doesn't support min/max aggregation"
+            raise ValueError(msg)
+
+        result = self.collection.aggregate.over_all(return_metrics=metrics)
+        field_metrics = result.properties.get(field_name)
+
+        return {
+            "min": getattr(field_metrics, "minimum", None) if field_metrics else None,
+            "max": getattr(field_metrics, "maximum", None) if field_metrics else None,
+        }
+
+    async def get_metadata_field_min_max_async(self, metadata_field: str) -> dict[str, Any]:
+        """
+        Asynchronously returns the minimum and maximum values for a numeric or date metadata field.
+
+        :param metadata_field: The metadata field name to get min/max for.
+            Can be prefixed with 'meta.' (e.g., 'meta.year' or 'year').
+        :returns: A dictionary with 'min' and 'max' keys containing the respective values.
+        :raises ValueError: If the field is not found or doesn't support min/max operations.
+        """
+        field_name = self._normalize_metadata_field_name(metadata_field)
+
+        # Get field type from schema
+        collection = await self.async_collection
+        config = await collection.config.get()
+        field_type = None
+        for prop in config.properties:
+            if prop.name == field_name:
+                field_type = prop.data_type
+                break
+
+        if field_type is None:
+            msg = f"Field '{field_name}' not found in collection schema"
+            raise ValueError(msg)
+
+        data_type_str = str(field_type.value) if hasattr(field_type, "value") else str(field_type)
+
+        # Build metrics based on type.
+        # Explicit type annotation needed for mypy compatibility with Python 3.10,
+        # which infers the type from the first assignment and fails on reassignment.
+        metrics: _MetricsInteger | _MetricsNumber | _MetricsDate
+        if data_type_str.lower() == "int":
+            metrics = Metrics(field_name).integer(minimum=True, maximum=True)
+        elif data_type_str.lower() == "number":
+            metrics = Metrics(field_name).number(minimum=True, maximum=True)
+        elif data_type_str.lower() == "date":
+            metrics = Metrics(field_name).date_(minimum=True, maximum=True)
+        else:
+            msg = f"Field type '{data_type_str}' doesn't support min/max aggregation"
+            raise ValueError(msg)
+
+        result = await collection.aggregate.over_all(return_metrics=metrics)
+        field_metrics = result.properties.get(field_name)
+
+        return {
+            "min": getattr(field_metrics, "minimum", None) if field_metrics else None,
+            "max": getattr(field_metrics, "maximum", None) if field_metrics else None,
+        }
+
+    def count_unique_metadata_by_filter(self, filters: dict[str, Any], metadata_fields: list[str]) -> dict[str, int]:
+        """
+        Returns the count of unique values for each specified metadata field.
+
+        :param filters: The filters to apply when counting unique values.
+            For filter syntax, see
+            [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering).
+        :param metadata_fields: List of metadata field names to count unique values for.
+            Field names can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
+        :returns: A dictionary mapping field names to counts of unique values.
+        :raises ValueError: If any of the requested fields don't exist in the collection schema.
+        """
+        validate_filters(filters)
+        weaviate_filter = convert_filters(filters)
+
+        normalized_fields = [self._normalize_metadata_field_name(f) for f in metadata_fields]
+
+        # Validate that all requested fields exist in the schema
+        config = self.collection.config.get()
+        schema_fields = {prop.name for prop in config.properties}
+        missing_fields = [f for f in normalized_fields if f not in schema_fields]
+        if missing_fields:
+            msg = f"Fields not found in collection schema: {missing_fields}"
+            raise ValueError(msg)
+
+        result = {}
+        for field in normalized_fields:
+            agg_result = self.collection.aggregate.over_all(
+                filters=weaviate_filter, group_by=GroupByAggregate(prop=field)
+            )
+            result[field] = len(agg_result.groups) if agg_result.groups else 0
+
+        return result
+
+    async def count_unique_metadata_by_filter_async(
+        self, filters: dict[str, Any], metadata_fields: list[str]
+    ) -> dict[str, int]:
+        """
+        Asynchronously returns the count of unique values for each specified metadata field.
+
+        :param filters: The filters to apply when counting unique values.
+            For filter syntax, see
+            [Haystack metadata filtering](https://docs.haystack.deepset.ai/docs/metadata-filtering).
+        :param metadata_fields: List of metadata field names to count unique values for.
+            Field names can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
+        :returns: A dictionary mapping field names to counts of unique values.
+        :raises ValueError: If any of the requested fields don't exist in the collection schema.
+        """
+        validate_filters(filters)
+        collection = await self.async_collection
+        weaviate_filter = convert_filters(filters)
+
+        normalized_fields = [self._normalize_metadata_field_name(f) for f in metadata_fields]
+
+        # Validate that all requested fields exist in the schema
+        config = await collection.config.get()
+        schema_fields = {prop.name for prop in config.properties}
+        missing_fields = [f for f in normalized_fields if f not in schema_fields]
+        if missing_fields:
+            msg = f"Fields not found in collection schema: {missing_fields}"
+            raise ValueError(msg)
+
+        result = {}
+        for field in normalized_fields:
+            agg_result = await collection.aggregate.over_all(
+                filters=weaviate_filter, group_by=GroupByAggregate(prop=field)
+            )
+            result[field] = len(agg_result.groups) if agg_result.groups else 0
+
+        return result
+
+    def get_metadata_field_unique_values(
+        self,
+        metadata_field: str,
+        search_term: str | None = None,
+        from_: int = 0,
+        size: int = 10000,
+    ) -> tuple[list[str], int]:
+        """
+        Returns unique values for a metadata field with pagination support.
+
+        :param metadata_field: The metadata field name to get unique values for.
+            Can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
+        :param search_term: Optional term to filter documents by content before
+            extracting unique values. If provided, only documents whose content
+            contains this term will be considered.
+            Note: Uses substring matching (case-sensitive, no stemming).
+        :param from_: The starting offset for pagination (0-indexed). Defaults to 0.
+        :param size: The maximum number of unique values to return. Defaults to 10000.
+        :returns: A tuple of (list of unique values, total count of unique values).
+        :raises ValueError: If the field is not found in the collection schema.
+        """
+        field_name = self._normalize_metadata_field_name(metadata_field)
+
+        config = self.collection.config.get()
+        schema_fields = {prop.name for prop in config.properties}
+        if field_name not in schema_fields:
+            msg = f"Field '{field_name}' not found in collection schema"
+            raise ValueError(msg)
+
+        weaviate_filter = None
+        if search_term:
+            weaviate_filter = convert_filters({"field": "content", "operator": "contains", "value": search_term})
+
+        # Weaviate's GroupByAggregate has a default limit of 100 groups.
+        # We use the document count as the limit to ensure all unique values are retrieved,
+        # since the number of unique values cannot exceed the number of documents.
+        doc_count = self.collection.aggregate.over_all(total_count=True).total_count or 0
+
+        agg_result = self.collection.aggregate.over_all(
+            filters=weaviate_filter, group_by=GroupByAggregate(prop=field_name, limit=doc_count)
+        )
+
+        all_values = [str(g.grouped_by.value) for g in agg_result.groups] if agg_result.groups else []
+        all_values.sort()  # Sort for consistent pagination
+        total_count = len(all_values)
+
+        paginated_values = all_values[from_ : from_ + size]
+
+        return paginated_values, total_count
+
+    async def get_metadata_field_unique_values_async(
+        self,
+        metadata_field: str,
+        search_term: str | None = None,
+        from_: int = 0,
+        size: int = 10000,
+    ) -> tuple[list[str], int]:
+        """
+        Asynchronously returns unique values for a metadata field with pagination support.
+
+        :param metadata_field: The metadata field name to get unique values for.
+            Can be prefixed with 'meta.' (e.g., 'meta.category' or 'category').
+        :param search_term: Optional term to filter documents by content before
+            extracting unique values. If provided, only documents whose content
+            contains this term will be considered.
+            Note: Uses substring matching (case-sensitive, no stemming).
+        :param from_: The starting offset for pagination (0-indexed). Defaults to 0.
+        :param size: The maximum number of unique values to return. Defaults to 10000.
+        :returns: A tuple of (list of unique values, total count of unique values).
+        :raises ValueError: If the field is not found in the collection schema.
+        """
+        field_name = self._normalize_metadata_field_name(metadata_field)
+
+        collection = await self.async_collection
+        config = await collection.config.get()
+        schema_fields = {prop.name for prop in config.properties}
+        if field_name not in schema_fields:
+            msg = f"Field '{field_name}' not found in collection schema"
+            raise ValueError(msg)
+
+        weaviate_filter = None
+        if search_term:
+            weaviate_filter = convert_filters({"field": "content", "operator": "contains", "value": search_term})
+
+        # Weaviate's GroupByAggregate has a default limit of 100 groups.
+        # We use the document count as the limit to ensure all unique values are retrieved,
+        # since the number of unique values cannot exceed the number of documents.
+        doc_count_result = await collection.aggregate.over_all(total_count=True)
+        doc_count = doc_count_result.total_count or 0
+
+        agg_result = await collection.aggregate.over_all(
+            filters=weaviate_filter, group_by=GroupByAggregate(prop=field_name, limit=doc_count)
+        )
+
+        all_values = [str(g.grouped_by.value) for g in agg_result.groups] if agg_result.groups else []
+        all_values.sort()  # Sort for consistent pagination
+        total_count = len(all_values)
+
+        paginated_values = all_values[from_ : from_ + size]
+
+        return paginated_values, total_count
+
     @staticmethod
     def _to_data_object(document: Document) -> dict[str, Any]:
         """
@@ -452,6 +840,10 @@ class WeaviateDocumentStore:
         For a detailed specification of the filters, refer to the
         DocumentStore.filter_documents() protocol documentation.
 
+        Note: The ``contains`` filter operator is case-sensitive (substring
+        matching). For case-insensitive matching, normalize the value before
+        building the filter.
+
         :param filters: The filters to apply to the document list.
         :returns: A list of Documents that match the given filters.
         """