haystack-ai 2.2.3__tar.gz → 2.2.4__tar.gz

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: haystack-ai
-Version: 2.2.3
+Version: 2.2.4
 Summary: LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data.
 Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack/actions
 Project-URL: Docs: RTD, https://haystack.deepset.ai/overview/intro

haystack_ai-2.2.4/VERSION.txt

@@ -0,0 +1 @@
+2.2.4

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/components/converters/azure.py

@@ -135,7 +135,8 @@ class AzureOCRDocumentConverter:
             result = poller.result()
             azure_output.append(result.to_dict())
 
-            docs = self._convert_tables_and_text(result=result, meta=metadata)
+            merged_metadata = {**bytestream.meta, **metadata}
+            docs = self._convert_tables_and_text(result=result, meta=merged_metadata)
             documents.extend(docs)
 
         return {"documents": documents, "raw_azure_response": azure_output}

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/components/retrievers/in_memory/bm25_retriever.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional
 
 from haystack import DeserializationError, Document, component, default_from_dict, default_to_dict
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.document_stores.types import FilterPolicy
 
 
 @component
@@ -40,6 +41,7 @@ class InMemoryBM25Retriever:
         filters: Optional[Dict[str, Any]] = None,
         top_k: int = 10,
         scale_score: bool = False,
+        filter_policy: FilterPolicy = FilterPolicy.REPLACE,
     ):
         """
         Create the InMemoryBM25Retriever component.
@@ -52,7 +54,7 @@ class InMemoryBM25Retriever:
             The maximum number of documents to retrieve.
         :param scale_score:
             Scales the BM25 score to a unit interval in the range of 0 to 1, where 1 means extremely relevant. If set to `False`, uses raw similarity scores.
-
+        :param filter_policy: The filter policy to apply during retrieval.
         :raises ValueError:
             If the specified `top_k` is not > 0.
         """
@@ -67,6 +69,7 @@ class InMemoryBM25Retriever:
         self.filters = filters
         self.top_k = top_k
         self.scale_score = scale_score
+        self.filter_policy = filter_policy
 
     def _get_telemetry_data(self) -> Dict[str, Any]:
         """
@@ -83,7 +86,12 @@ class InMemoryBM25Retriever:
         """
         docstore = self.document_store.to_dict()
         return default_to_dict(
-            self, document_store=docstore, filters=self.filters, top_k=self.top_k, scale_score=self.scale_score
+            self,
+            document_store=docstore,
+            filters=self.filters,
+            top_k=self.top_k,
+            scale_score=self.scale_score,
+            filter_policy=self.filter_policy.value,
         )
 
     @classmethod
@@ -101,6 +109,8 @@ class InMemoryBM25Retriever:
             raise DeserializationError("Missing 'document_store' in serialization data")
         if "type" not in init_params["document_store"]:
             raise DeserializationError("Missing 'type' in document store's serialization data")
+        if "filter_policy" in init_params:
+            init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
         data["init_parameters"]["document_store"] = InMemoryDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
@@ -132,8 +142,10 @@ class InMemoryBM25Retriever:
         :raises ValueError:
             If the specified DocumentStore is not found or is not a InMemoryDocumentStore instance.
         """
-        if filters is None:
-            filters = self.filters
+        if self.filter_policy == FilterPolicy.MERGE and filters:
+            filters = {**(self.filters or {}), **filters}
+        else:
+            filters = filters or self.filters
         if top_k is None:
             top_k = self.top_k
         if scale_score is None:

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/components/retrievers/in_memory/embedding_retriever.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional
 
 from haystack import DeserializationError, Document, component, default_from_dict, default_to_dict
 from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.document_stores.types import FilterPolicy
 
 
 @component
@@ -50,6 +51,7 @@ class InMemoryEmbeddingRetriever:
         top_k: int = 10,
         scale_score: bool = False,
         return_embedding: bool = False,
+        filter_policy: FilterPolicy = FilterPolicy.REPLACE,
     ):
         """
         Create the InMemoryEmbeddingRetriever component.
@@ -64,7 +66,7 @@ class InMemoryEmbeddingRetriever:
             Scales the BM25 score to a unit interval in the range of 0 to 1, where 1 means extremely relevant. If set to `False`, uses raw similarity scores.
         :param return_embedding:
             Whether to return the embedding of the retrieved Documents.
-
+        :param filter_policy: The filter policy to apply during retrieval.
         :raises ValueError:
             If the specified top_k is not > 0.
         """
@@ -80,6 +82,7 @@ class InMemoryEmbeddingRetriever:
         self.top_k = top_k
         self.scale_score = scale_score
         self.return_embedding = return_embedding
+        self.filter_policy = filter_policy
 
     def _get_telemetry_data(self) -> Dict[str, Any]:
         """
@@ -102,6 +105,7 @@ class InMemoryEmbeddingRetriever:
             top_k=self.top_k,
             scale_score=self.scale_score,
             return_embedding=self.return_embedding,
+            filter_policy=self.filter_policy.value,
         )
 
     @classmethod
@@ -119,6 +123,8 @@ class InMemoryEmbeddingRetriever:
             raise DeserializationError("Missing 'document_store' in serialization data")
         if "type" not in init_params["document_store"]:
             raise DeserializationError("Missing 'type' in document store's serialization data")
+        if "filter_policy" in init_params:
+            init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"])
         data["init_parameters"]["document_store"] = InMemoryDocumentStore.from_dict(
             data["init_parameters"]["document_store"]
         )
@@ -153,8 +159,10 @@ class InMemoryEmbeddingRetriever:
         :raises ValueError:
             If the specified DocumentStore is not found or is not an InMemoryDocumentStore instance.
         """
-        if filters is None:
-            filters = self.filters
+        if self.filter_policy == FilterPolicy.MERGE and filters:
+            filters = {**(self.filters or {}), **filters}
+        else:
+            filters = filters or self.filters
         if top_k is None:
             top_k = self.top_k
         if scale_score is None:

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/core/pipeline/base.py

@@ -719,7 +719,7 @@ class PipelineBase:
 
         return {**data}
 
-    def _init_to_run(self) -> List[Tuple[str, Component]]:
+    def _init_to_run(self, pipeline_inputs: Dict[str, Any]) -> List[Tuple[str, Component]]:
         to_run: List[Tuple[str, Component]] = []
         for node_name in self.graph.nodes:
             component = self.graph.nodes[node_name]["instance"]
@@ -729,6 +729,11 @@ class PipelineBase:
                 to_run.append((node_name, component))
                 continue
 
+            if node_name in pipeline_inputs:
+                # This component is in the input data, if it has enough inputs it can run right away
+                to_run.append((node_name, component))
+                continue
+
             for socket in component.__haystack_input__._sockets_dict.values():
                 if not socket.senders or socket.is_variadic:
                     # Component has at least one input not connected or is variadic, can run right away.

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/core/pipeline/pipeline.py

@@ -112,9 +112,12 @@ class Pipeline(PipelineBase):
         # Initialize the inputs state
         last_inputs: Dict[str, Dict[str, Any]] = self._init_inputs_state(data)
 
-        # Take all components that have at least 1 input not connected or is variadic,
-        # and all components that have no inputs at all
-        to_run: List[Tuple[str, Component]] = self._init_to_run()
+        # Take all components that:
+        # - have no inputs
+        # - receive input from the user
+        # - have at least one input not connected
+        # - have at least one input that is variadic
+        to_run: List[Tuple[str, Component]] = self._init_to_run(data)
 
         # These variables are used to detect when we're stuck in a loop.
         # Stuck loops can happen when one or more components are waiting for input but
@@ -232,8 +235,15 @@ class Pipeline(PipelineBase):
                         if name != sender_component_name:
                             continue
 
+                        pair = (receiver_component_name, self.graph.nodes[receiver_component_name]["instance"])
                         if edge_data["from_socket"].name not in res:
-                            # This output has not been produced by the component, skip it
+                            # The component didn't produce any output for this socket.
+                            # We can't run the receiver, let's remove it from the list of components to run
+                            # or we risk running it if it's in those lists.
+                            if pair in to_run:
+                                to_run.remove(pair)
+                            if pair in waiting_for_input:
+                                waiting_for_input.remove(pair)
                             continue
 
                         if receiver_component_name not in last_inputs:
@@ -249,7 +259,6 @@ class Pipeline(PipelineBase):
                         else:
                             last_inputs[receiver_component_name][edge_data["to_socket"].name] = value
 
-                        pair = (receiver_component_name, self.graph.nodes[receiver_component_name]["instance"])
                         is_greedy = pair[1].__haystack_is_greedy__
                         is_variadic = edge_data["to_socket"].is_variadic
                         if is_variadic and is_greedy:

{haystack_ai-2.2.3 → haystack_ai-2.2.4}/haystack/document_stores/types/__init__.py

@@ -2,7 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from .filter_policy import FilterPolicy
 from .policy import DuplicatePolicy
 from .protocol import DocumentStore
 
-__all__ = ["DocumentStore", "DuplicatePolicy"]
+__all__ = ["DocumentStore", "DuplicatePolicy", "FilterPolicy"]

haystack_ai-2.2.4/haystack/document_stores/types/filter_policy.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from typing import Any, Dict, Optional
+
+
+class FilterPolicy(Enum):
+    """
+    Policy to determine how filters are applied in retrievers interacting with document stores.
+    """
+
+    # Runtime filters replace init filters during retriever run invocation.
+    REPLACE = "replace"
+
+    # Runtime filters are merged with init filters, with runtime filters overwriting init values.
+    MERGE = "merge"
+
+    def __str__(self):
+        return self.value
+
+    @staticmethod
+    def from_str(filter_policy: str) -> "FilterPolicy":
+        """
+        Convert a string to a FilterPolicy enum.
+
+        :param filter_policy: The string to convert.
+        :return: The corresponding FilterPolicy enum.
+        """
+        enum_map = {e.value: e for e in FilterPolicy}
+        policy = enum_map.get(filter_policy)
+        if policy is None:
+            msg = f"Unknown FilterPolicy type '{filter_policy}'. Supported types are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return policy
+
+
+def apply_filter_policy(
+    filter_policy: FilterPolicy,
+    init_filters: Optional[Dict[str, Any]] = None,
+    runtime_filters: Optional[Dict[str, Any]] = None,
+) -> Optional[Dict[str, Any]]:
+    """
+    Apply the filter policy to the given initial and runtime filters to determine the final set of filters used.
+
+    The function combines or replaces the initial and runtime filters based on the specified filter policy.
+
+    :param filter_policy: The policy to apply when handling the filters. It can be one of the following:
+        - `FilterPolicy.REPLACE`: Runtime filters will replace the initial filters.
+        - `FilterPolicy.MERGE`: Runtime filters will be merged with the initial filters. If there are overlapping keys,
+          values from the runtime filters will overwrite those from the initial filters.
+    :param init_filters: The initial filters set during the initialization of the relevant retriever.
+    :param runtime_filters: The filters provided at runtime, usually during a query operation execution. These filters
+                            can change for each query/retreiver run invocation.
+    :returns: A dictionary containing the resulting filters based on the provided policy.
+    """
+    if filter_policy == FilterPolicy.MERGE and runtime_filters:
+        return {**(init_filters or {}), **runtime_filters}
+    else:
+        return runtime_filters or init_filters

haystack_ai-2.2.3/VERSION.txt

@@ -1 +0,0 @@
-2.2.3