lfx-nightly 0.1.11.dev0__py3-none-any.whl

lfx/components/chroma/chroma.py

@@ -0,0 +1,167 @@
+from copy import deepcopy
+from typing import TYPE_CHECKING
+
+from chromadb.config import Settings
+from langchain_chroma import Chroma
+from typing_extensions import override
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.base.vectorstores.utils import chroma_collection_to_data
+from lfx.inputs.inputs import BoolInput, DropdownInput, HandleInput, IntInput, StrInput
+from lfx.schema.data import Data
+
+if TYPE_CHECKING:
+    from lfx.schema.dataframe import DataFrame
+
+
+class ChromaVectorStoreComponent(LCVectorStoreComponent):
+    """Chroma Vector Store with search capabilities."""
+
+    display_name: str = "Chroma DB"
+    description: str = "Chroma Vector Store with search capabilities"
+    name = "Chroma"
+    icon = "Chroma"
+
+    inputs = [
+        StrInput(
+            name="collection_name",
+            display_name="Collection Name",
+            value="langflow",
+        ),
+        StrInput(
+            name="persist_directory",
+            display_name="Persist Directory",
+        ),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        StrInput(
+            name="chroma_server_cors_allow_origins",
+            display_name="Server CORS Allow Origins",
+            advanced=True,
+        ),
+        StrInput(
+            name="chroma_server_host",
+            display_name="Server Host",
+            advanced=True,
+        ),
+        IntInput(
+            name="chroma_server_http_port",
+            display_name="Server HTTP Port",
+            advanced=True,
+        ),
+        IntInput(
+            name="chroma_server_grpc_port",
+            display_name="Server gRPC Port",
+            advanced=True,
+        ),
+        BoolInput(
+            name="chroma_server_ssl_enabled",
+            display_name="Server SSL Enabled",
+            advanced=True,
+        ),
+        BoolInput(
+            name="allow_duplicates",
+            display_name="Allow Duplicates",
+            advanced=True,
+            info="If false, will not add documents that are already in the Vector Store.",
+        ),
+        DropdownInput(
+            name="search_type",
+            display_name="Search Type",
+            options=["Similarity", "MMR"],
+            value="Similarity",
+            advanced=True,
+        ),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            advanced=True,
+            value=10,
+        ),
+        IntInput(
+            name="limit",
+            display_name="Limit",
+            advanced=True,
+            info="Limit the number of records to compare when Allow Duplicates is False.",
+        ),
+    ]
+
+    @override
+    @check_cached_vector_store
+    def build_vector_store(self) -> Chroma:
+        """Builds the Chroma object."""
+        try:
+            from chromadb import Client
+            from langchain_chroma import Chroma
+        except ImportError as e:
+            msg = "Could not import Chroma integration package. Please install it with `pip install langchain-chroma`."
+            raise ImportError(msg) from e
+        # Chroma settings
+        chroma_settings = None
+        client = None
+        if self.chroma_server_host:
+            chroma_settings = Settings(
+                chroma_server_cors_allow_origins=self.chroma_server_cors_allow_origins or [],
+                chroma_server_host=self.chroma_server_host,
+                chroma_server_http_port=self.chroma_server_http_port or None,
+                chroma_server_grpc_port=self.chroma_server_grpc_port or None,
+                chroma_server_ssl_enabled=self.chroma_server_ssl_enabled,
+            )
+            client = Client(settings=chroma_settings)
+
+        # Check persist_directory and expand it if it is a relative path
+        persist_directory = self.resolve_path(self.persist_directory) if self.persist_directory is not None else None
+
+        chroma = Chroma(
+            persist_directory=persist_directory,
+            client=client,
+            embedding_function=self.embedding,
+            collection_name=self.collection_name,
+        )
+
+        self._add_documents_to_vector_store(chroma)
+        self.status = chroma_collection_to_data(chroma.get(limit=self.limit))
+        return chroma
+
+    def _add_documents_to_vector_store(self, vector_store: "Chroma") -> None:
+        """Adds documents to the Vector Store."""
+        ingest_data: list | Data | DataFrame = self.ingest_data
+        if not ingest_data:
+            self.status = ""
+            return
+
+        # Convert DataFrame to Data if needed using parent's method
+        ingest_data = self._prepare_ingest_data()
+
+        stored_documents_without_id = []
+        if self.allow_duplicates:
+            stored_data = []
+        else:
+            stored_data = chroma_collection_to_data(vector_store.get(limit=self.limit))
+            for value in deepcopy(stored_data):
+                del value.id
+                stored_documents_without_id.append(value)
+
+        documents = []
+        for _input in ingest_data or []:
+            if isinstance(_input, Data):
+                if _input not in stored_documents_without_id:
+                    documents.append(_input.to_lc_document())
+            else:
+                msg = "Vector Store Inputs must be Data objects."
+                raise TypeError(msg)
+
+        if documents and self.embedding is not None:
+            self.log(f"Adding {len(documents)} documents to the Vector Store.")
+            # Filter complex metadata to prevent ChromaDB errors
+            try:
+                from langchain_community.vectorstores.utils import filter_complex_metadata
+
+                filtered_documents = filter_complex_metadata(documents)
+                vector_store.add_documents(filtered_documents)
+            except ImportError:
+                self.log("Warning: Could not import filter_complex_metadata. Adding documents without filtering.")
+                vector_store.add_documents(documents)
+        else:
+            self.log("No documents to add to the Vector Store.")

lfx/components/cleanlab/__init__.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .cleanlab_evaluator import CleanlabEvaluator
+    from .cleanlab_rag_evaluator import CleanlabRAGEvaluator
+    from .cleanlab_remediator import CleanlabRemediator
+
+_dynamic_imports = {
+    "CleanlabEvaluator": "cleanlab_evaluator",
+    "CleanlabRAGEvaluator": "cleanlab_rag_evaluator",
+    "CleanlabRemediator": "cleanlab_remediator",
+}
+
+__all__ = [
+    "CleanlabEvaluator",
+    "CleanlabRAGEvaluator",
+    "CleanlabRemediator",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import cleanlab components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/cleanlab/cleanlab_evaluator.py

@@ -0,0 +1,155 @@
+from cleanlab_tlm import TLM
+
+from lfx.custom import Component
+from lfx.io import (
+    DropdownInput,
+    MessageTextInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.message import Message
+
+
+class CleanlabEvaluator(Component):
+    """A component that evaluates the trustworthiness of LLM responses using Cleanlab.
+
+    This component takes a prompt and response pair, along with optional system instructions,
+    and uses Cleanlab's evaluation algorithms to generate a trust score and explanation.
+
+    Inputs:
+        - system_prompt (MessageTextInput): Optional system-level instructions prepended to the user prompt.
+        - prompt (MessageTextInput): The user's prompt or query sent to the LLM.
+        - response (MessageTextInput): The response generated by the LLM to be evaluated. This should come from the
+          LLM component, i.e. OpenAI, Gemini, etc.
+        - api_key (SecretStrInput): Your Cleanlab API key.
+        - model (DropdownInput): The model used by Cleanlab to evaluate the response (can differ from the
+          generation model).
+        - quality_preset (DropdownInput): Tradeoff setting for accuracy vs. speed and cost. Higher presets are
+          slower but more accurate.
+
+    Outputs:
+        - response_passthrough (Message): The original response, passed through for downstream use.
+        - score (number): A float between 0 and 1 indicating Cleanlab's trustworthiness score for the response.
+        - explanation (Message): A textual explanation of why the response received its score.
+
+    This component works well in conjunction with the CleanlabRemediator to create a complete trust evaluation
+    and remediation pipeline.
+
+    More details on the evaluation metrics can be found here: https://help.cleanlab.ai/tlm/tutorials/tlm/
+    """
+
+    display_name = "Cleanlab Evaluator"
+    description = "Evaluates any LLM response using Cleanlab and outputs trust score and explanation."
+    icon = "Cleanlab"
+    name = "CleanlabEvaluator"
+
+    inputs = [
+        MessageTextInput(
+            name="system_prompt",
+            display_name="System Message",
+            info="System-level instructions prepended to the user query.",
+            value="",
+        ),
+        MessageTextInput(
+            name="prompt",
+            display_name="Prompt",
+            info="The user's query to the model.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="response",
+            display_name="Response",
+            info="The response to the user's query.",
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_key",
+            display_name="Cleanlab API Key",
+            info="Your Cleanlab API key.",
+            required=True,
+        ),
+        DropdownInput(
+            name="model",
+            display_name="Cleanlab Evaluation Model",
+            options=[
+                "gpt-4.1",
+                "gpt-4.1-mini",
+                "gpt-4.1-nano",
+                "o4-mini",
+                "o3",
+                "gpt-4.5-preview",
+                "gpt-4o-mini",
+                "gpt-4o",
+                "o3-mini",
+                "o1",
+                "o1-mini",
+                "gpt-4",
+                "gpt-3.5-turbo-16k",
+                "claude-3.7-sonnet",
+                "claude-3.5-sonnet-v2",
+                "claude-3.5-sonnet",
+                "claude-3.5-haiku",
+                "claude-3-haiku",
+                "nova-micro",
+                "nova-lite",
+                "nova-pro",
+            ],
+            info="The model Cleanlab uses to evaluate the response. This does NOT need to be the same model that "
+            "generated the response.",
+            value="gpt-4o-mini",
+            required=True,
+            advanced=True,
+        ),
+        DropdownInput(
+            name="quality_preset",
+            display_name="Quality Preset",
+            options=["base", "low", "medium", "high", "best"],
+            value="medium",
+            info="This determines the accuracy, latency, and cost of the evaluation. Higher quality is generally "
+            "slower but more accurate.",
+            required=True,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Response",
+            name="response_passthrough",
+            method="pass_response",
+            types=["Message"],
+        ),
+        Output(display_name="Trust Score", name="score", method="get_score", types=["number"]),
+        Output(
+            display_name="Explanation",
+            name="explanation",
+            method="get_explanation",
+            types=["Message"],
+        ),
+    ]
+
+    def _evaluate_once(self):
+        if not hasattr(self, "_cached_result"):
+            full_prompt = f"{self.system_prompt}\n\n{self.prompt}" if self.system_prompt else self.prompt
+            tlm = TLM(
+                api_key=self.api_key,
+                options={"log": ["explanation"], "model": self.model},
+                quality_preset=self.quality_preset,
+            )
+            self._cached_result = tlm.get_trustworthiness_score(full_prompt, self.response)
+        return self._cached_result
+
+    def get_score(self) -> float:
+        result = self._evaluate_once()
+        score = result.get("trustworthiness_score", 0.0)
+        self.status = f"Trust score: {score:.2f}"
+        return score
+
+    def get_explanation(self) -> Message:
+        result = self._evaluate_once()
+        explanation = result.get("log", {}).get("explanation", "No explanation returned.")
+        return Message(text=explanation)
+
+    def pass_response(self) -> Message:
+        self.status = "Passing through response."
+        return Message(text=self.response)

lfx/components/cleanlab/cleanlab_rag_evaluator.py

@@ -0,0 +1,254 @@
+from cleanlab_tlm import TrustworthyRAG, get_default_evals
+
+from lfx.custom import Component
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    MessageTextInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.message import Message
+
+
+class CleanlabRAGEvaluator(Component):
+    """A component that evaluates the quality of RAG (Retrieval-Augmented Generation) outputs using Cleanlab.
+
+    This component takes a query, retrieved context, and generated response from a RAG pipeline,
+    and uses Cleanlab's evaluation algorithms to assess various aspects of the RAG system's performance.
+
+    The component can evaluate:
+    - Overall trustworthiness of the LLM generated response
+    - Context sufficiency (whether the retrieved context contains information needed to answer the query)
+    - Response groundedness (whether the response is supported directly by the context)
+    - Response helpfulness (whether the response effectively addresses the user's query)
+    - Query ease (whether the user query seems easy for an AI system to properly handle, useful to diagnose
+      queries that are: complex, vague, tricky, or disgruntled-sounding)
+
+    Outputs:
+        - Trust Score: A score between 0-1 corresponding to the trustworthiness of the response. A higher score
+          indicates a higher confidence that the response is correct/good.
+        - Explanation: An LLM generated explanation of the trustworthiness assessment
+        - Other Evals: Additional evaluation metrics for selected evaluation types in the "Controls" tab
+        - Evaluation Summary: A comprehensive summary of context, query, response, and selected evaluation results
+
+    This component works well in conjunction with the CleanlabRemediator to create a complete trust evaluation
+    and remediation pipeline.
+
+    More details on the evaluation metrics can be found here: https://help.cleanlab.ai/tlm/use-cases/tlm_rag/
+    """
+
+    display_name = "Cleanlab RAG Evaluator"
+    description = "Evaluates context, query, and response from a RAG pipeline using Cleanlab and outputs trust metrics."
+    icon = "Cleanlab"
+    name = "CleanlabRAGEvaluator"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Cleanlab API Key",
+            info="Your Cleanlab API key.",
+            required=True,
+        ),
+        DropdownInput(
+            name="model",
+            display_name="Cleanlab Evaluation Model",
+            options=[
+                "gpt-4.1",
+                "gpt-4.1-mini",
+                "gpt-4.1-nano",
+                "o4-mini",
+                "o3",
+                "gpt-4.5-preview",
+                "gpt-4o-mini",
+                "gpt-4o",
+                "o3-mini",
+                "o1",
+                "o1-mini",
+                "gpt-4",
+                "gpt-3.5-turbo-16k",
+                "claude-3.7-sonnet",
+                "claude-3.5-sonnet-v2",
+                "claude-3.5-sonnet",
+                "claude-3.5-haiku",
+                "claude-3-haiku",
+                "nova-micro",
+                "nova-lite",
+                "nova-pro",
+            ],
+            info="The model Cleanlab uses to evaluate the context, query, and response. This does NOT need to be "
+            "the same model that generated the response.",
+            value="gpt-4o-mini",
+            required=True,
+            advanced=True,
+        ),
+        DropdownInput(
+            name="quality_preset",
+            display_name="Quality Preset",
+            options=["base", "low", "medium"],
+            value="medium",
+            info="This determines the accuracy, latency, and cost of the evaluation. Higher quality is generally "
+            "slower but more accurate.",
+            required=True,
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="context",
+            display_name="Context",
+            info="The context retrieved for the given query.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="query",
+            display_name="Query",
+            info="The user's query.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="response",
+            display_name="Response",
+            info="The response generated by the LLM.",
+            required=True,
+        ),
+        BoolInput(
+            name="run_context_sufficiency",
+            display_name="Run Context Sufficiency",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_response_groundedness",
+            display_name="Run Response Groundedness",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_response_helpfulness",
+            display_name="Run Response Helpfulness",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_query_ease",
+            display_name="Run Query Ease",
+            value=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Response", name="response_passthrough", method="pass_response", types=["Message"]),
+        Output(display_name="Trust Score", name="trust_score", method="get_trust_score", types=["number"]),
+        Output(display_name="Explanation", name="trust_explanation", method="get_trust_explanation", types=["Message"]),
+        Output(display_name="Other Evals", name="other_scores", method="get_other_scores", types=["Data"]),
+        Output(
+            display_name="Evaluation Summary",
+            name="evaluation_summary",
+            method="get_evaluation_summary",
+            types=["Message"],
+        ),
+    ]
+
+    def _evaluate_once(self):
+        if not hasattr(self, "_cached_result"):
+            try:
+                self.status = "Configuring selected evals..."
+                default_evals = get_default_evals()
+                enabled_names = []
+                if self.run_context_sufficiency:
+                    enabled_names.append("context_sufficiency")
+                if self.run_response_groundedness:
+                    enabled_names.append("response_groundedness")
+                if self.run_response_helpfulness:
+                    enabled_names.append("response_helpfulness")
+                if self.run_query_ease:
+                    enabled_names.append("query_ease")
+
+                selected_evals = [e for e in default_evals if e.name in enabled_names]
+
+                validator = TrustworthyRAG(
+                    api_key=self.api_key,
+                    quality_preset=self.quality_preset,
+                    options={"log": ["explanation"], "model": self.model},
+                    evals=selected_evals,
+                )
+
+                self.status = f"Running evals: {[e.name for e in selected_evals]}"
+                self._cached_result = validator.score(
+                    query=self.query,
+                    context=self.context,
+                    response=self.response,
+                )
+                self.status = "Evaluation complete."
+
+            except Exception as e:  # noqa: BLE001
+                self.status = f"Evaluation failed: {e!s}"
+                self._cached_result = {}
+        return self._cached_result
+
+    def pass_response(self) -> Message:
+        self.status = "Passing through response."
+        return Message(text=self.response)
+
+    def get_trust_score(self) -> float:
+        score = self._evaluate_once().get("trustworthiness", {}).get("score", 0.0)
+        self.status = f"Trust Score: {score:.3f}"
+        return score
+
+    def get_trust_explanation(self) -> Message:
+        explanation = self._evaluate_once().get("trustworthiness", {}).get("log", {}).get("explanation", "")
+        self.status = "Trust explanation extracted."
+        return Message(text=explanation)
+
+    def get_other_scores(self) -> dict:
+        result = self._evaluate_once()
+
+        selected = {
+            "context_sufficiency": self.run_context_sufficiency,
+            "response_groundedness": self.run_response_groundedness,
+            "response_helpfulness": self.run_response_helpfulness,
+            "query_ease": self.run_query_ease,
+        }
+
+        filtered_scores = {key: result[key]["score"] for key, include in selected.items() if include and key in result}
+
+        self.status = f"{len(filtered_scores)} other evals returned."
+        return filtered_scores
+
+    def get_evaluation_summary(self) -> Message:
+        result = self._evaluate_once()
+
+        query_text = self.query.strip()
+        context_text = self.context.strip()
+        response_text = self.response.strip()
+
+        trust = result.get("trustworthiness", {}).get("score", 0.0)
+        trust_exp = result.get("trustworthiness", {}).get("log", {}).get("explanation", "")
+
+        selected = {
+            "context_sufficiency": self.run_context_sufficiency,
+            "response_groundedness": self.run_response_groundedness,
+            "response_helpfulness": self.run_response_helpfulness,
+            "query_ease": self.run_query_ease,
+        }
+
+        other_scores = {key: result[key]["score"] for key, include in selected.items() if include and key in result}
+
+        metrics = f"Trustworthiness: {trust:.3f}"
+        if trust_exp:
+            metrics += f"\nExplanation: {trust_exp}"
+        if other_scores:
+            metrics += "\n" + "\n".join(f"{k.replace('_', ' ').title()}: {v:.3f}" for k, v in other_scores.items())
+
+        summary = (
+            f"Query:\n{query_text}\n"
+            "-----\n"
+            f"Context:\n{context_text}\n"
+            "-----\n"
+            f"Response:\n{response_text}\n"
+            "------------------------------\n"
+            f"{metrics}"
+        )
+
+        self.status = "Evaluation summary built."
+        return Message(text=summary)