lfx-nightly 0.2.0.dev25__py3-none-any.whl

lfx/components/cleanlab/cleanlab_rag_evaluator.py

@@ -0,0 +1,254 @@
+from cleanlab_tlm import TrustworthyRAG, get_default_evals
+
+from lfx.custom import Component
+from lfx.io import (
+    BoolInput,
+    DropdownInput,
+    MessageTextInput,
+    Output,
+    SecretStrInput,
+)
+from lfx.schema.message import Message
+
+
+class CleanlabRAGEvaluator(Component):
+    """A component that evaluates the quality of RAG (Retrieval-Augmented Generation) outputs using Cleanlab.
+
+    This component takes a query, retrieved context, and generated response from a RAG pipeline,
+    and uses Cleanlab's evaluation algorithms to assess various aspects of the RAG system's performance.
+
+    The component can evaluate:
+    - Overall trustworthiness of the LLM generated response
+    - Context sufficiency (whether the retrieved context contains information needed to answer the query)
+    - Response groundedness (whether the response is supported directly by the context)
+    - Response helpfulness (whether the response effectively addresses the user's query)
+    - Query ease (whether the user query seems easy for an AI system to properly handle, useful to diagnose
+      queries that are: complex, vague, tricky, or disgruntled-sounding)
+
+    Outputs:
+        - Trust Score: A score between 0-1 corresponding to the trustworthiness of the response. A higher score
+          indicates a higher confidence that the response is correct/good.
+        - Explanation: An LLM generated explanation of the trustworthiness assessment
+        - Other Evals: Additional evaluation metrics for selected evaluation types in the "Controls" tab
+        - Evaluation Summary: A comprehensive summary of context, query, response, and selected evaluation results
+
+    This component works well in conjunction with the CleanlabRemediator to create a complete trust evaluation
+    and remediation pipeline.
+
+    More details on the evaluation metrics can be found here: https://help.cleanlab.ai/tlm/use-cases/tlm_rag/
+    """
+
+    display_name = "Cleanlab RAG Evaluator"
+    description = "Evaluates context, query, and response from a RAG pipeline using Cleanlab and outputs trust metrics."
+    icon = "Cleanlab"
+    name = "CleanlabRAGEvaluator"
+
+    inputs = [
+        SecretStrInput(
+            name="api_key",
+            display_name="Cleanlab API Key",
+            info="Your Cleanlab API key.",
+            required=True,
+        ),
+        DropdownInput(
+            name="model",
+            display_name="Cleanlab Evaluation Model",
+            options=[
+                "gpt-4.1",
+                "gpt-4.1-mini",
+                "gpt-4.1-nano",
+                "o4-mini",
+                "o3",
+                "gpt-4.5-preview",
+                "gpt-4o-mini",
+                "gpt-4o",
+                "o3-mini",
+                "o1",
+                "o1-mini",
+                "gpt-4",
+                "gpt-3.5-turbo-16k",
+                "claude-3.7-sonnet",
+                "claude-3.5-sonnet-v2",
+                "claude-3.5-sonnet",
+                "claude-3.5-haiku",
+                "claude-3-haiku",
+                "nova-micro",
+                "nova-lite",
+                "nova-pro",
+            ],
+            info="The model Cleanlab uses to evaluate the context, query, and response. This does NOT need to be "
+            "the same model that generated the response.",
+            value="gpt-4o-mini",
+            required=True,
+            advanced=True,
+        ),
+        DropdownInput(
+            name="quality_preset",
+            display_name="Quality Preset",
+            options=["base", "low", "medium"],
+            value="medium",
+            info="This determines the accuracy, latency, and cost of the evaluation. Higher quality is generally "
+            "slower but more accurate.",
+            required=True,
+            advanced=True,
+        ),
+        MessageTextInput(
+            name="context",
+            display_name="Context",
+            info="The context retrieved for the given query.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="query",
+            display_name="Query",
+            info="The user's query.",
+            required=True,
+        ),
+        MessageTextInput(
+            name="response",
+            display_name="Response",
+            info="The response generated by the LLM.",
+            required=True,
+        ),
+        BoolInput(
+            name="run_context_sufficiency",
+            display_name="Run Context Sufficiency",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_response_groundedness",
+            display_name="Run Response Groundedness",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_response_helpfulness",
+            display_name="Run Response Helpfulness",
+            value=False,
+            advanced=True,
+        ),
+        BoolInput(
+            name="run_query_ease",
+            display_name="Run Query Ease",
+            value=False,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Response", name="response_passthrough", method="pass_response", types=["Message"]),
+        Output(display_name="Trust Score", name="trust_score", method="get_trust_score", types=["number"]),
+        Output(display_name="Explanation", name="trust_explanation", method="get_trust_explanation", types=["Message"]),
+        Output(display_name="Other Evals", name="other_scores", method="get_other_scores", types=["Data"]),
+        Output(
+            display_name="Evaluation Summary",
+            name="evaluation_summary",
+            method="get_evaluation_summary",
+            types=["Message"],
+        ),
+    ]
+
+    def _evaluate_once(self):
+        if not hasattr(self, "_cached_result"):
+            try:
+                self.status = "Configuring selected evals..."
+                default_evals = get_default_evals()
+                enabled_names = []
+                if self.run_context_sufficiency:
+                    enabled_names.append("context_sufficiency")
+                if self.run_response_groundedness:
+                    enabled_names.append("response_groundedness")
+                if self.run_response_helpfulness:
+                    enabled_names.append("response_helpfulness")
+                if self.run_query_ease:
+                    enabled_names.append("query_ease")
+
+                selected_evals = [e for e in default_evals if e.name in enabled_names]
+
+                validator = TrustworthyRAG(
+                    api_key=self.api_key,
+                    quality_preset=self.quality_preset,
+                    options={"log": ["explanation"], "model": self.model},
+                    evals=selected_evals,
+                )
+
+                self.status = f"Running evals: {[e.name for e in selected_evals]}"
+                self._cached_result = validator.score(
+                    query=self.query,
+                    context=self.context,
+                    response=self.response,
+                )
+                self.status = "Evaluation complete."
+
+            except Exception as e:  # noqa: BLE001
+                self.status = f"Evaluation failed: {e!s}"
+                self._cached_result = {}
+        return self._cached_result
+
+    def pass_response(self) -> Message:
+        self.status = "Passing through response."
+        return Message(text=self.response)
+
+    def get_trust_score(self) -> float:
+        score = self._evaluate_once().get("trustworthiness", {}).get("score", 0.0)
+        self.status = f"Trust Score: {score:.3f}"
+        return score
+
+    def get_trust_explanation(self) -> Message:
+        explanation = self._evaluate_once().get("trustworthiness", {}).get("log", {}).get("explanation", "")
+        self.status = "Trust explanation extracted."
+        return Message(text=explanation)
+
+    def get_other_scores(self) -> dict:
+        result = self._evaluate_once()
+
+        selected = {
+            "context_sufficiency": self.run_context_sufficiency,
+            "response_groundedness": self.run_response_groundedness,
+            "response_helpfulness": self.run_response_helpfulness,
+            "query_ease": self.run_query_ease,
+        }
+
+        filtered_scores = {key: result[key]["score"] for key, include in selected.items() if include and key in result}
+
+        self.status = f"{len(filtered_scores)} other evals returned."
+        return filtered_scores
+
+    def get_evaluation_summary(self) -> Message:
+        result = self._evaluate_once()
+
+        query_text = self.query.strip()
+        context_text = self.context.strip()
+        response_text = self.response.strip()
+
+        trust = result.get("trustworthiness", {}).get("score", 0.0)
+        trust_exp = result.get("trustworthiness", {}).get("log", {}).get("explanation", "")
+
+        selected = {
+            "context_sufficiency": self.run_context_sufficiency,
+            "response_groundedness": self.run_response_groundedness,
+            "response_helpfulness": self.run_response_helpfulness,
+            "query_ease": self.run_query_ease,
+        }
+
+        other_scores = {key: result[key]["score"] for key, include in selected.items() if include and key in result}
+
+        metrics = f"Trustworthiness: {trust:.3f}"
+        if trust_exp:
+            metrics += f"\nExplanation: {trust_exp}"
+        if other_scores:
+            metrics += "\n" + "\n".join(f"{k.replace('_', ' ').title()}: {v:.3f}" for k, v in other_scores.items())
+
+        summary = (
+            f"Query:\n{query_text}\n"
+            "-----\n"
+            f"Context:\n{context_text}\n"
+            "-----\n"
+            f"Response:\n{response_text}\n"
+            "------------------------------\n"
+            f"{metrics}"
+        )
+
+        self.status = "Evaluation summary built."
+        return Message(text=summary)

lfx/components/cleanlab/cleanlab_remediator.py

@@ -0,0 +1,131 @@
+from lfx.custom import Component
+from lfx.field_typing.range_spec import RangeSpec
+from lfx.io import BoolInput, FloatInput, HandleInput, MessageTextInput, Output, PromptInput
+from lfx.schema.message import Message
+
+
+class CleanlabRemediator(Component):
+    """Remediates potentially untrustworthy LLM responses based on trust scores computed by the Cleanlab Evaluator.
+
+    This component takes a response and its associated trust score,
+    and applies remediation strategies based on configurable thresholds and settings.
+
+    Inputs:
+        - response (MessageTextInput): The original LLM-generated response to be evaluated and possibly remediated.
+          The CleanlabEvaluator passes this response through.
+        - score (HandleInput): The trust score output from CleanlabEvaluator (expected to be a float between 0 and 1).
+        - explanation (MessageTextInput): Optional textual explanation for the trust score, to be included in the
+          output.
+        - threshold (Input[float]): Minimum trust score required to accept the response. If the score is lower, the
+          response is remediated.
+        - show_untrustworthy_response (BoolInput): If true, returns the original response with a warning; if false,
+          returns fallback text.
+        - untrustworthy_warning_text (PromptInput): Text warning to append to responses deemed untrustworthy (when
+          showing them).
+        - fallback_text (PromptInput): Replacement message returned if the response is untrustworthy and should be
+          hidden.
+
+    Outputs:
+        - remediated_response (Message): Either:
+            • the original response,
+            • the original response with appended warning, or
+            • the fallback response,
+          depending on the trust score and configuration.
+
+    This component is typically used downstream of CleanlabEvaluator or CleanlabRagValidator
+    to take appropriate action on low-trust responses and inform users accordingly.
+    """
+
+    display_name = "Cleanlab Remediator"
+    description = (
+        "Remediates an untrustworthy response based on trust score from the Cleanlab Evaluator, "
+        "score threshold, and message handling settings."
+    )
+    icon = "Cleanlab"
+    name = "CleanlabRemediator"
+
+    inputs = [
+        MessageTextInput(
+            name="response",
+            display_name="Response",
+            info="The response to the user's query.",
+            required=True,
+        ),
+        HandleInput(
+            name="score",
+            display_name="Trust Score",
+            info="The trustworthiness score output from the Cleanlab Evaluator.",
+            input_types=["number"],
+            required=True,
+        ),
+        MessageTextInput(
+            name="explanation",
+            display_name="Explanation",
+            info="The explanation from the Cleanlab Evaluator.",
+            required=False,
+        ),
+        FloatInput(
+            name="threshold",
+            display_name="Threshold",
+            field_type="float",
+            value=0.7,
+            range_spec=RangeSpec(min=0.0, max=1.0, step=0.05),
+            info="Minimum score required to show the response unmodified. Reponses with scores above this threshold "
+            "are considered trustworthy. Reponses with scores below this threshold are considered untrustworthy and "
+            "will be remediated based on the settings below.",
+            required=True,
+            show=True,
+        ),
+        BoolInput(
+            name="show_untrustworthy_response",
+            display_name="Show Untrustworthy Response",
+            info="If enabled, and the trust score is below the threshold, the original response is shown with the "
+            "added warning. If disabled, and the trust score is below the threshold, the fallback answer is returned.",
+            value=True,
+        ),
+        PromptInput(
+            name="untrustworthy_warning_text",
+            display_name="Warning for Untrustworthy Response",
+            info="Warning to append to the response if Show Untrustworthy Response is enabled and trust score is "
+            "below the threshold.",
+            value="⚠️ WARNING: The following response is potentially untrustworthy.",
+        ),
+        PromptInput(
+            name="fallback_text",
+            display_name="Fallback Answer",
+            info="Response returned if the trust score is below the threshold and 'Show Untrustworthy Response' is "
+            "disabled.",
+            value="Based on the available information, I cannot provide a complete answer to this question.",
+        ),
+    ]
+
+    outputs = [
+        Output(
+            display_name="Remediated Message",
+            name="remediated_response",
+            method="remediate_response",
+            types=["Message"],
+        ),
+    ]
+
+    def remediate_response(self) -> Message:
+        if self.score >= self.threshold:
+            self.status = f"Score {self.score:.2f} ≥ threshold {self.threshold:.2f} → accepted"
+            return Message(
+                text=f"{self.response}\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n**Trust Score:** {self.score:.2f}"
+            )
+
+        self.status = f"Score {self.score:.2f} < threshold {self.threshold:.2f} → flagged"
+
+        if self.show_untrustworthy_response:
+            parts = [
+                self.response,
+                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━",
+                f"**{self.untrustworthy_warning_text.strip()}**",
+                f"**Trust Score:** {self.score:.2f}",
+            ]
+            if self.explanation:
+                parts.append(f"**Explanation:** {self.explanation}")
+            return Message(text="\n\n".join(parts))
+
+        return Message(text=self.fallback_text)

lfx/components/clickhouse/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from .clickhouse import ClickhouseVectorStoreComponent
+
+_dynamic_imports = {
+    "ClickhouseVectorStoreComponent": "clickhouse",
+}
+
+__all__ = [
+    "ClickhouseVectorStoreComponent",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import ClickHouse components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/clickhouse/clickhouse.py

@@ -0,0 +1,135 @@
+from langchain_community.vectorstores import Clickhouse, ClickhouseSettings
+
+from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
+from lfx.helpers.data import docs_to_data
+from lfx.inputs.inputs import BoolInput, FloatInput
+from lfx.io import (
+    DictInput,
+    DropdownInput,
+    HandleInput,
+    IntInput,
+    SecretStrInput,
+    StrInput,
+)
+from lfx.schema.data import Data
+
+
+class ClickhouseVectorStoreComponent(LCVectorStoreComponent):
+    display_name = "ClickHouse"
+    description = "ClickHouse Vector Store with search capabilities"
+    name = "Clickhouse"
+    icon = "Clickhouse"
+
+    inputs = [
+        StrInput(name="host", display_name="hostname", required=True, value="localhost"),
+        IntInput(name="port", display_name="port", required=True, value=8123),
+        StrInput(name="database", display_name="database", required=True),
+        StrInput(name="table", display_name="Table name", required=True),
+        StrInput(name="username", display_name="The ClickHouse user name.", required=True),
+        SecretStrInput(name="password", display_name="Clickhouse Password", required=True),
+        DropdownInput(
+            name="index_type",
+            display_name="index_type",
+            options=["annoy", "vector_similarity"],
+            info="Type of the index.",
+            value="annoy",
+            advanced=True,
+        ),
+        DropdownInput(
+            name="metric",
+            display_name="metric",
+            options=["angular", "euclidean", "manhattan", "hamming", "dot"],
+            info="Metric to compute distance.",
+            value="angular",
+            advanced=True,
+        ),
+        BoolInput(
+            name="secure",
+            display_name="Use https/TLS. This overrides inferred values from the interface or port arguments.",
+            value=False,
+            advanced=True,
+        ),
+        StrInput(name="index_param", display_name="Param of the index", value="100,'L2Distance'", advanced=True),
+        DictInput(name="index_query_params", display_name="index query params", advanced=True),
+        *LCVectorStoreComponent.inputs,
+        HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
+        IntInput(
+            name="number_of_results",
+            display_name="Number of Results",
+            info="Number of results to return.",
+            value=4,
+            advanced=True,
+        ),
+        FloatInput(name="score_threshold", display_name="Score threshold", advanced=True),
+    ]
+
+    @check_cached_vector_store
+    def build_vector_store(self) -> Clickhouse:
+        try:
+            import clickhouse_connect
+        except ImportError as e:
+            msg = (
+                "Failed to import ClickHouse dependencies. "
+                "Install it using `uv pip install langflow[clickhouse-connect] --pre`"
+            )
+            raise ImportError(msg) from e
+
+        try:
+            client = clickhouse_connect.get_client(
+                host=self.host, port=self.port, username=self.username, password=self.password
+            )
+            client.command("SELECT 1")
+        except Exception as e:
+            msg = f"Failed to connect to Clickhouse: {e}"
+            raise ValueError(msg) from e
+
+        # Convert DataFrame to Data if needed using parent's method
+        self.ingest_data = self._prepare_ingest_data()
+
+        documents = []
+        for _input in self.ingest_data or []:
+            if isinstance(_input, Data):
+                documents.append(_input.to_lc_document())
+            else:
+                documents.append(_input)
+
+        kwargs = {}
+        if self.index_param:
+            kwargs["index_param"] = self.index_param.split(",")
+        if self.index_query_params:
+            kwargs["index_query_params"] = self.index_query_params
+
+        settings = ClickhouseSettings(
+            table=self.table,
+            database=self.database,
+            host=self.host,
+            index_type=self.index_type,
+            metric=self.metric,
+            password=self.password,
+            port=self.port,
+            secure=self.secure,
+            username=self.username,
+            **kwargs,
+        )
+        if documents:
+            clickhouse_vs = Clickhouse.from_documents(documents=documents, embedding=self.embedding, config=settings)
+
+        else:
+            clickhouse_vs = Clickhouse(embedding=self.embedding, config=settings)
+
+        return clickhouse_vs
+
+    def search_documents(self) -> list[Data]:
+        vector_store = self.build_vector_store()
+
+        if self.search_query and isinstance(self.search_query, str) and self.search_query.strip():
+            kwargs = {}
+            if self.score_threshold:
+                kwargs["score_threshold"] = self.score_threshold
+
+            docs = vector_store.similarity_search(query=self.search_query, k=self.number_of_results, **kwargs)
+
+            data = docs_to_data(docs)
+            self.status = data
+            return data
+        return []

lfx/components/cloudflare/__init__.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from lfx.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from lfx.components.cloudflare.cloudflare import CloudflareWorkersAIEmbeddingsComponent
+
+_dynamic_imports = {
+    "CloudflareWorkersAIEmbeddingsComponent": "cloudflare",
+}
+
+__all__ = ["CloudflareWorkersAIEmbeddingsComponent"]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import cloudflare components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

lfx/components/cloudflare/cloudflare.py

@@ -0,0 +1,81 @@
+from langchain_community.embeddings.cloudflare_workersai import CloudflareWorkersAIEmbeddings
+
+from lfx.base.models.model import LCModelComponent
+from lfx.field_typing import Embeddings
+from lfx.io import BoolInput, DictInput, IntInput, MessageTextInput, Output, SecretStrInput
+
+
+class CloudflareWorkersAIEmbeddingsComponent(LCModelComponent):
+    display_name: str = "Cloudflare Workers AI Embeddings"
+    description: str = "Generate embeddings using Cloudflare Workers AI models."
+    documentation: str = "https://python.langchain.com/docs/integrations/text_embedding/cloudflare_workersai/"
+    icon = "Cloudflare"
+    name = "CloudflareWorkersAIEmbeddings"
+
+    inputs = [
+        MessageTextInput(
+            name="account_id",
+            display_name="Cloudflare account ID",
+            info="Find your account ID https://developers.cloudflare.com/fundamentals/setup/find-account-and-zone-ids/#find-account-id-workers-and-pages",
+            required=True,
+        ),
+        SecretStrInput(
+            name="api_token",
+            display_name="Cloudflare API token",
+            info="Create an API token https://developers.cloudflare.com/fundamentals/api/get-started/create-token/",
+            required=True,
+        ),
+        MessageTextInput(
+            name="model_name",
+            display_name="Model Name",
+            info="List of supported models https://developers.cloudflare.com/workers-ai/models/#text-embeddings",
+            required=True,
+            value="@cf/baai/bge-base-en-v1.5",
+        ),
+        BoolInput(
+            name="strip_new_lines",
+            display_name="Strip New Lines",
+            advanced=True,
+            value=True,
+        ),
+        IntInput(
+            name="batch_size",
+            display_name="Batch Size",
+            advanced=True,
+            value=50,
+        ),
+        MessageTextInput(
+            name="api_base_url",
+            display_name="Cloudflare API base URL",
+            advanced=True,
+            value="https://api.cloudflare.com/client/v4/accounts",
+        ),
+        DictInput(
+            name="headers",
+            display_name="Headers",
+            info="Additional request headers",
+            is_list=True,
+            advanced=True,
+        ),
+    ]
+
+    outputs = [
+        Output(display_name="Embeddings", name="embeddings", method="build_embeddings"),
+    ]
+
+    def build_embeddings(self) -> Embeddings:
+        try:
+            embeddings = CloudflareWorkersAIEmbeddings(
+                account_id=self.account_id,
+                api_base_url=self.api_base_url,
+                api_token=self.api_token,
+                batch_size=self.batch_size,
+                headers=self.headers,
+                model_name=self.model_name,
+                strip_new_lines=self.strip_new_lines,
+            )
+        except Exception as e:
+            msg = f"Could not connect to CloudflareWorkersAIEmbeddings API: {e!s}"
+            raise ValueError(msg) from e
+
+        return embeddings