langfuse-haystack 3.3.0__tar.gz → 4.0.0__tar.gz

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [integrations/langfuse-v3.3.1] - 2025-12-10
+
+### 🚀 Features
+
+- *(langfuse)* Add embedder usage metrics for langfuse (#2542)
+
+### 🐛 Bug Fixes
+
+- Correct token usage accounting, fix trace polling in tests (#2594)
+
+### 🧹 Chores
+
+- Remove Readme API CI workflow and configs (#2573)
+
+
+## [integrations/langfuse-v3.3.0] - 2025-11-21
+
+### 🚀 Features
+
+- *(langfuse)* Embedder, retriever and generator as obs. type (#2497)
+
+### 🌀 Miscellaneous
+
+- Enhancement: Adopt PEP 585 type hinting (part 4) (#2527)
+- *(langfuse)* Log levels (#2522)
+
 ## [integrations/langfuse-v3.2.1] - 2025-11-07
 
 ### 🌀 Miscellaneous

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langfuse-haystack
-Version: 3.3.0
+Version: 4.0.0
 Summary: Langfuse integration for Haystack
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -10,15 +10,14 @@ License-Expression: Apache-2.0
 License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
-Requires-Dist: haystack-ai>=2.17.1
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.22.0
 Requires-Dist: langfuse<4.0.0,>=3.3.1
 Description-Content-Type: text/markdown
 

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/pyproject.toml

@@ -7,14 +7,13 @@ name = "langfuse-haystack"
 dynamic = ["version"]
 description = "Langfuse integration for Haystack"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
 classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -22,7 +21,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai>=2.17.1", "langfuse>=3.3.1, <4.0.0"]
+dependencies = ["haystack-ai>=2.22.0", "langfuse>=3.3.1, <4.0.0"]
 
 [project.urls]
 Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme"
@@ -46,8 +45,8 @@ installer = "uv"
 dependencies = ["haystack-pydoc-tools", "ruff"]
 
 [tool.hatch.envs.default.scripts]
-docs = ["pydoc-markdown pydoc/config.yml"]
-fmt = "ruff check --fix {args} && ruff format {args}"
+docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
+fmt = "ruff check --fix {args}; ruff format {args}"
 fmt-check = "ruff check {args} && ruff format --check {args}"
 
 [tool.hatch.envs.test]
@@ -82,7 +81,6 @@ allow-direct-references = true
 
 
 [tool.ruff]
-target-version = "py39"
 line-length = 120
 
 [tool.ruff.lint]
@@ -128,10 +126,6 @@ ignore = [
   # Asserts
   "S101",
 ]
-unfixable = [
-  # Don't touch unused imports
-  "F401",
-]
 
 [tool.ruff.lint.isort]
 known-first-party = ["haystack_integrations"]

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from typing import Any, Optional
+from typing import Any
 
 import httpx
 from haystack import component, default_from_dict, default_to_dict, logging, tracing
@@ -118,13 +118,13 @@ class LangfuseConnector:
         self,
         name: str,
         public: bool = False,
-        public_key: Optional[Secret] = Secret.from_env_var("LANGFUSE_PUBLIC_KEY"),  # noqa: B008
-        secret_key: Optional[Secret] = Secret.from_env_var("LANGFUSE_SECRET_KEY"),  # noqa: B008
-        httpx_client: Optional[httpx.Client] = None,
-        span_handler: Optional[SpanHandler] = None,
+        public_key: Secret | None = Secret.from_env_var("LANGFUSE_PUBLIC_KEY"),  # noqa: B008
+        secret_key: Secret | None = Secret.from_env_var("LANGFUSE_SECRET_KEY"),  # noqa: B008
+        httpx_client: httpx.Client | None = None,
+        span_handler: SpanHandler | None = None,
         *,
-        host: Optional[str] = None,
-        langfuse_client_kwargs: Optional[dict[str, Any]] = None,
+        host: str | None = None,
+        langfuse_client_kwargs: dict[str, Any] | None = None,
     ) -> None:
         """
         Initialize the LangfuseConnector component.
@@ -172,7 +172,7 @@ class LangfuseConnector:
         tracing.enable_tracing(self.tracer)
 
     @component.output_types(name=str, trace_url=str, trace_id=str)
-    def run(self, invocation_context: Optional[dict[str, Any]] = None) -> dict[str, str]:
+    def run(self, invocation_context: dict[str, Any] | None = None) -> dict[str, str]:
         """
         Runs the LangfuseConnector component.
 

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/src/haystack_integrations/tracing/langfuse/tracer.py

@@ -12,7 +12,7 @@ from contextlib import AbstractContextManager
 from contextvars import ContextVar
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Any, Literal, Optional, cast
+from typing import Any, Literal, cast
 
 from haystack import default_from_dict, default_to_dict, logging
 from haystack.dataclasses import ChatMessage
@@ -48,7 +48,7 @@ tracing_context_var: ContextVar[dict[Any, Any]] = ContextVar("tracing_context")
 
 # Internal span execution hierarchy for our tracer
 # Manages parent-child relationships and prevents cross-request span interleaving
-span_stack_var: ContextVar[Optional[list["LangfuseSpan"]]] = ContextVar("span_stack", default=None)
+span_stack_var: ContextVar[list["LangfuseSpan"] | None] = ContextVar("span_stack", default=None)
 
 
 class LangfuseSpan(Span):
@@ -150,9 +150,9 @@ class SpanContext:
 
     name: str
     operation_name: str
-    component_type: Optional[str]
+    component_type: str | None
     tags: dict[str, Any]
-    parent_span: Optional[Span]
+    parent_span: Span | None
     trace_name: str = "Haystack"
     public: bool = False
 
@@ -189,7 +189,7 @@ class SpanHandler(ABC):
     """
 
     def __init__(self) -> None:
-        self.tracer: Optional[langfuse.Langfuse] = None
+        self.tracer: langfuse.Langfuse | None = None
 
     def init_tracer(self, tracer: langfuse.Langfuse) -> None:
         """
@@ -215,7 +215,7 @@ class SpanHandler(ABC):
         pass
 
     @abstractmethod
-    def handle(self, span: LangfuseSpan, component_type: Optional[str]) -> None:
+    def handle(self, span: LangfuseSpan, component_type: str | None) -> None:
         """
         Process a span after component execution by attaching metadata and metrics.
 
@@ -242,41 +242,30 @@ class SpanHandler(ABC):
 
 def _sanitize_usage_data(usage: dict[str, Any]) -> dict[str, Any]:
     """
-    Sanitize usage data for Langfuse by flattening to a single-level dictionary.
+    Sanitize usage data for Langfuse by converting provider-specific keys to Langfuse standard keys.
 
-    Langfuse's usage_details must be a flat dictionary with only numeric values. This function:
-    - Flattens nested dictionaries using dot notation (e.g., cache_creation.input_tokens)
-    - Keeps int and float values
-    - Skips None, boolean, string, and other non-numeric types
+    Langfuse expects usage_details with standard keys: input_tokens, output_tokens, and total_tokens.
+    This function converts provider-specific keys to Langfuse's expected format:
+    - prompt_tokens -> input_tokens
+    - completion_tokens -> output_tokens
+    - total_tokens -> total_tokens (preserved as-is)
 
     :param usage: Raw usage dictionary from the provider.
-    :returns: Flat dictionary with only numeric values (int or float).
+    :returns: Dictionary with Langfuse standard keys (input_tokens, output_tokens, total_tokens).
     """
     if not isinstance(usage, dict):
         return {}
 
-    sanitized: dict[str, Any] = {}
-
-    def _flatten(data: dict[str, Any], prefix: str = "") -> None:
-        """Recursively flatten nested dictionaries."""
-        for key, value in data.items():
-            full_key = f"{prefix}.{key}" if prefix else key
-
-            if value is None:
-                # Skip None values (e.g., Anthropic's server_tool_use)
-                continue
-            elif isinstance(value, bool):
-                # Skip boolean values
-                continue
-            elif isinstance(value, (int, float)):
-                # Keep numeric values
-                sanitized[full_key] = value
-            elif isinstance(value, dict):
-                # Recursively flatten nested dicts
-                _flatten(value, full_key)
-            # Skip strings and other non-numeric types (e.g., Anthropic's service_tier)
-
-    _flatten(usage)
+    # Start with Langfuse standard keys from usage if present
+    sanitized: dict[str, Any] = {
+        k: v for k, v in usage.items() if k in ("input_tokens", "output_tokens", "total_tokens")
+    }
+    # Convert provider format to Langfuse standard keys if not already present
+    if "input_tokens" not in sanitized and "prompt_tokens" in usage:
+        sanitized["input_tokens"] = usage["prompt_tokens"]
+    if "output_tokens" not in sanitized and "completion_tokens" in usage:
+        sanitized["output_tokens"] = usage["completion_tokens"]
+
     return sanitized
 
 
@@ -349,7 +338,7 @@ class DefaultSpanHandler(SpanHandler):
         else:
             return LangfuseSpan(self.tracer.start_as_current_span(name=context.name))
 
-    def handle(self, span: LangfuseSpan, component_type: Optional[str]) -> None:
+    def handle(self, span: LangfuseSpan, component_type: str | None) -> None:
         # If the span is at the pipeline level, we add input and output keys to the span
         at_pipeline_level = span.get_data().get(_PIPELINE_INPUT_KEY) is not None
         if at_pipeline_level:
@@ -395,6 +384,30 @@ class DefaultSpanHandler(SpanHandler):
                 usage = meta[0].get("usage")
                 sanitized_usage = _sanitize_usage_data(usage) if usage else None
                 span.raw_span().update(usage_details=sanitized_usage, model=meta[0].get("model"))
+        elif component_type and component_type.endswith("Embedder"):
+            # Extract usage data from embedder output
+            output = span.get_data().get(_COMPONENT_OUTPUT_KEY, {})
+            meta = output.get("meta")
+
+            if meta and isinstance(meta, dict):
+                # Build update parameters with available data
+                update_params: dict[str, Any] = {}
+
+                # Try both common formats: 'usage' (OpenAI) or 'billed_units' (Cohere)
+                usage = meta.get("usage") or meta.get("billed_units")
+                if usage:
+                    sanitized_usage = _sanitize_usage_data(usage)
+                    if sanitized_usage:
+                        update_params["usage_details"] = sanitized_usage
+
+                # Some embedders may provide model information
+                model = meta.get("model")
+                if model and isinstance(model, str):
+                    update_params["model"] = model
+
+                # Single update call if we have data to update
+                if update_params:
+                    span.raw_span().update(**update_params)
 
 
 class LangfuseTracer(Tracer):
@@ -407,7 +420,7 @@ class LangfuseTracer(Tracer):
         tracer: langfuse.Langfuse,
         name: str = "Haystack",
         public: bool = False,
-        span_handler: Optional[SpanHandler] = None,
+        span_handler: SpanHandler | None = None,
     ) -> None:
         """
         Initialize a LangfuseTracer instance.
@@ -437,7 +450,7 @@ class LangfuseTracer(Tracer):
 
     @contextlib.contextmanager
     def trace(
-        self, operation_name: str, tags: Optional[dict[str, Any]] = None, parent_span: Optional[Span] = None
+        self, operation_name: str, tags: dict[str, Any] | None = None, parent_span: Span | None = None
     ) -> Iterator[Span]:
         tags = tags or {}
         span_name = tags.get(_COMPONENT_NAME_KEY, operation_name)
@@ -528,7 +541,7 @@ class LangfuseTracer(Tracer):
     def flush(self) -> None:
         self._tracer.flush()
 
-    def current_span(self) -> Optional[Span]:
+    def current_span(self) -> Span | None:
         """
         Return the current active span.
 

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/tests/test_tracer.py

@@ -6,7 +6,6 @@ import asyncio
 import datetime
 import logging
 import sys
-from typing import Optional
 from unittest.mock import MagicMock, Mock, patch
 
 import pytest
@@ -107,7 +106,7 @@ class MockLangfuseClient:
 
 
 class CustomSpanHandler(DefaultSpanHandler):
-    def handle(self, span: LangfuseSpan, component_type: Optional[str]) -> None:
+    def handle(self, span: LangfuseSpan, component_type: str | None) -> None:
         if component_type == "OpenAIChatGenerator":
             output = span.get_data().get(_COMPONENT_OUTPUT_KEY, {})
             replies = output.get("replies", [])
@@ -204,20 +203,16 @@ class TestSanitizeUsageData:
             "completion_tokens": 449,
         }
         result = _sanitize_usage_data(usage)
-        assert result == {
-            "cache_creation.ephemeral_1h_input_tokens": 0,
-            "cache_creation.ephemeral_5m_input_tokens": 0,
-            "cache_creation_input_tokens": 0,
-            "cache_read_input_tokens": 0,
-            "prompt_tokens": 25,
-            "completion_tokens": 449,
-        }
+        assert result["input_tokens"] == 25
+        assert result["output_tokens"] == 449
 
     def test_openai_usage_preserved(self):
         """Test OpenAI/Cohere flat dict with only numeric values works unchanged"""
         usage = {"prompt_tokens": 29, "completion_tokens": 267, "total_tokens": 296}
         result = _sanitize_usage_data(usage)
-        assert result == {"prompt_tokens": 29, "completion_tokens": 267, "total_tokens": 296}
+        assert result["input_tokens"] == 29
+        assert result["output_tokens"] == 267
+        assert result["total_tokens"] == 296
 
     def test_empty_and_invalid_input(self):
         """Test edge cases return empty dict"""
@@ -411,6 +406,91 @@ class TestDefaultSpanHandler:
         # Verify start_as_current_span was called for the actual span creation (not just parent)
         assert mock_client.start_as_current_span.call_count == 2  # Once for parent, once for the span
 
+    def test_handle_embedder_with_openai_format(self):
+        """Test that embedder usage is extracted in OpenAI format."""
+        mock_span = Mock()
+        mock_span.raw_span.return_value = mock_span
+        mock_span.get_data.return_value = {
+            "haystack.component.type": "OpenAITextEmbedder",
+            "haystack.component.output": {
+                "embedding": [0.1, 0.2, 0.3],
+                "meta": {"model": "custom-model", "usage": {"prompt_tokens": 15, "total_tokens": 15}},
+            },
+        }
+
+        handler = DefaultSpanHandler()
+        handler.handle(mock_span, component_type="OpenAITextEmbedder")
+
+        assert mock_span.update.call_count == 1
+        update_args = mock_span.update.call_args_list[0][1]
+        assert update_args["model"] == "custom-model"
+        assert update_args["usage_details"] == {"input_tokens": 15, "total_tokens": 15}
+
+    def test_handle_embedder_with_cohere_format(self):
+        """Test that embedder usage is extracted in Cohere billed_units format."""
+        mock_span = Mock()
+        mock_span.raw_span.return_value = mock_span
+        mock_span.get_data.return_value = {
+            "haystack.component.type": "CohereTextEmbedder",
+            "haystack.component.output": {
+                "embedding": [0.1, 0.2, 0.3],
+                "meta": {"api_version": {"version": "1"}, "billed_units": {"input_tokens": 4}},
+            },
+        }
+
+        handler = DefaultSpanHandler()
+        handler.handle(mock_span, component_type="CohereTextEmbedder")
+
+        assert mock_span.update.call_count == 1
+        assert mock_span.update.call_args_list[0][1] == {"usage_details": {"input_tokens": 4}}
+
+    def test_handle_embedder_without_usage(self):
+        """Test that embedders without usage data are handled gracefully."""
+        mock_span = Mock()
+        mock_span.raw_span.return_value = mock_span
+        mock_span.get_data.return_value = {
+            "haystack.component.type": "SentenceTransformersTextEmbedder",
+            "haystack.component.output": {
+                "embedding": [0.1, 0.2, 0.3],
+                "meta": {},  # No usage data
+            },
+        }
+
+        handler = DefaultSpanHandler()
+        handler.handle(mock_span, component_type="SentenceTransformersTextEmbedder")
+
+        # Should not call update when no usage data is available
+        assert mock_span.update.call_count == 0
+
+    def test_handle_embedder_with_nested_usage(self):
+        """Test that embedders with nested usage data are sanitized correctly."""
+        mock_span = Mock()
+        mock_span.raw_span.return_value = mock_span
+        mock_span.get_data.return_value = {
+            "haystack.component.type": "CustomEmbedder",
+            "haystack.component.output": {
+                "embedding": [0.1, 0.2, 0.3],
+                "meta": {
+                    "model": "custom-model",
+                    "usage": {
+                        "cache_creation": {"input_tokens": 10},
+                        "cache_read": {"input_tokens": 5},
+                        "total_tokens": 15,
+                    },
+                },
+            },
+        }
+
+        handler = DefaultSpanHandler()
+        handler.handle(mock_span, component_type="CustomEmbedder")
+
+        assert mock_span.update.call_count == 1
+        # Only adds total_tokens as Langfuse standard key (no prompt_tokens/completion_tokens to convert)
+        assert mock_span.update.call_args_list[0][1] == {
+            "usage_details": {"total_tokens": 15},
+            "model": "custom-model",
+        }
+
 
 class TestCustomSpanHandler:
     def test_handle(self):
@@ -454,13 +534,22 @@ class TestLangfuseTracer:
         mock_raw_span.metadata = {"tag1": "value1", "tag2": "value2"}
 
         with patch("haystack_integrations.tracing.langfuse.tracer.LangfuseSpan") as mock_langfuse_span:
+            mock_context_manager = MockContextManager()
+            mock_context_manager._span = mock_raw_span
+
             mock_span_instance = mock_langfuse_span.return_value
             mock_span_instance.raw_span.return_value = mock_raw_span
+            # Return a proper dict to prevent MagicMock from being truthy in handle() checks.
+            # When get_data() returns a MagicMock, `span.get_data().get(key) is not None` is True
+            # because MagicMock().get() returns another MagicMock (truthy). This triggers
+            # tracing_utils.coerce_tag_value() with MagicMock objects, which can hang on
+            # Linux Python 3.9/3.13 due to platform-specific MagicMock iteration behavior.
+            mock_span_instance.get_data.return_value = {}
+            mock_span_instance._context_manager = mock_context_manager
 
-            mock_context_manager = MockContextManager()
-            mock_context_manager._span = mock_raw_span
             mock_tracer = MagicMock()
             mock_tracer.start_as_current_span.return_value = mock_context_manager
+            mock_tracer.start_as_current_observation.return_value = mock_context_manager
 
             tracer = LangfuseTracer(tracer=mock_tracer, name="Haystack", public=False)
 

{langfuse_haystack-3.3.0 → langfuse_haystack-4.0.0}/tests/test_tracing.py

@@ -38,7 +38,7 @@ os.environ.setdefault("LANGFUSE_HOST", "https://cloud.langfuse.com")
 def poll_langfuse(url: str):
     """Utility function to poll Langfuse API until the trace is ready"""
     # Initial wait for trace creation
-    time.sleep(10)
+    time.sleep(30)
 
     auth = HTTPBasicAuth(os.environ["LANGFUSE_PUBLIC_KEY"], os.environ["LANGFUSE_SECRET_KEY"])
 
@@ -195,8 +195,8 @@ def test_tracing_with_sub_pipelines():
     # There should be two observations for the haystack.pipeline.run span: one for each sub pipeline
     # Main pipeline is stored under the name "Sub-pipeline example"
     assert len(haystack_pipeline_run_observations) == 2
-    assert "prompt_builder" in str(haystack_pipeline_run_observations[0])
-    assert "llm" in str(haystack_pipeline_run_observations[1])
+    # Verify both observations are pipeline runs (less brittle than checking for component names)
+    assert all(obs["name"] == "haystack.pipeline.run" for obs in haystack_pipeline_run_observations)
 
 
 @pytest.mark.skipif(

langfuse_haystack-3.3.0/pydoc/config.yml

@@ -1,30 +0,0 @@
-loaders:
-  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
-    search_path: [../src]
-    modules: [
-      "haystack_integrations.components.connectors.langfuse.langfuse_connector",
-      "haystack_integrations.tracing.langfuse.tracer",
-    ]
-    ignore_when_discovered: ["__init__"]
-processors:
-  - type: filter
-    expression:
-    documented_only: true
-    do_not_filter_modules: false
-    skip_empty_modules: true
-  - type: smart
-  - type: crossref
-renderer:
-  type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
-  excerpt: Langfuse integration for Haystack
-  category_slug: integrations-api
-  title: langfuse
-  slug: integrations-langfuse
-  order: 136
-  markdown:
-    descriptive_class_title: false
-    classdef_code_block: false
-    descriptive_module_title: true
-    add_method_class_prefix: true
-    add_member_class_prefix: false
-    filename: _readme_langfuse.md