langfuse-haystack 3.0.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/CHANGELOG.md +25 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/PKG-INFO +3 -3
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/example/basic_rag.py +2 -2
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/example/chat.py +2 -3
- langfuse_haystack-3.2.0/pydoc/config_docusaurus.yml +29 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/pyproject.toml +8 -29
- langfuse_haystack-3.2.0/src/haystack_integrations/components/connectors/py.typed +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/src/haystack_integrations/tracing/langfuse/tracer.py +59 -8
- langfuse_haystack-3.2.0/src/haystack_integrations/tracing/py.typed +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/tests/test_tracer.py +62 -35
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/tests/test_tracing.py +15 -6
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/.gitignore +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/LICENSE.txt +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/README.md +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/example/requirements.txt +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/pydoc/config.yml +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/src/haystack_integrations/components/connectors/__init__.py +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/src/haystack_integrations/components/connectors/langfuse/__init__.py +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/src/haystack_integrations/tracing/langfuse/__init__.py +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/tests/__init__.py +0 -0
- {langfuse_haystack-3.0.0 → langfuse_haystack-3.2.0}/tests/test_langfuse_connector.py +0 -0
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [integrations/langfuse-v3.1.0] - 2025-10-24
|
|
4
|
+
|
|
5
|
+
### 🐛 Bug Fixes
|
|
6
|
+
|
|
7
|
+
- Langfuse - add py.typed; fix testing with lowest deps (#2458)
|
|
8
|
+
|
|
9
|
+
### 📚 Documentation
|
|
10
|
+
|
|
11
|
+
- Add pydoc configurations for Docusaurus (#2411)
|
|
12
|
+
|
|
13
|
+
### ⚙️ CI
|
|
14
|
+
|
|
15
|
+
- Install dependencies in the `test` environment when testing with lowest direct dependencies and Haystack main (#2418)
|
|
16
|
+
|
|
17
|
+
### 🧹 Chores
|
|
18
|
+
|
|
19
|
+
- Remove ruff exclude and fix linting in Langfuse integration (#2257)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## [integrations/langfuse-v3.0.0] - 2025-09-19
|
|
23
|
+
|
|
24
|
+
### 🌀 Miscellaneous
|
|
25
|
+
|
|
26
|
+
- Migrate langfuse to v3 (#2247)
|
|
27
|
+
|
|
3
28
|
## [integrations/langfuse-v2.3.0] - 2025-08-25
|
|
4
29
|
|
|
5
30
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: langfuse-haystack
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: Langfuse integration for Haystack
|
|
5
5
|
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
@@ -18,8 +18,8 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
18
18
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
19
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
20
|
Requires-Python: >=3.9
|
|
21
|
-
Requires-Dist: haystack-ai>=2.
|
|
22
|
-
Requires-Dist: langfuse<4.0.0,>=3.
|
|
21
|
+
Requires-Dist: haystack-ai>=2.17.1
|
|
22
|
+
Requires-Dist: langfuse<4.0.0,>=3.3.0
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
24
|
|
|
25
25
|
# langfuse-haystack
|
|
@@ -62,5 +62,5 @@ if __name__ == "__main__":
|
|
|
62
62
|
question = "What does Rhodes Statue look like?"
|
|
63
63
|
response = pipeline.run({"text_embedder": {"text": question}, "prompt_builder": {"question": question}})
|
|
64
64
|
|
|
65
|
-
print(response["llm"]["replies"][0])
|
|
66
|
-
print(response["tracer"]["trace_url"])
|
|
65
|
+
print(response["llm"]["replies"][0]) # noqa: T201
|
|
66
|
+
print(response["tracer"]["trace_url"]) # noqa: T201
|
|
@@ -36,7 +36,6 @@ generators = {
|
|
|
36
36
|
selected_chat_generator = generators[selected_chat_generator]()
|
|
37
37
|
|
|
38
38
|
if __name__ == "__main__":
|
|
39
|
-
|
|
40
39
|
pipe = Pipeline()
|
|
41
40
|
pipe.add_component("tracer", LangfuseConnector("Chat example"))
|
|
42
41
|
pipe.add_component("prompt_builder", ChatPromptBuilder())
|
|
@@ -60,5 +59,5 @@ if __name__ == "__main__":
|
|
|
60
59
|
},
|
|
61
60
|
}
|
|
62
61
|
)
|
|
63
|
-
print(response["llm"]["replies"][0])
|
|
64
|
-
print(response["tracer"]["trace_url"])
|
|
62
|
+
print(response["llm"]["replies"][0]) # noqa: T201
|
|
63
|
+
print(response["tracer"]["trace_url"]) # noqa: T201
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- ignore_when_discovered:
|
|
3
|
+
- __init__
|
|
4
|
+
modules:
|
|
5
|
+
- haystack_integrations.components.connectors.langfuse.langfuse_connector
|
|
6
|
+
- haystack_integrations.tracing.langfuse.tracer
|
|
7
|
+
search_path:
|
|
8
|
+
- ../src
|
|
9
|
+
type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
10
|
+
processors:
|
|
11
|
+
- do_not_filter_modules: false
|
|
12
|
+
documented_only: true
|
|
13
|
+
expression: null
|
|
14
|
+
skip_empty_modules: true
|
|
15
|
+
type: filter
|
|
16
|
+
- type: smart
|
|
17
|
+
- type: crossref
|
|
18
|
+
renderer:
|
|
19
|
+
description: Langfuse integration for Haystack
|
|
20
|
+
id: integrations-langfuse
|
|
21
|
+
markdown:
|
|
22
|
+
add_member_class_prefix: false
|
|
23
|
+
add_method_class_prefix: true
|
|
24
|
+
classdef_code_block: false
|
|
25
|
+
descriptive_class_title: false
|
|
26
|
+
descriptive_module_title: true
|
|
27
|
+
filename: langfuse.md
|
|
28
|
+
title: langfuse
|
|
29
|
+
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
|
|
@@ -22,7 +22,7 @@ classifiers = [
|
|
|
22
22
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
23
23
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
24
24
|
]
|
|
25
|
-
dependencies = ["haystack-ai>=2.
|
|
25
|
+
dependencies = ["haystack-ai>=2.17.1", "langfuse>=3.3.0, <4.0.0"]
|
|
26
26
|
|
|
27
27
|
[project.urls]
|
|
28
28
|
Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme"
|
|
@@ -67,24 +67,15 @@ dependencies = [
|
|
|
67
67
|
unit = 'pytest -m "not integration" {args:tests}'
|
|
68
68
|
integration = 'pytest -m "integration" {args:tests}'
|
|
69
69
|
all = 'pytest {args:tests}'
|
|
70
|
-
cov-retry = '
|
|
70
|
+
cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
|
|
71
71
|
|
|
72
|
-
types = "mypy
|
|
72
|
+
types = "mypy -p haystack_integrations.components.connectors.langfuse -p haystack_integrations.tracing.langfuse {args}"
|
|
73
73
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
detached = true
|
|
80
|
-
dependencies = [
|
|
81
|
-
"pip",
|
|
82
|
-
"mypy>=1.0.0",
|
|
83
|
-
"ruff>=0.0.243",
|
|
84
|
-
]
|
|
85
|
-
|
|
86
|
-
[tool.hatch.envs.lint.scripts]
|
|
87
|
-
typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
|
|
74
|
+
[tool.mypy]
|
|
75
|
+
install_types = true
|
|
76
|
+
non_interactive = true
|
|
77
|
+
check_untyped_defs = true
|
|
78
|
+
disallow_incomplete_defs = true
|
|
88
79
|
|
|
89
80
|
[tool.hatch.metadata]
|
|
90
81
|
allow-direct-references = true
|
|
@@ -93,7 +84,6 @@ allow-direct-references = true
|
|
|
93
84
|
[tool.ruff]
|
|
94
85
|
target-version = "py38"
|
|
95
86
|
line-length = 120
|
|
96
|
-
exclude = ["example", "tests"]
|
|
97
87
|
|
|
98
88
|
[tool.ruff.lint]
|
|
99
89
|
select = [
|
|
@@ -167,17 +157,6 @@ omit = ["*/tests/*", "*/__init__.py"]
|
|
|
167
157
|
show_missing = true
|
|
168
158
|
exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
|
|
169
159
|
|
|
170
|
-
[[tool.mypy.overrides]]
|
|
171
|
-
module = [
|
|
172
|
-
"langfuse.*",
|
|
173
|
-
"haystack.*",
|
|
174
|
-
"haystack_integrations.*",
|
|
175
|
-
"pytest.*",
|
|
176
|
-
"numpy.*",
|
|
177
|
-
"httpx.*",
|
|
178
|
-
]
|
|
179
|
-
ignore_missing_imports = true
|
|
180
|
-
|
|
181
160
|
[tool.pytest.ini_options]
|
|
182
161
|
addopts = "--strict-markers"
|
|
183
162
|
markers = ["integration: integration tests"]
|
|
File without changes
|
|
@@ -10,7 +10,7 @@ from contextlib import AbstractContextManager
|
|
|
10
10
|
from contextvars import ContextVar
|
|
11
11
|
from dataclasses import dataclass
|
|
12
12
|
from datetime import datetime
|
|
13
|
-
from typing import Any, Dict, Iterator, List, Optional
|
|
13
|
+
from typing import Any, Dict, Iterator, List, Literal, Optional
|
|
14
14
|
|
|
15
15
|
from haystack import default_from_dict, default_to_dict, logging
|
|
16
16
|
from haystack.dataclasses import ChatMessage
|
|
@@ -256,6 +256,46 @@ class SpanHandler(ABC):
|
|
|
256
256
|
return default_to_dict(self)
|
|
257
257
|
|
|
258
258
|
|
|
259
|
+
def _sanitize_usage_data(usage: Dict[str, Any]) -> Dict[str, Any]:
|
|
260
|
+
"""
|
|
261
|
+
Sanitize usage data for Langfuse by flattening to a single-level dictionary.
|
|
262
|
+
|
|
263
|
+
Langfuse's usage_details must be a flat dictionary with only numeric values. This function:
|
|
264
|
+
- Flattens nested dictionaries using dot notation (e.g., cache_creation.input_tokens)
|
|
265
|
+
- Keeps int and float values
|
|
266
|
+
- Skips None, boolean, string, and other non-numeric types
|
|
267
|
+
|
|
268
|
+
:param usage: Raw usage dictionary from the provider.
|
|
269
|
+
:returns: Flat dictionary with only numeric values (int or float).
|
|
270
|
+
"""
|
|
271
|
+
if not isinstance(usage, dict):
|
|
272
|
+
return {}
|
|
273
|
+
|
|
274
|
+
sanitized: Dict[str, Any] = {}
|
|
275
|
+
|
|
276
|
+
def _flatten(data: Dict[str, Any], prefix: str = "") -> None:
|
|
277
|
+
"""Recursively flatten nested dictionaries."""
|
|
278
|
+
for key, value in data.items():
|
|
279
|
+
full_key = f"{prefix}.{key}" if prefix else key
|
|
280
|
+
|
|
281
|
+
if value is None:
|
|
282
|
+
# Skip None values (e.g., Anthropic's server_tool_use)
|
|
283
|
+
continue
|
|
284
|
+
elif isinstance(value, bool):
|
|
285
|
+
# Skip boolean values
|
|
286
|
+
continue
|
|
287
|
+
elif isinstance(value, (int, float)):
|
|
288
|
+
# Keep numeric values
|
|
289
|
+
sanitized[full_key] = value
|
|
290
|
+
elif isinstance(value, dict):
|
|
291
|
+
# Recursively flatten nested dicts
|
|
292
|
+
_flatten(value, full_key)
|
|
293
|
+
# Skip strings and other non-numeric types (e.g., Anthropic's service_tier)
|
|
294
|
+
|
|
295
|
+
_flatten(usage)
|
|
296
|
+
return sanitized
|
|
297
|
+
|
|
298
|
+
|
|
259
299
|
class DefaultSpanHandler(SpanHandler):
|
|
260
300
|
"""DefaultSpanHandler provides the default Langfuse tracing behavior for Haystack."""
|
|
261
301
|
|
|
@@ -271,10 +311,12 @@ class DefaultSpanHandler(SpanHandler):
|
|
|
271
311
|
# Get external tracing context for root trace creation (correlation metadata)
|
|
272
312
|
tracing_ctx = tracing_context_var.get({})
|
|
273
313
|
if not context.parent_span:
|
|
314
|
+
root_span_type: Literal["agent", "span"] = (
|
|
315
|
+
"agent" if context.operation_name == "haystack.agent.run" else "span"
|
|
316
|
+
)
|
|
274
317
|
# Create a new trace when there's no parent span
|
|
275
|
-
span_context_manager = self.tracer.
|
|
276
|
-
name=context.trace_name,
|
|
277
|
-
version=tracing_ctx.get("version"),
|
|
318
|
+
span_context_manager = self.tracer.start_as_current_observation(
|
|
319
|
+
name=context.trace_name, version=tracing_ctx.get("version"), as_type=root_span_type
|
|
278
320
|
)
|
|
279
321
|
|
|
280
322
|
# Create LangfuseSpan which will handle entering the context manager
|
|
@@ -289,6 +331,7 @@ class DefaultSpanHandler(SpanHandler):
|
|
|
289
331
|
"metadata": None,
|
|
290
332
|
"tags": tracing_ctx.get("tags"),
|
|
291
333
|
"public": context.public,
|
|
334
|
+
"release": None,
|
|
292
335
|
}
|
|
293
336
|
|
|
294
337
|
# Filter out None values and apply trace attributes
|
|
@@ -297,6 +340,10 @@ class DefaultSpanHandler(SpanHandler):
|
|
|
297
340
|
span._span.update_trace(**trace_attrs)
|
|
298
341
|
|
|
299
342
|
return span
|
|
343
|
+
elif context.component_type == "ToolInvoker":
|
|
344
|
+
return LangfuseSpan(self.tracer.start_as_current_observation(name=context.name, as_type="tool"))
|
|
345
|
+
elif context.operation_name == "haystack.agent.run":
|
|
346
|
+
return LangfuseSpan(self.tracer.start_as_current_observation(name=context.name, as_type="agent"))
|
|
300
347
|
elif context.component_type in _ALL_SUPPORTED_GENERATORS:
|
|
301
348
|
return LangfuseSpan(self.tracer.start_as_current_observation(name=context.name, as_type="generation"))
|
|
302
349
|
else:
|
|
@@ -327,7 +374,9 @@ class DefaultSpanHandler(SpanHandler):
|
|
|
327
374
|
if component_type in _SUPPORTED_GENERATORS:
|
|
328
375
|
meta = span.get_data().get(_COMPONENT_OUTPUT_KEY, {}).get("meta")
|
|
329
376
|
if meta:
|
|
330
|
-
|
|
377
|
+
usage = meta[0].get("usage")
|
|
378
|
+
sanitized_usage = _sanitize_usage_data(usage) if usage else None
|
|
379
|
+
span.raw_span().update(usage_details=sanitized_usage, model=meta[0].get("model"))
|
|
331
380
|
|
|
332
381
|
if component_type in _SUPPORTED_CHAT_GENERATORS:
|
|
333
382
|
replies = span.get_data().get(_COMPONENT_OUTPUT_KEY, {}).get("replies")
|
|
@@ -340,8 +389,10 @@ class DefaultSpanHandler(SpanHandler):
|
|
|
340
389
|
except ValueError:
|
|
341
390
|
logger.error(f"Failed to parse completion_start_time: {completion_start_time}")
|
|
342
391
|
completion_start_time = None
|
|
392
|
+
usage = meta.get("usage")
|
|
393
|
+
sanitized_usage = _sanitize_usage_data(usage) if usage else None
|
|
343
394
|
span.raw_span().update(
|
|
344
|
-
|
|
395
|
+
usage_details=sanitized_usage,
|
|
345
396
|
model=meta.get("model"),
|
|
346
397
|
completion_start_time=completion_start_time,
|
|
347
398
|
)
|
|
@@ -465,11 +516,11 @@ class LangfuseTracer(Tracer):
|
|
|
465
516
|
Return the URL to the tracing data.
|
|
466
517
|
:return: The URL to the tracing data.
|
|
467
518
|
"""
|
|
468
|
-
return self._tracer.get_trace_url()
|
|
519
|
+
return self._tracer.get_trace_url() or ""
|
|
469
520
|
|
|
470
521
|
def get_trace_id(self) -> str:
|
|
471
522
|
"""
|
|
472
523
|
Return the trace ID.
|
|
473
524
|
:return: The trace ID.
|
|
474
525
|
"""
|
|
475
|
-
return self._tracer.get_current_trace_id()
|
|
526
|
+
return self._tracer.get_current_trace_id() or ""
|
|
File without changes
|
|
@@ -6,12 +6,10 @@ import asyncio
|
|
|
6
6
|
import datetime
|
|
7
7
|
import logging
|
|
8
8
|
import sys
|
|
9
|
-
import json
|
|
10
9
|
from typing import Optional
|
|
11
10
|
from unittest.mock import MagicMock, Mock, patch
|
|
12
11
|
|
|
13
12
|
import pytest
|
|
14
|
-
from haystack import Pipeline, component
|
|
15
13
|
from haystack.dataclasses import ChatMessage, ToolCall
|
|
16
14
|
|
|
17
15
|
from haystack_integrations.tracing.langfuse.tracer import (
|
|
@@ -20,8 +18,8 @@ from haystack_integrations.tracing.langfuse.tracer import (
|
|
|
20
18
|
LangfuseSpan,
|
|
21
19
|
LangfuseTracer,
|
|
22
20
|
SpanContext,
|
|
21
|
+
_sanitize_usage_data,
|
|
23
22
|
)
|
|
24
|
-
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
25
23
|
|
|
26
24
|
|
|
27
25
|
# Mock functions for Langfuse v3 API
|
|
@@ -78,7 +76,7 @@ class MockSpan:
|
|
|
78
76
|
|
|
79
77
|
|
|
80
78
|
class MockTracer:
|
|
81
|
-
def trace(self, name, **kwargs):
|
|
79
|
+
def trace(self, name, **kwargs): # noqa: ARG002
|
|
82
80
|
# Return a unique mock span for each trace call
|
|
83
81
|
return MockSpan(name=name)
|
|
84
82
|
|
|
@@ -92,10 +90,10 @@ class MockLangfuseClient:
|
|
|
92
90
|
def __init__(self):
|
|
93
91
|
self._mock_context_manager = MockContextManager()
|
|
94
92
|
|
|
95
|
-
def start_as_current_span(self,
|
|
93
|
+
def start_as_current_span(self, _name=None, **_kwargs):
|
|
96
94
|
return self._mock_context_manager
|
|
97
95
|
|
|
98
|
-
def start_as_current_observation(self,
|
|
96
|
+
def start_as_current_observation(self, _name=None, _as_type=None, **_kwargs):
|
|
99
97
|
return self._mock_context_manager
|
|
100
98
|
|
|
101
99
|
def get_current_trace_id(self):
|
|
@@ -193,6 +191,41 @@ class TestSpanContext:
|
|
|
193
191
|
)
|
|
194
192
|
|
|
195
193
|
|
|
194
|
+
class TestSanitizeUsageData:
|
|
195
|
+
def test_anthropic_usage_flattens_and_filters(self):
|
|
196
|
+
"""Test Anthropic's nested dict with None and strings gets flattened and filtered"""
|
|
197
|
+
usage = {
|
|
198
|
+
"cache_creation": {"ephemeral_1h_input_tokens": 0, "ephemeral_5m_input_tokens": 0},
|
|
199
|
+
"cache_creation_input_tokens": 0,
|
|
200
|
+
"cache_read_input_tokens": 0,
|
|
201
|
+
"server_tool_use": None, # Should be filtered
|
|
202
|
+
"service_tier": "standard", # Should be filtered
|
|
203
|
+
"prompt_tokens": 25,
|
|
204
|
+
"completion_tokens": 449,
|
|
205
|
+
}
|
|
206
|
+
result = _sanitize_usage_data(usage)
|
|
207
|
+
assert result == {
|
|
208
|
+
"cache_creation.ephemeral_1h_input_tokens": 0,
|
|
209
|
+
"cache_creation.ephemeral_5m_input_tokens": 0,
|
|
210
|
+
"cache_creation_input_tokens": 0,
|
|
211
|
+
"cache_read_input_tokens": 0,
|
|
212
|
+
"prompt_tokens": 25,
|
|
213
|
+
"completion_tokens": 449,
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
def test_openai_usage_preserved(self):
|
|
217
|
+
"""Test OpenAI/Cohere flat dict with only numeric values works unchanged"""
|
|
218
|
+
usage = {"prompt_tokens": 29, "completion_tokens": 267, "total_tokens": 296}
|
|
219
|
+
result = _sanitize_usage_data(usage)
|
|
220
|
+
assert result == {"prompt_tokens": 29, "completion_tokens": 267, "total_tokens": 296}
|
|
221
|
+
|
|
222
|
+
def test_empty_and_invalid_input(self):
|
|
223
|
+
"""Test edge cases return empty dict"""
|
|
224
|
+
assert _sanitize_usage_data({}) == {}
|
|
225
|
+
assert _sanitize_usage_data(None) == {}
|
|
226
|
+
assert _sanitize_usage_data({"only_strings": "value", "only_none": None}) == {}
|
|
227
|
+
|
|
228
|
+
|
|
196
229
|
class TestDefaultSpanHandler:
|
|
197
230
|
def test_handle_generator(self):
|
|
198
231
|
mock_span = Mock()
|
|
@@ -206,7 +239,7 @@ class TestDefaultSpanHandler:
|
|
|
206
239
|
handler.handle(mock_span, component_type="OpenAIGenerator")
|
|
207
240
|
|
|
208
241
|
assert mock_span.update.call_count == 1
|
|
209
|
-
assert mock_span.update.call_args_list[0][1] == {"
|
|
242
|
+
assert mock_span.update.call_args_list[0][1] == {"usage_details": None, "model": "test_model"}
|
|
210
243
|
|
|
211
244
|
def test_handle_chat_generator(self):
|
|
212
245
|
mock_span = Mock()
|
|
@@ -228,9 +261,11 @@ class TestDefaultSpanHandler:
|
|
|
228
261
|
|
|
229
262
|
assert mock_span.update.call_count == 1
|
|
230
263
|
assert mock_span.update.call_args_list[0][1] == {
|
|
231
|
-
"
|
|
264
|
+
"usage_details": None,
|
|
232
265
|
"model": "test_model",
|
|
233
|
-
"completion_start_time": datetime.datetime(
|
|
266
|
+
"completion_start_time": datetime.datetime( # noqa: DTZ001
|
|
267
|
+
2021, 7, 27, 16, 2, 8, 12345
|
|
268
|
+
),
|
|
234
269
|
}
|
|
235
270
|
|
|
236
271
|
def test_handle_bad_completion_start_time(self, caplog):
|
|
@@ -255,7 +290,7 @@ class TestDefaultSpanHandler:
|
|
|
255
290
|
|
|
256
291
|
assert mock_span.update.call_count == 1
|
|
257
292
|
assert mock_span.update.call_args_list[0][1] == {
|
|
258
|
-
"
|
|
293
|
+
"usage_details": None,
|
|
259
294
|
"model": "test_model",
|
|
260
295
|
"completion_start_time": None,
|
|
261
296
|
}
|
|
@@ -302,18 +337,15 @@ class TestLangfuseTracer:
|
|
|
302
337
|
mock_raw_span.operation_name = "operation_name"
|
|
303
338
|
mock_raw_span.metadata = {"tag1": "value1", "tag2": "value2"}
|
|
304
339
|
|
|
305
|
-
with patch("haystack_integrations.tracing.langfuse.tracer.LangfuseSpan") as
|
|
306
|
-
|
|
307
|
-
) as mock_get_client:
|
|
308
|
-
mock_span_instance = MockLangfuseSpan.return_value
|
|
340
|
+
with patch("haystack_integrations.tracing.langfuse.tracer.LangfuseSpan") as mock_langfuse_span:
|
|
341
|
+
mock_span_instance = mock_langfuse_span.return_value
|
|
309
342
|
mock_span_instance.raw_span.return_value = mock_raw_span
|
|
310
343
|
|
|
311
|
-
mock_client = mock_get_client()
|
|
312
344
|
mock_context_manager = MockContextManager()
|
|
313
345
|
mock_context_manager._span = mock_raw_span
|
|
314
|
-
mock_client.start_as_current_span.return_value = mock_context_manager
|
|
315
|
-
|
|
316
346
|
mock_tracer = MagicMock()
|
|
347
|
+
mock_tracer.start_as_current_span.return_value = mock_context_manager
|
|
348
|
+
|
|
317
349
|
tracer = LangfuseTracer(tracer=mock_tracer, name="Haystack", public=False)
|
|
318
350
|
|
|
319
351
|
# check that the trace method is called on the tracer instance with the provided operation name and tags
|
|
@@ -329,9 +361,7 @@ class TestLangfuseTracer:
|
|
|
329
361
|
|
|
330
362
|
# check that update method is called on the span instance with the provided key value pairs
|
|
331
363
|
def test_update_span_with_pipeline_input_output_data(self):
|
|
332
|
-
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client")
|
|
333
|
-
mock_client = mock_get_client()
|
|
334
|
-
|
|
364
|
+
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client"):
|
|
335
365
|
tracer = LangfuseTracer(tracer=MockLangfuseClient(), name="Haystack", public=False)
|
|
336
366
|
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}) as span:
|
|
337
367
|
assert span.raw_span()._data["metadata"] == {"haystack.pipeline.input_data": "hello"}
|
|
@@ -340,9 +370,7 @@ class TestLangfuseTracer:
|
|
|
340
370
|
assert span.raw_span()._data["metadata"] == {"haystack.pipeline.output_data": "bye"}
|
|
341
371
|
|
|
342
372
|
def test_trace_generation(self):
|
|
343
|
-
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client")
|
|
344
|
-
mock_client = mock_get_client()
|
|
345
|
-
|
|
373
|
+
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client"):
|
|
346
374
|
tracer = LangfuseTracer(tracer=MockLangfuseClient(), name="Haystack", public=False)
|
|
347
375
|
tags = {
|
|
348
376
|
"haystack.component.type": "OpenAIChatGenerator",
|
|
@@ -356,9 +384,9 @@ class TestLangfuseTracer:
|
|
|
356
384
|
}
|
|
357
385
|
with tracer.trace(operation_name="operation_name", tags=tags) as span:
|
|
358
386
|
...
|
|
359
|
-
assert span.raw_span()._data["
|
|
387
|
+
assert span.raw_span()._data["usage_details"] is None
|
|
360
388
|
assert span.raw_span()._data["model"] == "test_model"
|
|
361
|
-
assert span.raw_span()._data["completion_start_time"] == datetime.datetime(2021, 7, 27, 16, 2, 8, 12345)
|
|
389
|
+
assert span.raw_span()._data["completion_start_time"] == datetime.datetime(2021, 7, 27, 16, 2, 8, 12345) # noqa: DTZ001
|
|
362
390
|
|
|
363
391
|
def test_handle_tool_invoker(self):
|
|
364
392
|
"""
|
|
@@ -402,7 +430,7 @@ class TestLangfuseTracer:
|
|
|
402
430
|
updated_name = name_update_call[1]["name"]
|
|
403
431
|
|
|
404
432
|
# verify the format of the updated span name to be: `original_component_name - [list_of_tool_names]`
|
|
405
|
-
assert updated_name != "tool_invoker",
|
|
433
|
+
assert updated_name != "tool_invoker", "Expected 'tool_invoker` to be upddated with tool names"
|
|
406
434
|
assert " - " in updated_name, f"Expected ' - ' in {updated_name}"
|
|
407
435
|
assert "[" in updated_name, f"Expected '[' in {updated_name}"
|
|
408
436
|
assert "]" in updated_name, f"Expected ']' in {updated_name}"
|
|
@@ -411,9 +439,7 @@ class TestLangfuseTracer:
|
|
|
411
439
|
assert "weather_tool" in updated_name, f"Expected 'weather_tool' in {updated_name}"
|
|
412
440
|
|
|
413
441
|
def test_trace_generation_invalid_start_time(self):
|
|
414
|
-
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client")
|
|
415
|
-
mock_client = mock_get_client()
|
|
416
|
-
|
|
442
|
+
with patch("haystack_integrations.tracing.langfuse.tracer.langfuse.get_client"):
|
|
417
443
|
tracer = LangfuseTracer(tracer=MockLangfuseClient(), name="Haystack", public=False)
|
|
418
444
|
tags = {
|
|
419
445
|
"haystack.component.type": "OpenAIChatGenerator",
|
|
@@ -425,7 +451,7 @@ class TestLangfuseTracer:
|
|
|
425
451
|
}
|
|
426
452
|
with tracer.trace(operation_name="operation_name", tags=tags) as span:
|
|
427
453
|
...
|
|
428
|
-
assert span.raw_span()._data["
|
|
454
|
+
assert span.raw_span()._data["usage_details"] is None
|
|
429
455
|
assert span.raw_span()._data["model"] == "test_model"
|
|
430
456
|
assert span.raw_span()._data["completion_start_time"] is None
|
|
431
457
|
|
|
@@ -434,7 +460,7 @@ class TestLangfuseTracer:
|
|
|
434
460
|
tracer_mock.flush = Mock() # Make flush a mock for assertions
|
|
435
461
|
|
|
436
462
|
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
437
|
-
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"})
|
|
463
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}):
|
|
438
464
|
pass
|
|
439
465
|
|
|
440
466
|
tracer_mock.flush.assert_called_once()
|
|
@@ -444,10 +470,11 @@ class TestLangfuseTracer:
|
|
|
444
470
|
tracer_mock = MockLangfuseClient()
|
|
445
471
|
tracer_mock.flush = Mock() # Make flush a mock for assertions
|
|
446
472
|
|
|
447
|
-
|
|
473
|
+
# Re-import LangfuseTracer to ensure it picks up the new environment variable
|
|
474
|
+
from haystack_integrations.tracing.langfuse.tracer import LangfuseTracer # noqa: PLC0415
|
|
448
475
|
|
|
449
476
|
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
450
|
-
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"})
|
|
477
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}):
|
|
451
478
|
pass
|
|
452
479
|
|
|
453
480
|
tracer_mock.flush.assert_not_called()
|
|
@@ -456,7 +483,7 @@ class TestLangfuseTracer:
|
|
|
456
483
|
tracer_mock = MockLangfuseClient()
|
|
457
484
|
|
|
458
485
|
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
459
|
-
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"})
|
|
486
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}):
|
|
460
487
|
pass
|
|
461
488
|
|
|
462
489
|
# Check behavioral state instead of internal _context list
|
|
@@ -471,7 +498,7 @@ class TestLangfuseTracer:
|
|
|
471
498
|
# Re-import LangfuseTracer and instantiate it with tracing disabled
|
|
472
499
|
with caplog.at_level(logging.WARNING):
|
|
473
500
|
monkeypatch.setenv("HAYSTACK_CONTENT_TRACING_ENABLED", "false")
|
|
474
|
-
from haystack_integrations.tracing.langfuse import LangfuseTracer
|
|
501
|
+
from haystack_integrations.tracing.langfuse import LangfuseTracer # noqa: PLC0415
|
|
475
502
|
|
|
476
503
|
LangfuseTracer(tracer=MockLangfuseClient(), name="Haystack", public=False)
|
|
477
504
|
assert "tracing is disabled" in caplog.text
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
#
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
|
4
4
|
|
|
5
|
+
import json
|
|
5
6
|
import os
|
|
6
7
|
import time
|
|
7
8
|
from typing import Any, Dict, List
|
|
8
9
|
from urllib.parse import urlparse
|
|
9
|
-
import json
|
|
10
10
|
|
|
11
11
|
import pytest
|
|
12
12
|
import requests
|
|
@@ -17,8 +17,17 @@ from haystack.dataclasses import ChatMessage
|
|
|
17
17
|
from requests.auth import HTTPBasicAuth
|
|
18
18
|
|
|
19
19
|
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
|
|
21
|
+
# We use the try/except block to prevent import errors when running unit tests with lowest direct dependencies,
|
|
22
|
+
# where the subdependencies of LLM integrations might not be respected.
|
|
23
|
+
try:
|
|
24
|
+
from haystack_integrations.components.generators.anthropic import AnthropicChatGenerator
|
|
25
|
+
except ImportError:
|
|
26
|
+
AnthropicChatGenerator = None
|
|
27
|
+
try:
|
|
28
|
+
from haystack_integrations.components.generators.cohere import CohereChatGenerator
|
|
29
|
+
except ImportError:
|
|
30
|
+
CohereChatGenerator = None
|
|
22
31
|
|
|
23
32
|
# don't remove (or move) this env var setting from here, it's needed to turn tracing on
|
|
24
33
|
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
@@ -38,7 +47,7 @@ def poll_langfuse(url: str):
|
|
|
38
47
|
|
|
39
48
|
res = None
|
|
40
49
|
while attempts > 0:
|
|
41
|
-
res = requests.get(url, auth=auth)
|
|
50
|
+
res = requests.get(url, auth=auth, timeout=60.0)
|
|
42
51
|
if res.status_code == 200:
|
|
43
52
|
return res
|
|
44
53
|
|
|
@@ -73,7 +82,7 @@ def basic_pipeline(llm_class, expected_trace):
|
|
|
73
82
|
(CohereChatGenerator, "COHERE_API_KEY", "Cohere"),
|
|
74
83
|
],
|
|
75
84
|
)
|
|
76
|
-
def test_tracing_integration(
|
|
85
|
+
def test_tracing_integration(env_var, expected_trace, basic_pipeline):
|
|
77
86
|
if not all([os.environ.get("LANGFUSE_SECRET_KEY"), os.environ.get("LANGFUSE_PUBLIC_KEY"), os.environ.get(env_var)]):
|
|
78
87
|
pytest.skip(f"Missing required environment variable: {env_var}")
|
|
79
88
|
|
|
@@ -242,7 +251,7 @@ def test_context_cleanup_after_nested_failures():
|
|
|
242
251
|
# Test 1: First run will fail and should clean up context
|
|
243
252
|
try:
|
|
244
253
|
main_pipeline.run({"nested_component": {"input_data": "invalid json"}})
|
|
245
|
-
except Exception:
|
|
254
|
+
except Exception: # noqa: S110
|
|
246
255
|
pass # Expected to fail
|
|
247
256
|
|
|
248
257
|
# Critical assertion: context should be empty after failed operation
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|