langfuse-haystack 0.0.3__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langfuse_haystack-0.1.0/PKG-INFO +114 -0
- langfuse_haystack-0.1.0/README.md +91 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/example/basic_rag.py +6 -8
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/example/chat.py +4 -8
- langfuse_haystack-0.1.0/pydoc/config.yml +30 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/pyproject.toml +5 -6
- langfuse_haystack-0.1.0/src/haystack_integrations/components/connectors/langfuse/__init__.py +6 -0
- langfuse_haystack-0.1.0/src/haystack_integrations/components/connectors/langfuse/langfuse_connector.py +107 -0
- langfuse_haystack-0.1.0/src/haystack_integrations/tracing/langfuse/tracer.py +171 -0
- langfuse_haystack-0.1.0/tests/test_langfuse_span.py +65 -0
- langfuse_haystack-0.1.0/tests/test_tracer.py +114 -0
- langfuse_haystack-0.1.0/tests/test_tracing.py +57 -0
- langfuse_haystack-0.0.3/PKG-INFO +0 -45
- langfuse_haystack-0.0.3/README.md +0 -22
- langfuse_haystack-0.0.3/src/haystack_integrations/components/others/langfuse/__about__.py +0 -1
- langfuse_haystack-0.0.3/src/haystack_integrations/components/others/langfuse/__init__.py +0 -3
- langfuse_haystack-0.0.3/src/haystack_integrations/components/others/langfuse/component.py +0 -16
- langfuse_haystack-0.0.3/src/haystack_integrations/tracing/langfuse/tracer.py +0 -89
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/.gitignore +0 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/LICENSE.txt +0 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/example/requirements.txt +0 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/src/haystack_integrations/tracing/langfuse/__init__.py +0 -0
- {langfuse_haystack-0.0.3 → langfuse_haystack-0.1.0}/tests/__init__.py +0 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: langfuse-haystack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme
|
|
5
|
+
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
6
|
+
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse
|
|
7
|
+
Author-email: deepset GmbH <info@deepset.ai>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
License-File: LICENSE.txt
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Requires-Dist: haystack-ai>=2.1.0
|
|
21
|
+
Requires-Dist: langfuse
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# langfuse-haystack
|
|
25
|
+
|
|
26
|
+
[](https://pypi.org/project/langfuse-haystack)
|
|
27
|
+
[](https://pypi.org/project/langfuse-haystack)
|
|
28
|
+
|
|
29
|
+
langfuse-haystack integrates tracing capabilities into [Haystack](https://github.com/deepset-ai/haystack) (2.x) pipelines using [Langfuse](https://langfuse.com/). This package enhances the visibility of pipeline runs by capturing comprehensive details of the execution traces, including API calls, context data, prompts, and more. Whether you're monitoring model performance, pinpointing areas for improvement, or creating datasets for fine-tuning and testing from your pipeline executions, langfuse-haystack is the right tool for you.
|
|
30
|
+
## Features
|
|
31
|
+
|
|
32
|
+
- Easy integration with Haystack pipelines
|
|
33
|
+
- Capture the full context of the execution
|
|
34
|
+
- Track model usage and cost
|
|
35
|
+
- Collect user feedback
|
|
36
|
+
- Identify low-quality outputs
|
|
37
|
+
- Build fine-tuning and testing datasets
|
|
38
|
+
|
|
39
|
+
## Installation
|
|
40
|
+
|
|
41
|
+
To install langfuse-haystack, run the following command:
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
pip install langfuse-haystack
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Usage
|
|
48
|
+
|
|
49
|
+
To enable tracing in your Haystack pipeline, add the `LangfuseConnector` to your pipeline.
|
|
50
|
+
You also need to set the `LANGFUSE_SECRET_KEY` and `LANGFUSE_PUBLIC_KEY` environment variables in order to connect to Langfuse account.
|
|
51
|
+
You can get these keys by signing up for an account on the Langfuse website.
|
|
52
|
+
|
|
53
|
+
Here's an example:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import os
|
|
57
|
+
|
|
58
|
+
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"
|
|
59
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
60
|
+
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
61
|
+
|
|
62
|
+
from haystack.components.builders import DynamicChatPromptBuilder
|
|
63
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
64
|
+
from haystack.dataclasses import ChatMessage
|
|
65
|
+
from haystack import Pipeline
|
|
66
|
+
|
|
67
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
68
|
+
|
|
69
|
+
if __name__ == "__main__":
|
|
70
|
+
pipe = Pipeline()
|
|
71
|
+
pipe.add_component("tracer", LangfuseConnector("Chat example"))
|
|
72
|
+
pipe.add_component("prompt_builder", DynamicChatPromptBuilder())
|
|
73
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-3.5-turbo"))
|
|
74
|
+
|
|
75
|
+
pipe.connect("prompt_builder.prompt", "llm.messages")
|
|
76
|
+
|
|
77
|
+
messages = [
|
|
78
|
+
ChatMessage.from_system("Always respond in German even if some input data is in other languages."),
|
|
79
|
+
ChatMessage.from_user("Tell me about {{location}}"),
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
response = pipe.run(
|
|
83
|
+
data={"prompt_builder": {"template_variables": {"location": "Berlin"}, "prompt_source": messages}}
|
|
84
|
+
)
|
|
85
|
+
print(response["llm"]["replies"][0])
|
|
86
|
+
print(response["tracer"]["trace_url"])
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
In this example, we add the `LangfuseConnector` to the pipeline with the name "tracer". Each run of the pipeline produces one trace viewable on the Langfuse website with a specific URL. The trace captures the entire execution context, including the prompts, completions, and metadata.
|
|
90
|
+
|
|
91
|
+
## Trace Visualization
|
|
92
|
+
|
|
93
|
+
Langfuse provides a user-friendly interface to visualize and analyze the traces generated by your Haystack pipeline. Login into your Langfuse account and navigate to the trace URL to view the trace details.
|
|
94
|
+
|
|
95
|
+
## Contributing
|
|
96
|
+
|
|
97
|
+
`hatch` is the best way to interact with this project. To install it, run:
|
|
98
|
+
```sh
|
|
99
|
+
pip install hatch
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
With `hatch` installed, run all the tests:
|
|
103
|
+
```
|
|
104
|
+
hatch run test
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Run the linters `ruff` and `mypy`:
|
|
108
|
+
```
|
|
109
|
+
hatch run lint:all
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
`langfuse-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# langfuse-haystack
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/langfuse-haystack)
|
|
4
|
+
[](https://pypi.org/project/langfuse-haystack)
|
|
5
|
+
|
|
6
|
+
langfuse-haystack integrates tracing capabilities into [Haystack](https://github.com/deepset-ai/haystack) (2.x) pipelines using [Langfuse](https://langfuse.com/). This package enhances the visibility of pipeline runs by capturing comprehensive details of the execution traces, including API calls, context data, prompts, and more. Whether you're monitoring model performance, pinpointing areas for improvement, or creating datasets for fine-tuning and testing from your pipeline executions, langfuse-haystack is the right tool for you.
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- Easy integration with Haystack pipelines
|
|
10
|
+
- Capture the full context of the execution
|
|
11
|
+
- Track model usage and cost
|
|
12
|
+
- Collect user feedback
|
|
13
|
+
- Identify low-quality outputs
|
|
14
|
+
- Build fine-tuning and testing datasets
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
To install langfuse-haystack, run the following command:
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
pip install langfuse-haystack
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Usage
|
|
25
|
+
|
|
26
|
+
To enable tracing in your Haystack pipeline, add the `LangfuseConnector` to your pipeline.
|
|
27
|
+
You also need to set the `LANGFUSE_SECRET_KEY` and `LANGFUSE_PUBLIC_KEY` environment variables in order to connect to Langfuse account.
|
|
28
|
+
You can get these keys by signing up for an account on the Langfuse website.
|
|
29
|
+
|
|
30
|
+
Here's an example:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
import os
|
|
34
|
+
|
|
35
|
+
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"
|
|
36
|
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
37
|
+
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
38
|
+
|
|
39
|
+
from haystack.components.builders import DynamicChatPromptBuilder
|
|
40
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
41
|
+
from haystack.dataclasses import ChatMessage
|
|
42
|
+
from haystack import Pipeline
|
|
43
|
+
|
|
44
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
45
|
+
|
|
46
|
+
if __name__ == "__main__":
|
|
47
|
+
pipe = Pipeline()
|
|
48
|
+
pipe.add_component("tracer", LangfuseConnector("Chat example"))
|
|
49
|
+
pipe.add_component("prompt_builder", DynamicChatPromptBuilder())
|
|
50
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-3.5-turbo"))
|
|
51
|
+
|
|
52
|
+
pipe.connect("prompt_builder.prompt", "llm.messages")
|
|
53
|
+
|
|
54
|
+
messages = [
|
|
55
|
+
ChatMessage.from_system("Always respond in German even if some input data is in other languages."),
|
|
56
|
+
ChatMessage.from_user("Tell me about {{location}}"),
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
response = pipe.run(
|
|
60
|
+
data={"prompt_builder": {"template_variables": {"location": "Berlin"}, "prompt_source": messages}}
|
|
61
|
+
)
|
|
62
|
+
print(response["llm"]["replies"][0])
|
|
63
|
+
print(response["tracer"]["trace_url"])
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
In this example, we add the `LangfuseConnector` to the pipeline with the name "tracer". Each run of the pipeline produces one trace viewable on the Langfuse website with a specific URL. The trace captures the entire execution context, including the prompts, completions, and metadata.
|
|
67
|
+
|
|
68
|
+
## Trace Visualization
|
|
69
|
+
|
|
70
|
+
Langfuse provides a user-friendly interface to visualize and analyze the traces generated by your Haystack pipeline. Login into your Langfuse account and navigate to the trace URL to view the trace details.
|
|
71
|
+
|
|
72
|
+
## Contributing
|
|
73
|
+
|
|
74
|
+
`hatch` is the best way to interact with this project. To install it, run:
|
|
75
|
+
```sh
|
|
76
|
+
pip install hatch
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
With `hatch` installed, run all the tests:
|
|
80
|
+
```
|
|
81
|
+
hatch run test
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Run the linters `ruff` and `mypy`:
|
|
85
|
+
```
|
|
86
|
+
hatch run lint:all
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## License
|
|
90
|
+
|
|
91
|
+
`langfuse-haystack` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -1,19 +1,16 @@
|
|
|
1
1
|
import os
|
|
2
2
|
|
|
3
|
-
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"
|
|
4
3
|
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
5
4
|
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
6
5
|
|
|
7
6
|
from datasets import load_dataset
|
|
8
|
-
|
|
9
|
-
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
|
10
7
|
from haystack import Document, Pipeline
|
|
8
|
+
from haystack.components.builders import PromptBuilder
|
|
11
9
|
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
|
12
|
-
from haystack.components.retrievers import InMemoryEmbeddingRetriever
|
|
13
10
|
from haystack.components.generators import OpenAIGenerator
|
|
14
|
-
from haystack.components.
|
|
15
|
-
|
|
16
|
-
from haystack_integrations.components.
|
|
11
|
+
from haystack.components.retrievers import InMemoryEmbeddingRetriever
|
|
12
|
+
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
|
13
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
17
14
|
|
|
18
15
|
|
|
19
16
|
def get_pipeline(document_store: InMemoryDocumentStore):
|
|
@@ -35,7 +32,7 @@ def get_pipeline(document_store: InMemoryDocumentStore):
|
|
|
35
32
|
|
|
36
33
|
basic_rag_pipeline = Pipeline()
|
|
37
34
|
# Add components to your pipeline
|
|
38
|
-
basic_rag_pipeline.add_component("tracer",
|
|
35
|
+
basic_rag_pipeline.add_component("tracer", LangfuseConnector("Basic RAG Pipeline"))
|
|
39
36
|
basic_rag_pipeline.add_component(
|
|
40
37
|
"text_embedder", SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
|
|
41
38
|
)
|
|
@@ -65,3 +62,4 @@ if __name__ == "__main__":
|
|
|
65
62
|
response = pipeline.run({"text_embedder": {"text": question}, "prompt_builder": {"question": question}})
|
|
66
63
|
|
|
67
64
|
print(response["llm"]["replies"][0])
|
|
65
|
+
print(response["tracer"]["trace_url"])
|
|
@@ -1,22 +1,17 @@
|
|
|
1
1
|
import os
|
|
2
2
|
|
|
3
|
-
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"
|
|
4
|
-
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
5
3
|
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
6
4
|
|
|
5
|
+
from haystack import Pipeline
|
|
7
6
|
from haystack.components.builders import DynamicChatPromptBuilder
|
|
8
7
|
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
9
8
|
from haystack.dataclasses import ChatMessage
|
|
10
|
-
from
|
|
11
|
-
from haystack.utils import Secret
|
|
12
|
-
|
|
13
|
-
from haystack_integrations.components.others.langfuse import LangfuseComponent
|
|
14
|
-
|
|
9
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
15
10
|
|
|
16
11
|
if __name__ == "__main__":
|
|
17
12
|
|
|
18
13
|
pipe = Pipeline()
|
|
19
|
-
pipe.add_component("tracer",
|
|
14
|
+
pipe.add_component("tracer", LangfuseConnector("Chat example"))
|
|
20
15
|
pipe.add_component("prompt_builder", DynamicChatPromptBuilder())
|
|
21
16
|
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-3.5-turbo"))
|
|
22
17
|
|
|
@@ -31,3 +26,4 @@ if __name__ == "__main__":
|
|
|
31
26
|
data={"prompt_builder": {"template_variables": {"location": "Berlin"}, "prompt_source": messages}}
|
|
32
27
|
)
|
|
33
28
|
print(response["llm"]["replies"][0])
|
|
29
|
+
print(response["tracer"]["trace_url"])
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
loaders:
|
|
2
|
+
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
|
|
3
|
+
search_path: [../src]
|
|
4
|
+
modules: [
|
|
5
|
+
"haystack_integrations.components.connectors.langfuse.langfuse_connector",
|
|
6
|
+
"haystack_integrations.tracing.langfuse.tracer",
|
|
7
|
+
]
|
|
8
|
+
ignore_when_discovered: ["__init__"]
|
|
9
|
+
processors:
|
|
10
|
+
- type: filter
|
|
11
|
+
expression:
|
|
12
|
+
documented_only: true
|
|
13
|
+
do_not_filter_modules: false
|
|
14
|
+
skip_empty_modules: true
|
|
15
|
+
- type: smart
|
|
16
|
+
- type: crossref
|
|
17
|
+
renderer:
|
|
18
|
+
type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
|
|
19
|
+
excerpt: Langfuse integration for Haystack
|
|
20
|
+
category_slug: integrations-api
|
|
21
|
+
title: langfuse
|
|
22
|
+
slug: integrations-langfuse
|
|
23
|
+
order: 136
|
|
24
|
+
markdown:
|
|
25
|
+
descriptive_class_title: false
|
|
26
|
+
classdef_code_block: false
|
|
27
|
+
descriptive_module_title: true
|
|
28
|
+
add_method_class_prefix: true
|
|
29
|
+
add_member_class_prefix: false
|
|
30
|
+
filename: _readme_langfuse.md
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["hatchling"]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
3
|
build-backend = "hatchling.build"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -25,7 +25,7 @@ classifiers = [
|
|
|
25
25
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
26
26
|
]
|
|
27
27
|
dependencies = [
|
|
28
|
-
"haystack-ai",
|
|
28
|
+
"haystack-ai>=2.1.0",
|
|
29
29
|
"langfuse"
|
|
30
30
|
]
|
|
31
31
|
|
|
@@ -38,10 +38,8 @@ Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/int
|
|
|
38
38
|
packages = ["src/haystack_integrations"]
|
|
39
39
|
|
|
40
40
|
[tool.hatch.version]
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
# source = "vcs"
|
|
44
|
-
# tag-pattern = 'integrations\/langfuse-v(?P<version>.*)'
|
|
41
|
+
source = "vcs"
|
|
42
|
+
tag-pattern = 'integrations\/langfuse-v(?P<version>.*)'
|
|
45
43
|
|
|
46
44
|
|
|
47
45
|
[tool.hatch.version.raw-options]
|
|
@@ -52,6 +50,7 @@ git_describe_command = 'git describe --tags --match="integrations/langfuse-v[0-9
|
|
|
52
50
|
dependencies = [
|
|
53
51
|
"coverage[toml]>=6.5",
|
|
54
52
|
"pytest",
|
|
53
|
+
"haystack-pydoc-tools",
|
|
55
54
|
]
|
|
56
55
|
[tool.hatch.envs.default.scripts]
|
|
57
56
|
test = "pytest {args:tests}"
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from haystack import component, tracing
|
|
2
|
+
from haystack_integrations.tracing.langfuse import LangfuseTracer
|
|
3
|
+
|
|
4
|
+
from langfuse import Langfuse
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@component
|
|
8
|
+
class LangfuseConnector:
|
|
9
|
+
"""
|
|
10
|
+
LangfuseConnector connects Haystack LLM framework with Langfuse in order to enable the tracing of operations
|
|
11
|
+
and data flow within various components of a pipeline.
|
|
12
|
+
|
|
13
|
+
Simply add this component to your pipeline, but *do not* connect it to any other component. The LangfuseConnector
|
|
14
|
+
will automatically trace the operations and data flow within the pipeline.
|
|
15
|
+
|
|
16
|
+
Note that you need to set the `LANGFUSE_SECRET_KEY` and `LANGFUSE_PUBLIC_KEY` environment variables in order
|
|
17
|
+
to use this component. The `LANGFUSE_SECRET_KEY` and `LANGFUSE_PUBLIC_KEY` are the secret and public keys provided
|
|
18
|
+
by Langfuse. You can get these keys by signing up for an account on the Langfuse website.
|
|
19
|
+
|
|
20
|
+
In addition, you need to set the `HAYSTACK_CONTENT_TRACING_ENABLED` environment variable to `true` in order to
|
|
21
|
+
enable Haystack tracing in your pipeline.
|
|
22
|
+
|
|
23
|
+
Lastly, you may disable flushing the data after each component by setting the `HAYSTACK_LANGFUSE_ENFORCE_FLUSH`
|
|
24
|
+
environent variable to `false`. By default, the data is flushed after each component and blocks the thread until
|
|
25
|
+
the data is sent to Langfuse. **Caution**: Disabling this feature may result in data loss if the program crashes
|
|
26
|
+
before the data is sent to Langfuse. Make sure you will call langfuse.flush() explicitly before the program exits.
|
|
27
|
+
E.g. by using tracer.actual_tracer.flush():
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from haystack.tracing import tracer
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
# your code here
|
|
34
|
+
finally:
|
|
35
|
+
tracer.actual_tracer.flush()
|
|
36
|
+
```
|
|
37
|
+
or in FastAPI by defining a shutdown event handler:
|
|
38
|
+
```python
|
|
39
|
+
from haystack.tracing import tracer
|
|
40
|
+
|
|
41
|
+
# ...
|
|
42
|
+
|
|
43
|
+
@app.on_event("shutdown")
|
|
44
|
+
async def shutdown_event():
|
|
45
|
+
tracer.actual_tracer.flush()
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Here is an example of how to use it:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import os
|
|
52
|
+
|
|
53
|
+
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
54
|
+
|
|
55
|
+
from haystack import Pipeline
|
|
56
|
+
from haystack.components.builders import DynamicChatPromptBuilder
|
|
57
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
58
|
+
from haystack.dataclasses import ChatMessage
|
|
59
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
|
|
63
|
+
pipe = Pipeline()
|
|
64
|
+
pipe.add_component("tracer", LangfuseConnector("Chat example"))
|
|
65
|
+
pipe.add_component("prompt_builder", DynamicChatPromptBuilder())
|
|
66
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-3.5-turbo"))
|
|
67
|
+
|
|
68
|
+
pipe.connect("prompt_builder.prompt", "llm.messages")
|
|
69
|
+
|
|
70
|
+
messages = [
|
|
71
|
+
ChatMessage.from_system("Always respond in German even if some input data is in other languages."),
|
|
72
|
+
ChatMessage.from_user("Tell me about {{location}}"),
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
response = pipe.run(
|
|
76
|
+
data={"prompt_builder": {"template_variables": {"location": "Berlin"}, "prompt_source": messages}}
|
|
77
|
+
)
|
|
78
|
+
print(response["llm"]["replies"][0])
|
|
79
|
+
print(response["tracer"]["trace_url"])
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(self, name: str, public: bool = False):
|
|
85
|
+
"""
|
|
86
|
+
Initialize the LangfuseConnector component.
|
|
87
|
+
|
|
88
|
+
:param name: The name of the pipeline or component. This name will be used to identify the tracing run on the
|
|
89
|
+
Langfuse dashboard.
|
|
90
|
+
:param public: Whether the tracing data should be public or private. If set to `True`, the tracing data will be
|
|
91
|
+
publicly accessible to anyone with the tracing URL. If set to `False`, the tracing data will be private and
|
|
92
|
+
only accessible to the Langfuse account owner. The default is `False`.
|
|
93
|
+
"""
|
|
94
|
+
self.name = name
|
|
95
|
+
self.tracer = LangfuseTracer(tracer=Langfuse(), name=name, public=public)
|
|
96
|
+
tracing.enable_tracing(self.tracer)
|
|
97
|
+
|
|
98
|
+
@component.output_types(name=str, trace_url=str)
|
|
99
|
+
def run(self):
|
|
100
|
+
"""
|
|
101
|
+
Runs the LangfuseConnector component.
|
|
102
|
+
|
|
103
|
+
:returns: A dictionary with the following keys:
|
|
104
|
+
- `name`: The name of the tracing component.
|
|
105
|
+
- `trace_url`: The URL to the tracing data.
|
|
106
|
+
"""
|
|
107
|
+
return {"name": self.name, "trace_url": self.tracer.get_trace_url()}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
import os
|
|
3
|
+
from typing import Any, Dict, Iterator, Optional, Union
|
|
4
|
+
|
|
5
|
+
from haystack.dataclasses import ChatMessage
|
|
6
|
+
from haystack.tracing import Span, Tracer, tracer
|
|
7
|
+
from haystack.tracing import utils as tracing_utils
|
|
8
|
+
|
|
9
|
+
import langfuse
|
|
10
|
+
|
|
11
|
+
HAYSTACK_LANGFUSE_ENFORCE_FLUSH_ENV_VAR = "HAYSTACK_LANGFUSE_ENFORCE_FLUSH"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LangfuseSpan(Span):
|
|
15
|
+
"""
|
|
16
|
+
Internal class representing a bridge between the Haystack span tracing API and Langfuse.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, span: "Union[langfuse.client.StatefulSpanClient, langfuse.client.StatefulTraceClient]") -> None:
|
|
20
|
+
"""
|
|
21
|
+
Initialize a LangfuseSpan instance.
|
|
22
|
+
|
|
23
|
+
:param span: The span instance managed by Langfuse.
|
|
24
|
+
"""
|
|
25
|
+
self._span = span
|
|
26
|
+
# locally cache tags
|
|
27
|
+
self._data: Dict[str, Any] = {}
|
|
28
|
+
|
|
29
|
+
def set_tag(self, key: str, value: Any) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Set a generic tag for this span.
|
|
32
|
+
|
|
33
|
+
:param key: The tag key.
|
|
34
|
+
:param value: The tag value.
|
|
35
|
+
"""
|
|
36
|
+
coerced_value = tracing_utils.coerce_tag_value(value)
|
|
37
|
+
self._span.update(metadata={key: coerced_value})
|
|
38
|
+
self._data[key] = value
|
|
39
|
+
|
|
40
|
+
def set_content_tag(self, key: str, value: Any) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Set a content-specific tag for this span.
|
|
43
|
+
|
|
44
|
+
:param key: The content tag key.
|
|
45
|
+
:param value: The content tag value.
|
|
46
|
+
"""
|
|
47
|
+
if not tracer.is_content_tracing_enabled:
|
|
48
|
+
return
|
|
49
|
+
if key.endswith(".input"):
|
|
50
|
+
if "messages" in value:
|
|
51
|
+
messages = [m.to_openai_format() for m in value["messages"]]
|
|
52
|
+
self._span.update(input=messages)
|
|
53
|
+
else:
|
|
54
|
+
self._span.update(input=value)
|
|
55
|
+
elif key.endswith(".output"):
|
|
56
|
+
if "replies" in value:
|
|
57
|
+
if all(isinstance(r, ChatMessage) for r in value["replies"]):
|
|
58
|
+
replies = [m.to_openai_format() for m in value["replies"]]
|
|
59
|
+
else:
|
|
60
|
+
replies = value["replies"]
|
|
61
|
+
self._span.update(output=replies)
|
|
62
|
+
else:
|
|
63
|
+
self._span.update(output=value)
|
|
64
|
+
|
|
65
|
+
self._data[key] = value
|
|
66
|
+
|
|
67
|
+
def raw_span(self) -> Any:
|
|
68
|
+
"""
|
|
69
|
+
Return the underlying span instance.
|
|
70
|
+
|
|
71
|
+
:return: The Langfuse span instance.
|
|
72
|
+
"""
|
|
73
|
+
return self._span
|
|
74
|
+
|
|
75
|
+
def get_correlation_data_for_logs(self) -> Dict[str, Any]:
|
|
76
|
+
return {}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class LangfuseTracer(Tracer):
|
|
80
|
+
"""
|
|
81
|
+
Internal class representing a bridge between the Haystack tracer and Langfuse.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(self, tracer: "langfuse.Langfuse", name: str = "Haystack", public: bool = False) -> None:
|
|
85
|
+
"""
|
|
86
|
+
Initialize a LangfuseTracer instance.
|
|
87
|
+
|
|
88
|
+
:param tracer: The Langfuse tracer instance.
|
|
89
|
+
:param name: The name of the pipeline or component. This name will be used to identify the tracing run on the
|
|
90
|
+
Langfuse dashboard.
|
|
91
|
+
:param public: Whether the tracing data should be public or private. If set to `True`, the tracing data will
|
|
92
|
+
be publicly accessible to anyone with the tracing URL. If set to `False`, the tracing data will be private
|
|
93
|
+
and only accessible to the Langfuse account owner.
|
|
94
|
+
"""
|
|
95
|
+
self._tracer = tracer
|
|
96
|
+
self._context: list[LangfuseSpan] = []
|
|
97
|
+
self._name = name
|
|
98
|
+
self._public = public
|
|
99
|
+
self.enforce_flush = os.getenv(HAYSTACK_LANGFUSE_ENFORCE_FLUSH_ENV_VAR, "true").lower() == "true"
|
|
100
|
+
|
|
101
|
+
@contextlib.contextmanager
|
|
102
|
+
def trace(self, operation_name: str, tags: Optional[Dict[str, Any]] = None) -> Iterator[Span]:
|
|
103
|
+
"""
|
|
104
|
+
Start and manage a new trace span.
|
|
105
|
+
:param operation_name: The name of the operation.
|
|
106
|
+
:param tags: A dictionary of tags to attach to the span.
|
|
107
|
+
:return: A context manager yielding the span.
|
|
108
|
+
"""
|
|
109
|
+
tags = tags or {}
|
|
110
|
+
span_name = tags.get("haystack.component.name", operation_name)
|
|
111
|
+
|
|
112
|
+
if tags.get("haystack.component.type") in ["OpenAIGenerator", "OpenAIChatGenerator"]:
|
|
113
|
+
span = LangfuseSpan(self.current_span().raw_span().generation(name=span_name))
|
|
114
|
+
else:
|
|
115
|
+
span = LangfuseSpan(self.current_span().raw_span().span(name=span_name))
|
|
116
|
+
|
|
117
|
+
self._context.append(span)
|
|
118
|
+
span.set_tags(tags)
|
|
119
|
+
|
|
120
|
+
yield span
|
|
121
|
+
|
|
122
|
+
if tags.get("haystack.component.type") == "OpenAIGenerator":
|
|
123
|
+
meta = span._data.get("haystack.component.output", {}).get("meta")
|
|
124
|
+
if meta:
|
|
125
|
+
# Haystack returns one meta dict for each message, but the 'usage' value
|
|
126
|
+
# is always the same, let's just pick the first item
|
|
127
|
+
m = meta[0]
|
|
128
|
+
span._span.update(usage=m.get("usage") or None, model=m.get("model"))
|
|
129
|
+
elif tags.get("haystack.component.type") == "OpenAIChatGenerator":
|
|
130
|
+
replies = span._data.get("haystack.component.output", {}).get("replies")
|
|
131
|
+
if replies:
|
|
132
|
+
meta = replies[0].meta
|
|
133
|
+
span._span.update(usage=meta.get("usage") or None, model=meta.get("model"))
|
|
134
|
+
|
|
135
|
+
pipeline_input = tags.get("haystack.pipeline.input_data", None)
|
|
136
|
+
if pipeline_input:
|
|
137
|
+
span._span.update(input=tags["haystack.pipeline.input_data"])
|
|
138
|
+
pipeline_output = tags.get("haystack.pipeline.output_data", None)
|
|
139
|
+
if pipeline_output:
|
|
140
|
+
span._span.update(output=tags["haystack.pipeline.output_data"])
|
|
141
|
+
|
|
142
|
+
span.raw_span().end()
|
|
143
|
+
self._context.pop()
|
|
144
|
+
|
|
145
|
+
if len(self._context) == 1:
|
|
146
|
+
# The root span has to be a trace, which need to be removed from the context after the pipeline run
|
|
147
|
+
self._context.pop()
|
|
148
|
+
|
|
149
|
+
if self.enforce_flush:
|
|
150
|
+
self.flush()
|
|
151
|
+
|
|
152
|
+
def flush(self):
|
|
153
|
+
self._tracer.flush()
|
|
154
|
+
|
|
155
|
+
def current_span(self) -> Span:
|
|
156
|
+
"""
|
|
157
|
+
Return the currently active span.
|
|
158
|
+
|
|
159
|
+
:return: The currently active span.
|
|
160
|
+
"""
|
|
161
|
+
if not self._context:
|
|
162
|
+
# The root span has to be a trace
|
|
163
|
+
self._context.append(LangfuseSpan(self._tracer.trace(name=self._name, public=self._public)))
|
|
164
|
+
return self._context[-1]
|
|
165
|
+
|
|
166
|
+
def get_trace_url(self) -> str:
|
|
167
|
+
"""
|
|
168
|
+
Return the URL to the tracing data.
|
|
169
|
+
:return: The URL to the tracing data.
|
|
170
|
+
"""
|
|
171
|
+
return self._tracer.get_trace_url()
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
4
|
+
|
|
5
|
+
from unittest.mock import Mock
|
|
6
|
+
from haystack.dataclasses import ChatMessage
|
|
7
|
+
from haystack_integrations.tracing.langfuse.tracer import LangfuseSpan
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TestLangfuseSpan:
|
|
11
|
+
|
|
12
|
+
# LangfuseSpan can be initialized with a span object
|
|
13
|
+
def test_initialized_with_span_object(self):
|
|
14
|
+
mock_span = Mock()
|
|
15
|
+
span = LangfuseSpan(mock_span)
|
|
16
|
+
assert span.raw_span() == mock_span
|
|
17
|
+
|
|
18
|
+
# set_tag method can update metadata of the span object
|
|
19
|
+
def test_set_tag_updates_metadata(self):
|
|
20
|
+
mock_span = Mock()
|
|
21
|
+
span = LangfuseSpan(mock_span)
|
|
22
|
+
|
|
23
|
+
span.set_tag("key", "value")
|
|
24
|
+
mock_span.update.assert_called_once_with(metadata={"key": "value"})
|
|
25
|
+
assert span._data["key"] == "value"
|
|
26
|
+
|
|
27
|
+
# set_content_tag method can update input and output of the span object
|
|
28
|
+
def test_set_content_tag_updates_input_and_output(self):
|
|
29
|
+
mock_span = Mock()
|
|
30
|
+
|
|
31
|
+
span = LangfuseSpan(mock_span)
|
|
32
|
+
span.set_content_tag("input_key", "input_value")
|
|
33
|
+
assert span._data["input_key"] == "input_value"
|
|
34
|
+
|
|
35
|
+
mock_span.reset_mock()
|
|
36
|
+
span.set_content_tag("output_key", "output_value")
|
|
37
|
+
assert span._data["output_key"] == "output_value"
|
|
38
|
+
|
|
39
|
+
# set_content_tag method can update input and output of the span object with messages/replies
|
|
40
|
+
def test_set_content_tag_updates_input_and_output_with_messages(self):
|
|
41
|
+
mock_span = Mock()
|
|
42
|
+
|
|
43
|
+
# test message input
|
|
44
|
+
span = LangfuseSpan(mock_span)
|
|
45
|
+
span.set_content_tag("key.input", {"messages": [ChatMessage.from_user("message")]})
|
|
46
|
+
assert mock_span.update.call_count == 1
|
|
47
|
+
# check we converted ChatMessage to OpenAI format
|
|
48
|
+
assert mock_span.update.call_args_list[0][1] == {"input": [{"role": "user", "content": "message"}]}
|
|
49
|
+
assert span._data["key.input"] == {"messages": [ChatMessage.from_user("message")]}
|
|
50
|
+
|
|
51
|
+
# test replies ChatMessage list
|
|
52
|
+
mock_span.reset_mock()
|
|
53
|
+
span.set_content_tag("key.output", {"replies": [ChatMessage.from_system("reply")]})
|
|
54
|
+
assert mock_span.update.call_count == 1
|
|
55
|
+
# check we converted ChatMessage to OpenAI format
|
|
56
|
+
assert mock_span.update.call_args_list[0][1] == {"output": [{"role": "system", "content": "reply"}]}
|
|
57
|
+
assert span._data["key.output"] == {"replies": [ChatMessage.from_system("reply")]}
|
|
58
|
+
|
|
59
|
+
# test replies string list
|
|
60
|
+
mock_span.reset_mock()
|
|
61
|
+
span.set_content_tag("key.output", {"replies": ["reply1", "reply2"]})
|
|
62
|
+
assert mock_span.update.call_count == 1
|
|
63
|
+
# check we handle properly string list replies
|
|
64
|
+
assert mock_span.update.call_args_list[0][1] == {"output": ["reply1", "reply2"]}
|
|
65
|
+
assert span._data["key.output"] == {"replies": ["reply1", "reply2"]}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from unittest.mock import Mock, MagicMock, patch
|
|
3
|
+
|
|
4
|
+
from haystack_integrations.tracing.langfuse.tracer import LangfuseTracer
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class TestLangfuseTracer:
|
|
8
|
+
|
|
9
|
+
# LangfuseTracer can be initialized with a Langfuse instance, a name and a boolean value for public.
|
|
10
|
+
def test_initialization(self):
|
|
11
|
+
langfuse_instance = Mock()
|
|
12
|
+
tracer = LangfuseTracer(tracer=langfuse_instance, name="Haystack", public=True)
|
|
13
|
+
assert tracer._tracer == langfuse_instance
|
|
14
|
+
assert tracer._context == []
|
|
15
|
+
assert tracer._name == "Haystack"
|
|
16
|
+
assert tracer._public
|
|
17
|
+
|
|
18
|
+
# check that the trace method is called on the tracer instance with the provided operation name and tags
|
|
19
|
+
# check that the span is added to the context and removed after the context manager exits
|
|
20
|
+
def test_create_new_span(self):
|
|
21
|
+
mock_raw_span = MagicMock()
|
|
22
|
+
mock_raw_span.operation_name = "operation_name"
|
|
23
|
+
mock_raw_span.metadata = {"tag1": "value1", "tag2": "value2"}
|
|
24
|
+
|
|
25
|
+
with patch("haystack_integrations.tracing.langfuse.tracer.LangfuseSpan") as MockLangfuseSpan:
|
|
26
|
+
mock_span_instance = MockLangfuseSpan.return_value
|
|
27
|
+
mock_span_instance.raw_span.return_value = mock_raw_span
|
|
28
|
+
|
|
29
|
+
mock_context_manager = MagicMock()
|
|
30
|
+
mock_context_manager.__enter__.return_value = mock_span_instance
|
|
31
|
+
|
|
32
|
+
mock_tracer = MagicMock()
|
|
33
|
+
mock_tracer.trace.return_value = mock_context_manager
|
|
34
|
+
|
|
35
|
+
tracer = LangfuseTracer(tracer=mock_tracer, name="Haystack", public=False)
|
|
36
|
+
|
|
37
|
+
with tracer.trace("operation_name", tags={"tag1": "value1", "tag2": "value2"}) as span:
|
|
38
|
+
assert len(tracer._context) == 2, "The trace span should have been added to the the root context span"
|
|
39
|
+
assert span.raw_span().operation_name == "operation_name"
|
|
40
|
+
assert span.raw_span().metadata == {"tag1": "value1", "tag2": "value2"}
|
|
41
|
+
|
|
42
|
+
assert (
|
|
43
|
+
len(tracer._context) == 0
|
|
44
|
+
), "The trace span should have been popped, and the root span is closed as well"
|
|
45
|
+
|
|
46
|
+
# check that update method is called on the span instance with the provided key value pairs
|
|
47
|
+
def test_update_span_with_pipeline_input_output_data(self):
|
|
48
|
+
class MockTracer:
|
|
49
|
+
|
|
50
|
+
def trace(self, name, **kwargs):
|
|
51
|
+
return MockSpan()
|
|
52
|
+
|
|
53
|
+
def flush(self):
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
class MockSpan:
|
|
57
|
+
def __init__(self):
|
|
58
|
+
self._data = {}
|
|
59
|
+
self._span = self
|
|
60
|
+
self.operation_name = "operation_name"
|
|
61
|
+
|
|
62
|
+
def raw_span(self):
|
|
63
|
+
return self
|
|
64
|
+
|
|
65
|
+
def span(self, name=None):
|
|
66
|
+
# assert correct operation name passed to the span
|
|
67
|
+
assert name == "operation_name"
|
|
68
|
+
return self
|
|
69
|
+
|
|
70
|
+
def update(self, **kwargs):
|
|
71
|
+
self._data.update(kwargs)
|
|
72
|
+
|
|
73
|
+
def generation(self, name=None):
|
|
74
|
+
return self
|
|
75
|
+
|
|
76
|
+
def end(self):
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
tracer = LangfuseTracer(tracer=MockTracer(), name="Haystack", public=False)
|
|
80
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}) as span:
|
|
81
|
+
assert span.raw_span()._data["metadata"] == {"haystack.pipeline.input_data": "hello"}
|
|
82
|
+
|
|
83
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.output_data": "bye"}) as span:
|
|
84
|
+
assert span.raw_span()._data["metadata"] == {"haystack.pipeline.output_data": "bye"}
|
|
85
|
+
|
|
86
|
+
def test_update_span_gets_flushed_by_default(self):
|
|
87
|
+
tracer_mock = Mock()
|
|
88
|
+
|
|
89
|
+
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
90
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}) as span:
|
|
91
|
+
pass
|
|
92
|
+
|
|
93
|
+
tracer_mock.flush.assert_called_once()
|
|
94
|
+
|
|
95
|
+
def test_update_span_flush_disable(self, monkeypatch):
|
|
96
|
+
monkeypatch.setenv("HAYSTACK_LANGFUSE_ENFORCE_FLUSH", "false")
|
|
97
|
+
tracer_mock = Mock()
|
|
98
|
+
|
|
99
|
+
from haystack_integrations.tracing.langfuse.tracer import LangfuseTracer
|
|
100
|
+
|
|
101
|
+
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
102
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}) as span:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
tracer_mock.flush.assert_not_called()
|
|
106
|
+
|
|
107
|
+
def test_context_is_empty_after_tracing(self):
|
|
108
|
+
tracer_mock = Mock()
|
|
109
|
+
|
|
110
|
+
tracer = LangfuseTracer(tracer=tracer_mock, name="Haystack", public=False)
|
|
111
|
+
with tracer.trace(operation_name="operation_name", tags={"haystack.pipeline.input_data": "hello"}) as span:
|
|
112
|
+
pass
|
|
113
|
+
|
|
114
|
+
assert tracer._context == []
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
# don't remove (or move) this env var setting from here, it's needed to turn tracing on
|
|
4
|
+
os.environ["HAYSTACK_CONTENT_TRACING_ENABLED"] = "true"
|
|
5
|
+
|
|
6
|
+
from urllib.parse import urlparse
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from haystack import Pipeline
|
|
12
|
+
from haystack.components.builders import DynamicChatPromptBuilder
|
|
13
|
+
from haystack.components.generators.chat import OpenAIChatGenerator
|
|
14
|
+
from haystack.dataclasses import ChatMessage
|
|
15
|
+
from requests.auth import HTTPBasicAuth
|
|
16
|
+
|
|
17
|
+
from haystack_integrations.components.connectors.langfuse import LangfuseConnector
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.mark.integration
|
|
21
|
+
@pytest.mark.skipif(
|
|
22
|
+
not os.environ.get("LANGFUSE_SECRET_KEY", None) and not os.environ.get("LANGFUSE_PUBLIC_KEY", None),
|
|
23
|
+
reason="Export an env var called LANGFUSE_SECRET_KEY and LANGFUSE_PUBLIC_KEY containing Langfuse credentials.",
|
|
24
|
+
)
|
|
25
|
+
def test_tracing_integration():
|
|
26
|
+
|
|
27
|
+
pipe = Pipeline()
|
|
28
|
+
pipe.add_component("tracer", LangfuseConnector(name="Chat example", public=True)) # public so anyone can verify run
|
|
29
|
+
pipe.add_component("prompt_builder", DynamicChatPromptBuilder())
|
|
30
|
+
pipe.add_component("llm", OpenAIChatGenerator(model="gpt-3.5-turbo"))
|
|
31
|
+
|
|
32
|
+
pipe.connect("prompt_builder.prompt", "llm.messages")
|
|
33
|
+
|
|
34
|
+
messages = [
|
|
35
|
+
ChatMessage.from_system("Always respond in German even if some input data is in other languages."),
|
|
36
|
+
ChatMessage.from_user("Tell me about {{location}}"),
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
response = pipe.run(
|
|
40
|
+
data={"prompt_builder": {"template_variables": {"location": "Berlin"}, "prompt_source": messages}}
|
|
41
|
+
)
|
|
42
|
+
assert "Berlin" in response["llm"]["replies"][0].content
|
|
43
|
+
assert response["tracer"]["trace_url"]
|
|
44
|
+
url = "https://cloud.langfuse.com/api/public/traces/"
|
|
45
|
+
trace_url = response["tracer"]["trace_url"]
|
|
46
|
+
parsed_url = urlparse(trace_url)
|
|
47
|
+
# trace id is the last part of the path (after the last '/')
|
|
48
|
+
uuid = os.path.basename(parsed_url.path)
|
|
49
|
+
try:
|
|
50
|
+
# GET request with Basic Authentication on the Langfuse API
|
|
51
|
+
response = requests.get(
|
|
52
|
+
url + uuid, auth=HTTPBasicAuth(os.environ.get("LANGFUSE_PUBLIC_KEY"), os.environ.get("LANGFUSE_SECRET_KEY"))
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
assert response.status_code == 200, f"Failed to retrieve data from Langfuse API: {response.status_code}"
|
|
56
|
+
except requests.exceptions.RequestException as e:
|
|
57
|
+
assert False, f"Failed to retrieve data from Langfuse API: {e}"
|
langfuse_haystack-0.0.3/PKG-INFO
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.3
|
|
2
|
-
Name: langfuse-haystack
|
|
3
|
-
Version: 0.0.3
|
|
4
|
-
Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse#readme
|
|
5
|
-
Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
|
|
6
|
-
Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/langfuse
|
|
7
|
-
Author-email: deepset GmbH <info@deepset.ai>
|
|
8
|
-
License-Expression: Apache-2.0
|
|
9
|
-
License-File: LICENSE.txt
|
|
10
|
-
Classifier: Development Status :: 4 - Beta
|
|
11
|
-
Classifier: Programming Language :: Python
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
-
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
-
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
19
|
-
Requires-Python: >=3.8
|
|
20
|
-
Requires-Dist: haystack-ai
|
|
21
|
-
Requires-Dist: langfuse
|
|
22
|
-
Description-Content-Type: text/markdown
|
|
23
|
-
|
|
24
|
-
# langfuse
|
|
25
|
-
|
|
26
|
-
[](https://pypi.org/project/langfuse-haystack)
|
|
27
|
-
[](https://pypi.org/project/langfuse-haystack)
|
|
28
|
-
|
|
29
|
-
-----
|
|
30
|
-
|
|
31
|
-
**Table of Contents**
|
|
32
|
-
|
|
33
|
-
- [langfuse](#langfuse)
|
|
34
|
-
- [Installation](#installation)
|
|
35
|
-
- [License](#license)
|
|
36
|
-
|
|
37
|
-
## Installation
|
|
38
|
-
|
|
39
|
-
```console
|
|
40
|
-
pip install langfuse-haystack
|
|
41
|
-
```
|
|
42
|
-
|
|
43
|
-
## License
|
|
44
|
-
|
|
45
|
-
`langfuse` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# langfuse
|
|
2
|
-
|
|
3
|
-
[](https://pypi.org/project/langfuse-haystack)
|
|
4
|
-
[](https://pypi.org/project/langfuse-haystack)
|
|
5
|
-
|
|
6
|
-
-----
|
|
7
|
-
|
|
8
|
-
**Table of Contents**
|
|
9
|
-
|
|
10
|
-
- [langfuse](#langfuse)
|
|
11
|
-
- [Installation](#installation)
|
|
12
|
-
- [License](#license)
|
|
13
|
-
|
|
14
|
-
## Installation
|
|
15
|
-
|
|
16
|
-
```console
|
|
17
|
-
pip install langfuse-haystack
|
|
18
|
-
```
|
|
19
|
-
|
|
20
|
-
## License
|
|
21
|
-
|
|
22
|
-
`langfuse` is distributed under the terms of the [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html) license.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.3"
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
from haystack import tracing, component
|
|
2
|
-
|
|
3
|
-
from haystack_integrations.tracing.langfuse import LangfuseTracer
|
|
4
|
-
|
|
5
|
-
from langfuse import Langfuse
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@component
|
|
9
|
-
class LangfuseComponent:
|
|
10
|
-
def __init__(self, name: str):
|
|
11
|
-
self.name = name
|
|
12
|
-
tracing.enable_tracing(LangfuseTracer(Langfuse(), name))
|
|
13
|
-
|
|
14
|
-
@component.output_types(name=str)
|
|
15
|
-
def run(self):
|
|
16
|
-
return {"name": self.name}
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
import contextlib
|
|
2
|
-
from typing import Any, Dict, Iterator, Optional, Union
|
|
3
|
-
|
|
4
|
-
from haystack.tracing import Span, Tracer, tracer
|
|
5
|
-
from haystack.tracing import utils as tracing_utils
|
|
6
|
-
|
|
7
|
-
import langfuse
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class LangfuseSpan(Span):
|
|
11
|
-
def __init__(self, span: "Union[langfuse.client.StatefulSpanClient, langfuse.client.StatefulTraceClient]") -> None:
|
|
12
|
-
self._span = span
|
|
13
|
-
# locally cache tags
|
|
14
|
-
self._data = {}
|
|
15
|
-
|
|
16
|
-
def set_tag(self, key: str, value: Any) -> None:
|
|
17
|
-
coerced_value = tracing_utils.coerce_tag_value(value)
|
|
18
|
-
self._span.update(metadata={key: coerced_value})
|
|
19
|
-
self._data[key] = value
|
|
20
|
-
|
|
21
|
-
def set_content_tag(self, key: str, value: Any) -> None:
|
|
22
|
-
if not tracer.is_content_tracing_enabled:
|
|
23
|
-
return
|
|
24
|
-
|
|
25
|
-
if key.endswith(".input"):
|
|
26
|
-
if "messages" in value:
|
|
27
|
-
messages = [m.to_openai_format() for m in value["messages"]]
|
|
28
|
-
self._span.update(input=messages)
|
|
29
|
-
else:
|
|
30
|
-
self._span.update(input=value)
|
|
31
|
-
elif key.endswith(".output"):
|
|
32
|
-
if "replies" in value:
|
|
33
|
-
replies = [m.to_openai_format() for m in value["replies"]]
|
|
34
|
-
self._span.update(output=replies)
|
|
35
|
-
else:
|
|
36
|
-
self._span.update(output=value)
|
|
37
|
-
|
|
38
|
-
self._data[key] = value
|
|
39
|
-
|
|
40
|
-
def raw_span(self) -> Any:
|
|
41
|
-
return self._span
|
|
42
|
-
|
|
43
|
-
def get_correlation_data_for_logs(self) -> Dict[str, Any]:
|
|
44
|
-
return {}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class LangfuseTracer(Tracer):
|
|
48
|
-
def __init__(self, tracer: "langfuse.Langfuse", name: str = "Haystack") -> None:
|
|
49
|
-
self._tracer = tracer
|
|
50
|
-
self._context: list[LangfuseSpan] = []
|
|
51
|
-
self._name = name
|
|
52
|
-
|
|
53
|
-
@contextlib.contextmanager
|
|
54
|
-
def trace(self, operation_name: str, tags: Optional[Dict[str, Any]] = None) -> Iterator[Span]:
|
|
55
|
-
tags = tags or {}
|
|
56
|
-
span_name = tags.get("haystack.component.name", operation_name)
|
|
57
|
-
|
|
58
|
-
if tags.get("haystack.component.type") in ["OpenAIGenerator", "OpenAIChatGenerator"]:
|
|
59
|
-
span = LangfuseSpan(self.current_span().raw_span().generation(name=span_name))
|
|
60
|
-
else:
|
|
61
|
-
span = LangfuseSpan(self.current_span().raw_span().span(name=span_name))
|
|
62
|
-
|
|
63
|
-
self._context.append(span)
|
|
64
|
-
span.set_tags(tags)
|
|
65
|
-
|
|
66
|
-
yield span
|
|
67
|
-
|
|
68
|
-
if tags.get("haystack.component.type") == "OpenAIGenerator":
|
|
69
|
-
meta = span._data.get("haystack.component.output", {}).get("meta")
|
|
70
|
-
if meta:
|
|
71
|
-
# Haystack returns one meta dict for each message, but the 'usage' value
|
|
72
|
-
# is always the same, let's just pick the first item
|
|
73
|
-
m = meta[0]
|
|
74
|
-
span._span.update(usage=m.get("usage"), model=m.get("model"))
|
|
75
|
-
elif tags.get("haystack.component.type") == "OpenAIChatGenerator":
|
|
76
|
-
replies = span._data.get("haystack.component.output", {}).get("replies")
|
|
77
|
-
if replies:
|
|
78
|
-
meta = replies[0].meta
|
|
79
|
-
span._span.update(usage=meta.get("usage"), model=meta.get("model"))
|
|
80
|
-
|
|
81
|
-
span.raw_span().end()
|
|
82
|
-
self._context.pop()
|
|
83
|
-
self._tracer.flush()
|
|
84
|
-
|
|
85
|
-
def current_span(self) -> Span:
|
|
86
|
-
if not self._context:
|
|
87
|
-
# The root span has to be a trace
|
|
88
|
-
self._context.append(LangfuseSpan(self._tracer.trace(name=self._name)))
|
|
89
|
-
return self._context[-1]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|