pi-langfuse 1.4.1 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.ts CHANGED
@@ -10,7 +10,7 @@
10
10
  import { basename } from "node:path";
11
11
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
12
12
 
13
- import { state, resetRunState } from "./src/state.js";
13
+ import { state, resetRunState, runWithSession, setCurrentSession } from "./src/state.js";
14
14
  import { ensureConfig, promptForConfig, loadConfig } from "./src/config.js";
15
15
  import { shutdownRuntime } from "./src/langfuse.js";
16
16
  import { getMessageFromEvent, extractAssistantOutput } from "./src/utils.js";
@@ -51,72 +51,79 @@ export default async function (pi: ExtensionAPI) {
51
51
  },
52
52
  });
53
53
 
54
- pi.on("session_start", async (_event, ctx) => {
54
+ const getSessionId = (ctx?: any) => {
55
+ try {
56
+ const sessionFile = ctx?.sessionManager?.getSessionFile?.();
57
+ return sessionFile ? basename(sessionFile, ".jsonl") : undefined;
58
+ } catch {
59
+ return undefined;
60
+ }
61
+ };
62
+
63
+ const withSession = <T>(ctx: any, fn: () => T): T => runWithSession(getSessionId(ctx) ?? state.currentSessionId, fn);
64
+
65
+ pi.on("session_start", async (_event, ctx) => withSession(ctx, async () => {
55
66
  state.setupAttemptedThisSession = false;
56
67
  await ensureConfig(ctx);
57
- const sessionFile = ctx.sessionManager.getSessionFile();
58
- if (sessionFile) {
59
- state.currentSessionId = basename(sessionFile, ".jsonl");
60
- }
61
68
  resetRunState();
62
- });
69
+ }));
63
70
 
64
- pi.on("model_select", async (event) => {
71
+ pi.on("model_select", async (event, ctx) => withSession(ctx, async () => {
65
72
  state.currentModel = event.model?.id || "";
66
73
  state.currentProvider = event.model?.provider || "";
67
- });
74
+ }));
68
75
 
69
- pi.on("before_agent_start", async (event, ctx) => {
76
+ pi.on("before_agent_start", async (event, ctx) => withSession(ctx, async () => {
70
77
  await startAgentRun(event, ctx);
71
- });
78
+ }));
72
79
 
73
- pi.on("agent_start", async (event, ctx) => {
80
+ pi.on("agent_start", async (event, ctx) => withSession(ctx, async () => {
74
81
  if (!state.agentState?.root) {
75
82
  await startAgentRun(event, ctx);
76
83
  }
77
- });
84
+ }));
78
85
 
79
- pi.on("turn_start", async (event) => {
86
+ pi.on("turn_start", async (event, ctx) => withSession(ctx, async () => {
80
87
  await startTurnObservation(event);
81
- });
88
+ }));
82
89
 
83
- pi.on("before_provider_request", async (event) => {
90
+ pi.on("before_provider_request", async (event, ctx) => withSession(ctx, async () => {
84
91
  await startGeneration(event);
85
- });
92
+ }));
86
93
 
87
- pi.on("after_provider_response", async (event) => {
94
+ pi.on("after_provider_response", async (event, ctx) => withSession(ctx, async () => {
88
95
  updateGenerationMetadata(event);
89
- });
96
+ }));
90
97
 
91
- pi.on("message_update", async (event) => {
98
+ pi.on("message_update", async (event, ctx) => withSession(ctx, async () => {
92
99
  recordTTFT(event);
93
100
  const message = getMessageFromEvent(event);
94
101
  if (message?.role === "assistant" && state.agentState) {
95
102
  state.agentState.latestAssistantOutput = extractAssistantOutput(message);
96
103
  }
97
- });
104
+ }));
98
105
 
99
- pi.on("message_end", async (event) => {
106
+ pi.on("message_end", async (event, ctx) => withSession(ctx, async () => {
100
107
  await finishGenerationFromMessage(event);
101
- });
108
+ }));
102
109
 
103
- pi.on("tool_execution_start", async (event) => {
110
+ pi.on("tool_execution_start", async (event, ctx) => withSession(ctx, async () => {
104
111
  await startToolObservation(event);
105
- });
112
+ }));
106
113
 
107
- pi.on("tool_call", async (event) => {
114
+ pi.on("tool_call", async (event, ctx) => withSession(ctx, async () => {
108
115
  await startToolObservation(event);
109
- });
116
+ }));
110
117
 
111
- pi.on("tool_result", async (event) => {
118
+ pi.on("tool_result", async (event, ctx) => withSession(ctx, async () => {
112
119
  await finishToolObservation(event);
113
- });
120
+ }));
114
121
 
115
- pi.on("tool_execution_end", async (event) => {
122
+ pi.on("tool_execution_end", async (event, ctx) => withSession(ctx, async () => {
116
123
  await finishToolObservation(event);
117
- });
124
+ }));
118
125
 
119
- pi.on("turn_end", async (event) => {
126
+ pi.on("turn_end", async (event, ctx) => withSession(ctx, async () => {
120
127
  state.turnCount++;
121
128
  const message = getMessageFromEvent(event);
122
129
  if (message?.role === "assistant") {
@@ -124,16 +131,16 @@ export default async function (pi: ExtensionAPI) {
124
131
  await finishGenerationFromMessage(event);
125
132
  }
126
133
  finishTurnObservation(event);
127
- });
134
+ }));
128
135
 
129
- pi.on("agent_end", async (event) => {
136
+ pi.on("agent_end", async (event, ctx) => withSession(ctx, async () => {
130
137
  await finishAgentRun(event);
131
138
  setTimeout(() => {
132
139
  shutdownRuntime().catch((error) => {
133
140
  console.warn("📊 Langfuse: Deferred shutdown failed", error);
134
141
  });
135
142
  }, 0);
136
- });
143
+ }));
137
144
 
138
145
  const handleSessionInterruption = (reason: string) => {
139
146
  if (state.agentState?.root) {
@@ -143,15 +150,21 @@ export default async function (pi: ExtensionAPI) {
143
150
  resetRunState();
144
151
  };
145
152
 
146
- pi.on("session_before_switch", async () => {
147
- handleSessionInterruption("Session switched");
153
+ pi.on("session_before_switch", async (_event, ctx) => {
154
+ const sessionId = getSessionId(ctx);
155
+ if (sessionId) {
156
+ setCurrentSession(sessionId);
157
+ }
148
158
  });
149
159
 
150
- pi.on("session_before_fork", async () => {
151
- handleSessionInterruption("Session forked");
160
+ pi.on("session_before_fork", async (_event, ctx) => {
161
+ const sessionId = getSessionId(ctx);
162
+ if (sessionId) {
163
+ setCurrentSession(sessionId);
164
+ }
152
165
  });
153
166
 
154
- pi.on("session_compact", async (event) => {
167
+ pi.on("session_compact", async (event, ctx) => withSession(ctx, async () => {
155
168
  if (state.agentState?.root) {
156
169
  const parent = state.agentState.activeTurn ?? state.agentState.root;
157
170
  try {
@@ -169,10 +182,10 @@ export default async function (pi: ExtensionAPI) {
169
182
  // ignore
170
183
  }
171
184
  }
172
- });
185
+ }));
173
186
 
174
- pi.on("session_shutdown", async () => {
187
+ pi.on("session_shutdown", async (_event, ctx) => withSession(ctx, async () => {
175
188
  handleSessionInterruption("Session shutdown before agent completed");
176
189
  await shutdownRuntime();
177
- });
190
+ }));
178
191
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-langfuse",
3
- "version": "1.4.1",
3
+ "version": "1.4.2",
4
4
  "description": "Langfuse extension for Pi coding agent",
5
5
  "repository": {
6
6
  "type": "git",
@@ -12,6 +12,16 @@
12
12
  "homepage": "https://github.com/gooyoung/pi-langfuse#readme",
13
13
  "type": "module",
14
14
  "main": "index.ts",
15
+ "files": [
16
+ "index.ts",
17
+ "src/",
18
+ "types/",
19
+ "README.md",
20
+ "README_CN.md",
21
+ "image.png",
22
+ "skills-lock.json",
23
+ "tsconfig.json"
24
+ ],
15
25
  "scripts": {
16
26
  "typecheck": "tsc --noEmit"
17
27
  },
package/src/state.ts CHANGED
@@ -1,38 +1,141 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
1
2
  import type { Config, AgentState } from "./types.js";
2
3
 
4
+ export interface SessionRunState {
5
+ currentModel: string;
6
+ currentProvider: string;
7
+ agentState: AgentState | null;
8
+ toolCallCount: number;
9
+ errorCount: number;
10
+ turnCount: number;
11
+ }
12
+
13
+ const DEFAULT_SESSION_ID = "__pi_langfuse_default_session__";
14
+
15
+ let activeSessionId = DEFAULT_SESSION_ID;
16
+ const sessionScope = new AsyncLocalStorage<string>();
17
+
18
+ function createSessionRunState(): SessionRunState {
19
+ return {
20
+ currentModel: "",
21
+ currentProvider: "",
22
+ agentState: null,
23
+ toolCallCount: 0,
24
+ errorCount: 0,
25
+ turnCount: 0,
26
+ };
27
+ }
28
+
29
+ function normalizeSessionId(sessionId?: string) {
30
+ return sessionId || DEFAULT_SESSION_ID;
31
+ }
32
+
33
+ function getActiveSessionId() {
34
+ return sessionScope.getStore() ?? activeSessionId;
35
+ }
36
+
37
+ export function getSessionRunState(sessionId = getActiveSessionId()): SessionRunState {
38
+ const normalizedSessionId = normalizeSessionId(sessionId);
39
+ let sessionState = state.sessionStates.get(normalizedSessionId);
40
+ if (!sessionState) {
41
+ sessionState = createSessionRunState();
42
+ state.sessionStates.set(normalizedSessionId, sessionState);
43
+ }
44
+ return sessionState;
45
+ }
46
+
47
+ export function setCurrentSession(sessionId?: string) {
48
+ activeSessionId = normalizeSessionId(sessionId);
49
+ getSessionRunState(activeSessionId);
50
+ }
51
+
52
+ export function runWithSession<T>(sessionId: string | undefined, fn: () => T): T {
53
+ const normalizedSessionId = normalizeSessionId(sessionId);
54
+ setCurrentSession(normalizedSessionId);
55
+ return sessionScope.run(normalizedSessionId, fn);
56
+ }
57
+
3
58
  export const state = {
4
59
  config: null as Config | null,
5
60
  setupAttemptedThisSession: false,
6
-
7
- currentSessionId: "",
8
- currentModel: "",
9
- currentProvider: "",
10
- agentState: null as AgentState | null,
11
-
12
- // Evaluation tracking state
13
- toolCallCount: 0,
14
- errorCount: 0,
15
- turnCount: 0,
61
+ sessionStates: new Map<string, SessionRunState>(),
62
+
63
+ get currentSessionId() {
64
+ const sessionId = getActiveSessionId();
65
+ return sessionId === DEFAULT_SESSION_ID ? "" : sessionId;
66
+ },
67
+ set currentSessionId(sessionId: string) {
68
+ setCurrentSession(sessionId);
69
+ },
70
+
71
+ get currentModel() {
72
+ return getSessionRunState().currentModel;
73
+ },
74
+ set currentModel(model: string) {
75
+ getSessionRunState().currentModel = model;
76
+ },
77
+
78
+ get currentProvider() {
79
+ return getSessionRunState().currentProvider;
80
+ },
81
+ set currentProvider(provider: string) {
82
+ getSessionRunState().currentProvider = provider;
83
+ },
84
+
85
+ get agentState() {
86
+ return getSessionRunState().agentState;
87
+ },
88
+ set agentState(agentState: AgentState | null) {
89
+ getSessionRunState().agentState = agentState;
90
+ },
91
+
92
+ get toolCallCount() {
93
+ return getSessionRunState().toolCallCount;
94
+ },
95
+ set toolCallCount(toolCallCount: number) {
96
+ getSessionRunState().toolCallCount = toolCallCount;
97
+ },
98
+
99
+ get errorCount() {
100
+ return getSessionRunState().errorCount;
101
+ },
102
+ set errorCount(errorCount: number) {
103
+ getSessionRunState().errorCount = errorCount;
104
+ },
105
+
106
+ get turnCount() {
107
+ return getSessionRunState().turnCount;
108
+ },
109
+ set turnCount(turnCount: number) {
110
+ getSessionRunState().turnCount = turnCount;
111
+ },
16
112
  };
17
113
 
18
- export function resetRunState() {
19
- state.agentState = null;
20
- state.toolCallCount = 0;
21
- state.errorCount = 0;
22
- state.turnCount = 0;
23
- state.currentModel = "";
24
- state.currentProvider = "";
114
+ export function resetRunState(sessionId = getActiveSessionId()) {
115
+ state.sessionStates.set(normalizeSessionId(sessionId), createSessionRunState());
25
116
  }
26
117
 
27
- export function computeEvaluationScores() {
28
- const toolSuccessRate = state.toolCallCount > 0 ? (state.toolCallCount - state.errorCount) / state.toolCallCount : 1;
29
- const sessionHadErrors = state.errorCount > 0;
118
+ export function clearAllSessionStates() {
119
+ state.sessionStates.clear();
120
+ activeSessionId = DEFAULT_SESSION_ID;
121
+ getSessionRunState();
122
+ }
123
+
124
+ export function computeEvaluationScores(sessionId = getActiveSessionId()) {
125
+ const sessionState = getSessionRunState(sessionId);
126
+ const toolSuccessRate =
127
+ sessionState.toolCallCount > 0
128
+ ? (sessionState.toolCallCount - sessionState.errorCount) / sessionState.toolCallCount
129
+ : 1;
130
+ const sessionHadErrors = sessionState.errorCount > 0;
30
131
 
31
132
  return {
32
- tool_call_count: state.toolCallCount,
33
- turn_count: state.turnCount,
34
- total_tool_errors: state.errorCount,
133
+ tool_call_count: sessionState.toolCallCount,
134
+ turn_count: sessionState.turnCount,
135
+ total_tool_errors: sessionState.errorCount,
35
136
  tool_success_rate: toolSuccessRate,
36
137
  session_had_errors: sessionHadErrors ? 1 : 0,
37
138
  };
38
139
  }
140
+
141
+ getSessionRunState();
@@ -1,140 +0,0 @@
1
- ---
2
- name: langfuse
3
- description: Interact with Langfuse and access its documentation. Use when needing to (1) query or modify Langfuse data programmatically via the CLI — traces, prompts, datasets, scores, sessions, and any other API resource, (2) look up Langfuse documentation, concepts, integration guides, or SDK usage, or (3) understand how any Langfuse feature works. This skill covers CLI-based API access (via npx) and multiple documentation retrieval methods.
4
- allowed-tools:
5
- - WebFetch(domain:langfuse.com)
6
- - Bash(curl *langfuse.com/*)
7
- - Bash(npx langfuse-cli api __schema *)
8
- - Bash(npx langfuse-cli api * --help *)
9
- - Bash(npx langfuse-cli api * list *)
10
- - Bash(npx langfuse-cli api * get *)
11
- - Bash(bunx langfuse-cli api __schema *)
12
- - Bash(bunx langfuse-cli api * --help *)
13
- - Bash(bunx langfuse-cli api * list *)
14
- - Bash(bunx langfuse-cli api * get *)
15
- ---
16
-
17
- # Langfuse
18
-
19
- This skill helps you use Langfuse effectively across all common workflows: instrumenting applications, migrating prompts, debugging traces, and accessing data programmatically.
20
-
21
- ## Core Principles
22
-
23
- Follow these principles for ALL Langfuse work:
24
-
25
- 1. **Documentation First**: NEVER implement based on memory. Always fetch current docs before writing code (Langfuse updates frequently) See the section below on how to access documentation.
26
- 2. **CLI for Data Access**: Use `langfuse-cli` when querying/modifying Langfuse data. See the section below on how to use the CLI.
27
- 3. **Best Practices by Use Case**: Check the relevant reference file below for use-case-specific guidelines before implementing
28
- 4. **Use latest Langfuse versions**: Unless the user specified otherwise or there's a good reason, always use the latest version of Langfuse SDKs/APIs.
29
-
30
-
31
- ## Use case specific references
32
-
33
- - instrumenting an existing function/application: references/instrumentation.md
34
- - migrating prompts from a codebase into Langfuse: references/prompt-migration.md
35
- - capturing user feedback (thumbs, ratings, implicit signals) as scores on traces: references/user-feedback.md
36
- - further tips on using the Langfuse CLI: references/cli.md
37
- - upgrading or migrating Langfuse SDKs to the latest version: references/sdk-upgrade.md
38
- - systematic error analysis — reading traces, building failure taxonomy, deciding what to fix: references/error-analysis.md
39
- - submitting feedback about this skill: references/skill-feedback.md
40
-
41
- ## 1. Langfuse API via CLI
42
-
43
- Use the `langfuse-cli` to interact with the full Langfuse REST API from the command line. Run via npx (no install required):
44
-
45
- Start by discovering the schema and available arguments:
46
-
47
- ```bash
48
- # Discover all available resources
49
- npx langfuse-cli api __schema
50
-
51
- # List actions for a resource
52
- npx langfuse-cli api <resource> --help
53
-
54
- # Show args/options for a specific action
55
- npx langfuse-cli api <resource> <action> --help
56
- ```
57
-
58
- ### Credentials
59
-
60
- Set environment variables before making calls:
61
-
62
- ```bash
63
- export LANGFUSE_PUBLIC_KEY=pk-lf-...
64
- export LANGFUSE_SECRET_KEY=sk-lf-...
65
- export LANGFUSE_HOST=https://cloud.langfuse.com # example for EU cloud. For US cloud it's us.cloud.langfuse.com, and can also be a self-hosted URL. The server must always be specified in order to access Langfuse.
66
- ```
67
-
68
- If not set, ask the user to set them in their shell or a `.env` file (do not ask them to paste keys into chat for security reasons). Keys are found in Langfuse UI → Settings → API Keys.
69
-
70
- ### Detailed CLI Reference
71
-
72
- For common workflows, tips, and full usage patterns, see [references/cli.md](references/cli.md).
73
-
74
- ## 2. Langfuse Documentation
75
-
76
- Three methods to access Langfuse docs, in order of preference. **Always prefer your application's native web fetch and search tools** (e.g., `WebFetch`, `WebSearch`, `mcp_fetch`, etc.) over `curl` when available. The URLs and patterns below work with any fetching method — the `curl` examples are just illustrative.
77
-
78
- ### 2a. Documentation Index (llms.txt)
79
-
80
- Fetch the full index of all documentation pages:
81
-
82
- ```bash
83
- curl -s https://langfuse.com/llms.txt
84
- ```
85
-
86
- Returns a structured list of every doc page with titles and URLs. Use this to discover the right page for a topic, then fetch that page directly.
87
-
88
- Alternatively, you can start on `https://langfuse.com/docs` and explore the site to find the page you need.
89
-
90
- ### 2b. Fetch Individual Pages as Markdown
91
-
92
- Any page listed in llms.txt can be fetched as markdown by appending `.md` to its path or by using `Accept: text/markdown` in the request headers. Use this when you know which page contains the information needed. Returns clean markdown with code examples and configuration details.
93
-
94
- ```bash
95
- curl -s "https://langfuse.com/docs/observability/overview.md"
96
- curl -s "https://langfuse.com/docs/observability/overview" -H "Accept: text/markdown"
97
- ```
98
-
99
- ### 2c. Search Documentation
100
-
101
- When you need to find information across all docs and github issues/discussions without knowing the specific page:
102
-
103
- ```bash
104
- curl -s "https://langfuse.com/api/search-docs?query=<url-encoded-query>"
105
- ```
106
-
107
- Example:
108
-
109
- ```bash
110
- curl -s "https://langfuse.com/api/search-docs?query=How+do+I+trace+LangGraph+agents"
111
- ```
112
-
113
- Returns a JSON response with:
114
-
115
- - `query`: the original query
116
- - `answer`: a JSON string containing an array of matching documents, each with:
117
- - `url`: link to the doc page
118
- - `title`: page title
119
- - `source.content`: array of relevant text excerpts from the page
120
-
121
- Search is a great fallback if you cannot find the relevant pages or need more context. Especially useful when debugging issues as all GitHub Issues and Discussions are also indexed. Responses can be large — extract only the relevant portions.
122
-
123
- ### Documentation Workflow
124
-
125
- 1. Start with **llms.txt** to orient — scan for relevant page titles
126
- 2. **Fetch specific pages** when you identify the right one
127
- 3. Fall back to **search** when the topic is unclear and you want more context
128
-
129
- ## Skill Feedback
130
-
131
- When the user expresses that something about this skill is not working as expected, gives incorrect guidance, is missing information, or could be improved — offer to submit feedback to the Langfuse skill maintainers. This includes when:
132
-
133
- - The skill gave wrong or outdated instructions
134
- - A workflow didn't produce the expected result
135
- - The user wishes the skill covered something it doesn't
136
- - The user explicitly says something like "this should work differently" or "this is wrong"
137
-
138
- **Do NOT trigger this** for issues with Langfuse itself (the product) — only for issues with this skill's instructions and behavior.
139
-
140
- When triggered, follow the process in [references/skill-feedback.md](references/skill-feedback.md).
@@ -1,51 +0,0 @@
1
- # Langfuse CLI Reference
2
-
3
- Documentation: https://langfuse.com/docs/api-and-data-platform/features/cli
4
-
5
- ## Install
6
-
7
- ```bash
8
- # Run directly (recommended)
9
- npx langfuse-cli api <resource> <action>
10
- bunx langfuse-cli api <resource> <action>
11
-
12
- # Or install globally
13
- npm i -g langfuse-cli
14
- langfuse api <resource> <action>
15
- ```
16
-
17
- ## Discovery
18
-
19
- ```bash
20
- # List all resources and auth info
21
- langfuse api __schema
22
-
23
- # List actions for a resource
24
- langfuse api <resource> --help
25
-
26
- # Show args/options for a specific action
27
- langfuse api <resource> <action> --help
28
-
29
- # Preview the curl command without executing
30
- langfuse api <resource> <action> --curl
31
- ```
32
-
33
- ## Credentials
34
-
35
- Set environment variables:
36
-
37
- ```bash
38
- export LANGFUSE_PUBLIC_KEY=pk-lf-...
39
- export LANGFUSE_SECRET_KEY=sk-lf-...
40
- export LANGFUSE_HOST=https://cloud.langfuse.com
41
- ```
42
-
43
- ## Tips
44
-
45
- - Use `--json` for machine-readable JSON output
46
- - Use `--curl` to preview the HTTP request without executing
47
- - Pagination: use `--limit` and `--page` on list endpoints
48
- - All list commands support filtering — check `<resource> <action> --help` for available options
49
- - Prefer `observations-v2s` over `observations` — the v2 endpoint returns richer data
50
- - Prefer `metrics-v2s` over `metrics` — the v2 endpoint returns richer data
51
- - Prefer `score-v2s` over `scores` — the v1 `scores` resource only supports create/delete; use `score-v2s` for list and get operations
@@ -1,100 +0,0 @@
1
- ---
2
- name: langfuse-error-analysis
3
- description: Deep-dive error analysis of an LLM pipeline or AI application using Langfuse traces.
4
- Use this skill whenever the user wants to understand why their AI system is producing
5
- bad outputs, where their pipeline is failing, how to categorise or label failures,
6
- what to prioritise fixing, or how to set up evaluators. Also trigger for "review my
7
- traces", "my outputs look wrong", "help me debug my LLM app", "I want to analyse
8
- errors", "build a failure taxonomy", "what's going wrong with my pipeline", or any
9
- request to systematically inspect, annotate, or score Langfuse traces. If the user
10
- is trying to understand or improve the quality of an AI system's outputs, use this skill.
11
- ---
12
-
13
- # Error Analysis
14
-
15
- ## Primary Guide
16
-
17
- **1. Fetch the guide in this blogpost**
18
-
19
- https://langfuse.com/guides/cookbook/error-analysis-llm-applications.md
20
-
21
- If fetch is not available query for langfuse.com error analysis guide
22
-
23
- Read it in full. It defines the authoritative 5-step process (sample selection → open coding → clustering → labelling → deciding what to fix).
24
-
25
- **2. Guide the user through this step by step**
26
-
27
- You as a coding agent and the user go through this together to perform a full error analysis with their data in langfuse. Do everything you can achieve via CLI (look up traces, create annotation queues, ...) for the user. Provide them with direct links to UI wherever their action is required. Be proactive and narrate what is going on for the user.
28
-
29
- ## Rules CRITICAL
30
- Use Langfuse CLI wherever possible
31
- Use charts where possible to display data
32
-
33
- ---
34
-
35
- ## Langfuse Implementation Notes
36
-
37
- The guide describes the process. These notes cover the Langfuse-specific API and CLI mechanics required to execute it.
38
-
39
- ### Credentials
40
-
41
- ```bash
42
- echo $LANGFUSE_PUBLIC_KEY # pk-lf-...
43
- echo $LANGFUSE_SECRET_KEY # sk-lf-...
44
- echo $LANGFUSE_HOST # https://cloud.langfuse.com (EU), https://us.cloud.langfuse.com (US), https://jp.cloud.langfuse.com (JP) or self-hosted
45
- ```
46
-
47
- If not set, check `.env` in the project root: `export $(grep -v '^#' .env | xargs)`. If `LANGFUSE_BASE_URL` is used instead of `LANGFUSE_HOST`, run `export LANGFUSE_HOST="$LANGFUSE_BASE_URL"`.
48
-
49
- ```bash
50
- AUTH=$(echo -n "${LANGFUSE_PUBLIC_KEY}:${LANGFUSE_SECRET_KEY}" | base64)
51
-
52
- # Verify before proceeding
53
- STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
54
- -H "Authorization: Basic $AUTH" \
55
- "${LANGFUSE_HOST}/api/public/projects")
56
- echo "Auth check: $STATUS"
57
- ```
58
-
59
- If status is not `200`, stop and ask the user to check their credentials and host before continuing.
60
-
61
- ### Annotation target: OBSERVATION not TRACE
62
-
63
- > **CRITICAL:** In OpenTelemetry-instrumented apps, trace-level `input`/`output` can be null — content lives in a GENERATION observation. Always add `objectType: OBSERVATION` pointing to the GENERATION observation ID to annotation queues. Adding `objectType: TRACE` shows nothing in the UI.
64
-
65
- ### Annotation queues
66
-
67
- > **CRITICAL:** Queues cannot be updated or deleted after creation. Create score configs first, then the queue with all config IDs. To add new configs later, create a new queue.
68
-
69
-
70
- **Always give the user a direct link immediately after creating a queue:**
71
-
72
- | Host | URL pattern |
73
- |------|-------------|
74
- | EU cloud | `https://cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |
75
- | US cloud | `https://us.cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |
76
- | Self-hosted | `<LANGFUSE_HOST>/project/<projectId>/annotation-queues/<queueId>` |
77
-
78
- Instruction to give: *"Please open code the first ~50 examples. For each trace, write what you observe in the `open_coding` field (describe behaviour, don't diagnose root causes), then set `pass_fail_assessment` to Pass or Fail."*
79
-
80
-
81
- ### Prompt fixes
82
-
83
- When a category warrants a prompt fix, always offer the user two options:
84
- 1. Create it as a versioned prompt in Langfuse (tracked, usable via the prompt API)
85
- 2. Draft the specific text change for them to review and apply
86
-
87
- ### Setup evaluators
88
-
89
- When a category warrants an evaluator setup, propose the type of evaluator and offer to set it up for user via CLI
90
-
91
-
92
- ### Common gotchas
93
-
94
- | Mistake | Fix |
95
- |---------|-----|
96
- | `objectType: TRACE` in queue | Use `objectType: OBSERVATION` with GENERATION obs ID |
97
- | Creating score config without checking existing | `GET /api/public/score-configs` first; can't delete |
98
- | Queue created before score configs | Create configs → collect IDs → create queue |
99
- | `--limit` > 100 on traces list | API hard cap; paginate with `--page` |
100
- | No rate limiting on queue item creation | `sleep 0.4` between calls to avoid 429 |