pi-langfuse 1.0.0 → 1.2.0

package/src/handlers/turn.ts

@@ -0,0 +1,53 @@
+import { state } from "../state.js";
+import { getRuntime } from "../langfuse.js";
+import { shapePayload } from "../utils.js";
+
+export async function startTurnObservation(event: Record<string, unknown>) {
+  if (!state.agentState?.root) {
+    return;
+  }
+
+  // If a turn is already active, close it (fallback safety)
+  if (state.agentState.activeTurn) {
+    state.agentState.activeTurn.end();
+    state.agentState.activeTurn = undefined;
+  }
+
+  try {
+    const turnIndex = event.turnIndex ?? state.turnCount;
+    const observation = state.agentState.root.startObservation
+      ? state.agentState.root.startObservation(
+          "turn",
+          {
+            input: shapePayload(event.context ?? event),
+            metadata: { turnIndex },
+          },
+          { asType: "span" },
+        )
+      : (await getRuntime()).startObservation(
+          "turn",
+          {
+            input: shapePayload(event.context ?? event),
+            metadata: { turnIndex },
+          },
+          { asType: "span" },
+        );
+
+    state.agentState.activeTurn = observation;
+  } catch (e) {
+    console.warn("📊 Langfuse: Failed to start turn observation", e);
+  }
+}
+
+export function finishTurnObservation(event?: Record<string, unknown>) {
+  if (!state.agentState?.activeTurn) {
+    return;
+  }
+
+  try {
+    state.agentState.activeTurn.end();
+    state.agentState.activeTurn = undefined;
+  } catch (e) {
+    console.warn("📊 Langfuse: Failed to finish turn observation", e);
+  }
+}

package/src/langfuse.ts

@@ -0,0 +1,75 @@
+import type { LangfuseRuntime, LangfuseScoreClient } from "./types.js";
+import { state } from "./state.js";
+
+let runtime: LangfuseRuntime | null = null;
+
+export async function getRuntime(): Promise<LangfuseRuntime> {
+  if (!state.config) {
+    throw new Error("Langfuse config is not set");
+  }
+
+  if (!runtime) {
+    const [{ NodeSDK }, { LangfuseSpanProcessor }, tracing, { LangfuseClient }] = await Promise.all([
+      import("@opentelemetry/sdk-node"),
+      import("@langfuse/otel"),
+      import("@langfuse/tracing"),
+      import("@langfuse/client"),
+    ]);
+
+    const spanProcessor = new LangfuseSpanProcessor({
+      publicKey: state.config.publicKey,
+      secretKey: state.config.secretKey,
+      baseUrl: state.config.host,
+    });
+    const sdk = new NodeSDK({ spanProcessors: [spanProcessor] });
+    sdk.start();
+
+    runtime = {
+      startObservation: tracing.startObservation as unknown as LangfuseRuntime["startObservation"],
+      propagateAttributes: tracing.propagateAttributes as unknown as LangfuseRuntime["propagateAttributes"],
+      scoreClient: new LangfuseClient({
+        publicKey: state.config.publicKey,
+        secretKey: state.config.secretKey,
+        baseUrl: state.config.host,
+      }) as LangfuseScoreClient,
+      spanProcessor,
+      sdk,
+    };
+  }
+
+  return runtime as LangfuseRuntime;
+}
+
+export async function shutdownRuntime(): Promise<void> {
+  if (!runtime) {
+    return;
+  }
+
+  try {
+    await runtime.scoreClient.flush?.();
+    await runtime.scoreClient.shutdown?.();
+    await runtime.spanProcessor?.forceFlush?.();
+    await runtime.spanProcessor?.shutdown?.();
+    await runtime.sdk?.shutdown?.();
+  } catch (e) {
+    console.warn("📊 Langfuse: Failed to flush/shutdown cleanly", e);
+  } finally {
+    runtime = null;
+  }
+}
+
+export async function sendScore(name: string, value: number, options: { traceId?: string; observationId?: string } = {}) {
+  try {
+    const rt = await getRuntime();
+    rt.scoreClient.score?.create({
+      name,
+      value,
+      dataType: name === "session_had_errors" || name === "tool_is_error" ? "BOOLEAN" : "NUMERIC",
+      traceId: options.traceId,
+      observationId: options.observationId,
+      sessionId: options.traceId ? undefined : state.currentSessionId || undefined,
+    });
+  } catch (e) {
+    console.warn(`📊 Langfuse: Failed to send score ${name}`, e);
+  }
+}

package/src/state.ts

@@ -0,0 +1,38 @@
+import type { Config, AgentState } from "./types.js";
+
+export const state = {
+  config: null as Config | null,
+  setupAttemptedThisSession: false,
+  
+  currentSessionId: "",
+  currentModel: "",
+  currentProvider: "",
+  agentState: null as AgentState | null,
+  
+  // Evaluation tracking state
+  toolCallCount: 0,
+  errorCount: 0,
+  turnCount: 0,
+};
+
+export function resetRunState() {
+  state.agentState = null;
+  state.toolCallCount = 0;
+  state.errorCount = 0;
+  state.turnCount = 0;
+  state.currentModel = "";
+  state.currentProvider = "";
+}
+
+export function computeEvaluationScores() {
+  const toolSuccessRate = state.toolCallCount > 0 ? (state.toolCallCount - state.errorCount) / state.toolCallCount : 1;
+  const sessionHadErrors = state.errorCount > 0;
+
+  return {
+    tool_call_count: state.toolCallCount,
+    turn_count: state.turnCount,
+    total_tool_errors: state.errorCount,
+    tool_success_rate: toolSuccessRate,
+    session_had_errors: sessionHadErrors ? 1 : 0,
+  };
+}

package/src/types.ts

@@ -0,0 +1,94 @@
+export interface Config {
+  publicKey: string;
+  secretKey: string;
+  host: string;
+}
+
+export interface LangfuseObservation {
+  id?: string;
+  traceId?: string;
+  update(body?: ObservationUpdate): LangfuseObservation;
+  end(body?: ObservationUpdate): void;
+  startObservation?(
+    name: string,
+    body?: ObservationUpdate,
+    options?: { asType?: "agent" | "generation" | "tool" | "span" },
+  ): LangfuseObservation;
+  setTraceIO?(body?: { input?: unknown; output?: unknown }): void;
+}
+
+export interface ObservationUpdate {
+  input?: unknown;
+  output?: unknown;
+  metadata?: Record<string, unknown>;
+  model?: string;
+  usageDetails?: Record<string, number>;
+  usage?: Record<string, number>;
+  costDetails?: Record<string, number>;
+  level?: "DEBUG" | "DEFAULT" | "WARNING" | "ERROR";
+  statusMessage?: string;
+  completionStartTime?: Date;
+}
+
+export interface LangfuseScoreClient {
+  score?: {
+    create(body: {
+      traceId?: string;
+      sessionId?: string;
+      observationId?: string;
+      name: string;
+      value: number;
+      dataType?: "NUMERIC" | "BOOLEAN";
+    }): unknown;
+  };
+  flush?: () => Promise<void>;
+  shutdown?: () => Promise<void>;
+}
+
+export interface LangfuseRuntime {
+  startObservation: (
+    name: string,
+    body?: ObservationUpdate,
+    options?: { asType?: "agent" | "generation" | "tool" | "span" },
+  ) => LangfuseObservation;
+  propagateAttributes: (
+    params: {
+      sessionId?: string;
+      traceName?: string;
+      metadata?: Record<string, string>;
+      tags?: string[];
+    },
+    fn: () => LangfuseObservation,
+  ) => LangfuseObservation;
+  scoreClient: LangfuseScoreClient;
+  spanProcessor?: { forceFlush?: () => Promise<void>; shutdown?: () => Promise<void> };
+  sdk?: { start?: () => void; shutdown?: () => Promise<void> };
+}
+
+export interface GenerationState {
+  observation: LangfuseObservation;
+  requestKey: string;
+  ended: boolean;
+  metadata: Record<string, unknown>;
+  ttftRecorded?: boolean;
+}
+
+export interface ToolState {
+  observation: LangfuseObservation;
+  toolName: string;
+  ended: boolean;
+}
+
+export interface AgentState {
+  root?: LangfuseObservation;
+  activeTurn?: LangfuseObservation;
+  traceId?: string;
+  promptInput?: unknown;
+  cwd?: string;
+  generationSeq: number;
+  activeGenerations: Map<string, GenerationState>;
+  generationOrder: string[];
+  activeTools: Map<string, ToolState>;
+  latestAssistantOutput?: unknown;
+  providerMetadataByRequest: Map<string, Record<string, unknown>>;
+}

package/src/utils.ts

@@ -0,0 +1,273 @@
+import {
+  MAX_ARRAY_ITEMS,
+  MAX_DEPTH,
+  MAX_OBJECT_KEYS,
+  MAX_STRING_LENGTH,
+  MAX_TOOL_PAYLOAD_LENGTH,
+} from "./constants.js";
+
+export function truncate(value: string, maxLength = MAX_STRING_LENGTH): string {
+  return value.length > maxLength ? `${value.slice(0, maxLength)}... [truncated]` : value;
+}
+
+export function tryParseJson(value: string): unknown {
+  const trimmed = value.trim();
+  if (!trimmed || !["{", "["].includes(trimmed[0])) {
+    return value;
+  }
+
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return value;
+  }
+}
+
+export function shapePayload(value: unknown, options: { maxString?: number; depth?: number } = {}): unknown {
+  const maxString = options.maxString ?? MAX_STRING_LENGTH;
+  const depth = options.depth ?? MAX_DEPTH;
+
+  function visit(item: unknown, remainingDepth: number, seen: WeakSet<object>): unknown {
+    if (typeof item === "string") {
+      const truncated = truncate(item, maxString);
+      const parsed = tryParseJson(truncated);
+      if (parsed === truncated) {
+        return truncated;
+      }
+      return visit(parsed, remainingDepth - 1, seen);
+    }
+
+    if (
+      item === null ||
+      typeof item === "undefined" ||
+      typeof item === "number" ||
+      typeof item === "boolean"
+    ) {
+      return item;
+    }
+
+    if (typeof item === "bigint") {
+      return item.toString();
+    }
+
+    if (typeof item === "function" || typeof item === "symbol") {
+      return `[${typeof item}]`;
+    }
+
+    if (remainingDepth <= 0) {
+      return `[max depth ${depth} reached]`;
+    }
+
+    if (Array.isArray(item)) {
+      return item.slice(0, MAX_ARRAY_ITEMS).map((entry) => visit(entry, remainingDepth - 1, seen));
+    }
+
+    if (item instanceof Error) {
+      return {
+        name: item.name,
+        message: item.message,
+        stack: item.stack ? truncate(item.stack, maxString) : undefined,
+      };
+    }
+
+    if (typeof item === "object") {
+      if (seen.has(item)) {
+        return "[circular]";
+      }
+      seen.add(item);
+
+      const output: Record<string, unknown> = {};
+      for (const [key, entry] of Object.entries(item as Record<string, unknown>).slice(0, MAX_OBJECT_KEYS)) {
+        output[key] = visit(entry, remainingDepth - 1, seen);
+      }
+      return output;
+    }
+
+    return String(item);
+  }
+
+  return visit(value, depth, new WeakSet<object>());
+}
+
+export function safeSerialize(value: unknown, maxLength = MAX_TOOL_PAYLOAD_LENGTH): string {
+  try {
+    return truncate(JSON.stringify(shapePayload(value, { maxString: maxLength }), null, 2), maxLength);
+  } catch {
+    return `[unserializable ${typeof value}]`;
+  }
+}
+
+export function extractTextContent(content: unknown, maxLength?: number): string | undefined {
+  if (typeof content === "string") {
+    return maxLength ? truncate(content, maxLength) : content;
+  }
+
+  if (!Array.isArray(content)) {
+    return undefined;
+  }
+
+  const text = content
+    .map((item) => {
+      if (!item || typeof item !== "object") return "";
+      const block = item as { type?: string; text?: string; thinking?: string };
+      return block.type === "text" && block.text ? block.text : "";
+    })
+    .filter(Boolean)
+    .join("\n");
+
+  if (!text) {
+    return undefined;
+  }
+
+  return maxLength ? truncate(text, maxLength) : text;
+}
+
+export function extractToolCalls(message: Record<string, unknown>): unknown | undefined {
+  return (
+    message.toolCalls ??
+    message.tool_calls ??
+    message.function_calls ??
+    (message.content && Array.isArray(message.content)
+      ? message.content.filter((block) => {
+          return block && typeof block === "object" && ["tool_use", "tool_call"].includes(String((block as { type?: string }).type));
+        })
+      : undefined)
+  );
+}
+
+export function extractAssistantOutput(message: unknown): unknown | undefined {
+  if (!message || typeof message !== "object") {
+    return undefined;
+  }
+
+  const msg = message as Record<string, unknown>;
+  const text = extractTextContent(msg.content);
+  if (text) {
+    return text;
+  }
+
+  const toolCalls = extractToolCalls(msg);
+  if (Array.isArray(toolCalls) && toolCalls.length > 0) {
+    return { toolCalls: shapePayload(toolCalls) };
+  }
+
+  if (toolCalls) {
+    return { toolCalls: shapePayload(toolCalls) };
+  }
+
+  return shapePayload(msg);
+}
+
+export function extractFinalAssistant(messages: unknown): Record<string, unknown> | undefined {
+  if (!Array.isArray(messages)) {
+    return undefined;
+  }
+  return messages.filter((message) => message?.role === "assistant").pop() as Record<string, unknown> | undefined;
+}
+
+export function getRequestKey(event: Record<string, unknown>, fallback: string): string {
+  return String(
+    event.requestId ??
+      event.providerRequestId ??
+      event.messageId ??
+      event.turnId ??
+      event.turnIndex ??
+      event.id ??
+      fallback,
+  );
+}
+
+export function getToolCallId(event: Record<string, unknown>): string | undefined {
+  const id = event.toolCallId ?? event.id ?? event.callId ?? event.tool_use_id ?? event.toolUseId;
+  return id === undefined || id === null ? undefined : String(id);
+}
+
+export function getToolName(event: Record<string, unknown>): string {
+  return String(
+    event.toolName ??
+      event.name ??
+      event.tool ??
+      event.functionName ??
+      (event.call && typeof event.call === "object" ? (event.call as Record<string, unknown>).name : undefined) ??
+      "tool",
+  );
+}
+
+export function getToolInput(event: Record<string, unknown>): unknown {
+  return (
+    event.input ??
+    event.args ??
+    event.arguments ??
+    event.params ??
+    (event.call && typeof event.call === "object" ? (event.call as Record<string, unknown>).input : undefined) ??
+    event
+  );
+}
+
+export function getProviderPayload(event: Record<string, unknown>): unknown {
+  return event.request ?? event.payload ?? event.body ?? event.providerPayload ?? event.messages ?? event;
+}
+
+export function getMessageFromEvent(event: Record<string, unknown>): Record<string, unknown> | undefined {
+  if (event.message && typeof event.message === "object") {
+    return event.message as Record<string, unknown>;
+  }
+  if (event.role || event.content) {
+    return event;
+  }
+  return undefined;
+}
+
+export function extractUsage(messageOrEvent: Record<string, unknown>): Record<string, number> | undefined {
+  const usage = (messageOrEvent.usage ??
+    (messageOrEvent.message && typeof messageOrEvent.message === "object"
+      ? (messageOrEvent.message as Record<string, unknown>).usage
+      : undefined)) as Record<string, unknown> | undefined;
+  if (!usage || typeof usage !== "object") {
+    return undefined;
+  }
+
+  const input = Number(usage.input ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens ?? 0);
+  const output = Number(usage.output ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens ?? 0);
+  const total = Number(usage.total ?? usage.totalTokens ?? usage.total_tokens ?? input + output);
+  const cacheRead = Number(usage.cacheRead ?? usage.cache_read ?? usage.cachedTokens ?? 0);
+  const cacheWrite = Number(usage.cacheWrite ?? usage.cache_write ?? 0);
+
+  return {
+    input,
+    output,
+    total,
+    ...(cacheRead ? { cacheRead } : {}),
+    ...(cacheWrite ? { cacheWrite } : {}),
+  };
+}
+
+export function extractCostDetails(messageOrEvent: Record<string, unknown>): Record<string, number> | undefined {
+  const usage = (messageOrEvent.usage ??
+    (messageOrEvent.message && typeof messageOrEvent.message === "object"
+      ? (messageOrEvent.message as Record<string, unknown>).usage
+      : undefined)) as Record<string, unknown> | undefined;
+  const cost = (messageOrEvent.cost ?? usage?.cost ?? messageOrEvent.costDetails) as Record<string, unknown> | undefined;
+  if (!cost || typeof cost !== "object") {
+    return undefined;
+  }
+
+  const input = Number(cost.input ?? cost.inputCost ?? 0);
+  const output = Number(cost.output ?? cost.outputCost ?? 0);
+  const total = Number(cost.total ?? cost.totalCost ?? input + output);
+
+  return { input, output, total };
+}
+
+export function extractResponseMetadata(event: Record<string, unknown>): Record<string, unknown> {
+  return shapePayload(
+    {
+      status: event.status ?? event.statusCode ?? event.httpStatus,
+      headers: event.headers,
+      responseHeaders: event.responseHeaders,
+      providerMetadata: event.providerMetadata ?? event.metadata,
+      requestId: event.requestId ?? event.providerRequestId,
+    },
+    { depth: 4, maxString: 4_000 },
+  ) as Record<string, unknown>;
+}

package/tsconfig.json

@@ -10,6 +10,7 @@
   },
   "include": [
     "index.ts",
+    "src/**/*.ts",
     "types/**/*.d.ts"
   ]
 }