promptlayer 1.0.59 → 1.0.61

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "promptlayer",
   "license": "MIT",
-  "version": "1.0.59",
+  "version": "1.0.61",
   "main": "dist/index.js",
   "module": "dist/esm/index.js",
   "types": "dist/index.d.ts",
@@ -13,6 +13,7 @@
   "scripts": {
     "build": "tsup-node",
     "lint": "tsc",
+    "test": "vitest run",
     "release": "npm run build && npm publish"
   },
   "devDependencies": {
@@ -20,13 +21,14 @@
     "@anthropic-ai/sdk": "^0.57.0",
     "@anthropic-ai/vertex-sdk": "^0.11.5",
     "@aws-sdk/client-bedrock-runtime": "^3.864.0",
-    "@google/genai": "^1.8.0",
+    "@google/genai": "^1.45.0",
     "@mistralai/mistralai": "^1.7.5",
     "@types/ably": "^1.0.0",
     "@types/node": "^20.8.0",
     "openai": "^6.7.0",
     "tsup": "^8.5.1",
     "typescript": "^5.2.2",
+    "vitest": "^2.1.9",
     "zod": "^3.25.0"
   },
   "dependencies": {

package/src/index.ts

@@ -27,6 +27,7 @@ import {
   utilLogRequest,
   vertexaiRequest,
 } from "@/utils/utils";
+import { categorizeError } from "@/utils/errors";
 import { streamResponse } from "@/utils/streaming";
 import * as opentelemetry from "@opentelemetry/api";
 
@@ -252,8 +253,6 @@ export class PromptLayer {
           );
         }
 
-        const response = await request_function(promptBlueprint!, kwargs);
-
         const _trackRequest = (body: object) => {
           const request_end_time = new Date().toISOString();
           return trackRequest(
@@ -280,6 +279,22 @@ export class PromptLayer {
           );
         };
 
+        let response: any;
+        try {
+          response = await request_function(promptBlueprint!, kwargs);
+        } catch (llmError: unknown) {
+          const errorType = categorizeError(llmError);
+          const errorMessage =
+            llmError instanceof Error ? llmError.message : String(llmError);
+          await _trackRequest({
+            request_response: {},
+            status: "ERROR",
+            error_type: errorType,
+            error_message: errorMessage,
+          });
+          throw llmError;
+        }
+
         if (stream)
           return streamResponse(
             response,

package/src/run-error-tracking.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect, vi, beforeEach, type Mock } from "vitest";
+
+// Mock modules before importing the subject
+vi.mock("@/utils/utils", () => ({
+  trackRequest: vi.fn().mockResolvedValue({ request_id: 1, prompt_blueprint: {} }),
+  configureProviderSettings: vi.fn().mockReturnValue({
+    provider_type: "openai",
+    kwargs: { model: "gpt-4" },
+  }),
+  getProviderConfig: vi.fn().mockReturnValue({
+    function_name: "openai.chat.completions.create",
+    stream_function: null,
+  }),
+  openaiRequest: vi.fn().mockResolvedValue({ choices: [{ message: { content: "hi" } }] }),
+  anthropicRequest: vi.fn(),
+  azureOpenAIRequest: vi.fn(),
+  googleRequest: vi.fn(),
+  mistralRequest: vi.fn(),
+  vertexaiRequest: vi.fn(),
+  amazonBedrockRequest: vi.fn(),
+  anthropicBedrockRequest: vi.fn(),
+  readEnv: vi.fn().mockReturnValue("test-api-key"),
+  runWorkflowRequest: vi.fn(),
+  utilLogRequest: vi.fn(),
+}));
+
+vi.mock("@/templates", () => {
+  return {
+    TemplateManager: class {
+      get = vi.fn().mockResolvedValue({
+        id: 1,
+        version: 1,
+        prompt_template: { type: "chat", messages: [] },
+        metadata: { model: { provider: "openai", name: "gpt-4", parameters: {} } },
+        llm_kwargs: { model: "gpt-4" },
+        custom_provider: null,
+      });
+    },
+  };
+});
+
+vi.mock("@/tracing", () => {
+  const fakeSpan = {
+    setAttribute: vi.fn(),
+    setStatus: vi.fn(),
+    end: vi.fn(),
+    spanContext: () => ({ spanId: "test-span-id" }),
+  };
+  return {
+    getTracer: () => ({
+      startActiveSpan: (_name: string, fn: (span: any) => any) => fn(fakeSpan),
+    }),
+    setupTracing: vi.fn(),
+  };
+});
+
+vi.mock("@/groups", () => ({
+  GroupManager: class {},
+}));
+
+vi.mock("@/track", () => ({
+  TrackManager: class {},
+}));
+
+vi.mock("@/span-wrapper", () => ({
+  wrapWithSpan: vi.fn(),
+}));
+
+vi.mock("@/utils/streaming", () => ({
+  streamResponse: vi.fn().mockReturnValue({ async *[Symbol.asyncIterator]() {} }),
+}));
+
+import { PromptLayer } from "@/index";
+import { trackRequest, openaiRequest } from "@/utils/utils";
+import { RateLimitError } from "openai";
+
+describe("run() error tracking", () => {
+  let client: PromptLayer;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    client = new PromptLayer({ apiKey: "test-api-key" });
+  });
+
+  it("tracks error with UNKNOWN_ERROR type and re-throws when LLM call fails", async () => {
+    const llmError = new Error("model overloaded");
+    (openaiRequest as Mock).mockRejectedValueOnce(llmError);
+
+    await expect(
+      client.run({ promptName: "test-prompt" })
+    ).rejects.toThrow("model overloaded");
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: {},
+        status: "ERROR",
+        error_type: "UNKNOWN_ERROR",
+        error_message: "model overloaded",
+      }),
+      true
+    );
+  });
+
+  it("tracks PROVIDER_RATE_LIMIT when LLM throws RateLimitError", async () => {
+    const rateLimitError = new RateLimitError(
+      429, undefined, "Too Many Requests", undefined
+    );
+    (openaiRequest as Mock).mockRejectedValueOnce(rateLimitError);
+
+    await expect(
+      client.run({ promptName: "test-prompt" })
+    ).rejects.toThrow("Too Many Requests");
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: {},
+        status: "ERROR",
+        error_type: "PROVIDER_RATE_LIMIT",
+        error_message: expect.stringContaining("Too Many Requests"),
+      }),
+      true
+    );
+  });
+
+  it("calls trackRequest without error fields on success", async () => {
+    const successResponse = { choices: [{ message: { content: "hello" } }] };
+    (openaiRequest as Mock).mockResolvedValueOnce(successResponse);
+
+    await client.run({ promptName: "test-prompt" });
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: successResponse,
+      }),
+      true
+    );
+
+    const callArgs = (trackRequest as Mock).mock.calls[0][1];
+    expect(callArgs).not.toHaveProperty("error_type");
+    expect(callArgs).not.toHaveProperty("error_message");
+    expect(callArgs).not.toHaveProperty("status");
+  });
+});

package/src/types.ts

@@ -34,6 +34,9 @@ export interface TrackRequest {
   return_data?: boolean;
   group_id?: number;
   span_id?: string;
+  status?: "SUCCESS" | "WARNING" | "ERROR";
+  error_type?: string;
+  error_message?: string;
   [k: string]: unknown;
 }
 
@@ -90,15 +93,59 @@ const templateFormat = ["f-string", "jinja2"] as const;
 
 export type TemplateFormat = (typeof templateFormat)[number];
 
+export type FileAnnotation = {
+  type: "file_citation";
+  index: number;
+  file_id: string;
+  filename: string;
+};
+
+export type WebAnnotation = {
+  type: "url_citation";
+  title: string;
+  url: string;
+  start_index: number;
+  end_index: number;
+  cited_text?: string;
+  encrypted_index?: string;
+};
+
+export type MapAnnotation = {
+  type: "map_citation";
+  title: string;
+  url: string;
+  place_id?: string;
+  start_index: number;
+  end_index: number;
+  cited_text?: string;
+};
+
+export type ContainerFileAnnotation = {
+  type: "container_file_citation";
+  container_id: string;
+  start_index?: number;
+  end_index?: number;
+  filename?: string;
+  file_id?: string;
+};
+
+export type Annotation =
+  | WebAnnotation
+  | FileAnnotation
+  | MapAnnotation
+  | ContainerFileAnnotation;
+
 export type ImageUrl = {
   url: string;
+  detail?: string;
 };
 
 export type TextContent = {
   id?: string;
   type: "text";
   text: string;
-  annotations?: Record<string, unknown>[];
+  annotations?: Annotation[];
+  thought_signature?: string;
 };
 
 export type ThinkingContent = {
@@ -113,17 +160,20 @@ export type CodeContent = {
   container_id?: string;
   type: "code";
   code: string;
+  language?: string;
 };
 
 export type ImageContent = {
   type: "image_url";
   image_url: ImageUrl;
+  image_variable?: string;
 };
 
 export type Media = {
   title: string;
   type: string;
-  url: string;
+  url?: string;
+  format?: "base64" | "url" | "neither";
 };
 
 export type MediaContent = {
@@ -136,32 +186,270 @@ export type MediaVariable = {
   name: string;
 };
 
+export type OutputMediaContent = {
+  type: "output_media";
+  id?: string;
+  url: string;
+  mime_type?: string;
+  media_type?: "image" | "video" | "audio";
+  provider_metadata?: Record<string, unknown>;
+};
+
+export type ServerToolUseContent = {
+  type: "server_tool_use";
+  id: string;
+  name: string;
+  input?: Record<string, unknown>;
+};
+
+export type WebSearchResult = {
+  type: "web_search_result";
+  url?: string;
+  title?: string;
+  encrypted_content?: string;
+  page_age?: string;
+};
+
+export type WebSearchToolResultContent = {
+  type: "web_search_tool_result";
+  tool_use_id: string;
+  content?: WebSearchResult[];
+};
+
+export type CodeExecutionResultContent = {
+  type: "code_execution_result";
+  output: string;
+  outcome?: string;
+};
+
+export type McpListToolsContent = {
+  type: "mcp_list_tools";
+  id?: string;
+  server_label?: string;
+  tools?: Record<string, unknown>[];
+  error?: string | Record<string, unknown>;
+};
+
+export type McpCallContent = {
+  type: "mcp_call";
+  id?: string;
+  name?: string;
+  server_label?: string;
+  arguments?: string;
+  output?: string;
+  error?: string | Record<string, unknown>;
+  approval_request_id?: string;
+};
+
+export type McpApprovalRequestContent = {
+  type: "mcp_approval_request";
+  id?: string;
+  name?: string;
+  arguments?: string;
+  server_label?: string;
+};
+
+export type McpApprovalResponseContent = {
+  type: "mcp_approval_response";
+  approval_request_id: string;
+  approve: boolean;
+};
+
+export type BashCodeExecutionToolResultContent = {
+  type: "bash_code_execution_tool_result";
+  tool_use_id: string;
+  content?: Record<string, unknown>;
+};
+
+export type TextEditorCodeExecutionToolResultContent = {
+  type: "text_editor_code_execution_tool_result";
+  tool_use_id: string;
+  content?: Record<string, unknown>;
+};
+
+export type ShellCallContent = {
+  type: "shell_call";
+  id?: string;
+  call_id?: string;
+  action?: Record<string, unknown>;
+  status?: string;
+};
+
+export type ShellCallOutputContent = {
+  type: "shell_call_output";
+  id?: string;
+  call_id?: string;
+  output?: Record<string, unknown>[];
+  status?: string;
+};
+
+export type ApplyPatchCallContent = {
+  type: "apply_patch_call";
+  id?: string;
+  call_id?: string;
+  operation?: Record<string, unknown>;
+  status?: string;
+};
+
+export type ApplyPatchCallOutputContent = {
+  type: "apply_patch_call_output";
+  id?: string;
+  call_id?: string;
+  output?: string;
+  status?: string;
+};
+
 export type Content =
   | TextContent
   | ThinkingContent
   | CodeContent
   | ImageContent
   | MediaContent
-  | MediaVariable;
+  | MediaVariable
+  | OutputMediaContent
+  | ServerToolUseContent
+  | WebSearchToolResultContent
+  | CodeExecutionResultContent
+  | McpListToolsContent
+  | McpCallContent
+  | McpApprovalRequestContent
+  | McpApprovalResponseContent
+  | BashCodeExecutionToolResultContent
+  | TextEditorCodeExecutionToolResultContent
+  | ShellCallContent
+  | ShellCallOutputContent
+  | ApplyPatchCallContent
+  | ApplyPatchCallOutputContent;
 
 export type Function_ = {
   name: string;
   description: string;
+  strict?: boolean;
   parameters: Record<string, unknown>;
 };
 
-export type Tool = {
-  id: string;
-  tool_id?: string;
+export type FunctionCall = {
+  name: string;
+  arguments: string;
+};
+
+export type WebSearchToolFilters = {
+  allowed_domains?: string[];
+};
+
+export type WebSearchToolUserLocation = {
+  city?: string;
+  country?: string;
+  region?: string;
+  timezone?: string;
+  type: "approximate";
+};
+
+export type OpenAIWebSearchToolConfig = {
+  type: "web_search" | "web_search_2025_08_26";
+  filters?: WebSearchToolFilters;
+  search_context_size?: "low" | "medium" | "high";
+  user_location?: WebSearchToolUserLocation;
+};
+
+export type ComparisonFilter = {
+  key: string;
+  value: string | number | boolean;
+  operation: "eq" | "ne" | "gt" | "gte" | "lt" | "lte";
+};
+
+export type CompoundFilter = {
+  operator: "and" | "or";
+  operands: (ComparisonFilter | CompoundFilter)[];
+};
+
+export type RankingOptions = {
+  ranker?: "auto" | "default-2024-11-15";
+  score_threshold?: number;
+};
+
+export type FileSearchToolConfig = {
+  type: "file_search";
+  vector_store_ids: string[];
+  filters?: ComparisonFilter | CompoundFilter;
+  max_num_results?: number;
+  ranking_options?: RankingOptions;
+};
+
+export type CodeInterpreterToolConfig = {
+  type: "code_interpreter";
+  container?: Record<string, unknown>;
+};
+
+export type ImageGenerationToolConfig = {
+  type: "image_generation";
+};
+
+export type ShellToolConfig = {
+  type: "shell";
+  environment?: Record<string, unknown>;
+};
+
+export type ApplyPatchToolConfig = {
+  type: "apply_patch";
+};
+
+export type McpToolApprovalFilter = {
+  tool_names?: string[];
+};
+
+export type McpToolApproval = {
+  never?: McpToolApprovalFilter;
+  always?: McpToolApprovalFilter;
+};
+
+export type McpToolConfig = {
+  type: "mcp";
+  server_label: string;
+  server_url?: string;
+  server_description?: string;
+  connector_id?: string;
+  authorization?: string;
+  allowed_tools?: string[];
+  require_approval?: string | McpToolApproval;
+};
+
+export type BuiltInToolConfig =
+  | OpenAIWebSearchToolConfig
+  | FileSearchToolConfig
+  | CodeInterpreterToolConfig
+  | ImageGenerationToolConfig
+  | McpToolConfig
+  | ShellToolConfig
+  | ApplyPatchToolConfig;
+
+export type FunctionTool = {
   type: "function";
   function: Function_;
 };
 
-export type FunctionCall = {
+export type BuiltInTool = {
+  id: string;
   name: string;
-  arguments: string;
+  description: string;
+  provider: string;
+  type:
+    | "web_search"
+    | "file_search"
+    | "code_interpreter"
+    | "image_generation"
+    | "google_maps"
+    | "url_context"
+    | "mcp"
+    | "bash"
+    | "shell"
+    | "apply_patch"
+    | "text_editor";
+  config: BuiltInToolConfig;
 };
 
+export type Tool = FunctionTool | BuiltInTool;
+
 export type SystemMessage = {
   role: "system";
   input_variables?: string[];
@@ -180,6 +468,7 @@ export type UserMessage = {
 
 export type ToolCall = {
   id: string;
+  tool_id?: string;
   type: "function";
   function: FunctionCall;
 };
@@ -206,7 +495,7 @@ export type ToolMessage = {
   role: "tool";
   input_variables?: string[];
   template_format?: TemplateFormat;
-  content: Content[];
+  content?: Content[];
   tool_call_id: string;
   name?: string;
 };
@@ -265,8 +554,11 @@ export type PromptTemplate = CompletionPromptTemplate | ChatPromptTemplate;
 export type Model = {
   api_type?: string;
   provider: string;
+  model_config_display_name?: string;
+  base_model?: string;
   name: string;
   parameters: Record<string, unknown>;
+  display_params?: Record<string, string | boolean | null>;
 };
 
 export type Metadata = {
@@ -282,6 +574,10 @@ export type PromptBlueprint = {
   prompt_template: PromptTemplate;
   commit_message?: string;
   metadata?: Metadata;
+  provider_base_url_name?: string;
+  report_id?: number;
+  inference_client_name?: string;
+  provider_id?: number;
 };
 
 export type PublishPromptTemplate = BasePromptTemplate &
@@ -353,6 +649,9 @@ export interface LogRequest {
   prompt_id?: number;
   score_name?: string;
   api_type?: string;
+  status?: "SUCCESS" | "WARNING" | "ERROR";
+  error_type?: string;
+  error_message?: string;
 }
 
 export interface RequestLog {