promptlayer 1.0.59 → 1.0.60

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "promptlayer",
   "license": "MIT",
-  "version": "1.0.59",
+  "version": "1.0.60",
   "main": "dist/index.js",
   "module": "dist/esm/index.js",
   "types": "dist/index.d.ts",
@@ -13,6 +13,7 @@
   "scripts": {
     "build": "tsup-node",
     "lint": "tsc",
+    "test": "vitest run",
     "release": "npm run build && npm publish"
   },
   "devDependencies": {
@@ -27,6 +28,7 @@
     "openai": "^6.7.0",
     "tsup": "^8.5.1",
     "typescript": "^5.2.2",
+    "vitest": "^2.1.9",
     "zod": "^3.25.0"
   },
   "dependencies": {

package/src/index.ts

@@ -27,6 +27,7 @@ import {
   utilLogRequest,
   vertexaiRequest,
 } from "@/utils/utils";
+import { categorizeError } from "@/utils/errors";
 import { streamResponse } from "@/utils/streaming";
 import * as opentelemetry from "@opentelemetry/api";
 
@@ -252,8 +253,6 @@ export class PromptLayer {
           );
         }
 
-        const response = await request_function(promptBlueprint!, kwargs);
-
         const _trackRequest = (body: object) => {
           const request_end_time = new Date().toISOString();
           return trackRequest(
@@ -280,6 +279,22 @@ export class PromptLayer {
           );
         };
 
+        let response: any;
+        try {
+          response = await request_function(promptBlueprint!, kwargs);
+        } catch (llmError: unknown) {
+          const errorType = categorizeError(llmError);
+          const errorMessage =
+            llmError instanceof Error ? llmError.message : String(llmError);
+          await _trackRequest({
+            request_response: {},
+            status: "ERROR",
+            error_type: errorType,
+            error_message: errorMessage,
+          });
+          throw llmError;
+        }
+
         if (stream)
           return streamResponse(
             response,

package/src/run-error-tracking.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect, vi, beforeEach, type Mock } from "vitest";
+
+// Mock modules before importing the subject
+vi.mock("@/utils/utils", () => ({
+  trackRequest: vi.fn().mockResolvedValue({ request_id: 1, prompt_blueprint: {} }),
+  configureProviderSettings: vi.fn().mockReturnValue({
+    provider_type: "openai",
+    kwargs: { model: "gpt-4" },
+  }),
+  getProviderConfig: vi.fn().mockReturnValue({
+    function_name: "openai.chat.completions.create",
+    stream_function: null,
+  }),
+  openaiRequest: vi.fn().mockResolvedValue({ choices: [{ message: { content: "hi" } }] }),
+  anthropicRequest: vi.fn(),
+  azureOpenAIRequest: vi.fn(),
+  googleRequest: vi.fn(),
+  mistralRequest: vi.fn(),
+  vertexaiRequest: vi.fn(),
+  amazonBedrockRequest: vi.fn(),
+  anthropicBedrockRequest: vi.fn(),
+  readEnv: vi.fn().mockReturnValue("test-api-key"),
+  runWorkflowRequest: vi.fn(),
+  utilLogRequest: vi.fn(),
+}));
+
+vi.mock("@/templates", () => {
+  return {
+    TemplateManager: class {
+      get = vi.fn().mockResolvedValue({
+        id: 1,
+        version: 1,
+        prompt_template: { type: "chat", messages: [] },
+        metadata: { model: { provider: "openai", name: "gpt-4", parameters: {} } },
+        llm_kwargs: { model: "gpt-4" },
+        custom_provider: null,
+      });
+    },
+  };
+});
+
+vi.mock("@/tracing", () => {
+  const fakeSpan = {
+    setAttribute: vi.fn(),
+    setStatus: vi.fn(),
+    end: vi.fn(),
+    spanContext: () => ({ spanId: "test-span-id" }),
+  };
+  return {
+    getTracer: () => ({
+      startActiveSpan: (_name: string, fn: (span: any) => any) => fn(fakeSpan),
+    }),
+    setupTracing: vi.fn(),
+  };
+});
+
+vi.mock("@/groups", () => ({
+  GroupManager: class {},
+}));
+
+vi.mock("@/track", () => ({
+  TrackManager: class {},
+}));
+
+vi.mock("@/span-wrapper", () => ({
+  wrapWithSpan: vi.fn(),
+}));
+
+vi.mock("@/utils/streaming", () => ({
+  streamResponse: vi.fn().mockReturnValue({ async *[Symbol.asyncIterator]() {} }),
+}));
+
+import { PromptLayer } from "@/index";
+import { trackRequest, openaiRequest } from "@/utils/utils";
+import { RateLimitError } from "openai";
+
+describe("run() error tracking", () => {
+  let client: PromptLayer;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    client = new PromptLayer({ apiKey: "test-api-key" });
+  });
+
+  it("tracks error with UNKNOWN_ERROR type and re-throws when LLM call fails", async () => {
+    const llmError = new Error("model overloaded");
+    (openaiRequest as Mock).mockRejectedValueOnce(llmError);
+
+    await expect(
+      client.run({ promptName: "test-prompt" })
+    ).rejects.toThrow("model overloaded");
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: {},
+        status: "ERROR",
+        error_type: "UNKNOWN_ERROR",
+        error_message: "model overloaded",
+      }),
+      true
+    );
+  });
+
+  it("tracks PROVIDER_RATE_LIMIT when LLM throws RateLimitError", async () => {
+    const rateLimitError = new RateLimitError(
+      429, undefined, "Too Many Requests", undefined
+    );
+    (openaiRequest as Mock).mockRejectedValueOnce(rateLimitError);
+
+    await expect(
+      client.run({ promptName: "test-prompt" })
+    ).rejects.toThrow("Too Many Requests");
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: {},
+        status: "ERROR",
+        error_type: "PROVIDER_RATE_LIMIT",
+        error_message: expect.stringContaining("Too Many Requests"),
+      }),
+      true
+    );
+  });
+
+  it("calls trackRequest without error fields on success", async () => {
+    const successResponse = { choices: [{ message: { content: "hello" } }] };
+    (openaiRequest as Mock).mockResolvedValueOnce(successResponse);
+
+    await client.run({ promptName: "test-prompt" });
+
+    expect(trackRequest).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({
+        request_response: successResponse,
+      }),
+      true
+    );
+
+    const callArgs = (trackRequest as Mock).mock.calls[0][1];
+    expect(callArgs).not.toHaveProperty("error_type");
+    expect(callArgs).not.toHaveProperty("error_message");
+    expect(callArgs).not.toHaveProperty("status");
+  });
+});

package/src/types.ts

@@ -34,6 +34,9 @@ export interface TrackRequest {
   return_data?: boolean;
   group_id?: number;
   span_id?: string;
+  status?: "SUCCESS" | "WARNING" | "ERROR";
+  error_type?: string;
+  error_message?: string;
   [k: string]: unknown;
 }
 
@@ -353,6 +356,9 @@ export interface LogRequest {
   prompt_id?: number;
   score_name?: string;
   api_type?: string;
+  status?: "SUCCESS" | "WARNING" | "ERROR";
+  error_type?: string;
+  error_message?: string;
 }
 
 export interface RequestLog {

package/src/utils/errors.test.ts

@@ -0,0 +1,68 @@
+import { describe, it, expect } from "vitest";
+import { categorizeError, ErrorType } from "@/utils/errors";
+import {
+  RateLimitError,
+  AuthenticationError,
+  APIConnectionTimeoutError,
+  BadRequestError,
+  InternalServerError,
+} from "openai";
+
+describe("categorizeError", () => {
+  // Branch: statusCode === 429 (+ className matches /ratelimit/i)
+  it("returns PROVIDER_RATE_LIMIT for OpenAI RateLimitError", () => {
+    const err = new RateLimitError(429, undefined, "Rate limit exceeded", undefined);
+    expect(categorizeError(err)).toBe(ErrorType.PROVIDER_RATE_LIMIT);
+  });
+
+  // Branch: className matches /timeout/i
+  it("returns PROVIDER_TIMEOUT for OpenAI APIConnectionTimeoutError", () => {
+    const err = new APIConnectionTimeoutError({ message: "Request timed out." });
+    expect(categorizeError(err)).toBe(ErrorType.PROVIDER_TIMEOUT);
+  });
+
+  // Branch: statusCode === 401 (+ className matches /authentication/i)
+  it("returns PROVIDER_AUTH_ERROR for OpenAI AuthenticationError", () => {
+    const err = new AuthenticationError(401, undefined, "Invalid API key", undefined);
+    expect(categorizeError(err)).toBe(ErrorType.PROVIDER_AUTH_ERROR);
+  });
+
+  // Branch: message includes "quota"
+  it("returns PROVIDER_QUOTA_LIMIT when message contains quota", () => {
+    expect(categorizeError(new Error("You exceeded your quota"))).toBe(
+      ErrorType.PROVIDER_QUOTA_LIMIT
+    );
+  });
+
+  // Branch: message includes "timeout"
+  it("returns PROVIDER_TIMEOUT when message contains timeout", () => {
+    expect(categorizeError(new Error("Request timeout"))).toBe(
+      ErrorType.PROVIDER_TIMEOUT
+    );
+  });
+
+  // Branch: message includes "timed out"
+  it("returns PROVIDER_TIMEOUT when message contains timed out", () => {
+    expect(categorizeError(new Error("Connection timed out"))).toBe(
+      ErrorType.PROVIDER_TIMEOUT
+    );
+  });
+
+  // Branch: statusCode defined but no earlier match → PROVIDER_ERROR
+  it("returns PROVIDER_ERROR for OpenAI InternalServerError", () => {
+    const err = new InternalServerError(500, undefined, "Internal server error", undefined);
+    expect(categorizeError(err)).toBe(ErrorType.PROVIDER_ERROR);
+  });
+
+  // Branch: fallthrough — no status, no matching class/message
+  it("returns UNKNOWN_ERROR for a plain Error", () => {
+    expect(categorizeError(new Error("something broke"))).toBe(
+      ErrorType.UNKNOWN_ERROR
+    );
+  });
+
+  // Branch: non-Error thrown → String(error) path, getClassName returns ""
+  it("returns UNKNOWN_ERROR for a non-Error value", () => {
+    expect(categorizeError("oops")).toBe(ErrorType.UNKNOWN_ERROR);
+  });
+});

package/src/utils/errors.ts

@@ -0,0 +1,62 @@
+export enum ErrorType {
+  PROVIDER_RATE_LIMIT = "PROVIDER_RATE_LIMIT",
+  PROVIDER_QUOTA_LIMIT = "PROVIDER_QUOTA_LIMIT",
+  PROVIDER_TIMEOUT = "PROVIDER_TIMEOUT",
+  PROVIDER_AUTH_ERROR = "PROVIDER_AUTH_ERROR",
+  PROVIDER_ERROR = "PROVIDER_ERROR",
+  UNKNOWN_ERROR = "UNKNOWN_ERROR",
+}
+
+function getStatusCode(error: unknown): number | undefined {
+  if (
+    error &&
+    typeof error === "object" &&
+    "status" in error &&
+    typeof (error as any).status === "number"
+  ) {
+    return (error as any).status;
+  }
+  return undefined;
+}
+
+function getClassName(error: unknown): string {
+  if (error && typeof error === "object" && error.constructor) {
+    return error.constructor.name;
+  }
+  return "";
+}
+
+export function categorizeError(error: unknown): ErrorType {
+  const statusCode = getStatusCode(error);
+  const className = getClassName(error);
+  const message =
+    error instanceof Error
+      ? error.message.toLowerCase()
+      : String(error).toLowerCase();
+
+  if (statusCode === 429 || /ratelimit/i.test(className)) {
+    return ErrorType.PROVIDER_RATE_LIMIT;
+  }
+
+  if (/timeout/i.test(className)) {
+    return ErrorType.PROVIDER_TIMEOUT;
+  }
+
+  if (statusCode === 401 || /authentication/i.test(className)) {
+    return ErrorType.PROVIDER_AUTH_ERROR;
+  }
+
+  if (message.includes("quota")) {
+    return ErrorType.PROVIDER_QUOTA_LIMIT;
+  }
+
+  if (message.includes("timeout") || message.includes("timed out")) {
+    return ErrorType.PROVIDER_TIMEOUT;
+  }
+
+  if (statusCode !== undefined) {
+    return ErrorType.PROVIDER_ERROR;
+  }
+
+  return ErrorType.UNKNOWN_ERROR;
+}

package/tsconfig.json

@@ -110,5 +110,6 @@
     "paths": {
       "@/*": ["*"]
     }
-  }
+  },
+  "exclude": ["**/*.test.ts"]
 }

package/vitest.config.ts

@@ -0,0 +1,6 @@
+import { defineConfig } from "vitest/config";
+import path from "path";
+
+export default defineConfig({
+  resolve: { alias: { "@": path.resolve(__dirname, "src") } },
+});