promptmetrics-sample 1.0.0

package/src/tests/logs.test.ts

@@ -0,0 +1,180 @@
+#!/usr/bin/env ts-node
+/**
+ * Logs Test Suite
+ * Tests all log-related SDK operations
+ */
+
+import "dotenv/config";
+import { PromptMetrics } from "@promptmetrics/sdk";
+import { runTestSuite, logger } from "../utils/logger";
+
+const pm = new PromptMetrics({
+  apiKey: process.env.PROMPTMETRICS_API_KEY!,
+});
+
+const TEST_TEMPLATE_ID = process.env.TEST_TEMPLATE_ID;
+const TEST_VERSION_ID = process.env.TEST_VERSION_ID;
+
+// Store log ID from first test for subsequent tests
+let fetchedLogId: string | null = null;
+
+const tests = [
+  {
+    name: "List logs with pagination",
+    fn: async () => {
+      const result = await pm.logs.list({
+        page: 1,
+        limit: 10,
+        sort_by: "created_at",
+        sort_order: "desc",
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("Logs Count", result.length);
+
+      if (result.length > 0) {
+        fetchedLogId = result[0]._id;
+        logger.data("First Log ID", result[0]._id);
+        logger.data("First Log Status", result[0].status);
+      }
+    },
+  },
+
+  {
+    name: "List logs filtered by template_id",
+    fn: async () => {
+      if (!TEST_TEMPLATE_ID) {
+        logger.info("Skipped - TEST_TEMPLATE_ID not set");
+        return;
+      }
+
+      const result = await pm.logs.list({
+        template_id: TEST_TEMPLATE_ID,
+        page: 1,
+        limit: 10,
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("Logs for Template", result.length);
+    },
+  },
+
+  {
+    name: "List logs filtered by version_id",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const result = await pm.logs.list({
+        template_version_id: TEST_VERSION_ID,
+        page: 1,
+        limit: 10,
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("Logs for Version", result.length);
+    },
+  },
+
+  {
+    name: "List logs filtered by status (SUCCESS)",
+    fn: async () => {
+      const result = await pm.logs.list({
+        status: "SUCCESS",
+        page: 1,
+        limit: 10,
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("Successful Logs", result.length);
+    },
+  },
+
+  {
+    name: "List logs filtered by status (ERROR)",
+    fn: async () => {
+      const result = await pm.logs.list({
+        status: "ERROR",
+        page: 1,
+        limit: 10,
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("Error Logs", result.length);
+    },
+  },
+
+  {
+    name: "List logs filtered by source (SDK)",
+    fn: async () => {
+      const result = await pm.logs.list({
+        source: "sdk",
+        page: 1,
+        limit: 10,
+      });
+
+      if (!result) throw new Error("Failed to list logs");
+      logger.data("SDK Logs", result.length);
+    },
+  },
+
+  {
+    name: "Get log by ID",
+    fn: async () => {
+      if (!fetchedLogId) {
+        logger.info("Skipped - No log ID available from previous test");
+        return;
+      }
+
+      const log = await pm.logs.get(fetchedLogId);
+      if (!log) throw new Error("Log not found");
+
+      logger.data("Log ID", log._id);
+      logger.data("Request ID", log.request_id);
+      logger.data("Status", log.status);
+      logger.data("Latency", `${log.latency}s`);
+      logger.data("Total Cost", `$${log.total_cost}`);
+      logger.data("Tags", log.tags || "No tags");
+    },
+  },
+
+  {
+    name: "Verify log has tags (from pm_tags)",
+    fn: async () => {
+      // Run a version with tags first
+      const versionId = TEST_VERSION_ID;
+      if (!versionId) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const runResult = await pm.versions.run(versionId, {
+        variables: { topic: "Tag Verification" },
+        pm_tags: ["verify-test", "log-check"],
+      });
+
+      // Get the log and verify tags
+      const logs = await pm.logs.list({
+        template_version_id: versionId,
+        limit: 1,
+        sort_by: "created_at",
+        sort_order: "desc",
+      });
+
+      if (logs.length > 0) {
+        const latestLog = logs[0];
+        logger.data("Log Tags", latestLog.tags);
+
+        if (latestLog.tags && latestLog.tags.includes("verify-test")) {
+          logger.success("Tags verified in log!");
+        } else {
+          logger.info("Tags may be stored but not returned in list");
+        }
+      }
+    },
+  },
+];
+
+runTestSuite("Logs Test Suite", tests);

package/src/tests/providers.test.ts

@@ -0,0 +1,99 @@
+#!/usr/bin/env ts-node
+/**
+ * Providers Test Suite
+ * Tests all provider-related SDK operations
+ */
+
+import "dotenv/config";
+import { PromptMetrics } from "@promptmetrics/sdk";
+import { runTestSuite, logger } from "../utils/logger";
+
+const pm = new PromptMetrics({
+  apiKey: process.env.PROMPTMETRICS_API_KEY!,
+});
+
+const tests = [
+  {
+    name: "List all LLM providers",
+    fn: async () => {
+      const providers = await pm.providers.list();
+      if (!providers) throw new Error("Failed to list providers");
+
+      logger.data("Providers Count", providers.length);
+
+      providers.forEach((provider) => {
+        logger.info(`Provider: ${provider.name} (${provider.slug})`);
+        logger.data("  Has Key", provider.has_key || false);
+      });
+    },
+  },
+
+  {
+    name: "Get models for OpenAI provider",
+    fn: async () => {
+      try {
+        const provider = await pm.providers.getModels("openai");
+        if (!provider) throw new Error("Provider not found");
+
+        logger.data("Provider Name", provider.name);
+        logger.data("Models Count", provider.models?.length || 0);
+
+        if (provider.models && provider.models.length > 0) {
+          logger.info("Sample models:");
+          provider.models.slice(0, 5).forEach((model) => {
+            logger.info(`  - ${model.name}`);
+          });
+        }
+      } catch (error: unknown) {
+        const err = error as { message?: string };
+        if (err.message?.includes("not found")) {
+          logger.info("OpenAI provider not configured - this is OK");
+        } else {
+          throw error;
+        }
+      }
+    },
+  },
+
+  {
+    name: "Get models for Anthropic/Claude provider",
+    fn: async () => {
+      try {
+        const provider = await pm.providers.getModels("anthropic");
+        if (!provider) throw new Error("Provider not found");
+
+        logger.data("Provider Name", provider.name);
+        logger.data("Models Count", provider.models?.length || 0);
+      } catch (error: unknown) {
+        const err = error as { message?: string };
+        if (err.message?.includes("not found")) {
+          logger.info("Anthropic provider not configured - this is OK");
+        } else {
+          throw error;
+        }
+      }
+    },
+  },
+
+  {
+    name: "Get models for OpenRouter provider",
+    fn: async () => {
+      try {
+        const provider = await pm.providers.getModels("openrouter");
+        if (!provider) throw new Error("Provider not found");
+
+        logger.data("Provider Name", provider.name);
+        logger.data("Models Count", provider.models?.length || 0);
+      } catch (error: unknown) {
+        const err = error as { message?: string };
+        if (err.message?.includes("not found")) {
+          logger.info("OpenRouter provider not configured - this is OK");
+        } else {
+          throw error;
+        }
+      }
+    },
+  },
+];
+
+runTestSuite("Providers Test Suite", tests);

package/src/tests/tags.test.ts

@@ -0,0 +1,237 @@
+#!/usr/bin/env ts-node
+/**
+ * Tags Test Suite
+ * Tests the pm_tags feature for prompt logs
+ */
+
+import "dotenv/config";
+import { PromptMetrics } from "@promptmetrics/sdk";
+import { runTestSuite, logger } from "../utils/logger";
+
+const pm = new PromptMetrics({
+  apiKey: process.env.PROMPTMETRICS_API_KEY!,
+});
+
+const TEST_VERSION_ID = process.env.TEST_VERSION_ID;
+
+const tests = [
+  {
+    name: "Run version with pm_tags array",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const tags = ["sdk-test", "automated", "tag-feature-test"];
+
+      const result = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "Tags Feature Test",
+        },
+        pm_tags: tags,
+      });
+
+      if (!result) throw new Error("Run failed");
+
+      logger.data("Request ID", result.request_id);
+      logger.data("Status", result.status);
+      logger.data("Tags Sent", tags);
+      logger.data("Tags in Response", result.tags || "Not returned");
+    },
+  },
+
+  {
+    name: "Run version with empty pm_tags (should inherit from template)",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const result = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "No Tags Test",
+        },
+        // No pm_tags - should inherit from template
+      });
+
+      if (!result) throw new Error("Run failed");
+
+      logger.data("Request ID", result.request_id);
+      logger.data("Status", result.status);
+      logger.data("Tags (inherited)", result.tags || "None");
+    },
+  },
+
+  {
+    name: "Run version with single tag",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const result = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "Single Tag Test",
+        },
+        pm_tags: ["single-tag"],
+      });
+
+      if (!result) throw new Error("Run failed");
+
+      logger.data("Request ID", result.request_id);
+      logger.data("Tags", result.tags || "Not returned");
+    },
+  },
+
+  {
+    name: "Run version with many tags",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const manyTags = [
+        "tag1",
+        "tag2",
+        "tag3",
+        "tag4",
+        "tag5",
+        "production",
+        "customer-support",
+        "high-priority",
+        "eu-region",
+        "compliance",
+      ];
+
+      const result = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "Many Tags Test",
+        },
+        pm_tags: manyTags,
+      });
+
+      if (!result) throw new Error("Run failed");
+
+      logger.data("Request ID", result.request_id);
+      logger.data("Tags Count", manyTags.length);
+      logger.data("Tags in Response", result.tags?.length || 0);
+    },
+  },
+
+  {
+    name: "Verify tags are stored in prompt log",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const uniqueTag = `verify-${Date.now()}`;
+
+      // Run with unique tag
+      const runResult = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "Tag Verification",
+        },
+        pm_tags: [uniqueTag, "verification-test"],
+      });
+
+      logger.data("Run Request ID", runResult.request_id);
+
+      // Wait a moment for the log to be created
+      await new Promise((resolve) => setTimeout(resolve, 500));
+
+      // Fetch recent logs and check for our tag
+      const logs = await pm.logs.list({
+        template_version_id: TEST_VERSION_ID,
+        limit: 5,
+        sort_by: "created_at",
+        sort_order: "desc",
+      });
+
+      if (logs.length === 0) {
+        logger.info("No logs found to verify");
+        return;
+      }
+
+      const latestLog = logs[0];
+      logger.data("Latest Log ID", latestLog._id);
+      logger.data("Latest Log Tags", latestLog.tags || "None");
+
+      // Check if our unique tag is in the log
+      if (latestLog.tags && latestLog.tags.includes(uniqueTag)) {
+        logger.success(`Tag '${uniqueTag}' verified in log!`);
+      } else {
+        logger.info("Tag may be stored but not returned in list response");
+      }
+    },
+  },
+
+  {
+    name: "Tags with special characters",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      const specialTags = [
+        "tag-with-dash",
+        "tag_with_underscore",
+        "tag.with.dots",
+        "CamelCaseTag",
+        "UPPERCASE",
+        "lowercase",
+        "tag123",
+      ];
+
+      const result = await pm.versions.run(TEST_VERSION_ID, {
+        variables: {
+          topic: "Special Tags Test",
+        },
+        pm_tags: specialTags,
+      });
+
+      if (!result) throw new Error("Run failed");
+
+      logger.data("Request ID", result.request_id);
+      logger.data("Special Tags Sent", specialTags.length);
+    },
+  },
+
+  {
+    name: "Tags inside @traceable function",
+    fn: async () => {
+      if (!TEST_VERSION_ID) {
+        logger.info("Skipped - TEST_VERSION_ID not set");
+        return;
+      }
+
+      // Create a traceable function that runs a version with tags
+      class TaggedService {
+        @pm.traceable({ name: "tagged_llm_call" })
+        async runWithTags(topic: string, tags: string[]): Promise<object> {
+          return pm.versions.run(TEST_VERSION_ID!, {
+            variables: { topic },
+            pm_tags: tags,
+          });
+        }
+      }
+
+      const service = new TaggedService();
+      const result = await service.runWithTags("Traceable Tags Test", [
+        "traceable-test",
+        "nested-tags",
+      ]);
+
+      logger.data("Result Status", (result as { status?: string }).status);
+      logger.data("Trace ID", pm.getCurrentTraceId() || "N/A");
+    },
+  },
+];
+
+runTestSuite("Tags (pm_tags) Test Suite", tests);