promptmetrics-sample 1.0.0

package/src/tests/template-run-load.test.ts

@@ -0,0 +1,332 @@
+#!/usr/bin/env ts-node
+/**
+ * Template Run Load Test
+ * Configurable load testing for canary deployments
+ *
+ * Usage:
+ *   npm run test:load -- --requests=1000 --concurrency=50
+ *   npm run test:load -- --template=my-template --requests=500
+ */
+
+import "dotenv/config";
+import { PromptMetrics } from "@promptmetrics/sdk";
+import { logger } from "../utils/logger";
+
+// Configuration interface
+interface LoadTestConfig {
+  templateIdentifier: string;
+  requestCount: number;
+  concurrency: number;
+  errorRate: number; // 0-1, percentage of requests that should fail
+  delayMs?: number; // Delay between batches
+  variables?: Record<string, any>;
+}
+
+// Parse command line arguments
+function parseArgs(): Partial<LoadTestConfig> {
+  const args = process.argv.slice(2);
+  const config: Partial<LoadTestConfig> = {};
+
+  args.forEach((arg) => {
+    const [key, value] = arg.replace("--", "").split("=");
+    switch (key) {
+      case "template":
+        config.templateIdentifier = value;
+        break;
+      case "requests":
+        config.requestCount = parseInt(value);
+        break;
+      case "concurrency":
+        config.concurrency = parseInt(value);
+        break;
+      case "error-rate":
+        config.errorRate = parseFloat(value);
+        break;
+      case "delay":
+        config.delayMs = parseInt(value);
+        break;
+    }
+  });
+
+  return config;
+}
+
+// Default configuration
+const defaultConfig: LoadTestConfig = {
+  templateIdentifier: process.env.TEST_TEMPLATE_NAME || "test-template",
+  requestCount: 100,
+  concurrency: 10,
+  errorRate: 0,
+  delayMs: 0,
+  variables: {},
+};
+
+// Merge with CLI args
+const cliConfig = parseArgs();
+const config: LoadTestConfig = { ...defaultConfig, ...cliConfig };
+
+// Initialize SDK
+const pm = new PromptMetrics({
+  apiKey: process.env.PROMPTMETRICS_API_KEY!,
+});
+
+// Metrics tracking
+interface TestMetrics {
+  totalRequests: number;
+  successfulRequests: number;
+  failedRequests: number;
+  canaryRequests: number;
+  legacyRequests: number;
+  errors: Array<{ message: string; timestamp: Date }>;
+  responseTimes: number[];
+  startTime: Date;
+  endTime?: Date;
+}
+
+const metrics: TestMetrics = {
+  totalRequests: 0,
+  successfulRequests: 0,
+  failedRequests: 0,
+  canaryRequests: 0,
+  legacyRequests: 0,
+  errors: [],
+  responseTimes: [],
+  startTime: new Date(),
+};
+
+// Progress bar
+function renderProgressBar(current: number, total: number): string {
+  const percentage = Math.floor((current / total) * 100);
+  const barLength = 20;
+  const filledLength = Math.floor((current / total) * barLength);
+  const bar = "█".repeat(filledLength) + "░".repeat(barLength - filledLength);
+  return `[${bar}] ${percentage}% (${current}/${total})`;
+}
+
+// Single request execution
+async function executeRequest(
+  requestId: number
+): Promise<{ success: boolean; responseTime: number; isCanary?: boolean }> {
+  const startTime = Date.now();
+
+  try {
+    // Simulate error based on error rate
+    const shouldFail = Math.random() < config.errorRate;
+
+    if (shouldFail) {
+      throw new Error(`Simulated error for request ${requestId}`);
+    }
+
+    // Execute template run
+    const result = await pm.templates.run(config.templateIdentifier, {
+      variables: config.variables,
+    });
+
+    const responseTime = Date.now() - startTime;
+
+    // Try to detect if this was a canary request
+    // This would need to be exposed by the API or inferred from metadata
+    const isCanary = (result as any).metadata?.is_canary;
+
+    return {
+      success: true,
+      responseTime,
+      isCanary,
+    };
+  } catch (error: any) {
+    const responseTime = Date.now() - startTime;
+    metrics.errors.push({
+      message: error.message || "Unknown error",
+      timestamp: new Date(),
+    });
+    return {
+      success: false,
+      responseTime,
+    };
+  }
+}
+
+// Execute batch of requests concurrently
+async function executeBatch(
+  batchSize: number,
+  startIndex: number
+): Promise<void> {
+  const promises = [];
+
+  for (let i = 0; i < batchSize; i++) {
+    const requestId = startIndex + i;
+    promises.push(executeRequest(requestId));
+  }
+
+  const results = await Promise.all(promises);
+
+  // Update metrics
+  results.forEach((result) => {
+    metrics.totalRequests++;
+    metrics.responseTimes.push(result.responseTime);
+
+    if (result.success) {
+      metrics.successfulRequests++;
+      if (result.isCanary === true) {
+        metrics.canaryRequests++;
+      } else if (result.isCanary === false) {
+        metrics.legacyRequests++;
+      }
+    } else {
+      metrics.failedRequests++;
+    }
+  });
+}
+
+// Main load test execution
+async function runLoadTest(): Promise<void> {
+  console.log("\n" + "=".repeat(60));
+  console.log("🚀 Template Run Load Test");
+  console.log("=".repeat(60));
+  console.log(`Template:     ${config.templateIdentifier}`);
+  console.log(`Requests:     ${config.requestCount}`);
+  console.log(`Concurrency:  ${config.concurrency}`);
+  console.log(`Error Rate:   ${(config.errorRate * 100).toFixed(1)}%`);
+  console.log(`Delay:        ${config.delayMs}ms`);
+  console.log("=".repeat(60) + "\n");
+
+  const batches = Math.ceil(config.requestCount / config.concurrency);
+
+  for (let i = 0; i < batches; i++) {
+    const startIndex = i * config.concurrency;
+    const remainingRequests = config.requestCount - startIndex;
+    const batchSize = Math.min(config.concurrency, remainingRequests);
+
+    // Execute batch
+    await executeBatch(batchSize, startIndex);
+
+    // Update progress
+    process.stdout.write(
+      `\rProgress: ${renderProgressBar(
+        metrics.totalRequests,
+        config.requestCount
+      )}`
+    );
+
+    // Delay between batches if configured
+    if (config.delayMs && i < batches - 1) {
+      await new Promise((resolve) => setTimeout(resolve, config.delayMs));
+    }
+  }
+
+  metrics.endTime = new Date();
+  console.log("\n");
+}
+
+// Calculate and display results
+function displayResults(): void {
+  const duration =
+    (metrics.endTime!.getTime() - metrics.startTime.getTime()) / 1000;
+  const avgResponseTime =
+    metrics.responseTimes.reduce((a, b) => a + b, 0) /
+      metrics.responseTimes.length || 0;
+  const minResponseTime = Math.min(...metrics.responseTimes) || 0;
+  const maxResponseTime = Math.max(...metrics.responseTimes) || 0;
+  const requestsPerSecond = metrics.totalRequests / duration;
+
+  console.log("=".repeat(60));
+  console.log("📊 Test Results");
+  console.log("=".repeat(60));
+
+  // Request stats
+  console.log("\n📈 Request Statistics:");
+  console.log(`  ✓ Total Requests:      ${metrics.totalRequests}`);
+  console.log(
+    `  ✓ Successful:          ${metrics.successfulRequests} (${(
+      (metrics.successfulRequests / metrics.totalRequests) *
+      100
+    ).toFixed(1)}%)`
+  );
+  console.log(
+    `  ✗ Failed:              ${metrics.failedRequests} (${(
+      (metrics.failedRequests / metrics.totalRequests) *
+      100
+    ).toFixed(1)}%)`
+  );
+
+  // Canary distribution (if available)
+  if (metrics.canaryRequests > 0 || metrics.legacyRequests > 0) {
+    const totalVersioned = metrics.canaryRequests + metrics.legacyRequests;
+    console.log("\n🔀 Traffic Distribution:");
+    console.log(
+      `  🐤 Canary Requests:    ${metrics.canaryRequests} (${(
+        (metrics.canaryRequests / totalVersioned) *
+        100
+      ).toFixed(1)}%)`
+    );
+    console.log(
+      `  🏛️  Legacy Requests:    ${metrics.legacyRequests} (${(
+        (metrics.legacyRequests / totalVersioned) *
+        100
+      ).toFixed(1)}%)`
+    );
+  }
+
+  // Performance stats
+  console.log("\n⚡ Performance:");
+  console.log(`  ⏱️  Duration:           ${duration.toFixed(2)}s`);
+  console.log(`  📊 Requests/sec:       ${requestsPerSecond.toFixed(2)}`);
+  console.log(`  📉 Avg Response Time:  ${avgResponseTime.toFixed(0)}ms`);
+  console.log(`  📊 Min Response Time:  ${minResponseTime.toFixed(0)}ms`);
+  console.log(`  📊 Max Response Time:  ${maxResponseTime.toFixed(0)}ms`);
+
+  // Error details
+  if (metrics.errors.length > 0) {
+    console.log("\n❌ Errors:");
+    const errorCounts = new Map<string, number>();
+    metrics.errors.forEach((error) => {
+      const count = errorCounts.get(error.message) || 0;
+      errorCounts.set(error.message, count + 1);
+    });
+
+    errorCounts.forEach((count, message) => {
+      console.log(`  • ${message}: ${count} occurrences`);
+    });
+
+    // Show last 3 errors with timestamps
+    console.log("\n  Recent Errors:");
+    metrics.errors.slice(-3).forEach((error) => {
+      console.log(`    ${error.timestamp.toISOString()}: ${error.message}`);
+    });
+  }
+
+  console.log("\n" + "=".repeat(60));
+
+  // Validation checks
+  console.log("\n✅ Validation:");
+
+  // Check counter accuracy (if we can query the backend)
+  console.log(`  ℹ️  Note: Verify backend counters match these numbers`);
+  console.log(`     Expected total_requests: ${metrics.totalRequests}`);
+  console.log(`     Expected canary_requests: ${metrics.canaryRequests}`);
+  console.log(`     Expected legacy_requests: ${metrics.legacyRequests}`);
+
+  console.log("\n" + "=".repeat(60) + "\n");
+}
+
+// Main execution
+async function main() {
+  try {
+    await runLoadTest();
+    displayResults();
+
+    // Exit with appropriate code
+    if (metrics.failedRequests > 0 && config.errorRate === 0) {
+      console.log("⚠️  Test completed with unexpected errors");
+      process.exit(1);
+    } else {
+      console.log("✅ Test completed successfully");
+      process.exit(0);
+    }
+  } catch (error: any) {
+    console.error("❌ Load test failed:", error.message);
+    process.exit(1);
+  }
+}
+
+main();

package/src/tests/templates.test.ts

@@ -0,0 +1,154 @@
+#!/usr/bin/env ts-node
+/**
+ * Templates Test Suite
+ * Tests all template-related SDK operations
+ */
+
+import "dotenv/config";
+import { PromptMetrics } from "@promptmetrics/sdk";
+import { runTestSuite, logger } from "../utils/logger";
+
+const pm = new PromptMetrics({
+  apiKey: process.env.PROMPTMETRICS_API_KEY!,
+});
+
+const TEST_TEMPLATE_NAME = process.env.TEST_TEMPLATE_NAME || "test-template";
+const TEST_TEMPLATE_ID = process.env.TEST_TEMPLATE_ID;
+
+const tests = [
+  {
+    name: "Get template by name",
+    fn: async () => {
+      const template = await pm.templates.get(TEST_TEMPLATE_NAME);
+      if (!template) throw new Error("Template not found");
+      logger.data("Template ID", template._id);
+      logger.data("Template Name", template.name);
+      logger.data("Versions Count", template.versions?.length || 0);
+    },
+  },
+
+  {
+    name: "Get template by ID",
+    fn: async () => {
+      if (!TEST_TEMPLATE_ID) {
+        logger.info("Skipped - TEST_TEMPLATE_ID not set");
+        return;
+      }
+      const template = await pm.templates.get(TEST_TEMPLATE_ID);
+      if (!template) throw new Error("Template not found");
+      logger.data("Template ID", template._id);
+      logger.data("Template Name", template.name);
+    },
+  },
+
+  {
+    name: "Get template with specific version number",
+    fn: async () => {
+      const template = await pm.templates.get(TEST_TEMPLATE_NAME, {
+        version: 1,
+      });
+      if (!template) throw new Error("Template not found");
+      logger.data("Template Name", template.name);
+      logger.data("Version", template.versions?.[0]?.version || "N/A");
+    },
+  },
+
+  {
+    name: "Get template with env_label (production)",
+    fn: async () => {
+      try {
+        const template = await pm.templates.get(TEST_TEMPLATE_NAME, {
+          env_label: "production",
+        });
+        if (!template) throw new Error("Template not found");
+        logger.data("Template Name", template.name);
+        logger.data("Env Label", template.versions?.[0]?.env_label || "N/A");
+      } catch (error: unknown) {
+        const err = error as { message?: string };
+        if (err.message?.includes("not found")) {
+          logger.info("No production version found - this is OK");
+        } else {
+          throw error;
+        }
+      }
+    },
+  },
+
+  {
+    name: "List templates with pagination",
+    fn: async () => {
+      const result = await pm.templates.list({
+        page: 1,
+        per_page: 10,
+      });
+      if (!result) throw new Error("Failed to list templates");
+      logger.data("Templates Count", result.prompts?.length || 0);
+      logger.data("Total", result.total || 0);
+    },
+  },
+
+  {
+    name: "List templates with search filter",
+    fn: async () => {
+      const result = await pm.templates.list({
+        search: TEST_TEMPLATE_NAME.substring(0, 4), // Search with partial name
+        page: 1,
+        per_page: 10,
+      });
+      if (!result) throw new Error("Failed to list templates");
+      logger.data("Search Results", result.prompts?.length || 0);
+    },
+  },
+
+  {
+    name: "Run template (canary-aware)",
+    fn: async () => {
+      const result = await pm.templates.run(TEST_TEMPLATE_NAME, {
+        variables: {
+          topic: "SDK Template Run Test",
+        },
+      });
+
+      if (!result) throw new Error("Run failed");
+      logger.data("Request ID", result.request_id);
+      logger.data("Status", result.status);
+      logger.data("Latency", `${result.latency}s`);
+      logger.data("Total Tokens", result.llm_total_tokens);
+    },
+  },
+
+  {
+    name: "Run template with pm_tags",
+    fn: async () => {
+      const result = await pm.templates.run(TEST_TEMPLATE_NAME, {
+        variables: {
+          topic: "Template Tags Test",
+        },
+        pm_tags: ["template-run-test", "sdk-sample"],
+      });
+
+      if (!result) throw new Error("Run failed");
+      logger.data("Request ID", result.request_id);
+      logger.data("Status", result.status);
+      logger.data("Tags", result.tags || "Not returned");
+    },
+  },
+
+  {
+    name: "Run template with specific version",
+    fn: async () => {
+      const result = await pm.templates.run(TEST_TEMPLATE_NAME, {
+        version: 1,
+        variables: {
+          topic: "Specific Version Test",
+        },
+      });
+
+      if (!result) throw new Error("Run failed");
+      logger.data("Request ID", result.request_id);
+      logger.data("Status", result.status);
+    },
+  },
+];
+
+runTestSuite("Templates Test Suite", tests);