retestkit 1.4.1 → 1.5.0

package/src/sampling/index.test.ts

@@ -1,414 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
-import { z } from "zod";
-import { createSamplingClient } from "./index.js";
-import type { Logger } from "../logger.js";
-import type { ClientCapabilities } from "../types/capabilities.js";
-import type { SamplingRequest, SamplingResponse } from "./types.js";
-
-describe("sampling", () => {
-  let mockLogger: Logger;
-
-  beforeEach(() => {
-    mockLogger = {
-      debug: vi.fn(),
-      info: vi.fn(),
-      warn: vi.fn(),
-      error: vi.fn(),
-      setLevel: vi.fn(),
-      withCorrelation: vi.fn().mockReturnThis(),
-    } as unknown as Logger;
-  });
-
-  const capabilitiesWithSampling: ClientCapabilities = {
-    sampling: true,
-    elicitation: false,
-    logging: false,
-    progress: true,
-    resourcesListChanged: false,
-    resourcesSubscribe: false,
-    protocolVersion: "2025-06-18",
-  };
-
-  const capabilitiesWithoutSampling: ClientCapabilities = {
-    sampling: false,
-    elicitation: false,
-    logging: false,
-    progress: false,
-    resourcesListChanged: false,
-    resourcesSubscribe: false,
-    protocolVersion: "2025-06-18",
-  };
-
-  describe("createSamplingClient", () => {
-    it("reports sampling availability correctly", () => {
-      const clientWith = createSamplingClient(
-        vi.fn(),
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const clientWithout = createSamplingClient(
-        vi.fn(),
-        capabilitiesWithoutSampling,
-        mockLogger
-      );
-
-      expect(clientWith.hasSampling()).toBe(true);
-      expect(clientWithout.hasSampling()).toBe(false);
-    });
-  });
-
-  describe("createMessage", () => {
-    const testSchema = z.object({
-      action: z.string(),
-      value: z.number(),
-    });
-
-    it("returns fallback when sampling unavailable", async () => {
-      const requestSampling = vi.fn();
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithoutSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.success).toBe(false);
-      expect(result.error).toBe("Sampling not available");
-      expect(result.promptResource).toBeDefined();
-      expect(requestSampling).not.toHaveBeenCalled();
-    });
-
-    it("parses valid JSON response successfully", async () => {
-      const mockResponse: SamplingResponse = {
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: JSON.stringify({ action: "test", value: 42 }),
-        },
-      };
-
-      const requestSampling = vi.fn().mockResolvedValue(mockResponse);
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.success).toBe(true);
-      expect(result.data).toEqual({ action: "test", value: 42 });
-    });
-
-    it("handles JSON wrapped in markdown code blocks", async () => {
-      const mockResponse: SamplingResponse = {
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: '```json\n{"action": "test", "value": 42}\n```',
-        },
-      };
-
-      const requestSampling = vi.fn().mockResolvedValue(mockResponse);
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.success).toBe(true);
-      expect(result.data).toEqual({ action: "test", value: 42 });
-    });
-
-    it("handles JSON wrapped in generic code blocks", async () => {
-      const mockResponse: SamplingResponse = {
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: '```\n{"action": "test", "value": 42}\n```',
-        },
-      };
-
-      const requestSampling = vi.fn().mockResolvedValue(mockResponse);
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.success).toBe(true);
-      expect(result.data).toEqual({ action: "test", value: 42 });
-    });
-
-    it("retries on invalid JSON when retryOnFailure is true", async () => {
-      const requestSampling = vi
-        .fn()
-        .mockResolvedValueOnce({
-          model: "test-model",
-          stopReason: "end_turn",
-          content: { type: "text", text: "not valid json" },
-        })
-        .mockResolvedValueOnce({
-          model: "test-model",
-          stopReason: "end_turn",
-          content: {
-            type: "text",
-            text: JSON.stringify({ action: "test", value: 42 }),
-          },
-        });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-        retryOnFailure: true,
-      });
-
-      expect(result.success).toBe(true);
-      expect(requestSampling).toHaveBeenCalledTimes(2);
-    });
-
-    it("does not retry when retryOnFailure is false", async () => {
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: { type: "text", text: "not valid json" },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-        retryOnFailure: false,
-      });
-
-      expect(result.success).toBe(false);
-      expect(result.error).toContain("Failed to parse response as JSON");
-      expect(requestSampling).toHaveBeenCalledTimes(1);
-    });
-
-    it("retries on schema validation failure", async () => {
-      const requestSampling = vi
-        .fn()
-        .mockResolvedValueOnce({
-          model: "test-model",
-          stopReason: "end_turn",
-          content: {
-            type: "text",
-            text: JSON.stringify({ action: "test", value: "not a number" }),
-          },
-        })
-        .mockResolvedValueOnce({
-          model: "test-model",
-          stopReason: "end_turn",
-          content: {
-            type: "text",
-            text: JSON.stringify({ action: "test", value: 42 }),
-          },
-        });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-        retryOnFailure: true,
-      });
-
-      expect(result.success).toBe(true);
-      expect(result.data).toEqual({ action: "test", value: 42 });
-    });
-
-    it("fails after retry exhausted", async () => {
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: { type: "text", text: "still not valid json" },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-        retryOnFailure: true,
-      });
-
-      expect(result.success).toBe(false);
-      expect(result.error).toContain("Retry failed");
-    });
-
-    it("handles request errors gracefully", async () => {
-      const requestSampling = vi
-        .fn()
-        .mockRejectedValue(new Error("Network error"));
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.success).toBe(false);
-      expect(result.error).toBe("Network error");
-    });
-
-    it("uses custom maxTokens and temperature", async () => {
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: JSON.stringify({ action: "test", value: 42 }),
-        },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-        maxTokens: 1000,
-        temperature: 0.5,
-      });
-
-      const request = requestSampling.mock.calls[0][0] as SamplingRequest;
-      expect(request.maxTokens).toBe(1000);
-      expect(request.temperature).toBe(0.5);
-    });
-
-    it("includes system prefix in the system prompt", async () => {
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: JSON.stringify({ action: "test", value: 42 }),
-        },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      const request = requestSampling.mock.calls[0][0] as SamplingRequest;
-      expect(request.systemPrompt).toContain("[WEBTEST-SYSTEM]");
-      expect(request.systemPrompt).toContain("You are a test assistant");
-    });
-
-    it("sets includeContext to thisServer", async () => {
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: {
-          type: "text",
-          text: JSON.stringify({ action: "test", value: 42 }),
-        },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      const request = requestSampling.mock.calls[0][0] as SamplingRequest;
-      expect(request.includeContext).toBe("thisServer");
-    });
-
-    it("returns rawResponse on success", async () => {
-      const rawJson = JSON.stringify({ action: "test", value: 42 });
-      const requestSampling = vi.fn().mockResolvedValue({
-        model: "test-model",
-        stopReason: "end_turn",
-        content: { type: "text", text: rawJson },
-      });
-
-      const client = createSamplingClient(
-        requestSampling,
-        capabilitiesWithSampling,
-        mockLogger
-      );
-
-      const result = await client.createMessage({
-        systemPrompt: "You are a test assistant",
-        userPrompt: "Do something",
-        schema: testSchema,
-      });
-
-      expect(result.rawResponse).toBe(rawJson);
-    });
-  });
-});

package/src/sampling/index.ts

@@ -1,286 +0,0 @@
-import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
-import type { z } from "zod";
-import type { Logger } from "../logger.js";
-import type { ClientCapabilities } from "../types/capabilities.js";
-import {
-  type SamplingRequest,
-  type SamplingResponse,
-  type SamplingResult,
-  type SamplingOptions,
-  type FallbackPromptResource,
-} from "./types.js";
-import { SYSTEM_PREFIX } from "./prompts.js";
-
-export * from "./types.js";
-export * from "./prompts.js";
-
-export interface SamplingClient {
-  createMessage<T>(options: SamplingOptions<T>): Promise<SamplingResult<T>>;
-  hasSampling(): boolean;
-}
-
-export function createSamplingClient(
-  requestSampling: (request: SamplingRequest) => Promise<SamplingResponse>,
-  capabilities: ClientCapabilities,
-  logger: Logger
-): SamplingClient {
-  const hasSamplingCapability = capabilities.sampling;
-
-  async function createMessage<T>(
-    options: SamplingOptions<T>
-  ): Promise<SamplingResult<T>> {
-    const {
-      systemPrompt,
-      userPrompt,
-      schema,
-      maxTokens = 4096,
-      temperature = 0.7,
-      retryOnFailure = true,
-      fallbackResourceId,
-    } = options;
-
-    // If sampling not available, return fallback
-    if (!hasSamplingCapability) {
-      logger.info("Sampling unavailable, returning fallback prompt resource");
-
-      const fallback: FallbackPromptResource = {
-        type: "fallback",
-        prompt: `${SYSTEM_PREFIX}\n\n${systemPrompt}\n\n${userPrompt}`,
-        expectedSchema: schema._def,
-        instructions:
-          "Execute this prompt with your LLM and provide the JSON response via manualNextActions input",
-      };
-
-      return {
-        success: false,
-        error: "Sampling not available",
-        promptResource: JSON.stringify(fallback, null, 2),
-      };
-    }
-
-    // Build the full system prompt with security prefix
-    const fullSystemPrompt = `${SYSTEM_PREFIX}\n\n${systemPrompt}`;
-
-    const request: SamplingRequest = {
-      messages: [
-        {
-          role: "user",
-          content: {
-            type: "text",
-            text: userPrompt,
-          },
-        },
-      ],
-      systemPrompt: fullSystemPrompt,
-      maxTokens,
-      temperature,
-      includeContext: "thisServer",
-    };
-
-    // Log sampling request for audit (with truncation)
-    logger.debug("Sampling request", {
-      systemPromptLength: fullSystemPrompt.length,
-      userPromptLength: userPrompt.length,
-      maxTokens,
-    });
-
-    try {
-      const response = await requestSampling(request);
-
-      // Log sampling response for audit
-      logger.debug("Sampling response received", {
-        model: response.model,
-        stopReason: response.stopReason,
-        responseLength: response.content.text.length,
-      });
-
-      const rawText = response.content.text;
-
-      // Try to parse and validate JSON response
-      let parsed: unknown;
-      try {
-        // Handle potential markdown code blocks
-        let jsonText = rawText.trim();
-        if (jsonText.startsWith("```json")) {
-          jsonText = jsonText.slice(7);
-        } else if (jsonText.startsWith("```")) {
-          jsonText = jsonText.slice(3);
-        }
-        if (jsonText.endsWith("```")) {
-          jsonText = jsonText.slice(0, -3);
-        }
-        jsonText = jsonText.trim();
-
-        parsed = JSON.parse(jsonText);
-      } catch (parseError) {
-        logger.warn("Failed to parse sampling response as JSON", {
-          error:
-            parseError instanceof Error ? parseError.message : "Parse error",
-          rawResponse: rawText.slice(0, 500),
-        });
-
-        // Retry if enabled
-        if (retryOnFailure) {
-          return retryWithFeedback(
-            options,
-            rawText,
-            "Response was not valid JSON. Please respond with valid JSON only.",
-            requestSampling,
-            logger
-          );
-        }
-
-        return {
-          success: false,
-          error: "Failed to parse response as JSON",
-          rawResponse: rawText,
-        };
-      }
-
-      // Validate against schema
-      const validationResult = schema.safeParse(parsed);
-
-      if (!validationResult.success) {
-        const errors = validationResult.error.issues
-          .map((i) => `${i.path.join(".")}: ${i.message}`)
-          .join("; ");
-
-        logger.warn("Sampling response failed schema validation", {
-          errors,
-          parsed,
-        });
-
-        // Retry if enabled
-        if (retryOnFailure) {
-          return retryWithFeedback(
-            options,
-            rawText,
-            `Response did not match expected schema: ${errors}`,
-            requestSampling,
-            logger
-          );
-        }
-
-        return {
-          success: false,
-          error: `Schema validation failed: ${errors}`,
-          rawResponse: rawText,
-        };
-      }
-
-      return {
-        success: true,
-        data: validationResult.data,
-        rawResponse: rawText,
-      };
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      logger.error("Sampling request failed", { error: message });
-
-      return {
-        success: false,
-        error: message,
-      };
-    }
-  }
-
-  return {
-    createMessage,
-    hasSampling: () => hasSamplingCapability,
-  };
-}
-
-async function retryWithFeedback<T>(
-  originalOptions: SamplingOptions<T>,
-  previousResponse: string,
-  feedbackMessage: string,
-  requestSampling: (request: SamplingRequest) => Promise<SamplingResponse>,
-  logger: Logger
-): Promise<SamplingResult<T>> {
-  logger.info("Retrying sampling with error feedback");
-
-  const {
-    systemPrompt,
-    userPrompt,
-    schema,
-    maxTokens = 4096,
-    temperature = 0.7,
-  } = originalOptions;
-
-  const fullSystemPrompt = `${SYSTEM_PREFIX}\n\n${systemPrompt}`;
-
-  const request: SamplingRequest = {
-    messages: [
-      {
-        role: "user",
-        content: {
-          type: "text",
-          text: userPrompt,
-        },
-      },
-      {
-        role: "assistant",
-        content: {
-          type: "text",
-          text: previousResponse,
-        },
-      },
-      {
-        role: "user",
-        content: {
-          type: "text",
-          text: `ERROR: ${feedbackMessage}\n\nPlease provide a corrected response with valid JSON matching the expected schema.`,
-        },
-      },
-    ],
-    systemPrompt: fullSystemPrompt,
-    maxTokens,
-    temperature,
-    includeContext: "thisServer",
-  };
-
-  try {
-    const response = await requestSampling(request);
-    const rawText = response.content.text;
-
-    let jsonText = rawText.trim();
-    if (jsonText.startsWith("```json")) {
-      jsonText = jsonText.slice(7);
-    } else if (jsonText.startsWith("```")) {
-      jsonText = jsonText.slice(3);
-    }
-    if (jsonText.endsWith("```")) {
-      jsonText = jsonText.slice(0, -3);
-    }
-    jsonText = jsonText.trim();
-
-    const parsed = JSON.parse(jsonText);
-    const validationResult = schema.safeParse(parsed);
-
-    if (!validationResult.success) {
-      const errors = validationResult.error.issues
-        .map((i) => `${i.path.join(".")}: ${i.message}`)
-        .join("; ");
-
-      return {
-        success: false,
-        error: `Retry also failed schema validation: ${errors}`,
-        rawResponse: rawText,
-      };
-    }
-
-    return {
-      success: true,
-      data: validationResult.data,
-      rawResponse: rawText,
-    };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : "Unknown error";
-    logger.error("Sampling retry failed", { error: message });
-
-    return {
-      success: false,
-      error: `Retry failed: ${message}`,
-    };
-  }
-}