pi-free 1.0.0

package/tests/usage-tracking.test.ts

@@ -0,0 +1,150 @@
+/**
+ * Usage Tracking Tests
+ */
+
+import { beforeEach, describe, expect, it } from "vitest";
+import {
+	getModelUsage,
+	getProviderModelUsage,
+	getSessionUsage,
+	getTopModels,
+	incrementModelRequestCount,
+	resetUsageStats,
+} from "../usage/tracking.ts";
+
+describe("Usage Tracking", () => {
+	beforeEach(() => {
+		resetUsageStats();
+	});
+
+	describe("incrementModelRequestCount", () => {
+		it("should track model requests", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 100, 50);
+
+			const usage = getModelUsage("kilo", "gpt-4");
+			expect(usage).toBeDefined();
+			expect(usage?.count).toBe(1);
+			expect(usage?.tokensIn).toBe(100);
+			expect(usage?.tokensOut).toBe(50);
+		});
+
+		it("should accumulate multiple requests", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 100, 50);
+			incrementModelRequestCount("kilo", "gpt-4", 200, 100);
+
+			const usage = getModelUsage("kilo", "gpt-4");
+			expect(usage?.count).toBe(2);
+			expect(usage?.tokensIn).toBe(300);
+			expect(usage?.tokensOut).toBe(150);
+		});
+
+		it("should track different models separately", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 100, 50);
+			incrementModelRequestCount("kilo", "claude-3", 200, 100);
+
+			expect(getModelUsage("kilo", "gpt-4")?.count).toBe(1);
+			expect(getModelUsage("kilo", "claude-3")?.count).toBe(1);
+		});
+
+		it("should track different providers separately", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 100, 50);
+			incrementModelRequestCount("openrouter", "gpt-4", 200, 100);
+
+			expect(getModelUsage("kilo", "gpt-4")?.count).toBe(1);
+			expect(getModelUsage("openrouter", "gpt-4")?.count).toBe(1);
+		});
+	});
+
+	describe("getProviderModelUsage", () => {
+		it("should return all models for provider", () => {
+			incrementModelRequestCount("kilo", "model-a", 100, 50);
+			incrementModelRequestCount("kilo", "model-b", 200, 100);
+			incrementModelRequestCount("openrouter", "model-c", 300, 150);
+
+			const kiloModels = getProviderModelUsage("kilo");
+			expect(kiloModels).toHaveLength(2);
+			expect(kiloModels.map((m) => m.modelId)).toContain("model-a");
+			expect(kiloModels.map((m) => m.modelId)).toContain("model-b");
+		});
+
+		it("should sort by count descending", () => {
+			incrementModelRequestCount("kilo", "popular", 100, 50);
+			incrementModelRequestCount("kilo", "popular", 100, 50);
+			incrementModelRequestCount("kilo", "popular", 100, 50);
+			incrementModelRequestCount("kilo", "unpopular", 100, 50);
+
+			const models = getProviderModelUsage("kilo");
+			expect(models[0].modelId).toBe("popular");
+			expect(models[0].count).toBe(3);
+		});
+	});
+
+	describe("getTopModels", () => {
+		it("should return top N models across providers", () => {
+			// Add many models
+			for (let i = 0; i < 5; i++) {
+				incrementModelRequestCount("kilo", `kilo-model-${i}`, 100, 50);
+			}
+			for (let i = 0; i < 5; i++) {
+				incrementModelRequestCount("openrouter", `or-model-${i}`, 100, 50);
+			}
+
+			const top5 = getTopModels(5);
+			expect(top5).toHaveLength(5);
+		});
+
+		it("should sort by total count", () => {
+			incrementModelRequestCount("kilo", "high-usage", 100, 50);
+			incrementModelRequestCount("kilo", "high-usage", 100, 50);
+			incrementModelRequestCount("kilo", "high-usage", 100, 50);
+			incrementModelRequestCount("kilo", "low-usage", 100, 50);
+
+			const top = getTopModels(2);
+			expect(top[0].modelId).toBe("high-usage");
+			expect(top[0].count).toBe(3);
+		});
+	});
+
+	describe("getSessionUsage", () => {
+		it("should return session stats", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 1000, 500);
+			incrementModelRequestCount("openrouter", "claude", 2000, 1000);
+
+			const session = getSessionUsage();
+			expect(session.totalRequests).toBe(2);
+			expect(session.totalTokensIn).toBe(3000);
+			expect(session.totalTokensOut).toBe(1500);
+			expect(session.providers).toHaveLength(2);
+		});
+
+		it("should format duration", () => {
+			const session = getSessionUsage();
+			expect(session.duration).toBeGreaterThanOrEqual(0);
+			expect(typeof session.durationFormatted).toBe("string");
+		});
+
+		it("should sort providers by request count", () => {
+			incrementModelRequestCount("kilo", "model", 100, 50);
+			incrementModelRequestCount("kilo", "model", 100, 50);
+			incrementModelRequestCount("kilo", "model", 100, 50);
+			incrementModelRequestCount("openrouter", "model", 100, 50);
+
+			const session = getSessionUsage();
+			expect(session.providers[0].name).toBe("kilo");
+			expect(session.providers[0].requests).toBe(3);
+		});
+	});
+
+	describe("resetUsageStats", () => {
+		it("should clear all stats", () => {
+			incrementModelRequestCount("kilo", "gpt-4", 100, 50);
+			resetUsageStats();
+
+			const usage = getModelUsage("kilo", "gpt-4");
+			expect(usage).toBeUndefined();
+
+			const session = getSessionUsage();
+			expect(session.totalRequests).toBe(0);
+		});
+	});
+});

package/tests/util.test.ts

@@ -0,0 +1,413 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+	cleanModelName,
+	fetchWithRetry,
+	fetchWithTimeout,
+	isUsableModel,
+	logWarning,
+	mapOpenRouterModel,
+	parseModelResponse,
+} from "../lib/util.ts";
+
+describe("Utility Functions", () => {
+	describe("logWarning", () => {
+		it("should log warning with provider and message", () => {
+			// logWarning now uses lib/logger.ts internally
+			// This test verifies it doesn't throw
+			expect(() =>
+				logWarning("test-provider", "Test warning message"),
+			).not.toThrow();
+		});
+
+		it("should include error details when provided", () => {
+			const testError = new Error("Test error");
+			expect(() =>
+				logWarning("test-provider", "Test warning", testError),
+			).not.toThrow();
+		});
+	});
+
+	describe("isUsableModel", () => {
+		it("should return true for normal model IDs", () => {
+			expect(isUsableModel("gpt-4")).toBe(true);
+			expect(isUsableModel("claude-3-opus")).toBe(true);
+			expect(isUsableModel("llama-3-70b")).toBe(true);
+		});
+
+		it("should return false for test models", () => {
+			expect(isUsableModel("gpt-4-test")).toBe(false);
+			expect(isUsableModel("test-model")).toBe(false);
+		});
+
+		it("should return false for debug models", () => {
+			expect(isUsableModel("gpt-4-debug")).toBe(false);
+			expect(isUsableModel("debug-llama")).toBe(false);
+		});
+
+		it("should handle case variations", () => {
+			expect(isUsableModel("GPT-4-Test")).toBe(true); // Case sensitive check
+			expect(isUsableModel("model-TEST")).toBe(true);
+		});
+
+		it("should filter by minimum size", () => {
+			// 70b model should pass 70B minimum
+			expect(isUsableModel("llama-3-70b", 70)).toBe(true);
+			// 8b model should fail 70B minimum
+			expect(isUsableModel("llama-3-8b", 70)).toBe(false);
+			// 405b model should pass
+			expect(isUsableModel("llama-3-405b", 70)).toBe(true);
+		});
+
+		it("should handle MoE model sizes", () => {
+			// 8x22b = 176b total, should pass 70B
+			expect(isUsableModel("mixtral-8x22b", 70)).toBe(true);
+			// 8x7b = 56b total, should fail 70B
+			expect(isUsableModel("mixtral-8x7b", 70)).toBe(false);
+		});
+
+		it("should skip size filter when minSizeB not provided", () => {
+			expect(isUsableModel("tiny-llama")).toBe(true);
+			expect(isUsableModel("llama-3-8b")).toBe(true);
+		});
+	});
+
+	describe("cleanModelName", () => {
+		it("should strip provider prefix with colon", () => {
+			expect(cleanModelName("QWEN : Qwen2.5 72B Instruct")).toBe(
+				"Qwen2.5 72B Instruct",
+			);
+			expect(cleanModelName("OpenAI : GPT-4")).toBe("GPT-4");
+			expect(cleanModelName("Anthropic : Claude 3 Opus")).toBe("Claude 3 Opus");
+		});
+
+		it("should strip provider prefix with slash", () => {
+			expect(cleanModelName("QWEN / Qwen2.5 Coder 32B Instruct")).toBe(
+				"Qwen2.5 Coder 32B Instruct",
+			);
+			expect(cleanModelName("Meta / Llama 3 70B")).toBe("Llama 3 70B");
+		});
+
+		it("should handle varying whitespace around separator", () => {
+			expect(cleanModelName("Provider:Model")).toBe("Model");
+			expect(cleanModelName("Provider: Model")).toBe("Model");
+			expect(cleanModelName("Provider :Model")).toBe("Model");
+			expect(cleanModelName("Provider  :  Model")).toBe("Model");
+		});
+
+		it("should return original name when no separator", () => {
+			expect(cleanModelName("GPT-4")).toBe("GPT-4");
+			expect(cleanModelName("Claude 3 Opus")).toBe("Claude 3 Opus");
+		});
+
+		it("should trim whitespace", () => {
+			expect(cleanModelName("  Model Name  ")).toBe("Model Name");
+		});
+	});
+
+	describe("mapOpenRouterModel", () => {
+		it("should map basic OpenRouter model", () => {
+			const input = {
+				id: "openai/gpt-4",
+				name: "GPT-4",
+				context_length: 8192,
+				max_completion_tokens: 4096,
+				pricing: {
+					prompt: "0.03",
+					completion: "0.06",
+				},
+				architecture: {
+					input_modalities: ["text"],
+					output_modalities: ["text"],
+				},
+			};
+
+			const result = mapOpenRouterModel(input);
+
+			expect(result.id).toBe("openai/gpt-4");
+			expect(result.name).toBe("GPT-4");
+			expect(result.cost.input).toBe(0.03);
+			expect(result.cost.output).toBe(0.06);
+			expect(result.cost.cacheRead).toBe(0);
+			expect(result.cost.cacheWrite).toBe(0);
+			expect(result.contextWindow).toBe(8192);
+			expect(result.maxTokens).toBe(4096);
+			expect(result.reasoning).toBe(false);
+			expect(result.input).toEqual(["text"]);
+		});
+
+		it("should clean provider prefix from model name", () => {
+			const input = {
+				id: "qwen/qwen-2.5-72b-instruct",
+				name: "QWEN : Qwen2.5 72B Instruct",
+				context_length: 128000,
+				pricing: {
+					prompt: "0",
+					completion: "0",
+				},
+				architecture: {
+					input_modalities: ["text"],
+					output_modalities: ["text"],
+				},
+			};
+
+			const result = mapOpenRouterModel(input);
+
+			expect(result.id).toBe("qwen/qwen-2.5-72b-instruct");
+			expect(result.name).toBe("Qwen2.5 72B Instruct");
+		});
+
+		it("should detect image input capability", () => {
+			const input = {
+				id: "openai/gpt-4-vision",
+				name: "GPT-4 Vision",
+				context_length: 128000,
+				pricing: {
+					prompt: "0.01",
+					completion: "0.03",
+				},
+				architecture: {
+					input_modalities: ["text", "image"],
+					output_modalities: ["text"],
+				},
+			};
+
+			const result = mapOpenRouterModel(input);
+
+			expect(result.input).toEqual(["text", "image"]);
+		});
+
+		it("should handle free models (zero pricing)", () => {
+			const input = {
+				id: "meta-llama/llama-3.1-8b",
+				name: "Llama 3.1 8B",
+				context_length: 128000,
+				pricing: {
+					prompt: "0",
+					completion: "0",
+				},
+				architecture: {
+					input_modalities: ["text"],
+					output_modalities: ["text"],
+				},
+			};
+
+			const result = mapOpenRouterModel(input);
+
+			expect(result.cost.input).toBe(0);
+			expect(result.cost.output).toBe(0);
+		});
+
+		it("should use default values when fields are missing", () => {
+			const input = {
+				id: "unknown/model",
+				name: "Unknown Model",
+			};
+
+			const result = mapOpenRouterModel(
+				input as unknown as Parameters<typeof mapOpenRouterModel>[0],
+			);
+
+			expect(result.contextWindow).toBe(4096);
+			expect(result.maxTokens).toBe(4096);
+			expect(result.cost.input).toBe(0);
+			expect(result.cost.output).toBe(0);
+		});
+
+		it("should use top_provider max tokens when available", () => {
+			const input = {
+				id: "anthropic/claude-3-opus",
+				name: "Claude 3 Opus",
+				context_length: 200000,
+				max_completion_tokens: null,
+				top_provider: {
+					max_completion_tokens: 4096,
+				},
+				pricing: {
+					prompt: "0.015",
+					completion: "0.075",
+				},
+				architecture: {
+					input_modalities: ["text"],
+					output_modalities: ["text"],
+				},
+			};
+
+			const result = mapOpenRouterModel(input);
+
+			expect(result.maxTokens).toBe(4096);
+		});
+	});
+
+	describe("parseModelResponse", () => {
+		it("should parse valid model response", async () => {
+			const mockResponse = {
+				ok: true,
+				json: async () => ({ data: [{ id: "model-1" }, { id: "model-2" }] }),
+			} as Response;
+
+			const result = await parseModelResponse(mockResponse, "test-provider");
+
+			expect(result.data).toHaveLength(2);
+			expect(result.data[0].id).toBe("model-1");
+		});
+
+		it("should throw on non-ok response", async () => {
+			const mockResponse = {
+				ok: false,
+				status: 500,
+				statusText: "Internal Server Error",
+			} as Response;
+
+			await expect(
+				parseModelResponse(mockResponse, "test-provider"),
+			).rejects.toThrow(
+				"Failed to fetch test-provider models: 500 Internal Server Error",
+			);
+		});
+
+		it("should throw on missing data array", async () => {
+			const mockResponse = {
+				ok: true,
+				json: async () => ({ models: [] }), // Wrong property name
+			} as Response;
+
+			await expect(
+				parseModelResponse(mockResponse, "test-provider"),
+			).rejects.toThrow(
+				"Invalid test-provider models response: missing data array",
+			);
+		});
+
+		it("should throw on non-array data", async () => {
+			const mockResponse = {
+				ok: true,
+				json: async () => ({ data: "not-an-array" }),
+			} as Response;
+
+			await expect(
+				parseModelResponse(mockResponse, "test-provider"),
+			).rejects.toThrow(
+				"Invalid test-provider models response: missing data array",
+			);
+		});
+	});
+
+	describe("fetchWithTimeout", () => {
+		it("should fetch successfully within timeout", async () => {
+			// Mock fetch to return immediately
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+			} as Response);
+
+			const result = await fetchWithTimeout(
+				"https://api.example.com/data",
+				{},
+				5000,
+			);
+
+			expect(result.ok).toBe(true);
+		});
+
+		it("should pass headers and options to fetch", async () => {
+			const fetchMock = vi.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+			} as Response);
+			global.fetch = fetchMock;
+
+			await fetchWithTimeout(
+				"https://api.example.com/data",
+				{
+					headers: { Authorization: "Bearer token" },
+					method: "POST",
+				},
+				5000,
+			);
+
+			expect(fetchMock).toHaveBeenCalledWith(
+				"https://api.example.com/data",
+				expect.objectContaining({
+					headers: { Authorization: "Bearer token" },
+					method: "POST",
+					signal: expect.any(AbortSignal),
+				}),
+			);
+		});
+	});
+
+	describe("fetchWithRetry", () => {
+		it("should succeed on first attempt", async () => {
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: true,
+				status: 200,
+			} as Response);
+
+			const result = await fetchWithRetry("https://api.example.com/data", {});
+
+			expect(result.ok).toBe(true);
+			expect(global.fetch).toHaveBeenCalledTimes(1);
+		});
+
+		it("should retry on server error (5xx)", async () => {
+			global.fetch = vi
+				.fn()
+				.mockResolvedValueOnce({
+					ok: false,
+					status: 503,
+				} as Response)
+				.mockResolvedValueOnce({
+					ok: true,
+					status: 200,
+				} as Response);
+
+			const result = await fetchWithRetry(
+				"https://api.example.com/data",
+				{},
+				3,
+				100,
+			);
+
+			expect(result.ok).toBe(true);
+			expect(global.fetch).toHaveBeenCalledTimes(2);
+		});
+
+		it("should throw immediately on 429 rate limit", async () => {
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: false,
+				status: 429,
+			} as Response);
+
+			await expect(
+				fetchWithRetry("https://api.example.com/data", {}),
+			).rejects.toThrow("Rate limited (429)");
+		});
+
+		it("should throw after max retries", async () => {
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: false,
+				status: 500,
+			} as Response);
+
+			await expect(
+				fetchWithRetry("https://api.example.com/data", {}, 2, 50),
+			).rejects.toThrow();
+
+			expect(global.fetch).toHaveBeenCalledTimes(2);
+		});
+
+		it("should return non-retryable error response", async () => {
+			// 400 Bad Request - should not retry
+			global.fetch = vi.fn().mockResolvedValue({
+				ok: false,
+				status: 400,
+			} as Response);
+
+			const result = await fetchWithRetry("https://api.example.com/data", {});
+
+			expect(result.ok).toBe(false);
+			expect(result.status).toBe(400);
+			expect(global.fetch).toHaveBeenCalledTimes(1);
+		});
+	});
+});