portos-ai-toolkit 0.1.0 → 0.3.0

package/README.md

@@ -16,6 +16,10 @@ npm install portos-ai-toolkit
 - **React Components**: Ready-to-use React components and hooks for AI provider management
 - **Express Routes**: Pre-built Express route handlers for provider, prompt, and run management
 
+## Screenshot
+
+![AI Providers UI](screenshot.png)
+
 ## Usage
 
 ### Server-side

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "portos-ai-toolkit",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Shared AI provider, model, and prompt template patterns for PortOS-style applications",
   "author": "Adam Eivy <adam@eivy.com> (https://atomantic.com)",
   "license": "MIT",
@@ -39,6 +39,7 @@
   "peerDependencies": {
     "express": "^4.21.2 || ^5.2.1",
     "socket.io": "^4.8.3",
+    "socket.io-client": "^4.8.3",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"
   },
@@ -49,6 +50,9 @@
     "socket.io": {
       "optional": true
     },
+    "socket.io-client": {
+      "optional": true
+    },
     "react": {
       "optional": true
     },

package/src/server/errorDetection.js

@@ -0,0 +1,305 @@
+/**
+ * Error Detection Utility
+ *
+ * Detects and categorizes errors from AI provider responses,
+ * particularly rate limits and usage limits that require fallback handling.
+ */
+
+/**
+ * Error categories and their characteristics
+ */
+export const ERROR_CATEGORIES = {
+  RATE_LIMIT: 'rate-limit',
+  USAGE_LIMIT: 'usage-limit',
+  AUTH_ERROR: 'auth-error',
+  MODEL_NOT_FOUND: 'model-not-found',
+  NETWORK_ERROR: 'network-error',
+  TIMEOUT: 'timeout',
+  QUOTA_EXCEEDED: 'quota-exceeded',
+  UNKNOWN: 'unknown'
+};
+
+/**
+ * Error patterns for detection
+ * NOTE: Order matters! More specific patterns should come before general ones.
+ */
+const ERROR_PATTERNS = [
+  // Quota exceeded (billing issues - check before usage limit since "quota" could match both)
+  {
+    pattern: /billing|payment|credit|insufficient funds/i,
+    category: ERROR_CATEGORIES.QUOTA_EXCEEDED,
+    requiresFallback: true,
+    actionable: true,
+    suggestedFix: 'Check billing status and add credits to the provider account'
+  },
+
+  // Rate limiting (temporary, short wait)
+  {
+    pattern: /API Error: 429|rate.?limit|too many requests/i,
+    category: ERROR_CATEGORIES.RATE_LIMIT,
+    requiresFallback: false, // Usually temporary
+    actionable: false,
+    suggestedFix: 'Wait and retry - temporary rate limiting'
+  },
+
+  // Usage limits (longer wait, fallback recommended)
+  {
+    pattern: /(?:hit your usage limit|You've hit your limit|usage limit|Upgrade to Pro)/i,
+    category: ERROR_CATEGORIES.USAGE_LIMIT,
+    requiresFallback: true,
+    actionable: true,
+    suggestedFix: 'Provider usage limit reached. Using fallback provider or wait for limit reset.',
+    extractWaitTime: true
+  },
+
+  // Authentication errors
+  {
+    pattern: /unauthorized|invalid.?api.?key|authentication|forbidden|401|403/i,
+    category: ERROR_CATEGORIES.AUTH_ERROR,
+    requiresFallback: true,
+    actionable: true,
+    suggestedFix: 'Check API key configuration for this provider'
+  },
+
+  // Model not found
+  {
+    pattern: /model.*(not found|does not exist|unavailable)|invalid model/i,
+    category: ERROR_CATEGORIES.MODEL_NOT_FOUND,
+    requiresFallback: true,
+    actionable: true,
+    suggestedFix: 'Check model name and availability in provider settings'
+  },
+
+  // Network errors
+  {
+    pattern: /ECONNREFUSED|ENOTFOUND|network error|connection refused|timeout|ETIMEDOUT/i,
+    category: ERROR_CATEGORIES.NETWORK_ERROR,
+    requiresFallback: false, // Often temporary
+    actionable: false,
+    suggestedFix: 'Check network connectivity and provider endpoint URL'
+  },
+
+  // Timeout
+  {
+    pattern: /timed out|timeout exceeded|SIGTERM/i,
+    category: ERROR_CATEGORIES.TIMEOUT,
+    requiresFallback: false,
+    actionable: false,
+    suggestedFix: 'Consider increasing timeout or reducing prompt complexity'
+  }
+];
+
+/**
+ * Wait time extraction patterns
+ */
+const WAIT_TIME_PATTERNS = [
+  // "resets 6am (America/Los_Angeles)" - extract timezone-aware time
+  /resets?\s+(\d{1,2}(?:am|pm)?)\s*\(([^)]+)\)/i,
+  // "try again in X day(s) X hour(s) X minute(s)"
+  /try again in\s+((?:\d+\s*(?:day|hour|minute|second)s?\s*)+)/i,
+  // "wait X minutes/hours/days"
+  /wait\s+((?:\d+\s*(?:day|hour|minute|second)s?\s*)+)/i,
+  // "in X hours", "in X minutes"
+  /in\s+(\d+)\s*(day|hour|minute|second)s?/i,
+  // Specific time format "1 day 1 hour 33 minutes"
+  /(\d+\s*day(?:s)?)?[,\s]*(\d+\s*hour(?:s)?)?[,\s]*(\d+\s*min(?:ute)?(?:s)?)?/i
+];
+
+/**
+ * Extract wait time from error message
+ * @param {string} text - Error text to search
+ * @returns {string|null} - Human-readable wait time or null
+ */
+export function extractWaitTime(text) {
+  if (!text) return null;
+
+  // Try each pattern
+  for (const pattern of WAIT_TIME_PATTERNS) {
+    const match = text.match(pattern);
+    if (match) {
+      // Clean up and return the matched time string
+      const timeStr = match.slice(1).filter(Boolean).join(' ').trim();
+      if (timeStr && timeStr !== ' ') {
+        return timeStr;
+      }
+    }
+  }
+
+  // Try to find any time-related content
+  const generalMatch = text.match(/(\d+)\s*(day|hour|min|sec)(?:ute)?(?:s)?/gi);
+  if (generalMatch) {
+    return generalMatch.join(' ');
+  }
+
+  return null;
+}
+
+/**
+ * Analyze error text and categorize it
+ *
+ * @param {string} errorText - The error message or output to analyze
+ * @param {number} exitCode - Optional exit code from CLI process
+ * @returns {Object} - Error analysis result
+ */
+export function analyzeError(errorText, exitCode = null) {
+  if (!errorText && exitCode === 0) {
+    return {
+      hasError: false,
+      category: null,
+      message: null,
+      waitTime: null,
+      requiresFallback: false,
+      actionable: false,
+      suggestedFix: null
+    };
+  }
+
+  const text = String(errorText || '');
+
+  // Check each pattern
+  for (const errorPattern of ERROR_PATTERNS) {
+    if (errorPattern.pattern.test(text)) {
+      const result = {
+        hasError: true,
+        category: errorPattern.category,
+        message: extractErrorMessage(text),
+        waitTime: errorPattern.extractWaitTime ? extractWaitTime(text) : null,
+        requiresFallback: errorPattern.requiresFallback,
+        actionable: errorPattern.actionable,
+        suggestedFix: errorPattern.suggestedFix
+      };
+
+      return result;
+    }
+  }
+
+  // If we have an error but couldn't categorize it
+  if (exitCode !== 0 && exitCode !== null) {
+    return {
+      hasError: true,
+      category: ERROR_CATEGORIES.UNKNOWN,
+      message: extractErrorMessage(text) || `Process exited with code ${exitCode}`,
+      waitTime: null,
+      requiresFallback: false,
+      actionable: false,
+      suggestedFix: null
+    };
+  }
+
+  return {
+    hasError: false,
+    category: null,
+    message: null,
+    waitTime: null,
+    requiresFallback: false,
+    actionable: false,
+    suggestedFix: null
+  };
+}
+
+/**
+ * Extract the most relevant error message from text
+ * @param {string} text - Full error text
+ * @returns {string} - Extracted error message
+ */
+function extractErrorMessage(text) {
+  if (!text) return '';
+
+  // Try to find common error message patterns
+  const patterns = [
+    /Error:\s*(.+?)(?:\n|$)/i,
+    /error":\s*"([^"]+)"/i,
+    /message":\s*"([^"]+)"/i,
+    /failed:\s*(.+?)(?:\n|$)/i
+  ];
+
+  for (const pattern of patterns) {
+    const match = text.match(pattern);
+    if (match) {
+      return match[1].trim();
+    }
+  }
+
+  // Return first meaningful line
+  const lines = text.split('\n').filter(line => line.trim());
+  return lines[0]?.substring(0, 200) || text.substring(0, 200);
+}
+
+/**
+ * Check if an HTTP status code indicates rate limiting
+ * @param {number} statusCode - HTTP status code
+ * @returns {boolean}
+ */
+export function isRateLimitStatus(statusCode) {
+  return statusCode === 429;
+}
+
+/**
+ * Check if an HTTP status code indicates auth error
+ * @param {number} statusCode - HTTP status code
+ * @returns {boolean}
+ */
+export function isAuthErrorStatus(statusCode) {
+  return statusCode === 401 || statusCode === 403;
+}
+
+/**
+ * Analyze an HTTP response for errors
+ * @param {Object} response - HTTP response object with status and body
+ * @returns {Object} - Error analysis result
+ */
+export function analyzeHttpError(response) {
+  const { status, statusText, body } = response;
+
+  if (status >= 200 && status < 300) {
+    return {
+      hasError: false,
+      category: null,
+      message: null,
+      waitTime: null,
+      requiresFallback: false,
+      actionable: false,
+      suggestedFix: null
+    };
+  }
+
+  // Check status code first
+  if (isRateLimitStatus(status)) {
+    return {
+      hasError: true,
+      category: ERROR_CATEGORIES.RATE_LIMIT,
+      message: `Rate limit exceeded (${status})`,
+      waitTime: extractWaitTime(body),
+      requiresFallback: false,
+      actionable: false,
+      suggestedFix: 'Wait and retry - temporary rate limiting'
+    };
+  }
+
+  if (isAuthErrorStatus(status)) {
+    return {
+      hasError: true,
+      category: ERROR_CATEGORIES.AUTH_ERROR,
+      message: `Authentication failed (${status})`,
+      waitTime: null,
+      requiresFallback: true,
+      actionable: true,
+      suggestedFix: 'Check API key configuration for this provider'
+    };
+  }
+
+  // Analyze body for more specific errors
+  if (body) {
+    return analyzeError(body);
+  }
+
+  return {
+    hasError: true,
+    category: ERROR_CATEGORIES.UNKNOWN,
+    message: statusText || `HTTP ${status}`,
+    waitTime: null,
+    requiresFallback: false,
+    actionable: false,
+    suggestedFix: null
+  };
+}

package/src/server/errorDetection.test.js

@@ -0,0 +1,180 @@
+import { describe, it, expect } from 'vitest';
+import {
+  analyzeError,
+  analyzeHttpError,
+  extractWaitTime,
+  ERROR_CATEGORIES
+} from './errorDetection.js';
+
+describe('Error Detection', () => {
+  describe('analyzeError', () => {
+    it('should detect rate limit errors', () => {
+      const result = analyzeError('API Error: 429 Too Many Requests');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.RATE_LIMIT);
+      expect(result.requiresFallback).toBe(false);
+    });
+
+    it('should detect rate limit from "rate limit" text', () => {
+      const result = analyzeError('Rate limit exceeded. Please try again later.');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.RATE_LIMIT);
+    });
+
+    it('should detect usage limit errors', () => {
+      const result = analyzeError("You've hit your usage limit. Upgrade to Pro or try again in 1 day 1 hour 33 minutes");
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.USAGE_LIMIT);
+      expect(result.requiresFallback).toBe(true);
+    });
+
+    it('should extract wait time from usage limit errors', () => {
+      const result = analyzeError("You've hit your usage limit. Upgrade to Pro or try again in 1 day 1 hour 33 minutes");
+      expect(result.waitTime).toBeTruthy();
+      expect(result.waitTime).toContain('day');
+    });
+
+    it('should detect authentication errors', () => {
+      const result = analyzeError('Error: 401 Unauthorized - Invalid API key');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.AUTH_ERROR);
+      expect(result.requiresFallback).toBe(true);
+    });
+
+    it('should detect model not found errors', () => {
+      const result = analyzeError('Error: model "claude-9" does not exist');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.MODEL_NOT_FOUND);
+      expect(result.requiresFallback).toBe(true);
+    });
+
+    it('should detect network errors', () => {
+      const result = analyzeError('Error: ECONNREFUSED 127.0.0.1:8080');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.NETWORK_ERROR);
+    });
+
+    it('should detect timeout errors', () => {
+      const result = analyzeError('Process timed out after 300000ms');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.TIMEOUT);
+    });
+
+    it('should detect quota exceeded errors', () => {
+      const result = analyzeError('Error: Billing quota exceeded. Please add credits.');
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.QUOTA_EXCEEDED);
+      expect(result.requiresFallback).toBe(true);
+    });
+
+    it('should return unknown for unrecognized errors with exit code', () => {
+      const result = analyzeError('Some unknown error occurred', 1);
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.UNKNOWN);
+    });
+
+    it('should return no error for success', () => {
+      const result = analyzeError('', 0);
+      expect(result.hasError).toBe(false);
+      expect(result.category).toBeNull();
+    });
+
+    it('should handle null/undefined input', () => {
+      const result = analyzeError(null, 0);
+      expect(result.hasError).toBe(false);
+    });
+  });
+
+  describe('extractWaitTime', () => {
+    it('should extract "X day X hour X minutes" format', () => {
+      const result = extractWaitTime('try again in 1 day 2 hours 30 minutes');
+      expect(result).toBeTruthy();
+      expect(result).toContain('day');
+      expect(result).toContain('hour');
+      expect(result).toContain('min');
+    });
+
+    it('should extract "in X hours" format', () => {
+      const result = extractWaitTime('Please wait, available in 3 hours');
+      expect(result).toBeTruthy();
+      expect(result).toMatch(/3\s*hour/i);
+    });
+
+    it('should extract "wait X minutes" format', () => {
+      const result = extractWaitTime('Wait 5 minutes before retrying');
+      expect(result).toBeTruthy();
+      expect(result).toMatch(/5\s*min/i);
+    });
+
+    it('should return null for no time found', () => {
+      const result = extractWaitTime('No time information here');
+      expect(result).toBeNull();
+    });
+
+    it('should handle null input', () => {
+      const result = extractWaitTime(null);
+      expect(result).toBeNull();
+    });
+  });
+
+  describe('analyzeHttpError', () => {
+    it('should detect 429 rate limit', () => {
+      const result = analyzeHttpError({
+        status: 429,
+        statusText: 'Too Many Requests',
+        body: ''
+      });
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.RATE_LIMIT);
+    });
+
+    it('should detect 401 auth error', () => {
+      const result = analyzeHttpError({
+        status: 401,
+        statusText: 'Unauthorized',
+        body: ''
+      });
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.AUTH_ERROR);
+    });
+
+    it('should detect 403 auth error', () => {
+      const result = analyzeHttpError({
+        status: 403,
+        statusText: 'Forbidden',
+        body: ''
+      });
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.AUTH_ERROR);
+    });
+
+    it('should return no error for 200 status', () => {
+      const result = analyzeHttpError({
+        status: 200,
+        statusText: 'OK',
+        body: ''
+      });
+      expect(result.hasError).toBe(false);
+    });
+
+    it('should analyze body for more specific errors', () => {
+      const result = analyzeHttpError({
+        status: 400,
+        statusText: 'Bad Request',
+        body: 'Error: model "invalid-model" does not exist'
+      });
+      expect(result.hasError).toBe(true);
+      expect(result.category).toBe(ERROR_CATEGORIES.MODEL_NOT_FOUND);
+    });
+
+    it('should extract wait time from 429 response body', () => {
+      const result = analyzeHttpError({
+        status: 429,
+        statusText: 'Too Many Requests',
+        body: 'Rate limit exceeded. Try again in 5 minutes.'
+      });
+      expect(result.hasError).toBe(true);
+      expect(result.waitTime).toBeTruthy();
+    });
+  });
+});

package/src/server/index.d.ts

@@ -1,5 +1,5 @@
 /**
- * TypeScript declarations for @portos/ai-toolkit/server
+ * TypeScript declarations for portos-ai-toolkit/server
  */
 
 export interface ProviderService {

package/src/server/index.js

@@ -6,13 +6,16 @@
 import { createProviderService } from './providers.js';
 import { createRunnerService } from './runner.js';
 import { createPromptsService } from './prompts.js';
+import { createProviderStatusService } from './providerStatus.js';
 import { createProvidersRoutes } from './routes/providers.js';
 import { createRunsRoutes } from './routes/runs.js';
 import { createPromptsRoutes } from './routes/prompts.js';
+import { createProviderStatusRoutes } from './routes/providerStatus.js';
 
 export * from './validation.js';
-export { createProviderService, createRunnerService, createPromptsService };
-export { createProvidersRoutes, createRunsRoutes, createPromptsRoutes };
+export * from './errorDetection.js';
+export { createProviderService, createRunnerService, createPromptsService, createProviderStatusService };
+export { createProvidersRoutes, createRunsRoutes, createPromptsRoutes, createProviderStatusRoutes };
 
 /**
  * Create a complete AI toolkit instance with services and routes
@@ -21,6 +24,7 @@ export function createAIToolkit(config = {}) {
   const {
     dataDir = './data',
     providersFile = 'providers.json',
+    statusFile = 'provider-status.json',
     runsDir = 'runs',
     promptsDir = 'prompts',
     screenshotsDir = './data/screenshots',
@@ -36,7 +40,11 @@ export function createAIToolkit(config = {}) {
     hooks = {},
 
     // Runner config
-    maxConcurrentRuns = 5
+    maxConcurrentRuns = 5,
+
+    // Provider status config
+    enableProviderStatus = true,
+    defaultFallbackPriority = ['claude-code', 'codex', 'lmstudio', 'local-lm-studio', 'ollama', 'gemini-cli']
   } = config;
 
   // Create services
@@ -46,12 +54,39 @@ export function createAIToolkit(config = {}) {
     sampleFile: sampleProvidersFile
   });
 
+  // Create provider status service if enabled
+  let providerStatusService = null;
+  if (enableProviderStatus) {
+    providerStatusService = createProviderStatusService({
+      dataDir,
+      statusFile,
+      defaultFallbackPriority,
+      onStatusChange: (eventData) => {
+        // Emit Socket.IO event if io is configured
+        io?.emit('provider:status:changed', eventData);
+      }
+    });
+
+    // Initialize status service
+    providerStatusService.init().catch(err => {
+      console.error(`❌ Failed to initialize provider status: ${err.message}`);
+    });
+  }
+
   const runnerService = createRunnerService({
     dataDir,
     runsDir,
     screenshotsDir,
     providerService,
-    hooks,
+    providerStatusService,
+    hooks: {
+      ...hooks,
+      // Add hook to emit provider error events
+      onProviderError: (providerId, errorAnalysis, output) => {
+        io?.emit('provider:error', { providerId, errorAnalysis });
+        hooks.onProviderError?.(providerId, errorAnalysis, output);
+      }
+    },
     maxConcurrentRuns
   });
 
@@ -70,19 +105,27 @@ export function createAIToolkit(config = {}) {
   const runsRouter = createRunsRoutes(runnerService, { asyncHandler, io });
   const promptsRouter = createPromptsRoutes(promptsService, { asyncHandler });
 
+  // Create provider status routes if enabled
+  let providerStatusRouter = null;
+  if (providerStatusService) {
+    providerStatusRouter = createProviderStatusRoutes(providerStatusService, { asyncHandler });
+  }
+
   return {
     // Services
     services: {
       providers: providerService,
       runner: runnerService,
-      prompts: promptsService
+      prompts: promptsService,
+      providerStatus: providerStatusService
     },
 
     // Routes
     routes: {
       providers: providersRouter,
       runs: runsRouter,
-      prompts: promptsRouter
+      prompts: promptsRouter,
+      providerStatus: providerStatusRouter
     },
 
     // Convenience method to mount all routes
@@ -90,6 +133,9 @@ export function createAIToolkit(config = {}) {
       app.use(`${basePath}/providers`, providersRouter);
       app.use(`${basePath}/runs`, runsRouter);
       app.use(`${basePath}/prompts`, promptsRouter);
+      if (providerStatusRouter) {
+        app.use(`${basePath}/providers/status`, providerStatusRouter);
+      }
     }
   };
 }