prepia 1.0.0

package/src/core/engine.mjs

@@ -0,0 +1,338 @@
+/**
+ * @fileoverview Main Prepia engine - orchestrates the full pipeline.
+ * @module core/engine
+ */
+
+import { EventEmitter } from 'node:events';
+import { decompose, partitionTasks } from './task-decomposer.mjs';
+import { packageContext } from './context-packager.mjs';
+import { generatePrompt, mergePrompts } from './prepimshot.mjs';
+import { classify, canHandleLocally, getLocalResponse } from '../models/local-model.mjs';
+import { Orchestrator } from '../tools/orchestrator.mjs';
+import { CacheManager } from '../cache/manager.mjs';
+import { Tracker } from '../analytics/tracker.mjs';
+import { StreamHandler } from '../stream/handler.mjs';
+import { sanitize, validateParams } from '../security/sanitizer.mjs';
+import { detectPersona } from '../persona/detector.mjs';
+import { checkQuality } from '../guard/checker.mjs';
+import { RateShield } from '../rate/shield.mjs';
+
+export class PrepiaEngine extends EventEmitter {
+  /**
+   * @param {Object} [options]
+   * @param {Object} [options.cache] - Cache options
+   * @param {Object} [options.rate] - Rate limiting options
+   * @param {Object} [options.model] - Model router options
+   */
+  constructor(options = {}) {
+    super();
+    this._cache = new CacheManager(options.cache || { enableDisk: false });
+    this._orchestrator = new Orchestrator();
+    this._tracker = new Tracker();
+    this._stream = new StreamHandler();
+    this._shield = new RateShield(options.rate || {});
+    this._modelRouter = options.modelRouter || null;
+    this._config = {
+      maxContextTokens: 4000,
+      defaultMode: 'shot',
+      enableLocalModel: true,
+      enableCache: true,
+      enableQualityCheck: true,
+      ...options.config,
+    };
+  }
+
+  /**
+   * Process a task through the full pipeline.
+   * @param {string} query - User query
+   * @param {Object} [options]
+   * @param {string} [options.mode='shot'] - Processing mode: flash, shot, stream
+   * @param {string} [options.workflow] - Workflow template to use
+   * @returns {Promise<Object>} Processing result
+   */
+  async process(query, options = {}) {
+    const start = Date.now();
+    const taskId = `task_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    const mode = options.mode || this._config.defaultMode;
+
+    this._stream.start(taskId, query);
+
+    try {
+      // Sanitize input
+      const sanitized = sanitize(query);
+      if (sanitized.modified) {
+        this.emit('input:sanitized', { issues: sanitized.issues });
+      }
+
+      // Check cache first
+      if (this._config.enableCache) {
+        const cacheKey = this._cache.generateKey({ query: sanitized.text, mode });
+        const cached = await this._cache.get(cacheKey);
+        if (cached) {
+          this._stream.complete(taskId, cached);
+          this._tracker.record({
+            id: taskId, type: 'cached', usedLLM: false,
+            duration: Date.now() - start, cached: true, success: true,
+          });
+          return { ...cached, fromCache: true };
+        }
+      }
+
+      // Flash mode: try local model first
+      if (mode === 'flash') {
+        const localResult = this._tryLocal(sanitized.text);
+        if (localResult) {
+          const duration = Date.now() - start;
+          const resultWithDuration = { ...localResult, duration, taskId };
+          this._stream.complete(taskId, resultWithDuration);
+          this._tracker.record({
+            id: taskId, type: 'flash', usedLLM: false,
+            duration, success: true,
+          });
+          return resultWithDuration;
+        }
+      }
+
+      // Decompose task
+      this._stream.update(taskId, 'analyzing', 'Analyzing task...');
+      const decomposed = decompose(sanitized.text);
+      const { local, llm } = partitionTasks(decomposed.subtasks);
+
+      // Execute local tasks
+      this._stream.update(taskId, 'processing', 'Processing locally...');
+      const localResults = await this._executeLocalTasks(local);
+
+      // If all tasks are local, no LLM needed
+      if (llm.length === 0 && localResults.length > 0) {
+        const answerParts = localResults.map(r => {
+          const res = r.result;
+          if (res === null || res === undefined) return null;
+          if (typeof res === 'string') return res;
+          if (Array.isArray(res)) {
+            return res.map(item => {
+              if (typeof item === 'object' && item !== null) {
+                if (item.value !== undefined) return String(item.value);
+                return JSON.stringify(item);
+              }
+              return String(item);
+            }).join('\n');
+          }
+          if (typeof res === 'object') {
+            if (res.value !== undefined) return String(res.value);
+            return JSON.stringify(res);
+          }
+          return String(res);
+        }).filter(Boolean);
+
+        const result = {
+          answer: answerParts.join('\n'),
+          mode: 'flash',
+          usedLLM: false,
+          taskId,
+          duration: Date.now() - start,
+        };
+        this._stream.complete(taskId, result);
+        this._tracker.record({
+          id: taskId, type: 'flash', usedLLM: false,
+          duration: Date.now() - start, success: true,
+        });
+        return result;
+      }
+
+      // Package context for LLM
+      this._stream.update(taskId, 'synthesizing', 'Preparing LLM prompt...');
+      const persona = detectPersona(sanitized.text);
+      const prompt = mode === 'shot'
+        ? mergePrompts(sanitized.text, localResults, { maxTokens: this._config.maxContextTokens })
+        : generatePrompt({
+            query: sanitized.text,
+            context: localResults.map(r => JSON.stringify(r.result)).join('\n'),
+            systemPrompt: persona.systemPrompt,
+            options: { maxTokens: this._config.maxContextTokens },
+          });
+
+      // Send to LLM
+      let llmResponse;
+      if (this._modelRouter) {
+        this._stream.update(taskId, 'synthesizing', 'Calling LLM...');
+        llmResponse = await this._modelRouter.send({
+          prompt: prompt.user,
+          system: prompt.system,
+          maxTokens: 2048,
+        });
+      } else {
+        // No model configured - return what we have
+        const formatResult = (res) => {
+          if (res === null || res === undefined) return null;
+          if (typeof res === 'string') return res;
+          if (Array.isArray(res)) return res.map(formatResult).filter(Boolean).join('\n');
+          if (typeof res === 'object') {
+            if (res.value !== undefined) return String(res.value);
+            return JSON.stringify(res);
+          }
+          return String(res);
+        };
+
+        llmResponse = {
+          content: localResults.length > 0
+            ? `Based on available information:\n${localResults.map(r => formatResult(r.result)).filter(Boolean).join('\n')}`
+            : 'No LLM configured. Please configure a model provider.',
+          provider: 'none',
+          usage: { prompt: 0, completion: 0, total: 0 },
+        };
+      }
+
+      // Quality check
+      let quality = null;
+      if (this._config.enableQualityCheck && llmResponse.content) {
+        quality = checkQuality(llmResponse.content, { query: sanitized.text });
+      }
+
+      const result = {
+        answer: llmResponse.content,
+        mode,
+        usedLLM: true,
+        taskId,
+        provider: llmResponse.provider,
+        tokens: llmResponse.usage,
+        quality,
+        duration: Date.now() - start,
+      };
+
+      // Cache result
+      if (this._config.enableCache && llmResponse.content) {
+        const cacheKey = this._cache.generateKey({ query: sanitized.text, mode });
+        await this._cache.set(cacheKey, result);
+      }
+
+      this._stream.complete(taskId, result);
+      this._tracker.record({
+        id: taskId, type: mode, usedLLM: true,
+        tokensUsed: llmResponse.usage?.total || 0,
+        tokensSaved: prompt.tokens || 0,
+        duration: Date.now() - start, success: true,
+      });
+
+      return result;
+    } catch (err) {
+      this._stream.error(taskId, err.message);
+      this._tracker.record({
+        id: taskId, type: mode, usedLLM: false,
+        duration: Date.now() - start, success: false,
+      });
+      throw err;
+    }
+  }
+
+  /**
+   * Try to handle a query with the local model.
+   * @param {string} query
+   * @returns {Object|null}
+   * @private
+   */
+  _tryLocal(query) {
+    const localResult = getLocalResponse(query);
+    if (localResult !== null) {
+      return {
+        answer: localResult,
+        mode: 'flash',
+        usedLLM: false,
+      };
+    }
+    return null;
+  }
+
+  /**
+   * Execute local (non-LLM) tasks.
+   * @param {Object[]} tasks
+   * @returns {Promise<Object[]>}
+   * @private
+   */
+  async _executeLocalTasks(tasks) {
+    const results = [];
+    for (const task of tasks) {
+      try {
+        const toolCalls = this._orchestrator.route(task.type, task.params);
+        if (toolCalls.length > 0) {
+          const toolResults = await this._orchestrator.executeMany(toolCalls);
+          results.push({
+            taskId: task.id,
+            result: toolResults.map(r => r.result).filter(Boolean),
+          });
+        }
+      } catch (err) {
+        results.push({ taskId: task.id, error: err.message });
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Get analytics data.
+   * @returns {Object}
+   */
+  getAnalytics() {
+    return {
+      metrics: this._tracker.getMetrics(),
+      savings: this._tracker.estimateSavings(),
+      cache: this._cache.stats(),
+    };
+  }
+
+  /**
+   * Clear all caches.
+   */
+  async clearCache() {
+    await this._cache.clear();
+  }
+
+  /**
+   * Get registered plugins.
+   * @returns {Object[]}
+   */
+  getPlugins() {
+    return this._orchestrator.listTools();
+  }
+
+  /**
+   * Update engine configuration.
+   * @param {Object} config
+   */
+  updateConfig(config) {
+    Object.assign(this._config, config);
+  }
+
+  /**
+   * Get the stream handler for progress tracking.
+   * @returns {StreamHandler}
+   */
+  getStream() {
+    return this._stream;
+  }
+
+  /**
+   * Get the tracker.
+   * @returns {Tracker}
+   */
+  getTracker() {
+    return this._tracker;
+  }
+
+  /**
+   * Get the cache manager.
+   * @returns {CacheManager}
+   */
+  getCache() {
+    return this._cache;
+  }
+
+  /**
+   * Get the orchestrator.
+   * @returns {Orchestrator}
+   */
+  getOrchestrator() {
+    return this._orchestrator;
+  }
+}
+
+export default PrepiaEngine;

package/src/core/event-bus.mjs

@@ -0,0 +1,84 @@
+/**
+ * @fileoverview Global event bus for inter-module communication.
+ * Centralizes all event handling across Prepia modules.
+ * @module core/event-bus
+ */
+
+import { EventEmitter } from 'node:events';
+
+/** @type {EventEmitter} Singleton event bus instance */
+const bus = new EventEmitter();
+
+/** Raise max listeners to avoid warnings in complex workflows */
+bus.setMaxListeners(100);
+
+/**
+ * Emit an event on the global bus.
+ * @param {string} event - Event name
+ * @param {...*} args - Event arguments
+ * @returns {boolean} Whether the event had listeners
+ */
+export function emit(event, ...args) {
+  return bus.emit(event, ...args);
+}
+
+/**
+ * Register a listener on the global bus.
+ * @param {string} event - Event name
+ * @param {Function} listener - Callback function
+ * @returns {EventEmitter} The bus instance for chaining
+ */
+export function on(event, listener) {
+  return bus.on(event, listener);
+}
+
+/**
+ * Register a one-time listener on the global bus.
+ * @param {string} event - Event name
+ * @param {Function} listener - Callback function
+ * @returns {EventEmitter} The bus instance for chaining
+ */
+export function once(event, listener) {
+  return bus.once(event, listener);
+}
+
+/**
+ * Remove a listener from the global bus.
+ * @param {string} event - Event name
+ * @param {Function} listener - Callback function to remove
+ * @returns {EventEmitter} The bus instance for chaining
+ */
+export function off(event, listener) {
+  return bus.off(event, listener);
+}
+
+/**
+ * Remove all listeners for a specific event, or all events.
+ * @param {string} [event] - Optional event name
+ */
+export function removeAll(event) {
+  if (event) {
+    bus.removeAllListeners(event);
+  } else {
+    bus.removeAllListeners();
+  }
+}
+
+/**
+ * Get the underlying EventEmitter for advanced usage.
+ * @returns {EventEmitter}
+ */
+export function getBus() {
+  return bus;
+}
+
+/**
+ * Get listener count for an event.
+ * @param {string} event - Event name
+ * @returns {number}
+ */
+export function listenerCount(event) {
+  return bus.listenerCount(event);
+}
+
+export default { emit, on, once, off, removeAll, getBus, listenerCount };

package/src/core/prepimshot.mjs

@@ -0,0 +1,120 @@
+/**
+ * @fileoverview PrepiShot - hyper-focused prompt generation for one-shot LLM calls.
+ * Strips all unnecessary context to get the absolute minimum the LLM needs.
+ * @module core/prepimshot
+ */
+
+import { estimateTokens } from './context-packager.mjs';
+
+/**
+ * Generate a minimal, focused prompt for a single LLM call.
+ * @param {Object} params
+ * @param {string} params.query - Original user query
+ * @param {string} [params.context] - Gathered context
+ * @param {string} [params.systemPrompt] - System prompt
+ * @param {Object} [params.options] - Additional options
+ * @param {number} [params.options.maxTokens=4000] - Max context tokens
+ * @param {string} [params.options.outputFormat='text'] - Desired output format
+ * @returns {Object} Optimized prompt
+ */
+export function generatePrompt(params) {
+  const { query, context = '', systemPrompt = '', options = {} } = params;
+  const { maxTokens = 4000, outputFormat = 'text' } = options;
+
+  if (!query) {
+    throw new Error('Query is required for prompt generation');
+  }
+
+  // Build the minimal system prompt
+  const system = buildSystemPrompt(systemPrompt, outputFormat);
+
+  // Build the user prompt with context
+  const userPrompt = buildUserPrompt(query, context, maxTokens);
+
+  const totalTokens = estimateTokens(system) + estimateTokens(userPrompt);
+
+  return {
+    system,
+    user: userPrompt,
+    tokens: totalTokens,
+    optimized: context.length > 0,
+    format: outputFormat,
+  };
+}
+
+/**
+ * Build a minimal system prompt.
+ * @param {string} custom - Custom system prompt
+ * @param {string} format - Output format
+ * @returns {string}
+ */
+function buildSystemPrompt(custom, format) {
+  const parts = [];
+
+  if (custom) {
+    parts.push(custom);
+  } else {
+    parts.push('You are a helpful assistant. Provide accurate, concise answers.');
+  }
+
+  if (format === 'json') {
+    parts.push('Respond in valid JSON format.');
+  } else if (format === 'markdown') {
+    parts.push('Use markdown formatting.');
+  } else if (format === 'list') {
+    parts.push('Respond as a bulleted list.');
+  }
+
+  return parts.join(' ');
+}
+
+/**
+ * Build the user prompt with context.
+ * @param {string} query
+ * @param {string} context
+ * @param {number} maxTokens
+ * @returns {string}
+ */
+function buildUserPrompt(query, context, maxTokens) {
+  if (!context || context.trim().length === 0) {
+    return query;
+  }
+
+  // Trim context to fit within token budget (leaving room for query)
+  const queryTokens = estimateTokens(query);
+  const availableTokens = maxTokens - queryTokens - 50; // 50 token buffer
+
+  let trimmedContext = context;
+  if (estimateTokens(context) > availableTokens) {
+    trimmedContext = context.substring(0, availableTokens * 4) + '\n[...]';
+  }
+
+  return `Context:\n${trimmedContext}\n\nQuestion: ${query}`;
+}
+
+/**
+ * Generate prompts for multiple sub-tasks and merge them into one.
+ * @param {string} query - Original query
+ * @param {Object[]} subtaskResults - Results from sub-tasks
+ * @param {Object} [options]
+ * @returns {Object} Merged prompt
+ */
+export function mergePrompts(query, subtaskResults, options = {}) {
+  const contextParts = [];
+
+  for (const result of subtaskResults) {
+    if (result.error) continue;
+    if (result.result) {
+      const text = typeof result.result === 'string' ? result.result : JSON.stringify(result.result);
+      contextParts.push(text);
+    }
+  }
+
+  return generatePrompt({
+    query,
+    context: contextParts.join('\n\n---\n\n'),
+    options,
+  });
+}
+
+export default { generatePrompt, mergePrompts };

package/src/core/task-decomposer.mjs

@@ -0,0 +1,158 @@
+/**
+ * @fileoverview Task decomposer - breaks complex tasks into sub-tasks.
+ * @module core/task-decomposer
+ */
+
+/**
+ * @typedef {Object} SubTask
+ * @property {string} id - Sub-task ID
+ * @property {string} type - Task type
+ * @property {string} description - What this sub-task does
+ * @property {boolean} needsLLM - Whether this requires an LLM call
+ * @property {string[]} dependencies - IDs of tasks this depends on
+ * @property {Object} params - Task parameters
+ */
+
+/**
+ * Decompose a complex task into sub-tasks.
+ * @param {string} task - The task description
+ * @param {Object} [options]
+ * @returns {Object} Decomposed task tree
+ */
+export function decompose(task, options = {}) {
+  if (!task || typeof task !== 'string') {
+    return { subtasks: [], root: null };
+  }
+
+  const subtasks = [];
+  let idCounter = 0;
+
+  const genId = () => `task_${++idCounter}`;
+
+  // Analyze the task to determine what's needed
+  const analysis = analyzeTask(task);
+
+  // Create sub-tasks based on analysis
+  if (analysis.needsSearch) {
+    const searchId = genId();
+    subtasks.push({
+      id: searchId,
+      type: 'search',
+      description: `Search for information about: ${analysis.topic}`,
+      needsLLM: false,
+      dependencies: [],
+      params: { query: analysis.searchQuery || analysis.topic },
+    });
+
+    if (analysis.needsScraping) {
+      subtasks.push({
+        id: genId(),
+        type: 'scrape',
+        description: 'Extract content from search results',
+        needsLLM: false,
+        dependencies: [searchId],
+        params: {},
+      });
+    }
+  }
+
+  if (analysis.needsCalculation) {
+    subtasks.push({
+      id: genId(),
+      type: 'calculate',
+      description: `Perform calculation: ${analysis.calcExpression || 'compute results'}`,
+      needsLLM: false,
+      dependencies: [],
+      params: { expression: analysis.calcExpression },
+    });
+  }
+
+  // Always have a synthesis step if there are sub-tasks
+  if (subtasks.length > 0) {
+    const synthDeps = subtasks.map(t => t.id);
+    subtasks.push({
+      id: genId(),
+      type: 'synthesize',
+      description: 'Combine results into final answer',
+      needsLLM: true,
+      dependencies: synthDeps,
+      params: { originalQuery: task },
+    });
+  } else {
+    // Simple task - just needs LLM
+    subtasks.push({
+      id: genId(),
+      type: 'llm',
+      description: task,
+      needsLLM: true,
+      dependencies: [],
+      params: { query: task },
+    });
+  }
+
+  return {
+    subtasks,
+    root: subtasks[subtasks.length - 1]?.id || null,
+    analysis,
+  };
+}
+
+/**
+ * Analyze a task to determine its components.
+ * @param {string} task
+ * @returns {Object}
+ */
+function analyzeTask(task) {
+  const lower = task.toLowerCase();
+
+  const needsSearch = /\b(search|find|look\s+up|what\s+is|who\s+is|when\s+did|latest|current|news)\b/i.test(lower);
+  const needsScraping = /\b(read|extract|scrape|get\s+content\s+from|fetch\s+page)\b/i.test(lower) ||
+    /https?:\/\//i.test(task);
+  const needsCalculation = /\b(calculate|compute|how\s+much|how\s+many|percentage|convert|math)\b/i.test(lower) ||
+    /\d+\s*[+\-*/^]\s*\d+/.test(task);
+  const needsFileOps = /\b(read\s+file|write\s+file|save|load|file)\b/i.test(lower);
+
+  // Extract topic
+  let topic = task;
+  topic = topic.replace(/^(please\s+)?(can\s+you\s+)?/i, '');
+  topic = topic.replace(/\?.*$/, '').trim();
+
+  // Extract search query
+  let searchQuery = null;
+  const searchMatch = task.match(/(?:search|find|look\s+up)\s+(?:for\s+)?(.+)/i);
+  if (searchMatch) searchQuery = searchMatch[1];
+
+  // Extract math expression
+  let calcExpression = null;
+  const pctMatch = task.match(/(\d+\.?\d*)\s*%\s*(?:of|off)?\s*(\d+\.?\d*)/);
+  if (pctMatch) {
+    calcExpression = `${pctMatch[1]}% of ${pctMatch[2]}`;
+  } else {
+    const mathMatch = task.match(/(\d+[\s+\-*/^()\d.]*\d+)/);
+    if (mathMatch) calcExpression = mathMatch[1];
+  }
+
+  return {
+    topic,
+    needsSearch,
+    needsScraping,
+    needsCalculation,
+    needsFileOps,
+    searchQuery,
+    calcExpression,
+    complexity: needsSearch && needsCalculation ? 'complex' : needsSearch || needsCalculation ? 'medium' : 'simple',
+  };
+}
+
+/**
+ * Check which sub-tasks can be handled locally (no LLM).
+ * @param {SubTask[]} subtasks
+ * @returns {{ local: SubTask[], llm: SubTask[] }}
+ */
+export function partitionTasks(subtasks) {
+  const local = subtasks.filter(t => !t.needsLLM);
+  const llm = subtasks.filter(t => t.needsLLM);
+  return { local, llm };
+}
+
+export default { decompose, partitionTasks };