profclaw 0.0.1-beta.1

package/dist/ai/embedding-service.js

@@ -0,0 +1,333 @@
+import OpenAI from 'openai';
+import { loadConfig } from '../utils/config-loader.js';
+import { logger } from '../utils/logger.js';
+const MODEL_LIMITS = {
+    'text-embedding-3-small': { maxInputTokens: 8191, maxBatchSize: 2048, dimensions: 1536, rateLimit: { requests: 3000, windowMs: 60_000 } },
+    'text-embedding-3-large': { maxInputTokens: 8191, maxBatchSize: 2048, dimensions: 3072, rateLimit: { requests: 3000, windowMs: 60_000 } },
+    'text-embedding-ada-002': { maxInputTokens: 8191, maxBatchSize: 2048, dimensions: 1536, rateLimit: { requests: 3000, windowMs: 60_000 } },
+    'voyage-3': { maxInputTokens: 32000, maxBatchSize: 128, dimensions: 1024, rateLimit: { requests: 300, windowMs: 60_000 } },
+    'voyage-3-lite': { maxInputTokens: 32000, maxBatchSize: 128, dimensions: 512, rateLimit: { requests: 300, windowMs: 60_000 } },
+    'voyage-code-3': { maxInputTokens: 32000, maxBatchSize: 128, dimensions: 1024, rateLimit: { requests: 300, windowMs: 60_000 } },
+    'text-embedding-004': { maxInputTokens: 2048, maxBatchSize: 100, dimensions: 768, rateLimit: { requests: 1500, windowMs: 60_000 } },
+};
+const config = loadConfig('settings.yml');
+// Embedding Service
+export class EmbeddingService {
+    openai;
+    usageStats = new Map();
+    constructor() {
+        const openaiKey = config.ai?.providers?.openai?.apiKey ?? process.env.OPENAI_API_KEY;
+        if (openaiKey && openaiKey !== '${OPENAI_API_KEY}') {
+            this.openai = new OpenAI({ apiKey: openaiKey });
+        }
+    }
+    /**
+     * Generate an embedding for a string of text
+     */
+    async generateEmbedding(text) {
+        const provider = config.ai?.defaultEmbeddingProvider ?? 'openai';
+        // Try primary provider first
+        if (provider === 'openai' && this.openai) {
+            try {
+                return await this.generateOpenAIEmbedding(text);
+            }
+            catch {
+                logger.warn('[AI] OpenAI embedding failed, trying fallback');
+            }
+        }
+        // Try Voyage
+        if (provider === 'voyage' || this.isVoyageConfigured()) {
+            try {
+                return await this.generateVoyageEmbedding(text);
+            }
+            catch {
+                logger.warn('[AI] Voyage embedding failed, trying next fallback');
+            }
+        }
+        // Try Gemini
+        if (provider === 'gemini' || this.isGeminiConfigured()) {
+            try {
+                return await this.generateGeminiEmbedding(text);
+            }
+            catch {
+                logger.warn('[AI] Gemini embedding failed, trying Ollama fallback');
+            }
+        }
+        // Try Ollama (always available as local fallback)
+        try {
+            const { getOllamaService } = await import('../intelligence/ollama.js');
+            const ollama = getOllamaService();
+            const isAvailable = await ollama.healthCheck();
+            if (isAvailable) {
+                return await ollama.generateEmbedding(text);
+            }
+        }
+        catch {
+            logger.warn('[AI] Ollama embedding failed');
+        }
+        logger.warn('[AI] No embedding provider available. Returning zero-vector.');
+        return new Array(1536).fill(0);
+    }
+    /**
+     * Batch embedding - process multiple texts at once
+     */
+    async generateBatchEmbeddings(texts) {
+        if (texts.length === 0)
+            return [];
+        if (texts.length === 1)
+            return [await this.generateEmbedding(texts[0])];
+        const provider = config.ai?.defaultEmbeddingProvider ?? 'openai';
+        // OpenAI batch API
+        if (provider === 'openai' && this.openai) {
+            try {
+                return await this.batchOpenAIEmbeddings(texts);
+            }
+            catch {
+                logger.warn('[AI] OpenAI batch embedding failed, falling back to sequential');
+            }
+        }
+        // Voyage batch API
+        if (provider === 'voyage' && this.isVoyageConfigured()) {
+            try {
+                return await this.batchVoyageEmbeddings(texts);
+            }
+            catch {
+                logger.warn('[AI] Voyage batch embedding failed, falling back to sequential');
+            }
+        }
+        // Gemini batch API
+        if (provider === 'gemini' && this.isGeminiConfigured()) {
+            try {
+                return await this.batchGeminiEmbeddings(texts);
+            }
+            catch {
+                logger.warn('[AI] Gemini batch embedding failed, falling back to sequential');
+            }
+        }
+        // Sequential fallback
+        return Promise.all(texts.map((t) => this.generateEmbedding(t)));
+    }
+    /**
+     * Get model limits for a specific model
+     */
+    getModelLimits(model) {
+        const effectiveModel = model ?? this.getActiveModel();
+        return MODEL_LIMITS[effectiveModel] ?? null;
+    }
+    /**
+     * Get current active embedding model name
+     */
+    getActiveModel() {
+        const provider = config.ai?.defaultEmbeddingProvider ?? 'openai';
+        switch (provider) {
+            case 'openai': return config.ai?.providers?.openai?.embeddingModel ?? 'text-embedding-3-small';
+            case 'voyage': return config.ai?.providers?.voyage?.model ?? 'voyage-3';
+            case 'gemini': return config.ai?.providers?.gemini?.embeddingModel ?? 'text-embedding-004';
+            case 'ollama': return config.ai?.providers?.ollama?.embeddingModel ?? 'nomic-embed-text';
+            default: return 'text-embedding-3-small';
+        }
+    }
+    /**
+     * Get usage statistics
+     */
+    getUsageStats() {
+        return Array.from(this.usageStats.values());
+    }
+    /**
+     * Get available embedding providers
+     */
+    getAvailableProviders() {
+        const providers = [];
+        if (this.openai)
+            providers.push('openai');
+        if (this.isVoyageConfigured())
+            providers.push('voyage');
+        if (this.isGeminiConfigured())
+            providers.push('gemini');
+        providers.push('ollama'); // always available
+        return providers;
+    }
+    // OpenAI Embeddings
+    async generateOpenAIEmbedding(text) {
+        if (!this.openai)
+            throw new Error('OpenAI not initialized');
+        const model = config.ai?.providers?.openai?.embeddingModel ?? 'text-embedding-3-small';
+        const response = await this.openai.embeddings.create({
+            model,
+            input: text.substring(0, 8000),
+        });
+        this.trackUsage('openai', model, 1, text.length);
+        return response.data[0].embedding;
+    }
+    async batchOpenAIEmbeddings(texts) {
+        if (!this.openai)
+            throw new Error('OpenAI not initialized');
+        const model = config.ai?.providers?.openai?.embeddingModel ?? 'text-embedding-3-small';
+        const limits = MODEL_LIMITS[model];
+        const batchSize = limits?.maxBatchSize ?? 2048;
+        const results = [];
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize).map((t) => t.substring(0, 8000));
+            const response = await this.openai.embeddings.create({ model, input: batch });
+            results.push(...response.data.map((d) => d.embedding));
+            this.trackUsage('openai', model, batch.length, batch.reduce((s, t) => s + t.length, 0));
+        }
+        return results;
+    }
+    // Voyage Embeddings
+    isVoyageConfigured() {
+        const key = config.ai?.providers?.voyage?.apiKey ?? process.env.VOYAGE_API_KEY;
+        return !!key && key !== '${VOYAGE_API_KEY}';
+    }
+    async generateVoyageEmbedding(text) {
+        const apiKey = config.ai?.providers?.voyage?.apiKey ?? process.env.VOYAGE_API_KEY;
+        if (!apiKey)
+            throw new Error('Voyage API key not configured');
+        const model = config.ai?.providers?.voyage?.model ?? 'voyage-3';
+        const response = await fetch('https://api.voyageai.com/v1/embeddings', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+            body: JSON.stringify({ model, input: [text.substring(0, 32000)] }),
+        });
+        if (!response.ok)
+            throw new Error(`Voyage API error: ${response.status}`);
+        const data = await response.json();
+        this.trackUsage('voyage', model, 1, text.length);
+        return data.data[0].embedding;
+    }
+    async batchVoyageEmbeddings(texts) {
+        const apiKey = config.ai?.providers?.voyage?.apiKey ?? process.env.VOYAGE_API_KEY;
+        if (!apiKey)
+            throw new Error('Voyage API key not configured');
+        const model = config.ai?.providers?.voyage?.model ?? 'voyage-3';
+        const batchSize = 128;
+        const results = [];
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize).map((t) => t.substring(0, 32000));
+            const response = await fetch('https://api.voyageai.com/v1/embeddings', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+                body: JSON.stringify({ model, input: batch }),
+            });
+            if (!response.ok)
+                throw new Error(`Voyage batch error: ${response.status}`);
+            const data = await response.json();
+            results.push(...data.data.map((d) => d.embedding));
+            this.trackUsage('voyage', model, batch.length, batch.reduce((s, t) => s + t.length, 0));
+        }
+        return results;
+    }
+    // Gemini Embeddings
+    isGeminiConfigured() {
+        const key = config.ai?.providers?.gemini?.apiKey ?? process.env.GOOGLE_API_KEY;
+        return !!key && key !== '${GOOGLE_API_KEY}';
+    }
+    async generateGeminiEmbedding(text) {
+        const apiKey = config.ai?.providers?.gemini?.apiKey ?? process.env.GOOGLE_API_KEY;
+        if (!apiKey)
+            throw new Error('Gemini API key not configured');
+        const model = config.ai?.providers?.gemini?.embeddingModel ?? 'text-embedding-004';
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1/models/${model}:embedContent?key=${apiKey}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ content: { parts: [{ text: text.substring(0, 2048) }] } }),
+        });
+        if (!response.ok)
+            throw new Error(`Gemini embedding error: ${response.status}`);
+        const data = await response.json();
+        this.trackUsage('gemini', model, 1, text.length);
+        return data.embedding.values;
+    }
+    async batchGeminiEmbeddings(texts) {
+        const apiKey = config.ai?.providers?.gemini?.apiKey ?? process.env.GOOGLE_API_KEY;
+        if (!apiKey)
+            throw new Error('Gemini API key not configured');
+        const model = config.ai?.providers?.gemini?.embeddingModel ?? 'text-embedding-004';
+        const requests = texts.map((text) => ({
+            model: `models/${model}`,
+            content: { parts: [{ text: text.substring(0, 2048) }] },
+        }));
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1/models/${model}:batchEmbedContents?key=${apiKey}`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ requests }),
+        });
+        if (!response.ok)
+            throw new Error(`Gemini batch embedding error: ${response.status}`);
+        const data = await response.json();
+        this.trackUsage('gemini', model, texts.length, texts.reduce((s, t) => s + t.length, 0));
+        return data.embeddings.map((e) => e.values);
+    }
+    // Streaming Embeddings
+    /**
+     * Generate embeddings with streaming progress callback.
+     * Processes texts in chunks and calls onProgress after each chunk completes.
+     */
+    async generateStreamingEmbeddings(texts, onProgress, options) {
+        const chunkSize = options?.chunkSize ?? 50;
+        const results = [];
+        const totalChunks = Math.ceil(texts.length / chunkSize);
+        for (let i = 0; i < texts.length; i += chunkSize) {
+            const chunk = texts.slice(i, i + chunkSize);
+            const chunkIndex = Math.floor(i / chunkSize);
+            try {
+                const embeddings = await this.generateBatchEmbeddings(chunk);
+                results.push(...embeddings);
+                onProgress({
+                    completed: Math.min(i + chunkSize, texts.length),
+                    total: texts.length,
+                    chunkIndex,
+                    totalChunks,
+                    percentage: Math.round((Math.min(i + chunkSize, texts.length) / texts.length) * 100),
+                    error: null,
+                });
+            }
+            catch (error) {
+                // On error, fill with zero vectors and report
+                const zeroVectors = chunk.map(() => new Array(1536).fill(0));
+                results.push(...zeroVectors);
+                onProgress({
+                    completed: Math.min(i + chunkSize, texts.length),
+                    total: texts.length,
+                    chunkIndex,
+                    totalChunks,
+                    percentage: Math.round((Math.min(i + chunkSize, texts.length) / texts.length) * 100),
+                    error: error instanceof Error ? error.message : 'Unknown error',
+                });
+                logger.warn(`[AI] Streaming embedding chunk ${chunkIndex} failed: ${error instanceof Error ? error.message : 'unknown'}`);
+            }
+        }
+        return results;
+    }
+    // Usage Tracking
+    trackUsage(provider, model, count, charCount) {
+        const key = `${provider}/${model}`;
+        const existing = this.usageStats.get(key);
+        if (existing) {
+            existing.totalRequests += count;
+            existing.totalTokens += Math.ceil(charCount / 4); // rough estimate
+            existing.totalBatches++;
+            existing.lastUsed = Date.now();
+        }
+        else {
+            this.usageStats.set(key, {
+                provider,
+                model,
+                totalRequests: count,
+                totalTokens: Math.ceil(charCount / 4),
+                totalBatches: 1,
+                errors: 0,
+                lastUsed: Date.now(),
+            });
+        }
+    }
+}
+// Singleton
+let instance = null;
+export function getEmbeddingService() {
+    if (!instance) {
+        instance = new EmbeddingService();
+    }
+    return instance;
+}
+//# sourceMappingURL=embedding-service.js.map

package/dist/auth/api-tokens.d.ts

@@ -0,0 +1,71 @@
+import type { Context } from 'hono';
+/**
+ * Token scopes for fine-grained permissions
+ */
+export type TokenScope = 'tasks:read' | 'tasks:write' | 'summaries:read' | 'summaries:write' | 'config:read' | 'config:write' | 'gateway:execute' | 'admin';
+/**
+ * API Token record
+ */
+export interface ApiToken {
+    id: string;
+    name: string;
+    /** Hashed token (never store plaintext) */
+    tokenHash: string;
+    /** First 8 chars for identification */
+    tokenPrefix: string;
+    scopes: TokenScope[];
+    createdAt: Date;
+    lastUsedAt?: Date;
+    expiresAt?: Date;
+    /** Rate limit: requests per minute */
+    rateLimit: number;
+    enabled: boolean;
+}
+/**
+ * Token creation result
+ */
+export interface CreateTokenResult {
+    token: ApiToken;
+    /** The actual token (only shown once!) */
+    plainTextToken: string;
+}
+/**
+ * Initialize API tokens table
+ */
+export declare function initApiTokensTable(): Promise<void>;
+/**
+ * Create a new API token
+ */
+export declare function createApiToken(name: string, scopes: TokenScope[], options?: {
+    expiresInDays?: number;
+    rateLimit?: number;
+}): Promise<CreateTokenResult>;
+/**
+ * Validate a token and return its info
+ */
+export declare function validateToken(plainTextToken: string): Promise<{
+    valid: boolean;
+    token?: ApiToken;
+    error?: string;
+}>;
+/**
+ * Check if a token has a specific scope
+ */
+export declare function hasScope(token: ApiToken, scope: TokenScope): boolean;
+/**
+ * List all tokens (without hashes)
+ */
+export declare function listApiTokens(): Promise<Omit<ApiToken, 'tokenHash'>[]>;
+/**
+ * Revoke a token
+ */
+export declare function revokeApiToken(id: string): Promise<boolean>;
+/**
+ * Hono middleware for token authentication
+ */
+export declare function tokenAuthMiddleware(requiredScopes?: TokenScope[]): (c: Context, next: () => Promise<void>) => Promise<void | (Response & import("hono").TypedResponse<{
+    error: string | undefined;
+}, 401, "json">) | (Response & import("hono").TypedResponse<{
+    error: string;
+}, 403, "json">)>;
+//# sourceMappingURL=api-tokens.d.ts.map

package/dist/auth/api-tokens.js

@@ -0,0 +1,235 @@
+import { randomBytes, createHash } from 'crypto';
+import { getStorage } from '../storage/index.js';
+/**
+ * In-memory token cache for fast validation
+ */
+const tokenCache = new Map();
+/**
+ * Rate limit tracking
+ */
+const rateLimitTracker = new Map();
+/**
+ * Generate a secure random token
+ */
+function generateToken() {
+    return `profclaw_${randomBytes(32).toString('base64url')}`;
+}
+/**
+ * Hash a token for storage
+ */
+function hashToken(token) {
+    return createHash('sha256').update(token).digest('hex');
+}
+/**
+ * Initialize API tokens table
+ */
+export async function initApiTokensTable() {
+    const storage = getStorage();
+    await storage.execute(`
+    CREATE TABLE IF NOT EXISTS api_tokens (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      token_hash TEXT NOT NULL UNIQUE,
+      token_prefix TEXT NOT NULL,
+      scopes TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      last_used_at TEXT,
+      expires_at TEXT,
+      rate_limit INTEGER NOT NULL DEFAULT 60,
+      enabled INTEGER NOT NULL DEFAULT 1
+    )
+  `);
+    // Migration for existing tables
+    try {
+        await storage.execute(`ALTER TABLE api_tokens ADD COLUMN token_prefix TEXT`);
+    }
+    catch {
+        // Column already exists
+    }
+    try {
+        await storage.execute(`ALTER TABLE api_tokens ADD COLUMN rate_limit INTEGER DEFAULT 60`);
+    }
+    catch {
+        // Column already exists
+    }
+    try {
+        await storage.execute(`ALTER TABLE api_tokens ADD COLUMN enabled INTEGER DEFAULT 1`);
+    }
+    catch {
+        // Column already exists
+    }
+}
+/**
+ * Create a new API token
+ */
+export async function createApiToken(name, scopes, options = {}) {
+    const plainTextToken = generateToken();
+    const tokenHash = hashToken(plainTextToken);
+    const tokenPrefix = plainTextToken.slice(0, 12);
+    const token = {
+        id: randomBytes(8).toString('hex'),
+        name,
+        tokenHash,
+        tokenPrefix,
+        scopes,
+        createdAt: new Date(),
+        expiresAt: options.expiresInDays
+            ? new Date(Date.now() + options.expiresInDays * 24 * 60 * 60 * 1000)
+            : undefined,
+        rateLimit: options.rateLimit || 60,
+        enabled: true,
+    };
+    // Store in database using generic execute
+    const storage = getStorage();
+    await storage.execute(`INSERT INTO api_tokens (id, name, token_hash, token_prefix, scopes, created_at, expires_at, rate_limit, enabled)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+        token.id,
+        token.name,
+        token.tokenHash,
+        token.tokenPrefix,
+        JSON.stringify(token.scopes),
+        token.createdAt.toISOString(),
+        token.expiresAt?.toISOString() || null,
+        token.rateLimit,
+        token.enabled ? 1 : 0,
+    ]);
+    // Cache for fast lookup
+    tokenCache.set(tokenHash, token);
+    return { token, plainTextToken };
+}
+/**
+ * Get token by hash from database
+ */
+async function getTokenByHash(hash) {
+    const storage = getStorage();
+    const rows = await storage.query('SELECT * FROM api_tokens WHERE token_hash = ?', [hash]);
+    if (rows.length === 0)
+        return null;
+    const row = rows[0];
+    return {
+        id: row.id,
+        name: row.name,
+        tokenHash: row.token_hash,
+        tokenPrefix: row.token_prefix,
+        scopes: JSON.parse(row.scopes),
+        createdAt: new Date(row.created_at),
+        lastUsedAt: row.last_used_at ? new Date(row.last_used_at) : undefined,
+        expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
+        rateLimit: row.rate_limit,
+        enabled: Boolean(row.enabled),
+    };
+}
+/**
+ * Validate a token and return its info
+ */
+export async function validateToken(plainTextToken) {
+    if (!plainTextToken || !plainTextToken.startsWith('profclaw_')) {
+        return { valid: false, error: 'Invalid token format' };
+    }
+    const tokenHash = hashToken(plainTextToken);
+    // Check cache first
+    let token = tokenCache.get(tokenHash);
+    // If not in cache, check database
+    if (!token) {
+        token = await getTokenByHash(tokenHash) || undefined;
+        if (token) {
+            tokenCache.set(tokenHash, token);
+        }
+    }
+    if (!token) {
+        return { valid: false, error: 'Token not found' };
+    }
+    if (!token.enabled) {
+        return { valid: false, error: 'Token disabled' };
+    }
+    if (token.expiresAt && token.expiresAt < new Date()) {
+        return { valid: false, error: 'Token expired' };
+    }
+    // Check rate limit
+    const rateLimitKey = token.id;
+    const now = Date.now();
+    const tracker = rateLimitTracker.get(rateLimitKey);
+    if (tracker) {
+        if (now < tracker.resetAt) {
+            if (tracker.count >= token.rateLimit) {
+                return { valid: false, error: 'Rate limit exceeded' };
+            }
+            tracker.count++;
+        }
+        else {
+            rateLimitTracker.set(rateLimitKey, { count: 1, resetAt: now + 60000 });
+        }
+    }
+    else {
+        rateLimitTracker.set(rateLimitKey, { count: 1, resetAt: now + 60000 });
+    }
+    // Update last used (async, don't block)
+    const storage = getStorage();
+    storage.execute('UPDATE api_tokens SET last_used_at = ? WHERE id = ?', [new Date().toISOString(), token.id]).catch(() => { });
+    return { valid: true, token };
+}
+/**
+ * Check if a token has a specific scope
+ */
+export function hasScope(token, scope) {
+    return token.scopes.includes('admin') || token.scopes.includes(scope);
+}
+/**
+ * List all tokens (without hashes)
+ */
+export async function listApiTokens() {
+    const storage = getStorage();
+    const rows = await storage.query('SELECT id, name, token_prefix, scopes, created_at, last_used_at, expires_at, rate_limit, enabled FROM api_tokens');
+    return rows.map(row => ({
+        id: row.id,
+        name: row.name,
+        tokenPrefix: row.token_prefix,
+        scopes: JSON.parse(row.scopes),
+        createdAt: new Date(row.created_at),
+        lastUsedAt: row.last_used_at ? new Date(row.last_used_at) : undefined,
+        expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
+        rateLimit: row.rate_limit,
+        enabled: Boolean(row.enabled),
+    }));
+}
+/**
+ * Revoke a token
+ */
+export async function revokeApiToken(id) {
+    const storage = getStorage();
+    await storage.execute('DELETE FROM api_tokens WHERE id = ?', [id]);
+    // Clear from cache
+    for (const [hash, token] of tokenCache.entries()) {
+        if (token.id === id) {
+            tokenCache.delete(hash);
+            break;
+        }
+    }
+    return true;
+}
+/**
+ * Hono middleware for token authentication
+ */
+export function tokenAuthMiddleware(requiredScopes = []) {
+    return async (c, next) => {
+        const authHeader = c.req.header('Authorization');
+        if (!authHeader || !authHeader.startsWith('Bearer ')) {
+            return c.json({ error: 'Missing or invalid Authorization header' }, 401);
+        }
+        const token = authHeader.slice(7);
+        const result = await validateToken(token);
+        if (!result.valid) {
+            return c.json({ error: result.error }, 401);
+        }
+        // Check required scopes
+        for (const scope of requiredScopes) {
+            if (!hasScope(result.token, scope)) {
+                return c.json({ error: `Missing required scope: ${scope}` }, 403);
+            }
+        }
+        // Attach token info to context
+        c.set('apiToken', result.token);
+        return next();
+    };
+}
+//# sourceMappingURL=api-tokens.js.map