thatcher 1.0.4

package/src/app/api/debug/workflow/route.js

@@ -0,0 +1,20 @@
+import { NextResponse } from '@/lib/next-polyfills';
+
+export async function GET() {
+  if (process.env.NODE_ENV === 'production') {
+    return NextResponse.json({ error: 'Not available in production' }, { status: 403 });
+  }
+  const { getConfigEngineSync } = await import('@/lib/config-generator-engine.js');
+  const engine = getConfigEngineSync();
+  const wf = engine.getConfig().workflows || {};
+  const summary = Object.fromEntries(
+    Object.entries(wf).map(([k, v]) => [k, {
+      state_field: v.state_field || 'status',
+      stages: v.stages ? Object.keys(v.stages) : [],
+      transitions: Array.isArray(v.transitions) ? v.transitions.length : null,
+    }])
+  );
+  return NextResponse.json({ workflows: summary });
+}
+
+export const config = { runtime: 'nodejs' };

package/src/app/api/domains/[domain]/route.js

@@ -0,0 +1,26 @@
+import { getDomainLoader } from '@/lib/domain-loader';
+import { getConfigEngine } from '@/lib/config-generator-engine';
+import { ok } from '@/lib/response-formatter';
+import { withErrorHandler } from '@/lib/with-error-handler';
+import { AppError, NotFoundError } from '@/lib/error-handler';
+import { HTTP } from '@/config/constants';
+
+export const GET = withErrorHandler(async (request, context) => {
+  const params = await context.params;
+  const { domain } = params;
+
+  if (!domain) {
+    throw new AppError('Domain parameter required', 'BAD_REQUEST', HTTP.BAD_REQUEST);
+  }
+
+  await getConfigEngine();
+  const domainLoader = getDomainLoader();
+  const validDomains = domainLoader.getValidDomains();
+
+  if (!validDomains.includes(domain.toLowerCase())) {
+    throw NotFoundError('domain', domain);
+  }
+
+  const domainInfo = domainLoader.getDomainInfo(domain);
+  return ok(domainInfo);
+}, 'Domains:Get');

package/src/app/api/domains/route.js

@@ -0,0 +1,21 @@
+import { getDomainLoader } from '@/lib/domain-loader';
+import { ok } from '@/lib/response-formatter';
+import { withErrorHandler } from '@/lib/with-error-handler';
+import { getConfigEngine } from '@/lib/config-generator-engine';
+
+export const GET = withErrorHandler(async (request) => {
+  await getConfigEngine();
+  const domainLoader = getDomainLoader();
+  const validDomains = domainLoader.getValidDomains();
+
+  const domains = validDomains.map(domainName => {
+    try {
+      return domainLoader.getDomainInfo(domainName);
+    } catch (error) {
+      console.error(`[Domains API] Error loading domain ${domainName}:`, error.message);
+      return null;
+    }
+  }).filter(Boolean);
+
+  return ok({ domains });
+}, 'Domains:List');

package/src/app/api/email/allocate/batch/route.js

@@ -0,0 +1,106 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { getDatabase, genId, now } from '@/lib/database-core';
+import { autoAllocateEmail } from '@/lib/email-parser';
+
+export async function POST(request) {
+  try {
+    const { requireUser, setCurrentRequest } = await import('@/engine.server');
+    setCurrentRequest(request);
+    const user = await requireUser();
+    if (user.role !== 'admin' && user.role !== 'partner') {
+      return NextResponse.json({ error: 'Permission denied' }, { status: 403 });
+    }
+
+    const body = await request.json();
+    const { min_confidence = 70, batch_size = 50 } = body;
+
+    const db = getDatabase();
+
+    const unallocatedEmails = db.prepare(`
+      SELECT * FROM email
+      WHERE allocated = 0 AND status = 'pending'
+      ORDER BY received_at DESC
+      LIMIT ?
+    `).all(batch_size);
+
+    const results = {
+      allocated: [],
+      skipped: [],
+      failed: [],
+      total: unallocatedEmails.length,
+    };
+
+    for (const email of unallocatedEmails) {
+      try {
+        const result = await autoAllocateEmail(email);
+
+        if (result.success && result.confidence >= min_confidence) {
+          const logId = genId();
+          const timestamp = now();
+
+          db.prepare(`
+            INSERT INTO activity_log (
+              id, entity_type, entity_id, action, message, details, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?)
+          `).run(
+            logId,
+            'email',
+            email.id,
+            'batch_allocated',
+            `Email batch-allocated to ${result.engagement_id ? 'engagement' : 'RFI'}`,
+            JSON.stringify({
+              engagement_id: result.engagement_id || null,
+              rfi_id: result.rfi_id || null,
+              confidence: result.confidence,
+              method: 'batch_automatic',
+            }),
+            timestamp
+          );
+
+          results.allocated.push({
+            email_id: email.id,
+            subject: email.subject,
+            engagement_id: result.engagement_id || null,
+            rfi_id: result.rfi_id || null,
+            confidence: result.confidence,
+          });
+        } else if (result.success && result.confidence < min_confidence) {
+          results.skipped.push({
+            email_id: email.id,
+            subject: email.subject,
+            confidence: result.confidence,
+            reason: `confidence ${result.confidence}% < ${min_confidence}%`,
+          });
+        } else {
+          results.failed.push({
+            email_id: email.id,
+            subject: email.subject,
+            reason: result.reason,
+          });
+        }
+      } catch (error) {
+        results.failed.push({
+          email_id: email.id,
+          subject: email.subject,
+          reason: error.message,
+        });
+
+        db.prepare(`
+          UPDATE email
+          SET processing_error = ?,
+              updated_at = ?
+          WHERE id = ?
+        `).run(error.message, now(), email.id);
+      }
+    }
+
+    return NextResponse.json(results);
+
+  } catch (error) {
+    console.error('[EMAIL_BATCH_ALLOCATE] Error:', error);
+    return NextResponse.json(
+      { error: 'Internal server error', message: error.message },
+      { status: 500 }
+    );
+  }
+}

package/src/app/api/email/allocate/route.js

@@ -0,0 +1,141 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { getDatabase, genId, now } from '@/lib/database-core';
+import {
+  allocateEmailToEntity,
+  autoAllocateEmail,
+  validateAllocation,
+  findEntityByAlternateId,
+} from '@/lib/email-parser';
+
+export async function POST(request) {
+  try {
+    const { requireUser, setCurrentRequest } = await import('@/engine.server');
+    setCurrentRequest(request);
+    await requireUser();
+
+    const body = await request.json();
+    const { email_id, engagement_id, rfi_id, auto = false } = body;
+
+    if (!email_id) {
+      return NextResponse.json(
+        { error: 'email_id is required' },
+        { status: 400 }
+      );
+    }
+
+    const db = getDatabase();
+    const email = db.prepare('SELECT * FROM email WHERE id = ?').get(email_id);
+
+    if (!email) {
+      return NextResponse.json(
+        { error: 'Email not found' },
+        { status: 404 }
+      );
+    }
+
+    if (email.allocated) {
+      return NextResponse.json(
+        { error: 'Email already allocated' },
+        { status: 400 }
+      );
+    }
+
+    let result;
+
+    if (auto) {
+      result = await autoAllocateEmail(email);
+
+      if (!result.success) {
+        return NextResponse.json(
+          {
+            success: false,
+            reason: result.reason,
+            confidence: result.confidence,
+            message: `Auto-allocation failed: ${result.reason}`,
+          },
+          { status: 400 }
+        );
+      }
+    } else {
+      if (!engagement_id && !rfi_id) {
+        return NextResponse.json(
+          { error: 'Either engagement_id or rfi_id must be provided for manual allocation' },
+          { status: 400 }
+        );
+      }
+
+      let resolvedEngagementId = engagement_id;
+      let resolvedRfiId = rfi_id;
+
+      if (engagement_id) {
+        resolvedEngagementId = findEntityByAlternateId('engagement', engagement_id);
+        if (!resolvedEngagementId) {
+          return NextResponse.json(
+            { error: 'Engagement not found' },
+            { status: 404 }
+          );
+        }
+      }
+
+      if (rfi_id) {
+        resolvedRfiId = findEntityByAlternateId('rfi', rfi_id);
+        if (!resolvedRfiId) {
+          return NextResponse.json(
+            { error: 'RFI not found' },
+            { status: 404 }
+          );
+        }
+      }
+
+      const updatedEmail = allocateEmailToEntity(
+        email_id,
+        resolvedEngagementId,
+        resolvedRfiId
+      );
+
+      result = {
+        success: true,
+        email: updatedEmail,
+        engagement_id: resolvedEngagementId,
+        rfi_id: resolvedRfiId,
+      };
+    }
+
+    const logId = genId();
+    const timestamp = now();
+
+    db.prepare(`
+      INSERT INTO activity_log (
+        id, entity_type, entity_id, action, message, details, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      logId,
+      'email',
+      email_id,
+      'allocated',
+      `Email allocated to ${result.engagement_id ? 'engagement' : 'RFI'}`,
+      JSON.stringify({
+        engagement_id: result.engagement_id || null,
+        rfi_id: result.rfi_id || null,
+        method: auto ? 'automatic' : 'manual',
+        confidence: result.confidence || 100,
+      }),
+      timestamp
+    );
+
+    return NextResponse.json({
+      success: true,
+      email: result.email,
+      engagement_id: result.engagement_id || null,
+      rfi_id: result.rfi_id || null,
+      confidence: result.confidence || 100,
+    });
+
+  } catch (error) {
+    console.error('[EMAIL_ALLOCATE] Error:', error);
+    return NextResponse.json(
+      { error: 'Internal server error', message: error.message },
+      { status: 500 }
+    );
+  }
+}

package/src/app/api/email/receive/route.js

@@ -0,0 +1,158 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { getDatabase, genId, now } from '@/lib/database-core';
+import path from 'path';
+import fs from 'fs';
+
+const TEMP_EMAIL_ATTACHMENTS_DIR = path.resolve(process.cwd(), 'data', 'temp_email_attachments');
+
+if (!fs.existsSync(TEMP_EMAIL_ATTACHMENTS_DIR)) {
+  fs.mkdirSync(TEMP_EMAIL_ATTACHMENTS_DIR, { recursive: true });
+}
+
+export async function POST(request) {
+  try {
+    const authHeader = request.headers.get('authorization');
+    const token = authHeader?.replace('Bearer ', '');
+    if (!token || token !== process.env.EMAIL_WEBHOOK_SECRET) {
+      return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+    }
+
+    const body = await request.json();
+
+    const {
+      from,
+      sender_email,
+      subject,
+      body: emailBody,
+      html_body,
+      text_body,
+      attachments = [],
+      message_id,
+      in_reply_to,
+      references,
+      received_date,
+    } = body;
+
+    if (!sender_email || !subject) {
+      return NextResponse.json(
+        { error: 'Missing required fields: sender_email and subject are required' },
+        { status: 400 }
+      );
+    }
+
+    const attachmentData = [];
+
+    for (const attachment of attachments) {
+      const { filename, content, content_type, size } = attachment;
+
+      if (filename && content) {
+        const attachmentId = genId();
+        const ext = path.extname(filename);
+        const safeFilename = `${attachmentId}${ext}`;
+        const filePath = path.join(TEMP_EMAIL_ATTACHMENTS_DIR, safeFilename);
+
+        const buffer = Buffer.isBuffer(content)
+          ? content
+          : Buffer.from(content, 'base64');
+
+        fs.writeFileSync(filePath, buffer);
+
+        attachmentData.push({
+          id: attachmentId,
+          filename,
+          path: filePath,
+          content_type,
+          size: size || buffer.length,
+        });
+      }
+    }
+
+    const emailRecord = {
+      sender_email,
+      sender_name: from || sender_email,
+      subject,
+      body: text_body || emailBody || '',
+      html_body: html_body || '',
+      message_id: message_id || genId(),
+      in_reply_to: in_reply_to || null,
+      references: references || null,
+      received_at: received_date ? Math.floor(new Date(received_date).getTime() / 1000) : now(),
+      allocated: false,
+      engagement_id: null,
+      rfi_id: null,
+      attachments: JSON.stringify(attachmentData),
+      status: 'pending',
+      processed: false,
+      processing_error: null,
+    };
+
+    const db = getDatabase();
+    const stmt = db.prepare(`
+      INSERT INTO email (
+        id, sender_email, sender_name, subject, body, html_body,
+        message_id, in_reply_to, references, received_at, allocated,
+        engagement_id, rfi_id, attachments, status, processed, processing_error,
+        created_at, updated_at
+      ) VALUES (
+        ?, ?, ?, ?, ?, ?,
+        ?, ?, ?, ?, ?,
+        ?, ?, ?, ?, ?, ?,
+        ?, ?
+      )
+    `);
+
+    const emailId = genId();
+    const timestamp = now();
+
+    stmt.run(
+      emailId,
+      emailRecord.sender_email,
+      emailRecord.sender_name,
+      emailRecord.subject,
+      emailRecord.body,
+      emailRecord.html_body,
+      emailRecord.message_id,
+      emailRecord.in_reply_to,
+      emailRecord.references,
+      emailRecord.received_at,
+      emailRecord.allocated ? 1 : 0,
+      emailRecord.engagement_id,
+      emailRecord.rfi_id,
+      emailRecord.attachments,
+      emailRecord.status,
+      emailRecord.processed ? 1 : 0,
+      emailRecord.processing_error,
+      timestamp,
+      timestamp
+    );
+
+    return NextResponse.json({
+      success: true,
+      email_id: emailId,
+      attachments_count: attachmentData.length,
+      message: 'Email received and stored successfully',
+    });
+
+  } catch (error) {
+    console.error('[EMAIL_RECEIVE] Error processing webhook:', error);
+
+    return NextResponse.json(
+      {
+        error: 'Internal server error',
+        message: error.message,
+      },
+      { status: 500 }
+    );
+  }
+}
+
+export async function GET(request) {
+  return NextResponse.json({
+    endpoint: 'Email Receive Webhook',
+    method: 'POST',
+    description: 'Accepts incoming emails from Gmail webhook',
+    required_fields: ['sender_email', 'subject'],
+    optional_fields: ['from', 'body', 'html_body', 'text_body', 'attachments', 'message_id', 'in_reply_to', 'references', 'received_date'],
+    attachment_format: 'Array of { filename, content (base64), content_type, size }',
+  });
+}

package/src/app/api/email/route.js

@@ -0,0 +1,3 @@
+import { createHttpMethods } from '@/lib/http-methods-factory';
+
+export const { GET, POST, PUT, PATCH, DELETE } = createHttpMethods('email');

package/src/app/api/email/send/route.js

@@ -0,0 +1,77 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { getDatabase, now } from '@/lib/database-core';
+import { getConfigEngine } from '@/lib/config-generator-engine';
+import { EMAIL_STATUS } from '@/config/constants';
+import { sendSingleEmail, checkFailureRate } from '@/lib/email-sender';
+
+let emailConfig = null;
+
+async function getEmailConfig() {
+  if (!emailConfig) {
+    const engine = await getConfigEngine();
+    emailConfig = engine.getConfig()?.thresholds?.email || {};
+  }
+  return emailConfig;
+}
+
+export async function POST(request) {
+  const token = request.headers.get('authorization')?.replace('Bearer ', '');
+  if (!token || token !== process.env.CRON_SECRET)
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+
+  const db = getDatabase();
+  try {
+    const emailCfg = await getEmailConfig();
+    const MAX_RETRIES = emailCfg.send_max_retries || 3;
+    const BATCH_SIZE = emailCfg.send_batch_size || 10;
+    const RATE_LIMIT_DELAY = emailCfg.rate_limit_delay_ms || 6000;
+    const MAX_DELAY_MS = emailCfg.retry_max_delay_ms || 30000;
+
+    const pendingEmails = db.prepare(`SELECT * FROM email WHERE status=? AND (retry_count IS NULL OR retry_count < ?) ORDER BY created_at ASC LIMIT ?`)
+      .all(EMAIL_STATUS.PENDING, MAX_RETRIES, BATCH_SIZE);
+
+    if (!pendingEmails.length)
+      return NextResponse.json({ success: true, message: 'No pending emails', processed: 0 });
+
+    const results = [];
+    let successCount = 0, failureCount = 0;
+
+    for (let i = 0; i < pendingEmails.length; i++) {
+      const email = pendingEmails[i];
+      db.prepare(`UPDATE email SET status=?, updated_at=? WHERE id=?`).run(EMAIL_STATUS.PROCESSING, now(), email.id);
+      const result = await sendSingleEmail(db, email, email.retry_count || 1, MAX_RETRIES, MAX_DELAY_MS);
+      results.push(result);
+      result.success ? successCount++ : failureCount++;
+      if (i < pendingEmails.length - 1)
+        await new Promise(r => setTimeout(r, RATE_LIMIT_DELAY));
+    }
+
+    checkFailureRate(db);
+    return NextResponse.json({ success: true, processed: pendingEmails.length, results: { success: successCount, failed: failureCount }, details: results });
+  } catch (error) {
+    console.error('[EMAIL] Queue processing error:', error);
+    return NextResponse.json({ error: 'Email queue processing failed', details: error.message }, { status: 500 });
+  }
+}
+
+export async function GET(request) {
+  const token = request.headers.get('authorization')?.replace('Bearer ', '');
+  if (!token || token !== process.env.CRON_SECRET)
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+
+  const db = getDatabase();
+  try {
+    const stats = db.prepare(`SELECT status, COUNT(*) as count FROM email GROUP BY status`).all();
+    const failureStats = db.prepare(`SELECT COUNT(*) as total, SUM(CASE WHEN status=? THEN 1 ELSE 0 END) as failed FROM email WHERE created_at >= ?`)
+      .get(EMAIL_STATUS.FAILED, now() - 86400);
+    const recentFailures = db.prepare(`SELECT id, recipient_email, subject, processing_error, retry_count, created_at FROM email WHERE status=? ORDER BY created_at DESC LIMIT 10`)
+      .all(EMAIL_STATUS.FAILED);
+    return NextResponse.json({
+      stats: stats.reduce((acc, r) => ({ ...acc, [r.status]: r.count }), {}),
+      failureRate: failureStats.total > 0 ? failureStats.failed / failureStats.total : 0,
+      recentFailures,
+    });
+  } catch (error) {
+    return NextResponse.json({ error: 'Failed to get email stats', details: error.message }, { status: 500 });
+  }
+}

package/src/app/api/email/unallocated/route.js

@@ -0,0 +1,47 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { getDatabase } from '@/lib/database-core';
+
+export async function GET(request) {
+  try {
+    const { requireUser, setCurrentRequest } = await import('@/engine.server');
+    setCurrentRequest(request);
+    await requireUser();
+
+    const { searchParams } = new URL(request.url);
+    const limit = parseInt(searchParams.get('limit') || '50', 10);
+    const offset = parseInt(searchParams.get('offset') || '0', 10);
+
+    const db = getDatabase();
+
+    const emails = db.prepare(`
+      SELECT *
+      FROM email
+      WHERE allocated = 0
+      ORDER BY received_at DESC
+      LIMIT ? OFFSET ?
+    `).all(limit, offset);
+
+    const total = db.prepare('SELECT COUNT(*) as count FROM email WHERE allocated = 0').get();
+
+    const emailsWithParsedAttachments = emails.map(email => ({
+      ...email,
+      attachments: email.attachments ? JSON.parse(email.attachments) : [],
+      allocated: Boolean(email.allocated),
+      processed: Boolean(email.processed),
+    }));
+
+    return NextResponse.json({
+      emails: emailsWithParsedAttachments,
+      total: total.count,
+      limit,
+      offset,
+    });
+
+  } catch (error) {
+    console.error('[EMAIL_UNALLOCATED] Error:', error);
+    return NextResponse.json(
+      { error: 'Internal server error', message: error.message },
+      { status: 500 }
+    );
+  }
+}

package/src/app/api/files/[id]/route.js

@@ -0,0 +1,38 @@
+import { NextResponse } from '@/lib/next-polyfills';
+import { requireUser } from '@/engine.server';
+import { permissionService } from '@/services/permission.service';
+import { get } from '@/engine';
+import { getSpec } from '@/config/spec-helpers';
+import { fileService } from '@/services/file.service';
+import { HTTP } from '@/config/constants';
+import { notFound } from '@/lib/response-formatter';
+
+export async function GET(request, { params }) {
+  try {
+    const user = await requireUser();
+    const { id } = params;
+
+    const spec = getSpec('file');
+    if (!permissionService.checkAccess(user, spec, 'view')) {
+      return NextResponse.json({ error: 'Permission denied' }, { status: HTTP.FORBIDDEN });
+    }
+
+    const fileRecord = get('file', id);
+    if (!fileRecord) {
+      return notFound('File not found');
+    }
+
+    const content = await fileService.download(fileRecord.drive_file_id);
+
+    const safeName = (fileRecord.file_name || 'download').replace(/[^\w.\-]/g, '_');
+    return new NextResponse(content, {
+      headers: {
+        'Content-Type': fileRecord.mime_type || 'application/octet-stream',
+        'Content-Disposition': `attachment; filename="${safeName}"`,
+      },
+    });
+  } catch (error) {
+    console.error('File download error:', error);
+    return NextResponse.json({ error: error.message }, { status: HTTP.INTERNAL_ERROR });
+  }
+}