takos-control 1.0.0

package/src/server/routes/index/index-context.ts

@@ -0,0 +1,80 @@
+import type { ExecutionContext } from 'hono';
+import type { AppContext, BaseVariables } from '../shared/route-auth';
+
+export type IndexContext = AppContext<BaseVariables>;
+
+export type VectorizeIndexBody = {
+  force_reindex?: boolean;
+};
+
+export type IndexFileBody = {
+  path: string;
+};
+
+export function scheduleBackground(c: IndexContext, task: Promise<unknown>): void {
+  const ctx = c.executionCtx;
+  if (ctx && 'waitUntil' in ctx) {
+    (ctx as ExecutionContext).waitUntil(task);
+  }
+}
+
+export function getR2Key(spaceId: string, fileId: string): string {
+  return `spaces/${spaceId}/files/${fileId}`;
+}
+
+export function chunkContent(
+  content: string,
+  maxChunkSize: number = 1000
+): Array<{ startLine: number; endLine: number; content: string }> {
+  const lines = content.split('\n');
+  const chunks: Array<{ startLine: number; endLine: number; content: string }> = [];
+  let currentChunk: string[] = [];
+  let startLine = 1;
+  let currentSize = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const lineSize = line.length + 1;
+
+    if (currentSize + lineSize > maxChunkSize && currentChunk.length > 0) {
+      chunks.push({
+        startLine,
+        endLine: startLine + currentChunk.length - 1,
+        content: currentChunk.join('\n'),
+      });
+      currentChunk = [line];
+      startLine = i + 1;
+      currentSize = lineSize;
+      continue;
+    }
+
+    currentChunk.push(line);
+    currentSize += lineSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push({
+      startLine,
+      endLine: startLine + currentChunk.length - 1,
+      content: currentChunk.join('\n'),
+    });
+  }
+
+  return chunks;
+}
+
+export function resolvePath(from: string, to: string): string {
+  const fromParts = from.split('/').slice(0, -1);
+  const toParts = to.split('/');
+
+  for (const part of toParts) {
+    if (part === '.') continue;
+    if (part === '..') {
+      fromParts.pop();
+      continue;
+    }
+    fromParts.push(part);
+  }
+
+  return fromParts.join('/');
+}

package/src/server/routes/index/index.ts

@@ -0,0 +1,50 @@
+import { Hono } from 'hono';
+import type { Context } from 'hono';
+import type { Env, User } from '../../../shared/types';
+import type { BaseVariables } from '../shared/route-auth';
+import { InMemoryRateLimiter } from '../../../shared/utils';
+import { parseJsonBody } from '../shared/route-auth';
+import {
+  handleIndexFile,
+  handleIndexStatus,
+  handleRebuildIndex,
+  handleVectorizeIndex,
+} from './handlers';
+import { handleGraphNeighbors } from './graph';
+import { BadRequestError } from 'takos-common/errors';
+
+const index = new Hono<{ Bindings: Env; Variables: BaseVariables }>();
+
+const expensiveIndexRateLimiter = new InMemoryRateLimiter({
+  maxRequests: 5,
+  windowMs: 15 * 60 * 1000,
+  message: 'Too many indexing requests. Please wait before triggering another reindex.',
+  keyGenerator: (c: Context) => {
+    const user = c.get('user') as User | undefined;
+    const spaceId = c.req.param('spaceId');
+    return `index:${user?.id || 'anon'}:${spaceId || 'unknown'}`;
+  },
+});
+
+index.get('/spaces/:spaceId/index/status', handleIndexStatus);
+
+index.post('/spaces/:spaceId/index/vectorize', expensiveIndexRateLimiter.middleware(), async (c) => {
+  const body = await parseJsonBody<{
+    force_reindex?: boolean;
+  }>(c, {});
+  if (body === null) {
+    throw new BadRequestError('Invalid JSON body');
+  }
+  return handleVectorizeIndex(c, body);
+});
+
+index.post('/spaces/:spaceId/index/rebuild', expensiveIndexRateLimiter.middleware(), handleRebuildIndex);
+
+index.post('/spaces/:spaceId/index/file', async (c) => {
+  const body = await parseJsonBody<{ path: string }>(c);
+  return handleIndexFile(c, body);
+});
+
+index.get('/spaces/:spaceId/graph/neighbors', handleGraphNeighbors);
+
+export default index;

package/src/server/routes/index/jobs.ts

@@ -0,0 +1,159 @@
+import type { D1Database, R2Bucket } from '../../../shared/types/bindings.ts';
+import type { SpaceFile } from '../../../shared/types';
+import { getDb } from '../../../infra/db';
+import { indexJobs, files, chunks, nodes } from '../../../infra/db/schema';
+import { eq, and, ne, inArray, asc } from 'drizzle-orm';
+import type { EmbeddingsService } from '../../../application/services/execution/embeddings';
+import { generateId, now } from '../../../shared/utils';
+import { extractAndCreateEdges } from './graph';
+import { chunkContent, getR2Key } from './index-context';
+import { logWarn } from '../../../shared/utils/logger';
+
+export async function runIndexJob(
+  db: D1Database,
+  storage: R2Bucket | undefined,
+  jobId: string,
+  embeddingsService?: EmbeddingsService | null
+): Promise<void> {
+  const drizzle = getDb(db);
+  const job = await drizzle.select().from(indexJobs).where(eq(indexJobs.id, jobId)).get();
+  if (!job || job.status !== 'queued') {
+    return;
+  }
+
+  const timestamp = now();
+  await drizzle.update(indexJobs).set({
+    status: 'running',
+    startedAt: timestamp,
+  }).where(eq(indexJobs.id, jobId));
+
+  try {
+    const fileRows = await drizzle.select().from(files).where(
+      and(
+        eq(files.accountId, job.accountId),
+        ne(files.origin, 'system'),
+        inArray(files.kind, ['source', 'config', 'doc']),
+      )
+    ).orderBy(asc(files.path)).all();
+
+    let processed = 0;
+    for (const file of fileRows) {
+      await indexFileContent(db, storage, job.accountId, file as unknown as SpaceFile, embeddingsService);
+      processed++;
+      await drizzle.update(indexJobs).set({ processedFiles: processed }).where(eq(indexJobs.id, jobId));
+    }
+
+    await drizzle.update(indexJobs).set({
+      status: 'completed',
+      completedAt: now(),
+    }).where(eq(indexJobs.id, jobId));
+  } catch (error) {
+    await drizzle.update(indexJobs).set({
+      status: 'failed',
+      error: String(error),
+      completedAt: now(),
+    }).where(eq(indexJobs.id, jobId));
+    throw error;
+  }
+}
+
+export async function indexFile(
+  db: D1Database,
+  storage: R2Bucket | undefined,
+  spaceId: string,
+  fileId: string,
+  jobId: string,
+  embeddingsService?: EmbeddingsService | null
+): Promise<void> {
+  const drizzle = getDb(db);
+  const timestamp = now();
+  await drizzle.update(indexJobs).set({
+    status: 'running',
+    startedAt: timestamp,
+  }).where(eq(indexJobs.id, jobId));
+
+  try {
+    const file = await drizzle.select().from(files).where(eq(files.id, fileId)).get();
+    if (!file) {
+      throw new Error('File not found');
+    }
+
+    await indexFileContent(db, storage, spaceId, file as unknown as SpaceFile, embeddingsService);
+    await drizzle.update(indexJobs).set({
+      status: 'completed',
+      processedFiles: 1,
+      completedAt: now(),
+    }).where(eq(indexJobs.id, jobId));
+  } catch (error) {
+    await drizzle.update(indexJobs).set({
+      status: 'failed',
+      error: String(error),
+      completedAt: now(),
+    }).where(eq(indexJobs.id, jobId));
+    throw error;
+  }
+}
+
+async function indexFileContent(
+  db: D1Database,
+  storage: R2Bucket | undefined,
+  spaceId: string,
+  file: SpaceFile,
+  embeddingsService?: EmbeddingsService | null
+): Promise<void> {
+  if (!storage) {
+    throw new Error('Storage not configured');
+  }
+
+  const drizzle = getDb(db);
+  const r2Key = getR2Key(spaceId, file.id);
+  const object = await storage.get(r2Key);
+  if (!object) {
+    throw new Error(`File content not found: ${file.path}`);
+  }
+
+  const content = await object.text();
+  await drizzle.delete(chunks).where(eq(chunks.fileId, file.id));
+  await drizzle.delete(nodes).where(
+    and(eq(nodes.accountId, spaceId), eq(nodes.type, 'file'), eq(nodes.refId, file.id))
+  );
+
+  const contentChunks = chunkContent(content);
+  const timestamp = now();
+  for (const chunk of contentChunks) {
+    const chunkId = generateId();
+    await drizzle.insert(chunks).values({
+      id: chunkId,
+      fileId: file.id,
+      accountId: spaceId,
+      startLine: chunk.startLine,
+      endLine: chunk.endLine,
+      content: chunk.content,
+      createdAt: now(),
+    });
+  }
+
+  if (embeddingsService) {
+    try {
+      await embeddingsService.removeFile(spaceId, file.id);
+      await embeddingsService.indexFile(spaceId, file, content);
+    } catch (err) {
+      logWarn('Vectorize indexing failed', { module: 'routes/index/jobs', error: err instanceof Error ? err.message : String(err) });
+    }
+  }
+
+  const nodeId = generateId();
+  await drizzle.insert(nodes).values({
+    id: nodeId,
+    accountId: spaceId,
+    type: 'file',
+    refId: file.id,
+    label: file.path,
+    metadata: '{}',
+    createdAt: timestamp,
+  });
+
+  await drizzle.update(files).set({ indexedAt: timestamp, updatedAt: timestamp }).where(eq(files.id, file.id));
+
+  await extractAndCreateEdges(db, spaceId, file, content, nodeId);
+}

package/src/server/routes/mcp.ts

@@ -0,0 +1,343 @@
+/**
+ * MCP Routes
+ *
+ * GET  /api/mcp/oauth/callback  - OAuth callback (no auth required, protected by state param)
+ * GET  /api/mcp/servers         - List registered MCP servers (auth required)
+ * DELETE /api/mcp/servers/:id   - Remove a registered MCP server (auth required)
+ * PATCH  /api/mcp/servers/:id   - Update a registered MCP server (auth required)
+ */
+
+import { Hono } from 'hono';
+import { z } from 'zod';
+import type { Env } from '../../shared/types';
+import {
+  consumeMcpOAuthPending,
+  completeMcpOAuthFlow,
+  registerExternalMcpServer,
+  listMcpServers,
+  deleteMcpServer,
+  updateMcpServer,
+  getMcpServerWithTokens,
+  decryptAccessToken,
+  refreshMcpToken,
+} from '../../application/services/platform/mcp';
+import { McpClient } from '../../application/tools/mcp-client';
+import { spaceAccess, type SpaceAccessRouteEnv } from './shared/route-auth';
+import { zValidator } from './zod-validator';
+import { escapeHtml } from './auth/html';
+import { logError, logWarn } from '../../shared/utils/logger';
+import { BadRequestError, NotFoundError, BadGatewayError, GatewayTimeoutError } from 'takos-common/errors';
+import { getSpaceOperationPolicy } from '../../application/tools/tool-policy';
+
+// ---------------------------------------------------------------------------
+// Zod schemas
+// ---------------------------------------------------------------------------
+
+const createServerSchema = z.object({
+  name: z.string(),
+  url: z.string(),
+  scope: z.string().optional(),
+});
+
+const updateServerSchema = z.object({
+  enabled: z.boolean().optional(),
+  name: z.string().optional(),
+});
+
+const mcpRoutes = new Hono<SpaceAccessRouteEnv>();
+
+const MCP_LIST_ROLES = getSpaceOperationPolicy('mcp_server.list').allowed_roles;
+const MCP_CREATE_ROLES = getSpaceOperationPolicy('mcp_server.create').allowed_roles;
+const MCP_UPDATE_ROLES = getSpaceOperationPolicy('mcp_server.update').allowed_roles;
+const MCP_DELETE_ROLES = getSpaceOperationPolicy('mcp_server.delete').allowed_roles;
+
+function serializeMcpServer(server: Awaited<ReturnType<typeof listMcpServers>>[number]) {
+  const sourceType = server.sourceType === 'worker' ? 'service' : server.sourceType;
+  return {
+    id: server.id,
+    name: server.name,
+    url: server.url,
+    transport: server.transport,
+    enabled: server.enabled,
+    source_type: sourceType,
+    auth_mode: server.authMode,
+    service_id: server.serviceId,
+    bundle_deployment_id: server.bundleDeploymentId,
+    managed: server.sourceType !== 'external',
+    scope: server.oauthScope,
+    issuer_url: server.oauthIssuerUrl,
+    token_expires_at: server.oauthTokenExpiresAt,
+    created_at: server.createdAt,
+    updated_at: server.updatedAt,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// OAuth callback — no authentication, protected by the state CSRF token
+// ---------------------------------------------------------------------------
+
+mcpRoutes.get('/oauth/callback', async (c) => {
+  const code = c.req.query('code');
+  const state = c.req.query('state');
+  const error = c.req.query('error');
+
+  if (error) {
+    return c.html(errorPage(`OAuth authorization failed: ${error}`), 400);
+  }
+
+  if (!code || !state) {
+    return c.html(errorPage('Missing code or state parameter'), 400);
+  }
+
+  let pending: Awaited<ReturnType<typeof consumeMcpOAuthPending>>;
+  try {
+    pending = await consumeMcpOAuthPending(c.env.DB, c.env, state);
+  } catch (err) {
+    logError('consumeMcpOAuthPending error', err, { module: 'mcp-oauth' });
+    return c.html(errorPage('Internal error processing OAuth callback'), 500);
+  }
+
+  if (!pending) {
+    return c.html(errorPage('Invalid or expired OAuth state. Please try again.'), 400);
+  }
+
+  const baseHostname = c.env.ADMIN_DOMAIN || c.env.TENANT_BASE_DOMAIN || 'localhost';
+  const redirectUri = `https://${baseHostname}/api/mcp/oauth/callback`;
+
+  try {
+    await completeMcpOAuthFlow(c.env.DB, c.env, {
+      spaceId: pending.spaceId,
+      serverName: pending.serverName,
+      serverUrl: pending.serverUrl,
+      tokenEndpoint: pending.tokenEndpoint,
+      code,
+      codeVerifier: pending.codeVerifier,
+      redirectUri,
+      scope: pending.scope,
+      issuerUrl: pending.issuerUrl ?? pending.serverUrl,
+    });
+  } catch (err) {
+    logError('completeMcpOAuthFlow error', err, { module: 'mcp-oauth' });
+    return c.html(errorPage('Failed to complete OAuth authorization. Please try again.'), 500);
+  }
+
+  return c.html(successPage(pending.serverName));
+});
+
+// ---------------------------------------------------------------------------
+// Authenticated MCP server management
+// ---------------------------------------------------------------------------
+
+/** POST /api/mcp/servers?spaceId=... */
+mcpRoutes.post('/servers',
+  zValidator('json', createServerSchema),
+  async (c) => {
+  const user = c.get('user');
+  if (!user) throw new AuthenticationError();
+
+  const spaceId = c.req.query('spaceId');
+  if (!spaceId) throw new BadRequestError('spaceId query param required');
+
+  const access = await requireSpaceAccess(c, spaceId, user.id, MCP_CREATE_ROLES);
+
+  const body = c.req.valid('json');
+
+  if (!body.name || !body.url) {
+    throw new BadRequestError('name and url are required');
+  }
+
+  const result = await registerExternalMcpServer(c.env.DB, c.env, {
+    spaceId,
+    name: body.name,
+    url: body.url,
+    scope: body.scope,
+  });
+
+  return c.json({
+    data: {
+      status: result.status,
+      name: result.name,
+      url: result.url,
+      auth_url: result.authUrl,
+      message: result.message,
+    },
+  });
+});
+
+/** GET /api/mcp/servers?spaceId=... */
+mcpRoutes.get('/servers', async (c) => {
+  const user = c.get('user');
+  if (!user) throw new AuthenticationError();
+
+  const spaceId = c.req.query('spaceId');
+  if (!spaceId) throw new BadRequestError('spaceId query param required');
+
+  const access = await requireSpaceAccess(c, spaceId, user.id, MCP_LIST_ROLES);
+
+  const servers = await listMcpServers(c.env.DB, spaceId);
+
+  return c.json({
+    data: servers.map(serializeMcpServer),
+  });
+});
+
+/** DELETE /api/mcp/servers/:id?spaceId=... */
+mcpRoutes.delete('/servers/:id', async (c) => {
+  const user = c.get('user');
+  if (!user) throw new AuthenticationError();
+
+  const serverId = c.req.param('id');
+  const spaceId = c.req.query('spaceId');
+  if (!spaceId) throw new BadRequestError('spaceId query param required');
+
+  const access = await requireSpaceAccess(c, spaceId, user.id, MCP_DELETE_ROLES);
+
+  const deleted = await deleteMcpServer(c.env.DB, spaceId, serverId);
+  if (!deleted) throw new NotFoundError('MCP server');
+
+  return c.json({ success: true });
+});
+
+/** PATCH /api/mcp/servers/:id?spaceId=... */
+mcpRoutes.patch('/servers/:id',
+  zValidator('json', updateServerSchema),
+  async (c) => {
+  const user = c.get('user');
+  if (!user) throw new AuthenticationError();
+
+  const serverId = c.req.param('id');
+  const spaceId = c.req.query('spaceId');
+  if (!spaceId) throw new BadRequestError('spaceId query param required');
+
+  const access = await requireSpaceAccess(c, spaceId, user.id, MCP_UPDATE_ROLES);
+
+  const body = c.req.valid('json');
+
+  const updated = await updateMcpServer(c.env.DB, spaceId, serverId, body);
+  if (!updated) throw new NotFoundError('MCP server');
+
+  return c.json({
+    data: serializeMcpServer(updated),
+  });
+});
+
+/** GET /api/mcp/servers/:id/tools?spaceId=... */
+mcpRoutes.get('/servers/:id/tools', async (c) => {
+  const user = c.get('user');
+  if (!user) throw new AuthenticationError();
+
+  const serverId = c.req.param('id');
+  const spaceId = c.req.query('spaceId');
+  if (!spaceId) throw new BadRequestError('spaceId query param required');
+
+  const access = await requireSpaceAccess(c, spaceId, user.id, MCP_LIST_ROLES);
+
+  const server = await getMcpServerWithTokens(c.env.DB, spaceId, serverId);
+  if (!server) throw new NotFoundError('MCP server');
+
+  let accessToken: string | null = null;
+
+  if (server.sourceType === 'external') {
+    // External server with OAuth tokens
+    accessToken = await decryptAccessToken(c.env.DB, c.env, {
+      id: server.id,
+      oauthAccessToken: server.oauthAccessToken,
+    });
+    // If token is expired, try to refresh
+    if (server.oauthTokenExpiresAt && new Date(server.oauthTokenExpiresAt) < new Date()) {
+      await refreshMcpToken(c.env.DB, c.env, {
+        id: server.id,
+        oauthRefreshToken: server.oauthRefreshToken,
+        oauthIssuerUrl: server.oauthIssuerUrl,
+      });
+      // Re-fetch after refresh
+      const refreshed = await getMcpServerWithTokens(c.env.DB, spaceId, serverId);
+      if (refreshed) {
+        accessToken = await decryptAccessToken(c.env.DB, c.env, {
+          id: refreshed.id,
+          oauthAccessToken: refreshed.oauthAccessToken,
+        });
+      }
+    }
+  } else {
+    // Managed server — no auth token needed
+    accessToken = null;
+  }
+
+  const client = new McpClient(server.url, accessToken, server.name);
+  try {
+    await client.connect();
+    const tools = await client.listTools();
+    return c.json({
+      data: {
+        tools: tools.map((t) => ({
+          name: t.sdkTool.name,
+          description: t.sdkTool.description ?? '',
+          inputSchema: t.sdkTool.inputSchema,
+        })),
+      },
+    });
+  } catch (err) {
+    const detail = err instanceof Error ? err.message : String(err);
+    logError('MCP tool listing failed', err, { module: 'mcp', serverId });
+    if (detail.includes('timeout') || detail.includes('Timeout')) {
+      throw new GatewayTimeoutError('MCP server connection timed out');
+    }
+    throw new BadGatewayError('Failed to connect to MCP server');
+  } finally {
+    // Best-effort cleanup; errors closing the client are non-fatal
+    await client.close().catch((e) => {
+      logWarn('MCP client close failed (non-critical)', { module: 'mcp', error: e instanceof Error ? e.message : String(e) });
+    });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// HTML templates
+// ---------------------------------------------------------------------------
+
+function successPage(serverName: string): string {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><title>MCP Server Connected</title>
+<style>
+  body { font-family: system-ui, sans-serif; display: flex; align-items: center; justify-content: center;
+         min-height: 100vh; margin: 0; background: #f0fdf4; }
+  .card { background: white; border-radius: 12px; padding: 2rem 3rem; box-shadow: 0 4px 24px rgba(0,0,0,.08);
+          text-align: center; max-width: 400px; }
+  h1 { color: #16a34a; margin-bottom: .5rem; }
+  p { color: #4b5563; }
+</style>
+</head>
+<body>
+<div class="card">
+  <h1>✓ Connected</h1>
+  <p>MCP server <strong>${escapeHtml(serverName)}</strong> has been successfully authorized.</p>
+  <p>You can close this window and continue in the agent.</p>
+</div>
+</body></html>`;
+}
+
+function errorPage(message: string): string {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><title>MCP Authorization Error</title>
+<style>
+  body { font-family: system-ui, sans-serif; display: flex; align-items: center; justify-content: center;
+         min-height: 100vh; margin: 0; background: #fef2f2; }
+  .card { background: white; border-radius: 12px; padding: 2rem 3rem; box-shadow: 0 4px 24px rgba(0,0,0,.08);
+          text-align: center; max-width: 400px; }
+  h1 { color: #dc2626; margin-bottom: .5rem; }
+  p { color: #4b5563; }
+</style>
+</head>
+<body>
+<div class="card">
+  <h1>Authorization Failed</h1>
+  <p>${escapeHtml(message)}</p>
+  <p>Please close this window and try again.</p>
+</div>
+</body></html>`;
+}
+
+export default mcpRoutes;