robodev 0.34.0 → 0.36.0

package/src/dev-server.ts

@@ -1,22 +1,43 @@
 import { Readable } from "node:stream";
 import cors from "@fastify/cors";
-import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
+import websocket from "@fastify/websocket";
+import Fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify";
+import type { WebSocket } from "ws";
 import {
+  SOCKET_CLOSE_POLICY,
+  SOCKET_IDLE_PING_MS,
+  bindSocketSession,
   buildOpenApi,
   handleLocalAuth,
   isLocalAuthPath,
+  isReservedStoragePath,
   matchProjectRoute,
+  matchSocketRoute,
+  parseLogicalKey,
+  resolveLocalObject,
   resolveLocalUser,
   runRoute,
   scalarHtml,
   swaggerHtml,
+  verifyStorageServeToken,
   type AuthQueryable,
   type RuntimeRoute,
+  type RuntimeSocket,
 } from "@robodev-ai/runtime";
-import type { JobEnqueueInput, RobodevDb } from "@robodev-ai/sdk";
+import type {
+  JobEnqueueInput,
+  RobodevDb,
+  SocketDefinition,
+  SocketHandlerContext,
+} from "@robodev-ai/sdk";
 import type { RuntimeJob } from "@robodev-ai/runtime";
 import { devRouteClients } from "./dev-clients.js";
-import { isInspectPath, type DevInspectStore } from "./dev-inspect.js";
+import {
+  buildDashboardState,
+  isInspectPath,
+  shouldCountRequestHistogram,
+  type DevInspectStore,
+} from "./dev-inspect.js";
 
 /** One compiled generation of the project. Swapped wholesale on reload. */
 export type DevRuntime = {
@@ -24,6 +45,7 @@ export type DevRuntime = {
   pool: AuthQueryable & { end: () => Promise<void> };
   routes: RuntimeRoute[];
   jobs: RuntimeJob[];
+  sockets: RuntimeSocket[];
   openapi: Record<string, unknown>;
 };
 
@@ -39,6 +61,7 @@ export type DevServerOptions = {
   brokerBaseUrl: string;
   googleClientId?: string;
   googleClientSecret?: string;
+  storageDir: string;
 };
 
 const SWAGGER_PATHS = new Set(["/_robodev/docs", "/_robodev/swagger", "/docs", "/swagger"]);
@@ -57,10 +80,242 @@ function rawBodyOf(request: FastifyRequest): Buffer | undefined {
   return (request as FastifyRequest & { rawBody?: Buffer }).rawBody;
 }
 
+function queryRecord(query: unknown): Record<string, string> {
+  if (!query || typeof query !== "object") return {};
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(query as Record<string, unknown>)) {
+    if (typeof value === "string") out[key] = value;
+    else if (Array.isArray(value) && typeof value[0] === "string") out[key] = value[0];
+  }
+  return out;
+}
+
+function socketAuthorization(request: FastifyRequest): string | undefined {
+  const header = request.headers.authorization;
+  if (typeof header === "string" && header.length > 0) return header;
+  const token = queryRecord(request.query).accessToken;
+  if (token) return `Bearer ${token}`;
+  return undefined;
+}
+
+function codedErrorBody(error: unknown): { status: number; body: Record<string, unknown> } | null {
+  if (!error || typeof error !== "object" || !("statusCode" in error)) return null;
+  const statusCode = (error as { statusCode: unknown }).statusCode;
+  if (typeof statusCode !== "number") return null;
+  const code = (error as { code?: unknown }).code;
+  const message = error instanceof Error ? error.message : String(error);
+  return {
+    status: statusCode,
+    body: {
+      error: typeof code === "string" ? code : message,
+      message,
+    },
+  };
+}
+
+function routeClients(options: DevServerOptions, getRuntime: () => DevRuntime | null) {
+  return devRouteClients({
+    env: options.env,
+    log: options.log,
+    enqueue: options.enqueue,
+    inspect: options.inspect,
+    query: (sql, values) => {
+      const runtime = getRuntime();
+      if (!runtime) throw new Error("dev runtime not ready");
+      return runtime.pool.query(sql, values);
+    },
+    objectsDir: options.storageDir,
+    publicBaseUrl: options.listenOrigin,
+    projectId: options.projectId,
+    jwtSecret: options.jwtSecret,
+  });
+}
+
+async function resolveSocketUser(
+  options: DevServerOptions,
+  def: SocketDefinition,
+  request: FastifyRequest,
+): Promise<{ ok: true; user: SocketHandlerContext["user"] } | { ok: false }> {
+  const auth = def.auth ?? false;
+  if (auth === false) return { ok: true, user: null };
+  if (typeof auth === "function") {
+    try {
+      const user = await auth({ headers: request.headers });
+      if (!user) return { ok: false };
+      return { ok: true, user };
+    } catch {
+      return { ok: false };
+    }
+  }
+  if (auth !== "required" && auth !== "optional") return { ok: false };
+  const user = await resolveLocalUser(
+    { projectId: options.projectId, jwtSecret: options.jwtSecret },
+    socketAuthorization(request),
+  );
+  if (auth === "required" && !user) return { ok: false };
+  return { ok: true, user };
+}
+
+async function handleLocalSocket(
+  socket: WebSocket,
+  request: FastifyRequest,
+  options: DevServerOptions,
+  getRuntime: () => DevRuntime | null,
+): Promise<void> {
+  const runtime = getRuntime();
+  if (!runtime) {
+    socket.close(SOCKET_CLOSE_POLICY, "not_found");
+    return;
+  }
+  const pathname = pathnameOf(request.url);
+  const matched = matchSocketRoute(pathname, runtime.sockets);
+  if (!matched) {
+    socket.close(SOCKET_CLOSE_POLICY, "not_found");
+    return;
+  }
+
+  const auth = await resolveSocketUser(options, matched.socket.def, request);
+  if (!auth.ok) {
+    socket.close(SOCKET_CLOSE_POLICY, "unauthorized");
+    return;
+  }
+
+  const clients = routeClients(options, getRuntime);
+  const session = await bindSocketSession({
+    projectId: options.projectId,
+    name: matched.socket.name,
+    socket: {
+      send: (data) => socket.send(data),
+      close: (code, reason) => socket.close(code, reason),
+      get readyState() {
+        return socket.readyState;
+      },
+    },
+    timeoutMs: matched.socket.def.timeoutMs,
+    createContext: ({ send, close, room }) => ({
+      db: runtime.db,
+      email: clients.email,
+      llm: clients.llm,
+      agent: clients.agent,
+      storage: clients.storage,
+      env: clients.env,
+      params: matched.params,
+      query: queryRecord(request.query),
+      headers: request.headers,
+      user: auth.user,
+      push: clients.push,
+      jobs: clients.jobs,
+      sockets: clients.sockets,
+      send,
+      close,
+      room,
+    }),
+    onConnect: matched.socket.def.onConnect,
+    onMessage: matched.socket.def.onMessage,
+    onClose: matched.socket.def.onClose,
+  });
+  if (!session.ok) {
+    if (socket.readyState === socket.OPEN) {
+      socket.close(session.code, session.reason);
+    }
+    return;
+  }
+
+  let alive = true;
+  const pingTimer = setInterval(() => {
+    if (socket.readyState !== socket.OPEN) {
+      clearInterval(pingTimer);
+      return;
+    }
+    if (!alive) {
+      socket.close(SOCKET_CLOSE_POLICY, "idle");
+      return;
+    }
+    alive = false;
+    try {
+      socket.ping();
+    } catch {
+      socket.close(SOCKET_CLOSE_POLICY, "idle");
+    }
+  }, SOCKET_IDLE_PING_MS);
+  pingTimer.unref?.();
+
+  socket.on("pong", () => {
+    alive = true;
+  });
+  socket.on("message", (data, isBinary) => {
+    if (isBinary) {
+      session.onBinary();
+      return;
+    }
+    const text = typeof data === "string" ? data : Buffer.from(data as Uint8Array).toString("utf8");
+    session.onText(text);
+  });
+  socket.on("close", () => {
+    clearInterval(pingTimer);
+    session.onClose();
+  });
+}
+
+async function serveLocalStorage(
+  request: FastifyRequest,
+  reply: FastifyReply,
+  options: DevServerOptions,
+  getRuntime: () => DevRuntime | null,
+  pathname: string,
+): Promise<void> {
+  const runtime = getRuntime();
+  if (!runtime) {
+    await reply.code(503).send({ error: "not_ready" });
+    return;
+  }
+  const encodedKey = pathname.slice("/storage/objects/".length);
+  let key: string;
+  try {
+    key = decodeURIComponent(encodedKey);
+    parseLogicalKey(key);
+  } catch (error) {
+    const mapped = codedErrorBody(error);
+    if (mapped) {
+      await reply.code(mapped.status).send(mapped.body);
+      return;
+    }
+    await reply.code(400).send({ error: "invalid_key" });
+    return;
+  }
+
+  const resolved = await resolveLocalObject({
+    query: (sql, values) => runtime.pool.query(sql, values),
+    objectsDir: options.storageDir,
+    key,
+  });
+  if (!resolved) {
+    await reply.code(404).send({ error: "not_found" });
+    return;
+  }
+
+  const user = await resolveLocalUser(
+    { projectId: options.projectId, jwtSecret: options.jwtSecret },
+    request.headers.authorization,
+  );
+  const signed = verifyStorageServeToken(
+    options.jwtSecret,
+    options.projectId,
+    key,
+    request.query as { exp?: unknown; sig?: unknown },
+  );
+  if (!resolved.row.public && !user && !signed) {
+    await reply.code(404).send({ error: "not_found" });
+    return;
+  }
+
+  await reply.type(resolved.row.content_type).send(resolved.body);
+}
+
 /**
  * A single-project HTTP host. Unlike the hosted project host there is no `Host` lookup
  * and no static SPA serving (local Vite does that). Fastify stays `logger: false`;
- * project/auth/docs requests are recorded on the process-local inspect page after reply.
+ * project/auth/docs requests are recorded on the process-local dashboard after reply.
  * Routes, validation, and OpenAPI all come from `@robodev-ai/runtime`.
  */
 export async function startDevServer(
@@ -74,6 +329,10 @@ export async function startDevServer(
     credentials: true,
     methods: ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
   });
+  await app.register(websocket);
+  app.get("/sockets/*", { websocket: true }, (socket, request) => {
+    void handleLocalSocket(socket, request, options, getRuntime);
+  });
 
   app.addContentTypeParser("application/json", { parseAs: "string" }, (_request, body, done) => {
     if (!body) {
@@ -103,13 +362,16 @@ export async function startDevServer(
   );
 
   // `rawBody` handlers and multipart parsing need the untouched bytes.
-  app.addHook("preParsing", async (_request, _reply, payload) => {
+  app.addHook("preParsing", async (request, _reply, payload) => {
+    if (String(request.headers.upgrade ?? "").toLowerCase() === "websocket") {
+      return payload;
+    }
     const chunks: Buffer[] = [];
     for await (const chunk of payload) {
       chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
     }
     const raw = Buffer.concat(chunks);
-    (_request as FastifyRequest & { rawBody?: Buffer }).rawBody = raw;
+    (request as FastifyRequest & { rawBody?: Buffer }).rawBody = raw;
     return Readable.from(raw);
   });
 
@@ -120,8 +382,18 @@ export async function startDevServer(
     const method = request.method.toLowerCase();
 
     if (isInspectPath(pathname)) {
-      if (pathname === "/_robodev/logs.json") {
-        return reply.send(options.inspect.list());
+      if (pathname === "/_robodev/logs") {
+        return reply.redirect("/_robodev");
+      }
+      if (pathname === "/_robodev/state.json" || pathname === "/_robodev/logs.json") {
+        return reply.send(
+          await buildDashboardState({
+            inspect: options.inspect,
+            runtime: getRuntime(),
+            googleClientId: options.googleClientId,
+            googleClientSecret: options.googleClientSecret,
+          }),
+        );
       }
       return reply.type("text/html; charset=utf-8").send(options.inspect.html());
     }
@@ -132,15 +404,20 @@ export async function startDevServer(
     } finally {
       const status = reply.statusCode;
       const durationMs = Date.now() - started;
-      options.inspect.record({
-        kind: "request",
-        method: request.method,
-        path: pathname,
-        status,
-        durationMs,
-      });
-      // Print here (not via `log`) so request lines stay off the Inspect Logs list.
-      console.log(`${request.method} ${pathname} ${status} ${durationMs}ms`);
+      if (!isInspectPath(pathname)) {
+        options.inspect.record({
+          kind: "request",
+          method: request.method,
+          path: pathname,
+          status,
+          durationMs,
+        });
+        // Print here (not via `log`) so request lines stay off compile/reload logs.
+        console.log(`${request.method} ${pathname} ${status} ${durationMs}ms`);
+        if (shouldCountRequestHistogram(pathname, getRuntime(), request.method)) {
+          options.inspect.incrementRequestHistogram(status);
+        }
+      }
     }
     return reply;
 
@@ -214,6 +491,11 @@ export async function startDevServer(
         return;
       }
 
+      if (request.method.toUpperCase() === "GET" && isReservedStoragePath(pathname)) {
+        await serveLocalStorage(request, reply, options, getRuntime, pathname);
+        return;
+      }
+
       if (!matchProjectRoute(pathname, method, runtime.routes)) {
         await reply.code(404).send({
           error: "not_found",
@@ -238,13 +520,8 @@ export async function startDevServer(
             { projectId: options.projectId, jwtSecret: options.jwtSecret },
             authorization,
           ),
-        clients: () =>
-          devRouteClients({
-            env: options.env,
-            log: options.log,
-            enqueue: options.enqueue,
-            inspect: options.inspect,
-          }),
+        clients: () => routeClients(options, getRuntime),
+        mapError: (error) => codedErrorBody(error),
       });
 
       if (outcome.kind === "error") {

package/src/dev.ts

@@ -9,9 +9,13 @@ import {
   assertJobCrons,
   buildPlan,
   compileApiTree,
+  closeProjectSockets,
   createLocalJobsEngine,
   ensureAuthSchema,
   ensureJobsSchema,
+  ensureStorageSchema,
+  setProjectSocketNames,
+  SOCKET_CLOSE_RESTART,
   isEsbuildFailure,
   loadModules,
   mapEsbuildErrors,
@@ -154,6 +158,8 @@ export async function main(args: string[]): Promise<void> {
   const postgres: DevPostgres = parsed.databaseUrl
     ? externalDevPostgres(parsed.databaseUrl)
     : await startDevPostgres({ dataDir: join(devDir, "pg"), port: parsed.dbPort, quiet: true });
+  const storageDir = join(devDir, "storage");
+  await mkdir(storageDir, { recursive: true });
 
   let current: DevRuntime | null = null;
   let lastSchemaSource = "";
@@ -172,7 +178,30 @@ export async function main(args: string[]): Promise<void> {
     getGeneration: () => current,
     log,
     clients: () =>
-      devRouteClients({ env, log, inspect, enqueue: (input) => engine.enqueue(input) }),
+      devRouteClients({
+        env,
+        log,
+        inspect,
+        enqueue: (input) => engine.enqueue(input),
+        query: (sql, values) => {
+          if (!current) throw new Error("dev runtime not ready");
+          return current.pool.query(sql, values);
+        },
+        objectsDir: storageDir,
+        publicBaseUrl: `http://localhost:${parsed.port}`,
+        projectId: DEV_PROJECT_ID,
+        jwtSecret: DEV_JWT_SECRET,
+      }),
+    onJobFinished: (event) => {
+      inspect.record({
+        kind: "job_run",
+        name: event.name,
+        status: event.outcome,
+        payload: event.payload,
+        error: event.error ?? undefined,
+      });
+      inspect.incrementJobHistogram(event.outcome);
+    },
   });
 
   async function compileAndActivate(input: { syncSchema: boolean }): Promise<void> {
@@ -222,15 +251,22 @@ export async function main(args: string[]): Promise<void> {
     await pool.query("SELECT 1");
     await ensureAuthSchema(pool);
     await ensureJobsSchema(pool);
+    await ensureStorageSchema(pool);
 
+    closeProjectSockets(DEV_PROJECT_ID, SOCKET_CLOSE_RESTART, "deployed");
     const previous = current;
     current = {
       db,
       pool,
       routes: loaded.routes,
       jobs: loaded.jobs,
+      sockets: loaded.sockets,
       openapi: buildDevOpenApi(loaded.database.name, loaded.routes),
     };
+    setProjectSocketNames(
+      DEV_PROJECT_ID,
+      loaded.sockets.map((socket) => socket.name),
+    );
     await engine.reconcile(loaded.jobs);
 
     for (const route of loaded.routes) {
@@ -243,10 +279,8 @@ export async function main(args: string[]): Promise<void> {
         log(`  job ${job.name}`);
       }
     }
-    if (loaded.sockets.length > 0) {
-      log(
-        `  ${loaded.sockets.length} socket(s) compiled but not served — sockets need hosted Starbase.`,
-      );
+    for (const socket of loaded.sockets) {
+      log(`  WS ${socket.path}`);
     }
     if (previous) {
       await previous.pool.end().catch(() => undefined);
@@ -275,6 +309,7 @@ export async function main(args: string[]): Promise<void> {
         brokerBaseUrl: starbaseUrl(),
         googleClientId: env.GOOGLE_CLIENT_ID,
         googleClientSecret: env.GOOGLE_CLIENT_SECRET,
+        storageDir,
       },
       () => current,
     );
@@ -295,9 +330,10 @@ export async function main(args: string[]): Promise<void> {
       `Postgres  ${postgres.host}:${postgres.port}/${postgres.database}  (--database-url)`,
     );
   }
+  console.log(`Storage   ${relative(project.root, storageDir) || storageDir}`);
   console.log(`API       ${apiUrl}`);
   console.log(`Swagger   ${apiUrl}/_robodev/docs`);
-  console.log(`Inspect   ${apiUrl}/_robodev/logs`);
+  console.log(`Dashboard  ${apiUrl}/_robodev`);
   console.log(
     `Project   ${relative(process.cwd(), project.backendDir) || "."} (${project.layout})`,
   );
@@ -352,6 +388,7 @@ export async function main(args: string[]): Promise<void> {
   async function shutdown(code: number): Promise<void> {
     if (closing) return;
     closing = true;
+    closeProjectSockets(DEV_PROJECT_ID);
     await engine.stop();
     for (const watcher of watchers) watcher.close();
     fe?.kill("SIGTERM");

package/templates/auth-chat/package.json

@@ -12,7 +12,7 @@
     "@robodev-ai/sdk": "^0.11.0"
   },
   "devDependencies": {
-    "robodev": "^0.34.0",
+    "robodev": "^0.36.0",
     "rulesync": "^8.18.0",
     "typescript": "^5.9.2"
   }

package/templates/backend/package.json

@@ -11,7 +11,7 @@
     "@robodev-ai/sdk": "^0.11.0"
   },
   "devDependencies": {
-    "robodev": "^0.34.0",
+    "robodev": "^0.36.0",
     "typescript": "^5.9.2"
   }
 }

package/templates/empty/package.json

@@ -11,7 +11,7 @@
     "@robodev-ai/sdk": "^0.11.0"
   },
   "devDependencies": {
-    "robodev": "^0.34.0",
+    "robodev": "^0.36.0",
     "typescript": "^5.9.2"
   }
 }

package/templates/marketplace/package.json

@@ -12,7 +12,7 @@
     "@robodev-ai/sdk": "^0.11.0"
   },
   "devDependencies": {
-    "robodev": "^0.34.0",
+    "robodev": "^0.36.0",
     "rulesync": "^8.18.0",
     "typescript": "^5.9.2"
   }

package/templates/space/package.json

@@ -12,7 +12,7 @@
     "@robodev-ai/sdk": "^0.11.0"
   },
   "devDependencies": {
-    "robodev": "^0.34.0",
+    "robodev": "^0.36.0",
     "rulesync": "^8.18.0",
     "typescript": "^5.9.2"
   }