robodev 0.29.0 → 0.31.0

package/src/dev-project.ts

@@ -0,0 +1,147 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { dirname, join, relative } from "node:path";
+import { isBackendTestPath } from "@robodev-ai/runtime";
+
+export type DevFile = { path: string; content: string };
+
+export type DevProject = {
+  /** Where `.robodev/dev/` and the root `package.json` live. */
+  root: string;
+  /** Absolute directory holding `database.ts` plus `api/`, `jobs/`, `hooks/`, `sockets/`. */
+  backendDir: string;
+  /** Backend layout that was matched, for the startup banner. */
+  layout: "root" | "robodev" | "packages/be";
+};
+
+/** Directories under the backend root that hold compilable project code. */
+export const BACKEND_DIRS = ["api", "jobs", "hooks", "sockets"] as const;
+
+const SKIP_DIRS = new Set(["node_modules", "dist", ".git", ".robodev"]);
+const MAX_BACKEND_FILES = 400;
+
+async function isFile(path: string): Promise<boolean> {
+  try {
+    return (await stat(path)).isFile();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Candidate backend roots for one directory, in the order `robodev deploy` prefers:
+ * `robodev/` first, then the project root, then Tiny's `packages/be`.
+ */
+export function backendCandidates(dir: string): { dir: string; layout: DevProject["layout"] }[] {
+  return [
+    { dir: join(dir, "robodev"), layout: "robodev" },
+    { dir, layout: "root" },
+    { dir: join(dir, "packages", "be"), layout: "packages/be" },
+  ];
+}
+
+/**
+ * Walks up from `cwd` looking for a created project: `database.ts` in `robodev/`, at the
+ * root, or in `packages/be` (Tiny). Walking up means `robodev dev` also works from
+ * `apps/fe`.
+ */
+export async function findDevProject(cwd: string): Promise<DevProject> {
+  let dir = cwd;
+  for (;;) {
+    for (const candidate of backendCandidates(dir)) {
+      if (await isFile(join(candidate.dir, "database.ts"))) {
+        return { root: dir, backendDir: candidate.dir, layout: candidate.layout };
+      }
+    }
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    "No database.ts found. Run robodev dev from a created project (database.ts, robodev/database.ts, or packages/be/database.ts).",
+  );
+}
+
+/**
+ * True for a path the runtime compiles: `database.ts` or a `.ts` file under a backend dir.
+ * Project tests are excluded, exactly as `robodev deploy` excludes them.
+ */
+export function isBackendDevPath(path: string): boolean {
+  const normalized = path.replaceAll("\\", "/");
+  if (normalized.includes("..")) return false;
+  if (isBackendTestPath(normalized)) return false;
+  if (normalized === "database.ts") return true;
+  if (!normalized.endsWith(".ts")) return false;
+  const top = normalized.split("/")[0] ?? "";
+  return (BACKEND_DIRS as readonly string[]).includes(top);
+}
+
+/**
+ * Reads the backend tree off disk as deploy files. Paths are the same ones hosted deploy
+ * uses (`database.ts`, `api/planets.ts`), so route mapping is identical.
+ */
+export async function collectBackendFiles(backendDir: string): Promise<DevFile[]> {
+  const files: DevFile[] = [];
+
+  async function walk(dir: string): Promise<void> {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const abs = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (SKIP_DIRS.has(entry.name)) continue;
+        await walk(abs);
+        continue;
+      }
+      if (!entry.isFile()) continue;
+      const path = relative(backendDir, abs).replaceAll("\\", "/");
+      if (!isBackendDevPath(path)) continue;
+      files.push({ path, content: await readFile(abs, "utf8") });
+      if (files.length > MAX_BACKEND_FILES) {
+        throw new Error(`Backend may include at most ${MAX_BACKEND_FILES} files`);
+      }
+    }
+  }
+
+  if (await isFile(join(backendDir, "database.ts"))) {
+    files.push({
+      path: "database.ts",
+      content: await readFile(join(backendDir, "database.ts"), "utf8"),
+    });
+  }
+  for (const name of BACKEND_DIRS) {
+    await walk(join(backendDir, name));
+  }
+
+  if (!files.some((file) => file.path === "database.ts")) {
+    throw new Error(`database.ts not found in ${backendDir}`);
+  }
+  return files;
+}
+
+/**
+ * Where the local Postgres cluster and build output live. Legacy-layout projects keep
+ * `.robodev` as a link *file* at the root, so fall back beside it rather than failing.
+ */
+export async function devDirFor(root: string): Promise<string> {
+  const dotRobodev = join(root, ".robodev");
+  try {
+    if (!(await stat(dotRobodev)).isDirectory()) {
+      return join(root, ".robodev-dev");
+    }
+  } catch {
+    /* absent: safe to create as a directory */
+  }
+  return join(dotRobodev, "dev");
+}
+
+export async function hasAppsFe(root: string): Promise<boolean> {
+  try {
+    return (await stat(join(root, "apps/fe/package.json"))).isFile();
+  } catch {
+    return false;
+  }
+}

package/src/dev-server.ts

@@ -0,0 +1,201 @@
+import { Readable } from "node:stream";
+import cors from "@fastify/cors";
+import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
+import {
+  buildOpenApi,
+  handleLocalAuth,
+  isLocalAuthPath,
+  matchProjectRoute,
+  resolveLocalUser,
+  runRoute,
+  scalarHtml,
+  swaggerHtml,
+  type AuthQueryable,
+  type RuntimeRoute,
+} from "@robodev-ai/runtime";
+import type { RobodevDb } from "@robodev-ai/sdk";
+import { devRouteClients } from "./dev-clients.js";
+
+/** One compiled generation of the project. Swapped wholesale on reload. */
+export type DevRuntime = {
+  db: RobodevDb;
+  pool: AuthQueryable & { end: () => Promise<void> };
+  routes: RuntimeRoute[];
+  openapi: Record<string, unknown>;
+};
+
+export type DevServerOptions = {
+  port: number;
+  projectId: string;
+  jwtSecret: string;
+  env: Record<string, string>;
+  log: (line: string) => void;
+};
+
+const SWAGGER_PATHS = new Set(["/_robodev/docs", "/_robodev/swagger", "/docs", "/swagger"]);
+const SCALAR_PATHS = new Set(["/_robodev/scalar", "/scalar"]);
+
+export function buildDevOpenApi(projectName: string, routes: RuntimeRoute[]) {
+  return buildOpenApi({ title: `${projectName} APIs`, projectId: "local", routes });
+}
+
+function pathnameOf(url: string): string {
+  const raw = decodeURIComponent(url.split("?")[0] ?? "/").replace(/\\/g, "/");
+  return raw.replace(/\/$/, "") || "/";
+}
+
+function rawBodyOf(request: FastifyRequest): Buffer | undefined {
+  return (request as FastifyRequest & { rawBody?: Buffer }).rawBody;
+}
+
+/**
+ * A single-project HTTP host. Unlike the hosted project host there is no `Host` lookup,
+ * no static SPA serving (local Vite does that), and no request logging — the routes,
+ * validation, and OpenAPI all come from `@robodev-ai/runtime`.
+ */
+export async function startDevServer(
+  options: DevServerOptions,
+  getRuntime: () => DevRuntime | null,
+): Promise<{ app: FastifyInstance; close: () => Promise<void> }> {
+  const app = Fastify({ logger: false, bodyLimit: 25 * 1024 * 1024 });
+
+  await app.register(cors, {
+    origin: true,
+    credentials: true,
+    methods: ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
+  });
+
+  app.addContentTypeParser("application/json", { parseAs: "string" }, (_request, body, done) => {
+    if (!body) {
+      done(null, {});
+      return;
+    }
+    try {
+      done(null, JSON.parse(body as string));
+    } catch (error) {
+      done(error as Error, undefined);
+    }
+  });
+  app.addContentTypeParser(
+    "application/x-www-form-urlencoded",
+    { parseAs: "string" },
+    (_request, body, done) => {
+      done(null, Object.fromEntries(new URLSearchParams(String(body ?? "")).entries()));
+    },
+  );
+  app.addContentTypeParser(
+    "application/octet-stream",
+    { parseAs: "buffer" },
+    (_request, body, done) => done(null, body),
+  );
+  app.addContentTypeParser("multipart/form-data", { parseAs: "buffer" }, (_request, body, done) =>
+    done(null, body),
+  );
+
+  // `rawBody` handlers and multipart parsing need the untouched bytes.
+  app.addHook("preParsing", async (_request, _reply, payload) => {
+    const chunks: Buffer[] = [];
+    for await (const chunk of payload) {
+      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    }
+    const raw = Buffer.concat(chunks);
+    (_request as FastifyRequest & { rawBody?: Buffer }).rawBody = raw;
+    return Readable.from(raw);
+  });
+
+  // The catch-all lives in the not-found handler: `@fastify/cors` already owns `OPTIONS /*`,
+  // and every project route is resolved by the runtime rather than registered with Fastify.
+  app.setNotFoundHandler(async (request, reply) => {
+    const pathname = pathnameOf(request.url);
+    const method = request.method.toLowerCase();
+
+    if (pathname === "/_robodev/ready") {
+      const runtime = getRuntime();
+      if (!runtime) return reply.code(503).send({ error: "not_ready" });
+      try {
+        await runtime.pool.query("SELECT 1");
+        return reply.send({ ok: true });
+      } catch {
+        return reply.code(503).send({ error: "not_ready" });
+      }
+    }
+
+    if (SWAGGER_PATHS.has(pathname)) {
+      return reply.type("text/html; charset=utf-8").send(swaggerHtml("/openapi.json"));
+    }
+    if (SCALAR_PATHS.has(pathname)) {
+      return reply.type("text/html; charset=utf-8").send(scalarHtml("/openapi.json"));
+    }
+
+    const runtime = getRuntime();
+    if (!runtime) {
+      return reply.code(503).send({
+        error: "not_ready",
+        message: "The project is still compiling. Retry in a moment.",
+      });
+    }
+
+    if (pathname === "/openapi.json") {
+      return reply.send(runtime.openapi);
+    }
+
+    if (isLocalAuthPath(pathname)) {
+      const result = await handleLocalAuth(
+        {
+          method: request.method,
+          pathname,
+          body: request.body,
+          query: request.query,
+          authorization: request.headers.authorization,
+        },
+        {
+          db: runtime.pool,
+          projectId: options.projectId,
+          jwtSecret: options.jwtSecret,
+          deliverCode: ({ to, type, code }) => options.log(`${type} code for ${to}: ${code}`),
+        },
+      );
+      return reply.code(result.status).send(result.body);
+    }
+
+    if (!matchProjectRoute(pathname, method, runtime.routes)) {
+      return reply.code(404).send({
+        error: "not_found",
+        message: `${request.method} ${pathname} is not defined`,
+        routes: runtime.routes.map((route) => `${route.method.toUpperCase()} ${route.path}`),
+      });
+    }
+
+    const outcome = await runRoute({
+      db: runtime.db,
+      routes: runtime.routes,
+      method,
+      pathname,
+      query: request.query ?? {},
+      body: request.body,
+      rawBody: rawBodyOf(request),
+      headers: request.headers as Record<string, string | string[] | undefined>,
+      authorization: request.headers.authorization,
+      resolveUser: (authorization) =>
+        resolveLocalUser(
+          { projectId: options.projectId, jwtSecret: options.jwtSecret },
+          authorization,
+        ),
+      clients: () => devRouteClients({ env: options.env, log: options.log }),
+    });
+
+    if (outcome.kind === "error") {
+      return reply.code(outcome.status).send(outcome.body);
+    }
+    for (const [name, value] of Object.entries(outcome.encoded.headers)) {
+      reply.header(name, value);
+    }
+    return reply
+      .code(outcome.encoded.status)
+      .type(outcome.encoded.contentType)
+      .send(outcome.encoded.payload);
+  });
+
+  await app.listen({ port: options.port, host: "0.0.0.0" });
+  return { app, close: () => app.close() };
+}

package/src/dev.ts

@@ -0,0 +1,348 @@
+import { watch, type FSWatcher } from "node:fs";
+import { mkdir, readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { join, relative } from "node:path";
+import { spawn, type ChildProcess } from "node:child_process";
+import {
+  apiBundleExternals,
+  applyPlan,
+  buildPlan,
+  compileApiTree,
+  ensureAuthSchema,
+  isEsbuildFailure,
+  loadModules,
+  mapEsbuildErrors,
+  writeDeploySources,
+  type SchemaPlan,
+} from "@robodev-ai/runtime";
+import { createDb as fallbackCreateDb, type DatabaseDef } from "@robodev-ai/sdk";
+import { commandAvailable } from "./frontend-build.js";
+import {
+  collectBackendFiles,
+  devDirFor,
+  findDevProject,
+  hasAppsFe,
+  isBackendDevPath,
+  type DevProject,
+} from "./dev-project.js";
+import { externalDevPostgres, startDevPostgres, type DevPostgres } from "./dev-postgres.js";
+import { buildDevOpenApi, startDevServer, type DevRuntime } from "./dev-server.js";
+
+export type DevArgs = {
+  port: number;
+  dbPort: number;
+  force: boolean;
+  watch: boolean;
+  withFe: boolean;
+  databaseUrl?: string;
+};
+
+const DEFAULT_PORT = 4000;
+const RELOAD_DEBOUNCE_MS = 150;
+/** Local-only signing key. Tokens never leave the machine. */
+const DEV_JWT_SECRET = "robodev-dev-local-secret";
+const DEV_PROJECT_ID = "prj-local";
+
+function numberFlag(args: string[], name: string, fallback: number): number {
+  const index = args.indexOf(name);
+  if (index === -1) return fallback;
+  const raw = args[index + 1];
+  const value = Number(raw);
+  if (raw === undefined || !Number.isInteger(value) || value < 0) {
+    throw new Error(`${name} needs a non-negative integer`);
+  }
+  return value;
+}
+
+function stringFlag(args: string[], name: string): string | undefined {
+  const index = args.indexOf(name);
+  if (index === -1) return undefined;
+  const value = args[index + 1];
+  if (!value || value.startsWith("--")) {
+    throw new Error(`${name} needs a value`);
+  }
+  return value;
+}
+
+export function parseDevArgs(args: string[]): DevArgs {
+  return {
+    port: numberFlag(args, "--port", DEFAULT_PORT),
+    dbPort: numberFlag(args, "--db-port", 0),
+    force: args.includes("--force"),
+    watch: !args.includes("--no-watch"),
+    withFe: args.includes("--with-fe"),
+    databaseUrl: stringFlag(args, "--database-url"),
+  };
+}
+
+/** Minimal `.env` reader for `ctx.env`. Hosted projects get this from project secrets. */
+export function parseEnvFile(content: string): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eq = trimmed.indexOf("=");
+    if (eq <= 0) continue;
+    const key = trimmed.slice(0, eq).trim();
+    let value = trimmed.slice(eq + 1).trim();
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+    env[key] = value;
+  }
+  return env;
+}
+
+async function loadProjectEnv(root: string): Promise<Record<string, string>> {
+  try {
+    return parseEnvFile(await readFile(join(root, ".env"), "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Uses the project's own `@robodev-ai/sdk` when it has one, so `ctx.db` and the Drizzle
+ * tables from `database.ts` come from the same drizzle-orm copy — the way hosted deploy
+ * compiles them.
+ */
+async function resolveCreateDb(backendDir: string, root: string): Promise<typeof fallbackCreateDb> {
+  for (const base of [backendDir, root]) {
+    try {
+      const require = createRequire(join(base, "package.json"));
+      const entry = require.resolve("@robodev-ai/sdk");
+      const mod = (await import(entry)) as { createDb?: typeof fallbackCreateDb };
+      if (mod.createDb) return mod.createDb;
+    } catch {
+      /* fall through to the CLI's own copy */
+    }
+  }
+  return fallbackCreateDb;
+}
+
+function printPlan(plan: SchemaPlan): void {
+  for (const op of plan.operations) {
+    const mark = op.destructive ? "!" : " ";
+    console.log(` ${mark} ${op.type} ${op.table}${op.column ? `.${op.column}` : ""}`);
+  }
+}
+
+export type DevCompileResult = {
+  database: DatabaseDef;
+  runtime: DevRuntime;
+};
+
+export async function main(args: string[]): Promise<void> {
+  const parsed = parseDevArgs(args);
+  const project = await findDevProject(process.cwd());
+  const devDir = await devDirFor(project.root);
+  const buildDir = join(devDir, "build");
+  await mkdir(buildDir, { recursive: true });
+
+  const createDb = await resolveCreateDb(project.backendDir, project.root);
+  const env = await loadProjectEnv(project.root);
+
+  const postgres: DevPostgres = parsed.databaseUrl
+    ? externalDevPostgres(parsed.databaseUrl)
+    : await startDevPostgres({ dataDir: join(devDir, "pg"), port: parsed.dbPort, quiet: true });
+
+  let current: DevRuntime | null = null;
+  let lastSchemaSource = "";
+  let closing = false;
+
+  const log = (line: string) => console.log(line);
+
+  async function compileAndActivate(input: { syncSchema: boolean }): Promise<void> {
+    const files = await collectBackendFiles(project.backendDir);
+    await writeDeploySources(join(buildDir, "src"), files);
+
+    let distDir: string;
+    try {
+      distDir = await compileApiTree({
+        dir: buildDir,
+        files,
+        external: apiBundleExternals(),
+        nodePaths: [join(project.backendDir, "node_modules"), join(project.root, "node_modules")],
+      });
+    } catch (error) {
+      if (isEsbuildFailure(error)) {
+        for (const issue of mapEsbuildErrors(error.errors, join(buildDir, "src"))) {
+          const where = issue.path ? `${issue.path}${issue.line ? `:${issue.line}` : ""}: ` : "";
+          console.error(`  ${where}${issue.text}`);
+        }
+        throw new Error("Compile failed");
+      }
+      throw error;
+    }
+
+    const loaded = await loadModules(distDir, files);
+    // Any edit to database.ts can change columns, not just tables, so compare its source.
+    const schemaSource = files.find((file) => file.path === "database.ts")?.content ?? "";
+
+    if (input.syncSchema || schemaSource !== lastSchemaSource) {
+      const plan = await buildPlan(postgres.connectionString, loaded.database);
+      if (plan.destructive && !parsed.force) {
+        console.log("Destructive schema changes (skipped, rerun with --force):");
+        printPlan(plan);
+      } else {
+        if (plan.operations.length > 0) {
+          console.log("Schema changes:");
+          printPlan(plan);
+        }
+        await applyPlan(postgres.connectionString, plan);
+        lastSchemaSource = schemaSource;
+      }
+    }
+
+    const { db, pool } = createDb(postgres.connectionString, loaded.database.tables);
+    await pool.query("SELECT 1");
+    await ensureAuthSchema(pool);
+
+    const previous = current;
+    current = {
+      db,
+      pool,
+      routes: loaded.routes,
+      openapi: buildDevOpenApi(loaded.database.name, loaded.routes),
+    };
+    if (previous) {
+      await previous.pool.end().catch(() => undefined);
+    }
+
+    for (const route of loaded.routes) {
+      console.log(`  ${route.method.toUpperCase()} ${route.path}`);
+    }
+    if (loaded.jobs.length > 0) {
+      console.log(
+        `  ${loaded.jobs.length} job(s) compiled but not run — jobs need hosted Starbase.`,
+      );
+    }
+    if (loaded.sockets.length > 0) {
+      console.log(
+        `  ${loaded.sockets.length} socket(s) compiled but not served — sockets need hosted Starbase.`,
+      );
+    }
+  }
+
+  async function releaseDatabase(): Promise<void> {
+    await current?.pool.end().catch(() => undefined);
+    await postgres.stop().catch(() => undefined);
+  }
+
+  let server: Awaited<ReturnType<typeof startDevServer>>;
+  try {
+    await compileAndActivate({ syncSchema: true });
+    server = await startDevServer(
+      { port: parsed.port, projectId: DEV_PROJECT_ID, jwtSecret: DEV_JWT_SECRET, env, log },
+      () => current,
+    );
+  } catch (error) {
+    // Nothing is watching yet, so shut the cluster down instead of orphaning it.
+    await releaseDatabase();
+    throw error;
+  }
+
+  const apiUrl = `http://localhost:${parsed.port}`;
+  console.log("");
+  if (postgres.dataDir) {
+    console.log(
+      `Postgres  localhost:${postgres.port}  (data: ${relative(project.root, postgres.dataDir) || postgres.dataDir})`,
+    );
+  } else {
+    console.log(
+      `Postgres  ${postgres.host}:${postgres.port}/${postgres.database}  (--database-url)`,
+    );
+  }
+  console.log(`API       ${apiUrl}`);
+  console.log(`Swagger   ${apiUrl}/_robodev/docs`);
+  console.log(
+    `Project   ${relative(process.cwd(), project.backendDir) || "."} (${project.layout})`,
+  );
+  console.log("");
+
+  const fe = parsed.withFe ? await startFrontend(project, apiUrl) : null;
+  const watchers = parsed.watch ? startWatch() : [];
+
+  function startWatch(): FSWatcher[] {
+    let timer: NodeJS.Timeout | undefined;
+    let running = false;
+    let pending = false;
+
+    async function reload(): Promise<void> {
+      if (running) {
+        pending = true;
+        return;
+      }
+      running = true;
+      try {
+        console.log("Change detected, reloading…");
+        await compileAndActivate({ syncSchema: false });
+      } catch (error) {
+        console.error(error instanceof Error ? error.message : error);
+        console.error("Keeping the previous build. Fix the error and save again.");
+      } finally {
+        running = false;
+        if (pending) {
+          pending = false;
+          void reload();
+        }
+      }
+    }
+
+    try {
+      return [
+        watch(project.backendDir, { recursive: true }, (_event, filename) => {
+          // Ignore editor scratch files and anything the runtime would not compile.
+          if (filename && !isBackendDevPath(String(filename))) return;
+          if (timer) clearTimeout(timer);
+          timer = setTimeout(() => void reload(), RELOAD_DEBOUNCE_MS);
+        }),
+      ];
+    } catch (error) {
+      console.error(
+        `Could not watch ${project.backendDir} (${error instanceof Error ? error.message : error}). Restart after edits, or pass --no-watch.`,
+      );
+      return [];
+    }
+  }
+
+  async function shutdown(code: number): Promise<void> {
+    if (closing) return;
+    closing = true;
+    for (const watcher of watchers) watcher.close();
+    fe?.kill("SIGTERM");
+    await server.close().catch(() => undefined);
+    await releaseDatabase();
+    process.exit(code);
+  }
+
+  process.on("SIGINT", () => void shutdown(0));
+  process.on("SIGTERM", () => void shutdown(0));
+}
+
+/** Runs the starter's `apps/fe` dev server with the local API pre-wired. */
+async function startFrontend(project: DevProject, apiUrl: string): Promise<ChildProcess | null> {
+  if (!(await hasAppsFe(project.root))) {
+    console.log("--with-fe: no apps/fe/package.json, serving APIs only.");
+    return null;
+  }
+  const command = (await commandAvailable("bun")) ? "bun" : "npm";
+  const args = command === "bun" ? ["run", "dev"] : ["run", "dev"];
+  const child = spawn(command, args, {
+    cwd: join(project.root, "apps", "fe"),
+    stdio: "inherit",
+    env: {
+      ...process.env,
+      VITE_API_URL: apiUrl,
+      VITE_PUBLIC_API_URL: apiUrl,
+      APP_PUBLIC_API_URL: apiUrl,
+    },
+  });
+  child.on("error", (error) => {
+    console.error(`Could not start apps/fe with ${command}: ${error.message}`);
+  });
+  return child;
+}