veryfront 0.1.74 → 0.1.75

package/src/cli/commands/knowledge/command.ts

@@ -1,6 +1,6 @@
 import * as dntShim from "../../../_dnt.shims.js";
 import { z } from "zod";
-import { createFileSystem } from "../../../src/platform/index.js";
+import { createFileSystem, getEnv } from "../../../src/platform/index.js";
 import { basename, extname, join, normalize, relative } from "../../../src/platform/compat/path/index.js";
 import { withSpan } from "../../../src/observability/tracing/otlp-setup.js";
 import { cliLogger } from "../../utils/index.js";
@@ -9,6 +9,7 @@ import type { ParsedArgs } from "../../shared/types.js";
 import { downloadUploadToFile, listAllUploads, type UploadItem } from "../uploads/command.js";
 import { putRemoteFileFromLocal } from "../files/command.js";
 import { knowledgeIngestPythonSource } from "./parser-source.js";
+import { createJobUserLogger, type Logger, serverLogger } from "../../../src/utils/index.js";
 
 const SUPPORTED_EXTENSIONS = new Set([
   ".pdf",
@@ -59,6 +60,8 @@ type KnowledgeSource =
 
 type DownloadResult = { uploadPath: string; localPath: string; bytes?: number };
 
+const knowledgeJobLogger = serverLogger.component("knowledge-ingest");
+
 const KnowledgeIngestArgsSchema = z.object({
   projectSlug: z.string().optional(),
   projectDir: z.string().optional(),
@@ -130,6 +133,27 @@ function printJson(value: unknown): void {
   console.log(JSON.stringify(value, null, 2));
 }
 
+function createKnowledgeIngestEventLogger(): Logger | null {
+  const projectId = getEnv("TENANT_PROJECT_ID");
+  const jobId = getEnv("JOB_ID");
+
+  if (!projectId || !jobId) {
+    return null;
+  }
+
+  return createJobUserLogger(knowledgeJobLogger, {
+    projectId,
+    jobId,
+    batchId: getEnv("JOB_BATCH_ID") ?? undefined,
+    jobTarget: getEnv("JOB_TARGET") ?? undefined,
+    task: "knowledge-ingest",
+  });
+}
+
+function buildKnowledgeSourceName(source: KnowledgeSource): string {
+  return basename(source.kind === "upload" ? source.uploadPath : source.localPath);
+}
+
 function showKnowledgeUsage(): void {
   console.log(`
 Veryfront Knowledge
@@ -484,6 +508,7 @@ export async function ingestResolvedSources(
     outputDir: string;
     runParser: typeof runKnowledgeParser;
     uploadKnowledgeFile: (remotePath: string, localPath: string) => Promise<{ path: string }>;
+    eventLogger?: Logger | null;
   },
 ): Promise<KnowledgeIngestFileResult[]> {
   if (options.slug && sources.length !== 1) {
@@ -494,6 +519,13 @@ export async function ingestResolvedSources(
   const results: KnowledgeIngestFileResult[] = [];
 
   for (const [index, source] of sources.entries()) {
+    deps.eventLogger?.info("Processing knowledge source", {
+      phase: "file_processing",
+      progress_current: index + 1,
+      progress_total: sources.length,
+      source_name: buildKnowledgeSourceName(source),
+    });
+
     const parser = await deps.runParser({
       filePath: source.localPath,
       outputDir: deps.outputDir,
@@ -507,6 +539,26 @@ export async function ingestResolvedSources(
       options.knowledgePath,
     );
     const uploaded = await deps.uploadKnowledgeFile(remotePath, parser.sandbox_output_path);
+
+    deps.eventLogger?.info("Knowledge source ingested", {
+      phase: "file_completed",
+      progress_current: index + 1,
+      progress_total: sources.length,
+      source_name: buildKnowledgeSourceName(source),
+      remote_path: uploaded.path,
+      warning_count: parser.warnings.length,
+    });
+
+    if (parser.warnings.length > 0) {
+      deps.eventLogger?.warn("Knowledge source emitted warnings", {
+        phase: "file_warning",
+        progress_current: index + 1,
+        progress_total: sources.length,
+        source_name: buildKnowledgeSourceName(source),
+        warning_count: parser.warnings.length,
+      });
+    }
+
     results.push(
       createKnowledgeIngestResult({
         source: buildSourceReference(source),
@@ -545,8 +597,14 @@ export async function knowledgeCommand(args: ParsedArgs): Promise<void> {
         const outputDir = options.outputDir ?? await defaultOutputRoot();
         const shouldCleanupOutputDir = options.outputDir === undefined;
         const downloadOutputDir = resolveKnowledgeDownloadOutputDir(outputDir);
+        const eventLogger = createKnowledgeIngestEventLogger();
 
         try {
+          eventLogger?.info("Starting knowledge ingest", {
+            phase: "started",
+            mode: options.path ? "path_prefix" : "explicit_sources",
+          });
+
           const sources = await collectKnowledgeSources(options, {
             client,
             projectSlug: config.projectSlug,
@@ -558,15 +616,27 @@ export async function knowledgeCommand(args: ParsedArgs): Promise<void> {
               ),
           });
 
+          eventLogger?.info("Resolved knowledge sources", {
+            phase: "sources_resolved",
+            progress_total: sources.length,
+          });
+
           const results = await ingestResolvedSources(sources, options, {
             client,
             projectSlug: config.projectSlug,
             outputDir,
             runParser: runKnowledgeParser,
+            eventLogger,
             uploadKnowledgeFile: (remotePath, localPath) =>
               putRemoteFileFromLocal(client, config.projectSlug, remotePath, localPath),
           });
 
+          eventLogger?.info("Completed knowledge ingest", {
+            phase: "completed",
+            progress_current: results.length,
+            progress_total: results.length,
+          });
+
           if (options.json) {
             printJson(results);
             return;
@@ -578,6 +648,11 @@ export async function knowledgeCommand(args: ParsedArgs): Promise<void> {
               cliLogger.info(`  ${result.summary}`);
             }
           }
+        } catch (error) {
+          eventLogger?.error("Knowledge ingest failed", {
+            phase: "failed",
+          });
+          throw error;
         } finally {
           if (shouldCleanupOutputDir) {
             await Promise.all([

package/src/deno.js

@@ -1,6 +1,6 @@
 export default {
   "name": "veryfront",
-  "version": "0.1.74",
+  "version": "0.1.75",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "exclude": [
@@ -34,6 +34,7 @@ export default {
     "./resource": "./src/resource/index.ts",
     "./mcp": "./src/mcp/index.ts",
     "./middleware": "./src/middleware/index.ts",
+    "./utils": "./src/utils/index.ts",
     "./oauth": "./src/oauth/index.ts",
     "./provider": "./src/provider/index.ts",
     "./fs": "./src/fs/index.ts",
@@ -71,6 +72,7 @@ export default {
     "veryfront/workflow/claude-code": "./src/workflow/claude-code/index.ts",
     "veryfront/workflow/claude-code/react": "./src/workflow/claude-code/react/index.ts",
     "veryfront/workflow/discovery": "./src/workflow/discovery/index.ts",
+    "veryfront/utils": "./src/utils/index.ts",
     "veryfront/utils/box": "./src/utils/box.ts",
     "veryfront/utils/case-utils": "./src/utils/case-utils.ts",
     "veryfront/utils/constants/server": "./src/utils/constants/server.ts",

package/src/src/data/data-fetcher.ts

@@ -1,11 +1,22 @@
 import { withSpan } from "../observability/tracing/otlp-setup.js";
 import { SpanNames } from "../observability/tracing/span-names.js";
 import { CacheManager } from "./data-fetching-cache.js";
-import { ServerDataFetcher } from "./server-data-fetcher.js";
+import { ServerDataFetcher, type ServerDataFetchOptions } from "./server-data-fetcher.js";
 import { StaticDataFetcher } from "./static-data-fetcher.js";
 import { StaticPathsFetcher } from "./static-paths-fetcher.js";
 import type { DataContext, DataResult, PageWithData, StaticPathsResult } from "./types.js";
 
+/**
+ * Options for isolated data fetching. Passed through to ServerDataFetcher
+ * when worker isolation is enabled.
+ */
+export interface FetchDataOptions {
+  /** Absolute path to the module containing getServerData */
+  modulePath?: string;
+  /** Project directory for worker scoping */
+  projectDir?: string;
+}
+
 export class DataFetcher {
   private cacheManager: CacheManager;
   private serverFetcher: ServerDataFetcher;
@@ -23,6 +34,7 @@ export class DataFetcher {
     pageModule: PageWithData,
     context: DataContext,
     mode: "development" | "production" = "development",
+    options?: FetchDataOptions,
   ): Promise<DataResult> {
     const preferServerData = mode === "development" || !pageModule.getStaticData;
     const useServer = preferServerData && !!pageModule.getServerData;
@@ -34,10 +46,14 @@ export class DataFetcher {
       ? "static"
       : "none";
 
+    const isolationOptions: ServerDataFetchOptions | undefined = options
+      ? { modulePath: options.modulePath, projectDir: options.projectDir }
+      : undefined;
+
     return withSpan(
       SpanNames.DATA_FETCH,
       () => {
-        if (useServer) return this.serverFetcher.fetch(pageModule, context);
+        if (useServer) return this.serverFetcher.fetch(pageModule, context, isolationOptions);
         if (useStatic) return this.staticFetcher.fetch(pageModule, context);
         return Promise.resolve({ props: {} });
       },

package/src/src/data/index.ts

@@ -13,5 +13,5 @@ export type {
   PageWithData,
   StaticPathsResult,
 } from "./types.js";
-export { DataFetcher } from "./data-fetcher.js";
+export { DataFetcher, type FetchDataOptions } from "./data-fetcher.js";
 export { notFound, redirect } from "./helpers.js";

package/src/src/data/server-data-fetcher.ts

@@ -1,12 +1,29 @@
+import * as dntShim from "../../_dnt.shims.js";
 import type { DataContext, DataResult, PageWithData } from "./types.js";
 import { serverLogger } from "../utils/index.js";
 import { DATA_FETCH_TIMEOUT_MS } from "../config/defaults.js";
 import { TimeoutError, withTimeoutThrow } from "../rendering/utils/stream-utils.js";
 import { withSpan } from "../observability/tracing/otlp-setup.js";
 import { CircuitBreakerOpen, getCircuitBreaker } from "../utils/circuit-breaker.js";
+import { getWorkerPool, isDataIsolationEnabled } from "../security/sandbox/worker-pool.js";
+import type { WorkerResponse } from "../security/sandbox/worker-types.js";
+
+/**
+ * Options for isolated data fetching through Worker pool.
+ */
+export interface ServerDataFetchOptions {
+  /** Absolute path to the module containing getServerData */
+  modulePath?: string;
+  /** Project directory for worker scoping */
+  projectDir?: string;
+}
 
 export class ServerDataFetcher {
-  fetch(pageModule: PageWithData, context: DataContext): Promise<DataResult> {
+  fetch(
+    pageModule: PageWithData,
+    context: DataContext,
+    options?: ServerDataFetchOptions,
+  ): Promise<DataResult> {
     if (typeof pageModule.getServerData !== "function") {
       return Promise.resolve({ props: {} });
     }
@@ -20,6 +37,11 @@ export class ServerDataFetcher {
       successThreshold: 2,
     });
 
+    // Choose isolated or direct execution
+    const useIsolation = isDataIsolationEnabled() &&
+      !!options?.modulePath &&
+      !!options?.projectDir;
+
     return withSpan(
       "data.fetch_server",
       async () => {
@@ -28,7 +50,9 @@ export class ServerDataFetcher {
         try {
           const result = await circuitBreaker.execute(() =>
             withTimeoutThrow(
-              Promise.resolve(pageModule.getServerData!(context)),
+              useIsolation
+                ? this.fetchIsolated(options!.modulePath!, options!.projectDir!, context)
+                : Promise.resolve(pageModule.getServerData!(context)),
               DATA_FETCH_TIMEOUT_MS,
               `getServerData for ${pathname}`,
             )
@@ -59,7 +83,11 @@ export class ServerDataFetcher {
             throw error;
           }
 
-          this.logError("DATA_FETCH_ERROR getServerData failed", error, { pathname, durationMs });
+          this.logError("DATA_FETCH_ERROR getServerData failed", error, {
+            pathname,
+            durationMs,
+            isolated: useIsolation,
+          });
           throw error;
         }
       },
@@ -68,8 +96,55 @@ export class ServerDataFetcher {
         "data.pathname": pathname,
         "data.timeout_ms": DATA_FETCH_TIMEOUT_MS,
         "data.project_id": projectId,
+        "data.isolated": useIsolation,
+      },
+    );
+  }
+
+  /**
+   * Execute getServerData in a per-project Worker.
+   */
+  private async fetchIsolated(
+    modulePath: string,
+    projectDir: string,
+    context: DataContext,
+  ): Promise<DataResult> {
+    const pool = getWorkerPool();
+    const body = context.request?.body ? new Uint8Array(await context.request.arrayBuffer()) : null;
+
+    const workerResponse: WorkerResponse = await pool.execute(
+      projectDir,
+      [projectDir],
+      {
+        type: "fetch-data",
+        id: dntShim.crypto.randomUUID(),
+        modulePath,
+        context: {
+          params: context.params,
+          query: context.query?.toString() ?? "",
+          request: {
+            url: context.request?.url ?? context.url?.toString() ?? "http://localhost",
+            method: context.request?.method ?? "GET",
+            headers: context.request ? [...context.request.headers.entries()] : [],
+            body,
+          },
+          url: context.url?.toString() ?? "http://localhost",
+        },
       },
     );
+
+    if (workerResponse.type === "error") {
+      const err = new Error(workerResponse.error.message);
+      err.name = workerResponse.error.name;
+      throw err;
+    }
+
+    if (workerResponse.type === "data-result") {
+      return workerResponse.result as DataResult;
+    }
+
+    // Unexpected response type — shouldn't happen but be defensive
+    throw new Error(`Unexpected worker response type: ${workerResponse.type}`);
   }
 
   /**

package/src/src/rendering/orchestrator/lifecycle.ts

@@ -38,6 +38,8 @@ export interface LifecycleOptions {
   projectId?: string;
   /** Content source identifier for cache isolation (branch or release) */
   contentSourceId?: string;
+  /** Injectable factory for testing — bypasses real service construction */
+  servicesFactory?: (adapter: RuntimeAdapter) => RendererServices;
 }
 
 export interface RendererServices {
@@ -62,6 +64,7 @@ export class RendererLifecycle {
   private contentSourceId?: string;
   private services?: RendererServices;
   private adapter!: RuntimeAdapter;
+  private servicesFactory?: (adapter: RuntimeAdapter) => RendererServices;
 
   constructor(options: LifecycleOptions) {
     this.configManager = options.configManager;
@@ -69,6 +72,7 @@ export class RendererLifecycle {
     this.moduleServerUrl = options.moduleServerUrl;
     this.projectId = options.projectId;
     this.contentSourceId = options.contentSourceId;
+    this.servicesFactory = options.servicesFactory;
   }
 
   async initialize(): Promise<RendererServices> {
@@ -83,6 +87,13 @@ export class RendererLifecycle {
       this.adapter = await runtime.get();
     }
 
+    // Allow tests to bypass the full service graph construction
+    if (this.servicesFactory) {
+      this.services = this.servicesFactory(this.adapter);
+      logger.debug("Renderer services initialized via injected factory");
+      return this.services;
+    }
+
     const projectDir = this.configManager.getProjectDir();
     const mode = this.configManager.getMode();
     const debugMode = this.configManager.isDebugMode();

package/src/src/rendering/orchestrator/pipeline.ts

@@ -36,7 +36,7 @@ import type { PageResolver } from "../page-resolution/index.js";
 import type { LayoutOrchestrator } from "./layout.js";
 import type { SSROrchestrator } from "./ssr-orchestrator.js";
 import type { PageDataResponse, RenderOptions, RenderResult } from "./types.js";
-import { DataFetcher } from "../../data/index.js";
+import { DataFetcher, type FetchDataOptions } from "../../data/index.js";
 import type { DataContext, PageWithData } from "../../data/types.js";
 import { clearSSRModuleCacheForProject } from "../../modules/react-loader/index.js";
 import { setupSSRGlobals } from "../ssr-globals.js";
@@ -303,8 +303,13 @@ export class RenderPipeline {
           Promise.all(
             dataJobs.map(async (job) => {
               try {
+                const jobPath = (job as LoadedModule & { path?: string }).path;
+                const fetchOptions: FetchDataOptions = {
+                  modulePath: jobPath,
+                  projectDir: this.config.projectDir,
+                };
                 const result = await this.dataFetcher
-                  .fetchData(job.mod as PageWithData, dataContext, this.config.mode);
+                  .fetchData(job.mod as PageWithData, dataContext, this.config.mode, fetchOptions);
                 return { ...job, result, error: null as Error | null };
               } catch (error) {
                 return { ...job, result: null, error: error as Error };

package/src/src/rendering/orchestrator/ssr-orchestrator.ts

@@ -10,6 +10,8 @@ import { computeHash } from "../utils/index.js";
 import type { HTMLGenerationContext, HTMLGenerator } from "./html.js";
 import type { RenderOptions } from "./types.js";
 import { runWithHeadCollector } from "../../react/head-collector.js";
+import { getWorkerPool, isSSRIsolationEnabled } from "../../security/sandbox/worker-pool.js";
+import type { WorkerResponse } from "../../security/sandbox/worker-types.js";
 
 const logger = rendererLogger.component("ssr-orchestrator");
 
@@ -27,6 +29,24 @@ export interface SSRRenderingResult {
   ssrHash: string;
 }
 
+/**
+ * Options for isolated SSR rendering through the Worker pool.
+ * When provided and SSR isolation is enabled, the rendering happens
+ * in a per-project Worker instead of the main process.
+ */
+export interface SSRIsolationOptions {
+  /** Temp file path for the page component module */
+  pageModulePath: string;
+  /** Ordered layout module temp paths (innermost → outermost) */
+  layoutModulePaths: string[];
+  /** Page component props */
+  pageProps: Record<string, unknown>;
+  /** Layout props (one entry per layout, matching layoutModulePaths order) */
+  layoutProps: Record<string, unknown>[];
+  /** Project directory for worker scoping */
+  projectDir: string;
+}
+
 function getElementTypeName(el: React.ReactElement | null | undefined): string {
   if (!el?.type) return "unknown";
   if (typeof el.type === "string") return el.type;
@@ -46,7 +66,18 @@ export class SSROrchestrator {
     pageElement: React.ReactElement,
     generationContext: Omit<HTMLGenerationContext, "html" | "ssrHash">,
     options?: RenderOptions,
+    isolationOptions?: SSRIsolationOptions,
   ): Promise<SSRRenderingResult> {
+    // Isolated SSR path: render in per-project Worker
+    if (
+      isSSRIsolationEnabled() &&
+      isolationOptions?.pageModulePath &&
+      isolationOptions?.projectDir
+    ) {
+      return this.performIsolatedSSR(generationContext, options, isolationOptions);
+    }
+
+    // Default path: render in main process
     logger.debug("performSSRRendering called", {
       elementType: getElementTypeName(pageElement),
       hasChildren: !!(pageElement.props as Record<string, unknown>)?.children,
@@ -133,6 +164,94 @@ export class SSROrchestrator {
     };
   }
 
+  /**
+   * Perform SSR rendering in an isolated per-project Worker.
+   *
+   * The Worker imports user modules from their temp file paths,
+   * constructs the React element tree, and renders to HTML.
+   * For streaming, the Worker sends chunks via postMessage.
+   */
+  private async performIsolatedSSR(
+    generationContext: Omit<HTMLGenerationContext, "html" | "ssrHash">,
+    options: RenderOptions | undefined,
+    isolation: SSRIsolationOptions,
+  ): Promise<SSRRenderingResult> {
+    const wantsStream = options?.delivery === "stream";
+    const pool = getWorkerPool();
+    const requestId = dntShim.crypto.randomUUID();
+
+    return withSpan(
+      "ssr.isolated_render",
+      async () => {
+        const worker = pool.getOrCreateWorker(isolation.projectDir, [isolation.projectDir]);
+
+        if (wantsStream) {
+          // Streaming mode: get a ReadableStream of chunks from the Worker
+          const stream = worker.executeStream({
+            type: "render-ssr",
+            id: requestId,
+            pageModulePath: isolation.pageModulePath,
+            layoutModulePaths: isolation.layoutModulePaths,
+            pageProps: isolation.pageProps,
+            layoutProps: isolation.layoutProps,
+            delivery: "stream",
+          });
+
+          const ssrHash = `stream-isolated-${Date.now()}`;
+
+          // Generate HTML stream using the framework's HTML generator
+          const finalStream = await this.config.htmlGenerator.generateHTMLStream(stream, {
+            ...generationContext,
+            ssrHash,
+            options: { ...generationContext.options, ...options },
+            collectedHead: undefined,
+          });
+
+          return { fullHtml: "", finalStream, ssrHash };
+        }
+
+        // String mode: render to HTML in Worker, get result back
+        const workerResponse: WorkerResponse = await worker.execute({
+          type: "render-ssr",
+          id: requestId,
+          pageModulePath: isolation.pageModulePath,
+          layoutModulePaths: isolation.layoutModulePaths,
+          pageProps: isolation.pageProps,
+          layoutProps: isolation.layoutProps,
+          delivery: "string",
+        });
+
+        if (workerResponse.type === "error") {
+          const err = new Error(workerResponse.error.message);
+          err.name = workerResponse.error.name;
+          throw err;
+        }
+
+        if (workerResponse.type !== "ssr-result") {
+          throw new Error(`Unexpected worker response type: ${workerResponse.type}`);
+        }
+
+        const html = workerResponse.html;
+        const ssrHash = await computeHash(html);
+
+        const fullHtml = await this.config.htmlGenerator.generateFullHTML({
+          ...generationContext,
+          html,
+          ssrHash,
+          options: { ...generationContext.options, ...options },
+          collectedHead: undefined,
+        });
+
+        return { fullHtml, finalStream: null, ssrHash };
+      },
+      {
+        "ssr.isolated": true,
+        "ssr.wants_stream": wantsStream,
+        "ssr.project_dir": isolation.projectDir,
+      },
+    );
+  }
+
   private createStream(html: string): ReadableStream | null {
     try {
       return new dntShim.Response(html).body ?? null;

package/src/src/routing/api/handler.ts

@@ -13,7 +13,7 @@ import { ApiRouteMatcher, type RouteMatch } from "./api-route-matcher.js";
 import type { APIRoute } from "./module-loader/types.js";
 import { loadHandlerModule } from "./module-loader/loader.js";
 import { discoverAppRoutes, discoverPagesRoutes } from "./route-discovery.js";
-import { executeAppRoute, executePagesRoute } from "./route-executor.js";
+import { executeAppRoute, executePagesRoute, type ExecuteRouteOptions } from "./route-executor.js";
 import { withSpan } from "../../observability/tracing/otlp-setup.js";
 
 /** Max entries in the loaded-handler LRU cache */
@@ -188,9 +188,22 @@ export class APIRouteHandler {
         // Note: Cannot use path-based detection (/app/) as projectDir may be '/app' in production
         const isAppRoute = /\/route\.(ts|js|tsx|jsx)$/.test(match.route.page);
 
+        const isolationOptions: ExecuteRouteOptions = {
+          modulePath: match.route.page,
+          projectDir: this.projectDir,
+        };
+
         const response = isAppRoute
-          ? await executeAppRoute(handler, request, match, pathname, adapter)
-          : await executePagesRoute(handler, request, match, pathname, adapter, this.projectDir);
+          ? await executeAppRoute(handler, request, match, pathname, adapter, isolationOptions)
+          : await executePagesRoute(
+            handler,
+            request,
+            match,
+            pathname,
+            adapter,
+            this.projectDir,
+            isolationOptions,
+          );
 
         const corsResponse = await applyCORSHeaders({
           request,