veryfront 0.1.95 → 0.1.97

package/src/src/transforms/mdx/esm-module-loader/module-fetcher/dependency-recovery.ts

@@ -0,0 +1,173 @@
+/**
+ * Distributed recovery for missing MDX ESM module dependencies.
+ *
+ * Restores missing vfmod files on fresh pods from the distributed transform
+ * cache, scoped to the current project and content source.
+ *
+ * @module transforms/mdx/esm-module-loader/module-fetcher/dependency-recovery
+ */
+
+import { basename, dirname } from "../../../../platform/compat/path/index.js";
+import { detokenizeAllCachePaths } from "../../../../cache/index.js";
+import type { CacheBackend } from "../../../../cache/types.js";
+import type { Logger } from "../../../../utils/logger/logger.js";
+import { getDistributedTransformBackend } from "../../../esm/transform-cache.js";
+import { ensureHttpBundlesExist } from "../../../esm/http-cache.js";
+import { extractHttpBundlePaths } from "../../../../modules/react-loader/ssr-module-loader/http-bundle-helpers.js";
+import { getHttpBundleCacheDir } from "../../../../utils/cache-dir.js";
+import { LOG_PREFIX_MDX_LOADER } from "../constants.js";
+import { getLocalFs } from "../cache/index.js";
+import { buildMdxEsmModuleRecoveryCacheKey } from "../cache-format.js";
+
+const MDX_VFMOD_FILE_URL_PATTERN = /file:\/\/([^"'\s]+veryfront-mdx-esm\/[^"'\s]+\.mjs)/gi;
+
+interface EnsureMdxModuleDependenciesOptions {
+  projectId: string;
+  contentSourceId: string;
+  log: Logger;
+  distributedCache?: CacheBackend | null;
+}
+
+interface EnsureMdxModuleDependenciesResult {
+  recovered: string[];
+  missing: string[];
+}
+
+function extractMdxModuleDependencyPaths(code: string): string[] {
+  const paths: string[] = [];
+  const seen = new Set<string>();
+  let match: RegExpExecArray | null;
+  while ((match = MDX_VFMOD_FILE_URL_PATTERN.exec(code)) !== null) {
+    const rawPath = match[1];
+    if (!rawPath) continue;
+    const cleanPath = rawPath.replace(/\?.*$/, "");
+    if (seen.has(cleanPath)) continue;
+    seen.add(cleanPath);
+    paths.push(cleanPath);
+  }
+  return paths;
+}
+
+async function ensureHttpBundleDependencies(code: string, log: Logger): Promise<boolean> {
+  const bundlePaths = extractHttpBundlePaths(code);
+  if (bundlePaths.length === 0) return true;
+
+  const failed = await ensureHttpBundlesExist(bundlePaths, getHttpBundleCacheDir());
+  if (failed.length === 0) return true;
+
+  log.warn(`${LOG_PREFIX_MDX_LOADER} Failed to recover HTTP bundles for vfmod dependency`, {
+    failed,
+    totalBundles: bundlePaths.length,
+  });
+  return false;
+}
+
+async function ensureModuleFileAndDeps(
+  absolutePath: string,
+  distributedCache: CacheBackend,
+  options: EnsureMdxModuleDependenciesOptions,
+  visited: Set<string>,
+  recovered: Set<string>,
+): Promise<boolean> {
+  if (visited.has(absolutePath)) return true;
+  visited.add(absolutePath);
+
+  const localFs = getLocalFs();
+
+  try {
+    const stat = await localFs.stat(absolutePath);
+    if (stat?.isFile) {
+      const existingCode = await localFs.readTextFile(absolutePath);
+      if (!(await ensureHttpBundleDependencies(existingCode, options.log))) return false;
+
+      for (const nestedPath of extractMdxModuleDependencyPaths(existingCode)) {
+        if (
+          !(await ensureModuleFileAndDeps(
+            nestedPath,
+            distributedCache,
+            options,
+            visited,
+            recovered,
+          ))
+        ) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+  } catch (_) {
+    /* expected: dependency may not exist on this pod yet */
+  }
+
+  const recoveryKey = buildMdxEsmModuleRecoveryCacheKey(
+    options.projectId,
+    options.contentSourceId,
+    basename(absolutePath),
+  );
+
+  const portableCode = await distributedCache.get(recoveryKey);
+  if (!portableCode) {
+    options.log.debug(`${LOG_PREFIX_MDX_LOADER} No distributed vfmod recovery entry`, {
+      dependencyPath: absolutePath,
+      recoveryKey,
+    });
+    return false;
+  }
+
+  const recoveredCode = detokenizeAllCachePaths(portableCode);
+  if (!(await ensureHttpBundleDependencies(recoveredCode, options.log))) return false;
+
+  for (const nestedPath of extractMdxModuleDependencyPaths(recoveredCode)) {
+    if (
+      !(await ensureModuleFileAndDeps(
+        nestedPath,
+        distributedCache,
+        options,
+        visited,
+        recovered,
+      ))
+    ) {
+      return false;
+    }
+  }
+
+  await localFs.mkdir(dirname(absolutePath), { recursive: true });
+  await localFs.writeTextFile(absolutePath, recoveredCode);
+  recovered.add(absolutePath);
+
+  options.log.debug(`${LOG_PREFIX_MDX_LOADER} Recovered vfmod dependency from distributed cache`, {
+    dependencyPath: absolutePath,
+    recoveryKey,
+  });
+
+  return true;
+}
+
+export async function ensureMdxModuleDependencies(
+  code: string,
+  options: EnsureMdxModuleDependenciesOptions,
+): Promise<EnsureMdxModuleDependenciesResult> {
+  const distributedCache = options.distributedCache ?? (await getDistributedTransformBackend());
+  if (!distributedCache) return { recovered: [], missing: extractMdxModuleDependencyPaths(code) };
+
+  const visited = new Set<string>();
+  const recovered = new Set<string>();
+  const missing: string[] = [];
+
+  for (const dependencyPath of extractMdxModuleDependencyPaths(code)) {
+    const ok = await ensureModuleFileAndDeps(
+      dependencyPath,
+      distributedCache,
+      options,
+      visited,
+      recovered,
+    );
+    if (!ok) missing.push(dependencyPath);
+  }
+
+  return {
+    recovered: [...recovered],
+    missing,
+  };
+}

package/src/src/transforms/mdx/esm-module-loader/module-fetcher/distributed-cache.ts

@@ -23,6 +23,9 @@ import {
   findMissingFileDependenciesInCode,
   hasIncompatibleFrameworkPaths,
 } from "./framework-validator.js";
+import { ensureMdxModuleDependencies } from "./dependency-recovery.js";
+import { buildMdxEsmModuleFileName, buildMdxEsmModuleRecoveryCacheKey } from "../cache-format.js";
+import { hashString } from "../utils/hash.js";
 
 /** TTL for cached transforms (uses centralized config) */
 const TRANSFORM_CACHE_TTL_SECONDS = TRANSFORM_DISTRIBUTED_TTL_SEC;
@@ -59,6 +62,8 @@ interface DistributedCacheReadResult {
  */
 export async function readDistributedCache(
   transformCacheKey: string,
+  projectId: string,
+  contentSourceId: string | undefined,
   normalizedPath: string,
   projectSlug: string,
   projectDir: string,
@@ -129,10 +134,26 @@ export async function readDistributedCache(
     // that don't exist on this machine.
     if (moduleCode) {
       const missingDeps = await findMissingFileDependenciesInCode(moduleCode, log);
-      if (missingDeps.length > 0) {
+      if (missingDeps.length > 0 && contentSourceId) {
+        const recovered = await ensureMdxModuleDependencies(moduleCode, {
+          distributedCache,
+          projectId,
+          contentSourceId,
+          log,
+        });
+        if (recovered.recovered.length > 0) {
+          log.debug(`${LOG_PREFIX_MDX_LOADER} Recovered missing vfmod dependencies from cache`, {
+            normalizedPath,
+            recovered: recovered.recovered.slice(0, 5),
+          });
+        }
+      }
+
+      const unresolvedDeps = await findMissingFileDependenciesInCode(moduleCode, log);
+      if (unresolvedDeps.length > 0) {
         log.warn(
-          `${LOG_PREFIX_MDX_LOADER} Cached code has ${missingDeps.length} missing file dependencies, invalidating`,
-          { normalizedPath, missingDeps: missingDeps.slice(0, 5) },
+          `${LOG_PREFIX_MDX_LOADER} Cached code has ${unresolvedDeps.length} missing file dependencies, invalidating`,
+          { normalizedPath, missingDeps: unresolvedDeps.slice(0, 5) },
         );
         moduleCode = null;
       }
@@ -177,6 +198,8 @@ export async function readDistributedCache(
 export function writeDistributedCache(
   distributedCache: DistributedCache,
   transformCacheKey: string,
+  projectId: string,
+  contentSourceId: string,
   moduleCode: string,
   normalizedPath: string,
   log: Logger,
@@ -195,6 +218,23 @@ export function writeDistributedCache(
       });
     });
 
+  const moduleFileName = buildMdxEsmModuleFileName(hashString(normalizedPath + moduleCode));
+  const moduleRecoveryKey = buildMdxEsmModuleRecoveryCacheKey(
+    projectId,
+    contentSourceId,
+    moduleFileName,
+  );
+
+  distributedCache
+    .set(moduleRecoveryKey, portableCode, TRANSFORM_CACHE_TTL_SECONDS)
+    .catch((error) => {
+      log.debug(`${LOG_PREFIX_MDX_LOADER} Distributed vfmod recovery set failed`, {
+        normalizedPath,
+        moduleRecoveryKey,
+        error,
+      });
+    });
+
   // Create and store bundle manifest companion key for atomic validation
   const bundlePaths = extractHttpBundlePaths(moduleCode);
   if (bundlePaths.length > 0) {

package/src/src/transforms/mdx/esm-module-loader/module-fetcher/framework-validator.ts

@@ -14,6 +14,12 @@ import { getLocalFs } from "../cache/index.js";
 import { extractHttpBundlePaths } from "../../../../modules/react-loader/ssr-module-loader/http-bundle-helpers.js";
 import { ensureHttpBundlesExist } from "../../../esm/http-cache.js";
 import { MDX_ESM_MJS_FILE_URL_PATTERN_SOURCE } from "../cache-format.js";
+import { ensureMdxModuleDependencies } from "./dependency-recovery.js";
+
+interface MdxRecoveryOptions {
+  projectId: string;
+  contentSourceId: string;
+}
 
 /**
  * Check if cached code has file:// paths that are incompatible with this environment.
@@ -157,6 +163,7 @@ export async function validateCachedModule(
   log: Logger,
   pathCache: Map<string, string>,
   versionedKey: string,
+  recoveryOptions?: MdxRecoveryOptions,
 ): Promise<boolean> {
   // Reject caches with raw HTTP URLs - all modules should use file:// paths.
   // This ensures consistency between compiled and non-compiled modes.
@@ -192,6 +199,36 @@ export async function validateCachedModule(
     }
   }
 
+  if (recoveryOptions) {
+    const recovered = await ensureMdxModuleDependencies(cachedCode, {
+      ...recoveryOptions,
+      log,
+    });
+    if (recovered.recovered.length > 0) {
+      log.debug(`${LOG_PREFIX_MDX_LOADER} Recovered cached module vfmod dependencies`, {
+        normalizedPath,
+        cachedPath,
+        recovered: recovered.recovered.slice(0, 5),
+      });
+    }
+  }
+
+  const missingDeps = await findMissingFileDependenciesInCode(cachedCode, log);
+  if (missingDeps.length > 0) {
+    log.warn(`${LOG_PREFIX_MDX_LOADER} Cached module has missing vfmod dependencies`, {
+      normalizedPath,
+      cachedPath,
+      missingDeps: missingDeps.slice(0, 5),
+    });
+    pathCache.delete(versionedKey);
+    try {
+      await getLocalFs().remove(cachedPath);
+    } catch (_) {
+      /* expected: cached file may already be removed */
+    }
+    return false;
+  }
+
   if (!(await hasIncompatibleFrameworkPaths(cachedCode, log))) return true;
 
   log.warn(`${LOG_PREFIX_MDX_LOADER} Cached module has incompatible framework paths`, {

package/src/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts

@@ -190,7 +190,7 @@ async function doFetchAndCacheModule(
   parentModulePath?: string,
 ): Promise<string | null> {
   const log = getLog(context);
-  const { esmCacheDir, adapter, projectDir, projectId } = context;
+  const { esmCacheDir, adapter, projectDir, projectId, contentSourceId } = context;
 
   const pathCache = await getModulePathCache(esmCacheDir);
   const versionedKey = getVersionedPathCacheKey(normalizedPath);
@@ -209,6 +209,12 @@ async function doFetchAndCacheModule(
             log,
             pathCache,
             versionedKey,
+            context.contentSourceId
+              ? {
+                projectId: context.projectId,
+                contentSourceId: context.contentSourceId,
+              }
+              : undefined,
           )
         ) {
           recordModuleToSession(normalizedPath);
@@ -252,7 +258,9 @@ async function doFetchAndCacheModule(
     const { sourceCode, actualFilePath } = resolved;
 
     const contentHash = hashString(sourceCode);
-    const transformCacheKey = getTransformCacheKey(projectId, normalizedPath, contentHash);
+    const transformCacheKey = contentSourceId
+      ? getTransformCacheKey(projectId, contentSourceId, normalizedPath, contentHash)
+      : null;
 
     let moduleCode: string | null = null;
     let needsDistributedCacheWrite = false;
@@ -260,14 +268,18 @@ async function doFetchAndCacheModule(
     // Try distributed cache read with full validation.
     // Returns null only if no distributed backend is configured.
     // Otherwise returns { code, distributedCache } where code may be null (miss).
-    const distResult = await readDistributedCache(
-      transformCacheKey,
-      normalizedPath,
-      projectSlug,
-      projectDir,
-      context.reactVersion,
-      log,
-    );
+    const distResult = transformCacheKey
+      ? await readDistributedCache(
+        transformCacheKey,
+        projectId,
+        contentSourceId,
+        normalizedPath,
+        projectSlug,
+        projectDir,
+        context.reactVersion,
+        log,
+      )
+      : null;
     if (distResult?.code) {
       moduleCode = distResult.code;
     }
@@ -402,10 +414,12 @@ async function doFetchAndCacheModule(
 
     // Write to distributed cache AFTER nested imports are resolved.
     // This ensures other pods get fully-resolved code without /_vf_modules/ paths.
-    if (needsDistributedCacheWrite && distResult?.distributedCache) {
+    if (needsDistributedCacheWrite && distResult?.distributedCache && transformCacheKey) {
       writeDistributedCache(
         distResult.distributedCache,
         transformCacheKey,
+        projectId,
+        contentSourceId!,
         moduleCode,
         normalizedPath,
         log,
@@ -449,6 +463,7 @@ export function createModuleFetcherContext(
   projectDir: string,
   projectId: string,
   options?: {
+    contentSourceId?: string;
     isLocalProject?: boolean;
     projectSlug?: string;
     reactVersion?: string;

package/src/src/transforms/mdx/esm-module-loader/types.ts

@@ -51,6 +51,7 @@ export interface ModuleFetcherContext {
   adapter: RuntimeAdapter;
   projectDir: string;
   projectId: string;
+  contentSourceId?: string;
   projectSlug?: string;
   isLocalProject?: boolean;
   /**

package/src/src/utils/version.ts

@@ -3,7 +3,7 @@ import { getEnv } from "../platform/compat/process.js";
 
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
-export const VERSION = "0.1.95";
+export const VERSION = "0.1.97";
 
 export function normalizeVeryfrontVersion(version: string | undefined): string | undefined {
   if (!version) return undefined;

package/src/src/workflow/executor/workflow-executor.ts

@@ -27,6 +27,8 @@ import type {
 import { generateId, parseDuration } from "../types.js";
 import { hasLockSupport, type WorkflowBackend } from "../backends/types.js";
 import { getCurrentRequestContext } from "../../platform/adapters/fs/veryfront/multi-project-adapter.js";
+import { env as getProcessEnv } from "../../platform/compat/process.js";
+import { mergeInjectedWorkflowEnv } from "../../jobs/runtime-env.js";
 import { DAGExecutor } from "./dag-executor.js";
 import { CheckpointManager } from "./checkpoint-manager.js";
 import { runWithWorkflowTenant, StepExecutor, type StepExecutorConfig } from "./step-executor.js";
@@ -196,6 +198,7 @@ export class WorkflowExecutor {
         releaseId: requestCtx.releaseId ?? null,
       }
       : undefined;
+    const injectedProjectEnv = mergeInjectedWorkflowEnv(undefined, getProcessEnv());
 
     const run: WorkflowRun<TInput, TOutput> = {
       id: options?.runId ?? generateId("run"),
@@ -205,7 +208,10 @@ export class WorkflowExecutor {
       input,
       nodeStates: {},
       currentNodes: [],
-      context: { input },
+      context: {
+        input,
+        ...(injectedProjectEnv ? { env: injectedProjectEnv } : {}),
+      },
       checkpoints: [],
       pendingApprovals: [],
       createdAt: new Date(),

package/src/src/workflow/types.ts

@@ -50,6 +50,7 @@ type DurationString = string;
  */
 export interface WorkflowContext {
   input: unknown;
+  env?: Record<string, string>;
   _tenant?: CapturedTenantContext;
   [nodeId: string]: unknown;
 }

package/src/src/workflow/worker/dynamic-job-entrypoint.ts

@@ -27,9 +27,11 @@
 
 import { logger as baseLogger } from "../../utils/index.js";
 import { getEnv } from "../../platform/compat/process.js";
+import { env as getProcessEnv } from "../../platform/compat/process.js";
 import { runWithRequestContext } from "../../platform/adapters/fs/veryfront/multi-project-adapter.js";
 import { enhanceAdapterWithFS } from "../../platform/adapters/fs/integration.js";
 import { denoAdapter } from "../../platform/adapters/runtime/deno/index.js";
+import { mergeInjectedWorkflowEnv } from "../../jobs/runtime-env.js";
 import { discoverWorkflows } from "../discovery/index.js";
 import type { VeryfrontConfig } from "../../config/index.js";
 import type { WorkflowBackend } from "../backends/types.js";
@@ -109,12 +111,28 @@ export async function runDynamicWorkflowJob(
 
   try {
     // Fetch the workflow run
-    const run = await backend.getRun(runId);
+    let run = await backend.getRun(runId);
     if (!run) {
       logger.error(`Workflow run not found: ${runId}`);
       return DYNAMIC_EXIT_CODES.NOT_FOUND;
     }
 
+    const injectedEnv = mergeInjectedWorkflowEnv(run.context.env, getProcessEnv());
+    if (injectedEnv) {
+      const currentEnv = run.context.env;
+      const currentSerialized = currentEnv ? JSON.stringify(currentEnv) : "";
+      const nextSerialized = JSON.stringify(injectedEnv);
+      if (currentSerialized !== nextSerialized) {
+        await backend.updateRun(runId, {
+          context: {
+            ...run.context,
+            env: injectedEnv,
+          },
+        });
+        run = (await backend.getRun(runId)) ?? run;
+      }
+    }
+
     // Get tenant context (from env or from stored run)
     const tenant = getTenantFromEnv() ?? run._tenant;
 

package/src/src/workflow/worker/job-entrypoint.ts

@@ -22,7 +22,9 @@
 
 import { logger as baseLogger } from "../../utils/index.js";
 import { getEnv } from "../../platform/compat/process.js";
+import { env as getProcessEnv } from "../../platform/compat/process.js";
 import { runWithRequestContext } from "../../platform/adapters/fs/veryfront/multi-project-adapter.js";
+import { mergeInjectedWorkflowEnv } from "../../jobs/runtime-env.js";
 import type { WorkflowBackend } from "../backends/types.js";
 import type { WorkflowExecutor } from "../executor/workflow-executor.js";
 import type { CapturedTenantContext, WorkflowDefinition } from "../types.js";
@@ -116,12 +118,28 @@ export async function runWorkflowJob(config: JobEntrypointConfig): Promise<numbe
 
   try {
     // Fetch the workflow run
-    const run = await backend.getRun(runId);
+    let run = await backend.getRun(runId);
     if (!run) {
       logger.error(`Workflow run not found: ${runId}`);
       return EXIT_CODES.NOT_FOUND;
     }
 
+    const injectedEnv = mergeInjectedWorkflowEnv(run.context.env, getProcessEnv());
+    if (injectedEnv) {
+      const currentEnv = run.context.env;
+      const currentSerialized = currentEnv ? JSON.stringify(currentEnv) : "";
+      const nextSerialized = JSON.stringify(injectedEnv);
+      if (currentSerialized !== nextSerialized) {
+        await backend.updateRun(runId, {
+          context: {
+            ...run.context,
+            env: injectedEnv,
+          },
+        });
+        run = (await backend.getRun(runId)) ?? run;
+      }
+    }
+
     // Get tenant context (from env or from stored run)
     const tenant = getTenantFromEnv() ?? run._tenant;