veryfront 0.0.80 → 0.0.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/deno.js +1 -1
- package/esm/src/cache/backend.d.ts +20 -0
- package/esm/src/cache/backend.d.ts.map +1 -1
- package/esm/src/cache/backend.js +57 -0
- package/esm/src/cache/hash.d.ts +107 -0
- package/esm/src/cache/hash.d.ts.map +1 -0
- package/esm/src/cache/hash.js +166 -0
- package/esm/src/cache/index.d.ts +3 -0
- package/esm/src/cache/index.d.ts.map +1 -1
- package/esm/src/cache/index.js +3 -0
- package/esm/src/cache/module-cache.d.ts +82 -0
- package/esm/src/cache/module-cache.d.ts.map +1 -0
- package/esm/src/cache/module-cache.js +214 -0
- package/esm/src/cache/multi-tier.d.ts +177 -0
- package/esm/src/cache/multi-tier.d.ts.map +1 -0
- package/esm/src/cache/multi-tier.js +352 -0
- package/esm/src/cli/templates/integration-loader.d.ts.map +1 -1
- package/esm/src/cli/templates/integration-loader.js +2 -4
- package/esm/src/modules/react-loader/ssr-module-loader/loader.d.ts.map +1 -1
- package/esm/src/modules/react-loader/ssr-module-loader/loader.js +121 -14
- package/esm/src/observability/tracing/span-names.d.ts +2 -0
- package/esm/src/observability/tracing/span-names.d.ts.map +1 -1
- package/esm/src/observability/tracing/span-names.js +2 -0
- package/esm/src/rendering/orchestrator/module-loader/cache.d.ts +10 -2
- package/esm/src/rendering/orchestrator/module-loader/cache.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/module-loader/cache.js +11 -6
- package/esm/src/rendering/orchestrator/module-loader/index.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/module-loader/index.js +72 -77
- package/esm/src/transforms/esm/http-cache.d.ts.map +1 -1
- package/esm/src/transforms/esm/http-cache.js +6 -29
- package/esm/src/transforms/esm/transform-cache.d.ts +25 -0
- package/esm/src/transforms/esm/transform-cache.d.ts.map +1 -1
- package/esm/src/transforms/esm/transform-cache.js +45 -0
- package/esm/src/transforms/mdx/esm-module-loader/module-fetcher/index.d.ts.map +1 -1
- package/esm/src/transforms/mdx/esm-module-loader/module-fetcher/index.js +2 -36
- package/esm/src/utils/constants/cache.d.ts +4 -0
- package/esm/src/utils/constants/cache.d.ts.map +1 -1
- package/esm/src/utils/constants/cache.js +14 -1
- package/package.json +1 -1
- package/src/deno.js +1 -1
- package/src/src/cache/backend.ts +62 -0
- package/src/src/cache/hash.ts +205 -0
- package/src/src/cache/index.ts +3 -0
- package/src/src/cache/module-cache.ts +252 -0
- package/src/src/cache/multi-tier.ts +503 -0
- package/src/src/cli/templates/integration-loader.ts +2 -8
- package/src/src/modules/react-loader/ssr-module-loader/loader.ts +137 -18
- package/src/src/observability/tracing/span-names.ts +2 -0
- package/src/src/rendering/orchestrator/module-loader/cache.ts +14 -8
- package/src/src/rendering/orchestrator/module-loader/index.ts +94 -89
- package/src/src/transforms/esm/http-cache.ts +12 -32
- package/src/src/transforms/esm/transform-cache.ts +53 -0
- package/src/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts +2 -40
- package/src/src/utils/constants/cache.ts +21 -1
|
@@ -50,7 +50,31 @@ import {
|
|
|
50
50
|
transformSemaphore,
|
|
51
51
|
} from "./cache/index.js";
|
|
52
52
|
import type { ModuleCacheEntry, SSRModuleLoaderOptions } from "./types.js";
|
|
53
|
-
import { getCacheBaseDir } from "../../../utils/cache-dir.js";
|
|
53
|
+
import { getCacheBaseDir, getHttpBundleCacheDir } from "../../../utils/cache-dir.js";
|
|
54
|
+
import { ensureHttpBundlesExist } from "../../../transforms/esm/http-cache.js";
|
|
55
|
+
|
|
56
|
+
/** Pattern to match HTTP bundle file:// paths in transformed code */
|
|
57
|
+
const HTTP_BUNDLE_PATTERN = /file:\/\/([^"'\s]+veryfront-http-bundle\/http-([a-f0-9]+)\.mjs)/gi;
|
|
58
|
+
|
|
59
|
+
/** Extract HTTP bundle paths from transformed code for proactive recovery */
|
|
60
|
+
function extractHttpBundlePaths(code: string): Array<{ path: string; hash: string }> {
|
|
61
|
+
const bundles: Array<{ path: string; hash: string }> = [];
|
|
62
|
+
const seen = new Set<string>();
|
|
63
|
+
let match;
|
|
64
|
+
while ((match = HTTP_BUNDLE_PATTERN.exec(code)) !== null) {
|
|
65
|
+
const path = match[1] as string;
|
|
66
|
+
const hash = match[2] as string;
|
|
67
|
+
if (!seen.has(hash)) {
|
|
68
|
+
seen.add(hash);
|
|
69
|
+
bundles.push({ path, hash });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
HTTP_BUNDLE_PATTERN.lastIndex = 0;
|
|
73
|
+
return bundles;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Track temp paths that have been verified for HTTP bundles to avoid redundant I/O */
|
|
77
|
+
const verifiedHttpBundlePaths = new Set<string>();
|
|
54
78
|
|
|
55
79
|
/**
|
|
56
80
|
* SSR Module Loader with Redis Support.
|
|
@@ -100,11 +124,45 @@ export class SSRModuleLoader {
|
|
|
100
124
|
);
|
|
101
125
|
}
|
|
102
126
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
127
|
+
let mod: Record<string, unknown>;
|
|
128
|
+
try {
|
|
129
|
+
mod = await withSpan(
|
|
130
|
+
SpanNames.SSR_DYNAMIC_IMPORT,
|
|
131
|
+
() => import(`file://${cacheEntry.tempPath}?v=${cacheEntry.contentHash}`),
|
|
132
|
+
{ "ssr.file": fileName },
|
|
133
|
+
) as Record<string, unknown>;
|
|
134
|
+
} catch (importError) {
|
|
135
|
+
// If import fails due to missing HTTP bundle, try to recover and retry once
|
|
136
|
+
const errorMsg = importError instanceof Error
|
|
137
|
+
? importError.message
|
|
138
|
+
: String(importError);
|
|
139
|
+
const bundleMatch = errorMsg.match(/veryfront-http-bundle\/http-([a-f0-9]+)\.mjs/);
|
|
140
|
+
if (bundleMatch) {
|
|
141
|
+
const hash = bundleMatch[1]!;
|
|
142
|
+
logger.warn(
|
|
143
|
+
"[SSR-MODULE-LOADER] Import failed due to missing HTTP bundle, attempting recovery",
|
|
144
|
+
{
|
|
145
|
+
file: filePath.slice(-40),
|
|
146
|
+
hash,
|
|
147
|
+
},
|
|
148
|
+
);
|
|
149
|
+
const { recoverHttpBundleByHash } = await import(
|
|
150
|
+
"../../../transforms/esm/http-cache.js"
|
|
151
|
+
);
|
|
152
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
153
|
+
const recovered = await recoverHttpBundleByHash(hash, cacheDir);
|
|
154
|
+
if (recovered) {
|
|
155
|
+
logger.info("[SSR-MODULE-LOADER] HTTP bundle recovered, retrying import", { hash });
|
|
156
|
+
mod = await import(
|
|
157
|
+
`file://${cacheEntry.tempPath}?v=${cacheEntry.contentHash}&retry=1`
|
|
158
|
+
) as Record<string, unknown>;
|
|
159
|
+
} else {
|
|
160
|
+
throw importError;
|
|
161
|
+
}
|
|
162
|
+
} else {
|
|
163
|
+
throw importError;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
108
166
|
|
|
109
167
|
failedComponents.delete(circuitKey);
|
|
110
168
|
return extractComponent(mod, filePath);
|
|
@@ -372,9 +430,44 @@ export class SSRModuleLoader {
|
|
|
372
430
|
|
|
373
431
|
const cachedEntry = globalModuleCache.get(contentCacheKey);
|
|
374
432
|
if (cachedEntry) {
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
433
|
+
// Verify HTTP bundles exist for in-memory cached transforms (once per path)
|
|
434
|
+
if (!verifiedHttpBundlePaths.has(cachedEntry.tempPath)) {
|
|
435
|
+
try {
|
|
436
|
+
const cachedCode = await this.fs.readTextFile(cachedEntry.tempPath);
|
|
437
|
+
const bundlePaths = extractHttpBundlePaths(cachedCode);
|
|
438
|
+
if (bundlePaths.length > 0) {
|
|
439
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
440
|
+
const failed = await ensureHttpBundlesExist(bundlePaths, cacheDir);
|
|
441
|
+
if (failed.length > 0) {
|
|
442
|
+
logger.warn(
|
|
443
|
+
"[SSR-MODULE-LOADER] In-memory cached module has unrecoverable HTTP bundles, re-transforming",
|
|
444
|
+
{
|
|
445
|
+
file: filePath.slice(-40),
|
|
446
|
+
failed,
|
|
447
|
+
},
|
|
448
|
+
);
|
|
449
|
+
globalModuleCache.delete(contentCacheKey);
|
|
450
|
+
globalModuleCache.delete(filePathCacheKey);
|
|
451
|
+
// Fall through to Redis or fresh transform
|
|
452
|
+
} else {
|
|
453
|
+
verifiedHttpBundlePaths.add(cachedEntry.tempPath);
|
|
454
|
+
}
|
|
455
|
+
} else {
|
|
456
|
+
verifiedHttpBundlePaths.add(cachedEntry.tempPath);
|
|
457
|
+
}
|
|
458
|
+
} catch {
|
|
459
|
+
// File doesn't exist or unreadable, invalidate cache
|
|
460
|
+
globalModuleCache.delete(contentCacheKey);
|
|
461
|
+
globalModuleCache.delete(filePathCacheKey);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// Re-check after potential invalidation
|
|
466
|
+
if (globalModuleCache.has(contentCacheKey)) {
|
|
467
|
+
globalModuleCache.set(filePathCacheKey, cachedEntry);
|
|
468
|
+
await this.ensureDependenciesExist(code, filePath, depth);
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
378
471
|
}
|
|
379
472
|
|
|
380
473
|
const redisEnabled = getRedisEnabled();
|
|
@@ -382,18 +475,44 @@ export class SSRModuleLoader {
|
|
|
382
475
|
if (redisEnabled && redisClient) {
|
|
383
476
|
const redisCode = await getFromRedis(contentCacheKey);
|
|
384
477
|
if (redisCode) {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
478
|
+
// Proactively ensure HTTP bundles exist before using cached transform.
|
|
479
|
+
// The cached code may reference file:// paths to HTTP bundles that were
|
|
480
|
+
// created on a different pod and may not exist locally.
|
|
481
|
+
let httpBundlesOk = true;
|
|
482
|
+
const bundlePaths = extractHttpBundlePaths(redisCode);
|
|
483
|
+
if (bundlePaths.length > 0) {
|
|
484
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
485
|
+
const failed = await ensureHttpBundlesExist(bundlePaths, cacheDir);
|
|
486
|
+
if (failed.length > 0) {
|
|
487
|
+
logger.warn(
|
|
488
|
+
"[SSR-MODULE-LOADER] Redis cached code has unrecoverable HTTP bundles, re-transforming",
|
|
489
|
+
{
|
|
490
|
+
file: filePath.slice(-40),
|
|
491
|
+
failed,
|
|
492
|
+
},
|
|
493
|
+
);
|
|
494
|
+
httpBundlesOk = false;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
388
497
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
498
|
+
if (httpBundlesOk) {
|
|
499
|
+
const tempPath = await this.getTempPath(filePath, contentHash);
|
|
500
|
+
await this.fs.mkdir(tempPath.substring(0, tempPath.lastIndexOf("/")), {
|
|
501
|
+
recursive: true,
|
|
502
|
+
});
|
|
503
|
+
await this.fs.writeTextFile(tempPath, redisCode);
|
|
504
|
+
verifiedHttpBundlePaths.add(tempPath);
|
|
392
505
|
|
|
393
|
-
|
|
506
|
+
const entry: ModuleCacheEntry = { tempPath, contentHash };
|
|
507
|
+
globalModuleCache.set(contentCacheKey, entry);
|
|
508
|
+
globalModuleCache.set(filePathCacheKey, entry);
|
|
394
509
|
|
|
395
|
-
|
|
396
|
-
|
|
510
|
+
logger.debug("[SSR-MODULE-LOADER] Redis cache hit", { file: filePath.slice(-40) });
|
|
511
|
+
|
|
512
|
+
await this.ensureDependenciesExist(code, filePath, depth);
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
// Fall through to re-transform, which will create HTTP bundles locally
|
|
397
516
|
}
|
|
398
517
|
}
|
|
399
518
|
|
|
@@ -98,6 +98,8 @@ export const SpanNames = {
|
|
|
98
98
|
CACHE_REGISTRY_DELETE_REDIS_KEYS: "cache.registry.delete_redis_keys",
|
|
99
99
|
CACHE_KEYS_GET_ALL_ASYNC: "cache.keys.get_all_async",
|
|
100
100
|
CACHE_KEYS_DELETE_ALL_ASYNC: "cache.keys.delete_all_async",
|
|
101
|
+
CACHE_MULTI_TIER_GET: "cache.multi_tier.get",
|
|
102
|
+
CACHE_MULTI_TIER_SET: "cache.multi_tier.set",
|
|
101
103
|
|
|
102
104
|
HTML_GENERATE_SHELL_PARTS: "html.generate_shell_parts",
|
|
103
105
|
HTML_WRAP_IN_SHELL: "html.wrap_in_shell",
|
|
@@ -1,4 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module Loader Cache Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides hash generation and cache factory functions.
|
|
5
|
+
* Module caches are now pod-level singletons (see src/cache/module-cache.ts)
|
|
6
|
+
* to ensure caches persist across requests within the same pod.
|
|
7
|
+
*
|
|
8
|
+
* @module rendering/orchestrator/module-loader/cache
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
// Re-export pod-level cache factories
|
|
1
12
|
import * as dntShim from "../../../../_dnt.shims.js";
|
|
13
|
+
|
|
14
|
+
export { createEsmCache, createModuleCache } from "../../../cache/module-cache.js";
|
|
15
|
+
|
|
2
16
|
const HEX_CHARS = "0123456789abcdef";
|
|
3
17
|
|
|
4
18
|
export async function generateHash(str: string): Promise<string> {
|
|
@@ -13,11 +27,3 @@ export async function generateHash(str: string): Promise<string> {
|
|
|
13
27
|
}
|
|
14
28
|
return hex;
|
|
15
29
|
}
|
|
16
|
-
|
|
17
|
-
export function createModuleCache(): Map<string, string> {
|
|
18
|
-
return new Map();
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export function createEsmCache(): Map<string, string> {
|
|
22
|
-
return new Map();
|
|
23
|
-
}
|
|
@@ -7,80 +7,52 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { parallelMap, rendererLogger as logger } from "../../../utils/index.js";
|
|
10
|
-
import { Singleflight } from "../../../utils/singleflight.js";
|
|
11
10
|
import type { RuntimeAdapter } from "../../../platform/adapters/base.js";
|
|
12
|
-
import type { CacheBackend } from "../../../cache/backend.js";
|
|
13
11
|
import { getLocalAdapter } from "../../../platform/adapters/registry.js";
|
|
14
12
|
import { generateHash } from "./cache.js";
|
|
15
13
|
import { findLocalLibFile, findSourceFile } from "../file-resolver/index.js";
|
|
16
14
|
import { transformToESM } from "../../../transforms/esm-transform.js";
|
|
17
15
|
import { getProjectTmpDir } from "../../../modules/react-loader/index.js";
|
|
18
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
generateCacheKey as generateTransformCacheKey,
|
|
18
|
+
getOrComputeTransform,
|
|
19
|
+
initializeTransformCache,
|
|
20
|
+
setCachedTransformAsync,
|
|
21
|
+
} from "../../../transforms/esm/transform-cache.js";
|
|
22
|
+
import { hashString } from "../../../cache/hash.js";
|
|
23
|
+
import { TRANSFORM_DISTRIBUTED_TTL_SEC } from "../../../utils/constants/cache.js";
|
|
24
|
+
import { ensureHttpBundlesExist } from "../../../transforms/esm/http-cache.js";
|
|
25
|
+
import { getHttpBundleCacheDir } from "../../../utils/cache-dir.js";
|
|
19
26
|
|
|
20
27
|
// Re-export utilities
|
|
21
28
|
export { createEsmCache, createModuleCache, generateHash } from "./cache.js";
|
|
22
29
|
export { fetchEsmModule, rewriteEsmPaths } from "./esm-rewriter.js";
|
|
23
30
|
|
|
24
|
-
/**
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
let
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (distributedTransformCache !== undefined) {
|
|
39
|
-
return Promise.resolve(distributedTransformCache);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return distributedCacheInit.do("init", async () => {
|
|
43
|
-
try {
|
|
44
|
-
const { CacheBackends } = await import("../../../cache/backend.js");
|
|
45
|
-
const backend = await CacheBackends.transform();
|
|
46
|
-
|
|
47
|
-
// Only use distributed cache if API or Redis (not memory - that's per-process)
|
|
48
|
-
if (backend.type === "memory") {
|
|
49
|
-
distributedTransformCache = null;
|
|
50
|
-
logger.debug("[ModuleLoader] No distributed transform cache (memory only)");
|
|
51
|
-
return null;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
distributedTransformCache = backend;
|
|
55
|
-
logger.debug("[ModuleLoader] Distributed transform cache initialized", {
|
|
56
|
-
type: backend.type,
|
|
57
|
-
});
|
|
58
|
-
return backend;
|
|
59
|
-
} catch (error) {
|
|
60
|
-
logger.debug("[ModuleLoader] Failed to init distributed transform cache", { error });
|
|
61
|
-
distributedTransformCache = null;
|
|
62
|
-
return null;
|
|
31
|
+
/** Pattern to match HTTP bundle file:// paths in transformed code */
|
|
32
|
+
const HTTP_BUNDLE_PATTERN = /file:\/\/([^"'\s]+veryfront-http-bundle\/http-([a-f0-9]+)\.mjs)/gi;
|
|
33
|
+
|
|
34
|
+
/** Extract HTTP bundle paths from transformed code for proactive recovery */
|
|
35
|
+
function extractHttpBundlePaths(code: string): Array<{ path: string; hash: string }> {
|
|
36
|
+
const bundles: Array<{ path: string; hash: string }> = [];
|
|
37
|
+
const seen = new Set<string>();
|
|
38
|
+
let match;
|
|
39
|
+
while ((match = HTTP_BUNDLE_PATTERN.exec(code)) !== null) {
|
|
40
|
+
const path = match[1] as string;
|
|
41
|
+
const hash = match[2] as string;
|
|
42
|
+
if (!seen.has(hash)) {
|
|
43
|
+
seen.add(hash);
|
|
44
|
+
bundles.push({ path, hash });
|
|
63
45
|
}
|
|
64
|
-
}
|
|
46
|
+
}
|
|
47
|
+
HTTP_BUNDLE_PATTERN.lastIndex = 0;
|
|
48
|
+
return bundles;
|
|
65
49
|
}
|
|
66
50
|
|
|
67
|
-
/**
|
|
68
|
-
|
|
69
|
-
* Includes content hash so cache invalidates when source changes.
|
|
70
|
-
*/
|
|
71
|
-
function getTransformCacheKey(projectId: string, filePath: string, contentHash: string): string {
|
|
72
|
-
return `v${TRANSFORM_CACHE_VERSION}:${projectId}:${filePath}:${contentHash}`;
|
|
73
|
-
}
|
|
51
|
+
/** Cache for created directories to avoid repeated mkdir calls */
|
|
52
|
+
const createdDirs = new Set<string>();
|
|
74
53
|
|
|
75
|
-
/**
|
|
76
|
-
|
|
77
|
-
let hash = 0;
|
|
78
|
-
for (let i = 0; i < str.length; i++) {
|
|
79
|
-
hash = ((hash << 5) - hash) + str.charCodeAt(i);
|
|
80
|
-
hash &= hash;
|
|
81
|
-
}
|
|
82
|
-
return Math.abs(hash).toString(36);
|
|
83
|
-
}
|
|
54
|
+
/** TTL for cached transforms (uses centralized config) */
|
|
55
|
+
const TRANSFORM_CACHE_TTL_SECONDS = TRANSFORM_DISTRIBUTED_TTL_SEC;
|
|
84
56
|
|
|
85
57
|
export interface ModuleLoaderConfig {
|
|
86
58
|
projectDir: string;
|
|
@@ -223,39 +195,54 @@ export async function transformModuleWithDeps(
|
|
|
223
195
|
|
|
224
196
|
const contentHash = hashString(fileContent);
|
|
225
197
|
const effectiveProjectId = projectId ?? projectDir;
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
198
|
+
const scopedPath = `${effectiveProjectId}:${filePath}`;
|
|
199
|
+
const transformCacheKey = generateTransformCacheKey(scopedPath, contentHash, true); // ssr=true
|
|
200
|
+
|
|
201
|
+
// Initialize transform cache (lazy, only once per pod)
|
|
202
|
+
await initializeTransformCache();
|
|
203
|
+
|
|
204
|
+
// Use consolidated transform cache with getOrCompute pattern
|
|
205
|
+
let transformedCode = await getOrComputeTransform(
|
|
206
|
+
transformCacheKey,
|
|
207
|
+
() => {
|
|
208
|
+
logger.debug("[ModuleLoader] Transform cache miss, transforming", { filePath });
|
|
209
|
+
return transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
210
|
+
projectId: effectiveProjectId,
|
|
211
|
+
dev: mode === "development",
|
|
212
|
+
ssr: true,
|
|
213
|
+
});
|
|
214
|
+
},
|
|
215
|
+
TRANSFORM_CACHE_TTL_SECONDS,
|
|
216
|
+
);
|
|
230
217
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
218
|
+
// Proactively ensure HTTP bundles exist before writing the module.
|
|
219
|
+
// Cached transforms from a different pod may reference file:// paths
|
|
220
|
+
// to HTTP bundles that don't exist locally.
|
|
221
|
+
const bundlePaths = extractHttpBundlePaths(transformedCode);
|
|
222
|
+
if (bundlePaths.length > 0) {
|
|
223
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
224
|
+
const failed = await ensureHttpBundlesExist(bundlePaths, cacheDir);
|
|
225
|
+
if (failed.length > 0) {
|
|
226
|
+
logger.warn("[ModuleLoader] HTTP bundle recovery failed, re-transforming", {
|
|
227
|
+
filePath,
|
|
228
|
+
failed,
|
|
229
|
+
});
|
|
230
|
+
transformedCode = await transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
231
|
+
projectId: effectiveProjectId,
|
|
232
|
+
dev: mode === "development",
|
|
233
|
+
ssr: true,
|
|
234
|
+
});
|
|
235
|
+
setCachedTransformAsync(
|
|
236
|
+
transformCacheKey,
|
|
237
|
+
transformedCode,
|
|
238
|
+
contentHash,
|
|
239
|
+
TRANSFORM_CACHE_TTL_SECONDS,
|
|
240
|
+
).catch((error) => {
|
|
241
|
+
logger.debug("[ModuleLoader] Failed to update transform cache after re-transform", {
|
|
237
242
|
filePath,
|
|
238
|
-
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
} catch (error) {
|
|
242
|
-
logger.debug("[ModuleLoader] Distributed cache get failed", { filePath, error });
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if (!transformedCode) {
|
|
247
|
-
transformedCode = await transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
248
|
-
projectId: effectiveProjectId,
|
|
249
|
-
dev: mode === "development",
|
|
250
|
-
ssr: true,
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
if (distributedCache) {
|
|
254
|
-
distributedCache
|
|
255
|
-
.set(transformCacheKey, transformedCode, TRANSFORM_CACHE_TTL_SECONDS)
|
|
256
|
-
.catch((error) => {
|
|
257
|
-
logger.debug("[ModuleLoader] Distributed cache set failed", { filePath, error });
|
|
243
|
+
error,
|
|
258
244
|
});
|
|
245
|
+
});
|
|
259
246
|
}
|
|
260
247
|
}
|
|
261
248
|
|
|
@@ -296,6 +283,24 @@ export async function loadModule(filePath: string, config: ModuleLoaderConfig):
|
|
|
296
283
|
try {
|
|
297
284
|
return await import(moduleUrl);
|
|
298
285
|
} catch (error) {
|
|
286
|
+
// If import fails due to missing HTTP bundle, try to recover and retry once
|
|
287
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
288
|
+
const bundleMatch = errorMsg.match(/veryfront-http-bundle\/http-([a-f0-9]+)\.mjs/);
|
|
289
|
+
if (bundleMatch) {
|
|
290
|
+
const hash = bundleMatch[1]!;
|
|
291
|
+
logger.warn("[ModuleLoader] Import failed due to missing HTTP bundle, attempting recovery", {
|
|
292
|
+
filePath,
|
|
293
|
+
hash,
|
|
294
|
+
});
|
|
295
|
+
const { recoverHttpBundleByHash } = await import("../../../transforms/esm/http-cache.js");
|
|
296
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
297
|
+
const recovered = await recoverHttpBundleByHash(hash, cacheDir);
|
|
298
|
+
if (recovered) {
|
|
299
|
+
logger.info("[ModuleLoader] HTTP bundle recovered, retrying import", { hash });
|
|
300
|
+
return await import(`file://${tempFilePath}?t=${Date.now()}&retry=1`);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
299
304
|
logger.error("[ModuleLoader] Failed to import module:", {
|
|
300
305
|
filePath,
|
|
301
306
|
tempFilePath,
|
|
@@ -14,7 +14,6 @@ import { isAbsolute, join } from "../../platform/compat/path/index.js";
|
|
|
14
14
|
import { cwd } from "../../platform/compat/process.js";
|
|
15
15
|
import { rendererLogger as logger } from "../../utils/index.js";
|
|
16
16
|
import { simpleHash } from "../../utils/hash-utils.js";
|
|
17
|
-
import { Singleflight } from "../../utils/singleflight.js";
|
|
18
17
|
import { LRUCache } from "../../utils/lru-wrapper.js";
|
|
19
18
|
import { withSpan } from "../../observability/tracing/otlp-setup.js";
|
|
20
19
|
import { SpanNames } from "../../observability/tracing/span-names.js";
|
|
@@ -23,39 +22,20 @@ import type { ImportMapConfig } from "../../modules/import-map/types.js";
|
|
|
23
22
|
import { isDeno } from "../../platform/compat/runtime.js";
|
|
24
23
|
import { getDenoNpmReactMap, getReactImportMap, REACT_VERSION } from "./package-registry.js";
|
|
25
24
|
import { parseImports, replaceSpecifiers } from "./lexer.js";
|
|
26
|
-
import
|
|
25
|
+
import { CacheBackends, createDistributedCacheAccessor } from "../../cache/backend.js";
|
|
26
|
+
import {
|
|
27
|
+
HTTP_MODULE_CACHE_MAX_ENTRIES,
|
|
28
|
+
HTTP_MODULE_DISTRIBUTED_TTL_SEC,
|
|
29
|
+
} from "../../utils/constants/cache.js";
|
|
27
30
|
|
|
28
31
|
/** Lazy-loaded distributed cache backend for cross-pod sharing */
|
|
29
|
-
|
|
30
|
-
|
|
32
|
+
const getDistributedCache = createDistributedCacheAccessor(
|
|
33
|
+
() => CacheBackends.httpModule(),
|
|
34
|
+
"HTTP-CACHE",
|
|
35
|
+
);
|
|
31
36
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return distributedCacheInit.do("init", async () => {
|
|
36
|
-
try {
|
|
37
|
-
const { CacheBackends } = await import("../../cache/backend.js");
|
|
38
|
-
const backend = await CacheBackends.httpModule();
|
|
39
|
-
|
|
40
|
-
if (backend.type === "memory") {
|
|
41
|
-
distributedCache = null;
|
|
42
|
-
logger.debug("[HTTP-CACHE] No distributed cache available (memory only)");
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
distributedCache = backend;
|
|
47
|
-
logger.debug("[HTTP-CACHE] Distributed cache initialized", { type: backend.type });
|
|
48
|
-
return backend;
|
|
49
|
-
} catch (error) {
|
|
50
|
-
logger.debug("[HTTP-CACHE] Failed to initialize distributed cache", { error });
|
|
51
|
-
distributedCache = null;
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** TTL for cached modules in distributed cache (24 hours) */
|
|
58
|
-
const DISTRIBUTED_CACHE_TTL_SECONDS = 86400;
|
|
37
|
+
/** TTL for cached modules in distributed cache (uses centralized config) */
|
|
38
|
+
const DISTRIBUTED_CACHE_TTL_SECONDS = HTTP_MODULE_DISTRIBUTED_TTL_SEC;
|
|
59
39
|
|
|
60
40
|
type CacheOptions = {
|
|
61
41
|
cacheDir: string;
|
|
@@ -64,7 +44,7 @@ type CacheOptions = {
|
|
|
64
44
|
reactVersion?: string;
|
|
65
45
|
};
|
|
66
46
|
|
|
67
|
-
const cachedPaths = new LRUCache<string, string>({ maxEntries:
|
|
47
|
+
const cachedPaths = new LRUCache<string, string>({ maxEntries: HTTP_MODULE_CACHE_MAX_ENTRIES });
|
|
68
48
|
const processingStack = new Set<string>();
|
|
69
49
|
|
|
70
50
|
function ensureAbsoluteDir(path: string): string {
|
|
@@ -156,6 +156,59 @@ export function destroyTransformCache(): void {
|
|
|
156
156
|
localFallback.clear();
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
/**
|
|
160
|
+
* Get the underlying distributed cache backend.
|
|
161
|
+
*
|
|
162
|
+
* This is exposed for callers that need direct access to the distributed
|
|
163
|
+
* cache (e.g., MDX module-fetcher that stores raw code strings instead of
|
|
164
|
+
* TransformCacheEntry JSON). Ensures initialization happens only once.
|
|
165
|
+
*
|
|
166
|
+
* Returns null if distributed cache is not available (memory-only mode).
|
|
167
|
+
*/
|
|
168
|
+
export async function getDistributedTransformBackend(): Promise<CacheBackend | null> {
|
|
169
|
+
await initializeTransformCache();
|
|
170
|
+
if (!cacheBackend || cacheBackend.type === "memory") return null;
|
|
171
|
+
return cacheBackend;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Get a cached transform or compute it if not found.
|
|
176
|
+
*
|
|
177
|
+
* This is the preferred way to use the transform cache - it handles:
|
|
178
|
+
* - Cache lookup (distributed first, then local fallback)
|
|
179
|
+
* - Compute on miss
|
|
180
|
+
* - Cache storage on compute
|
|
181
|
+
*
|
|
182
|
+
* @param key - Cache key (use generateCacheKey to build it)
|
|
183
|
+
* @param computeFn - Function to compute the transform if not cached
|
|
184
|
+
* @param ttlSeconds - TTL for the cached entry
|
|
185
|
+
* @returns The cached or computed code
|
|
186
|
+
*/
|
|
187
|
+
export async function getOrComputeTransform(
|
|
188
|
+
key: string,
|
|
189
|
+
computeFn: () => Promise<string>,
|
|
190
|
+
ttlSeconds: number = DEFAULT_TTL_SECONDS,
|
|
191
|
+
): Promise<string> {
|
|
192
|
+
// Try to get from cache first
|
|
193
|
+
const cached = await getCachedTransformAsync(key);
|
|
194
|
+
if (cached) {
|
|
195
|
+
logger.debug("[TransformCache] Cache hit", { key });
|
|
196
|
+
return cached.code;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Compute on miss
|
|
200
|
+
logger.debug("[TransformCache] Cache miss, computing", { key });
|
|
201
|
+
const code = await computeFn();
|
|
202
|
+
|
|
203
|
+
// Store in cache (fire-and-forget for performance)
|
|
204
|
+
const hash = String(Date.now()); // Simple hash for now
|
|
205
|
+
setCachedTransformAsync(key, code, hash, ttlSeconds).catch((error) => {
|
|
206
|
+
logger.debug("[TransformCache] Failed to cache computed transform", { key, error });
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
return code;
|
|
210
|
+
}
|
|
211
|
+
|
|
159
212
|
export function getTransformCacheStats(): {
|
|
160
213
|
fallbackEntries: number;
|
|
161
214
|
maxFallbackEntries: number;
|
|
@@ -16,11 +16,9 @@ import * as dntShim from "../../../../../_dnt.shims.js";
|
|
|
16
16
|
|
|
17
17
|
import { join, posix } from "../../../../../deps/deno.land/std@0.220.0/path/mod.js";
|
|
18
18
|
import { rendererLogger as logger } from "../../../../utils/index.js";
|
|
19
|
-
import { Singleflight } from "../../../../utils/singleflight.js";
|
|
20
19
|
import { withSpan } from "../../../../observability/tracing/otlp-setup.js";
|
|
21
20
|
import { SpanNames } from "../../../../observability/tracing/span-names.js";
|
|
22
21
|
import type { RuntimeAdapter } from "../../../../platform/adapters/base.js";
|
|
23
|
-
import type { CacheBackend } from "../../../../cache/backend.js";
|
|
24
22
|
import { transformToESM } from "../../../esm-transform.js";
|
|
25
23
|
import { TRANSFORM_CACHE_VERSION } from "../../../esm/package-registry.js";
|
|
26
24
|
import { ensureHttpBundlesExist } from "../../../esm/http-cache.js";
|
|
@@ -37,48 +35,12 @@ import { hashString } from "../utils/hash.js";
|
|
|
37
35
|
import { createStubModule } from "../utils/stub-module.js";
|
|
38
36
|
import { resolveModuleFile } from "../resolution/file-finder.js";
|
|
39
37
|
import { recordSSRModules } from "../../../../modules/manifest/route-module-manifest.js";
|
|
38
|
+
import { getDistributedTransformBackend } from "../../../esm/transform-cache.js";
|
|
40
39
|
import { TRANSFORM_DISTRIBUTED_TTL_SEC } from "../../../../utils/constants/cache.js";
|
|
41
40
|
|
|
42
|
-
/**
|
|
43
|
-
* Distributed transform cache for cross-pod sharing.
|
|
44
|
-
* Caches transformed module code in Redis/API so other pods don't need to re-transform.
|
|
45
|
-
*/
|
|
46
|
-
let distributedTransformCache: CacheBackend | null | undefined;
|
|
47
|
-
const distributedCacheInit = new Singleflight<CacheBackend | null>();
|
|
48
|
-
|
|
49
41
|
/** TTL for cached transforms (uses centralized config) */
|
|
50
42
|
const TRANSFORM_CACHE_TTL_SECONDS = TRANSFORM_DISTRIBUTED_TTL_SEC;
|
|
51
43
|
|
|
52
|
-
function getDistributedTransformCache(): Promise<CacheBackend | null> {
|
|
53
|
-
if (distributedTransformCache !== undefined) return Promise.resolve(distributedTransformCache);
|
|
54
|
-
|
|
55
|
-
return distributedCacheInit.do("init", async () => {
|
|
56
|
-
try {
|
|
57
|
-
const { CacheBackends } = await import("../../../../cache/backend.js");
|
|
58
|
-
const backend = await CacheBackends.transform();
|
|
59
|
-
|
|
60
|
-
// Only use distributed cache if API or Redis (not memory - that's per-process)
|
|
61
|
-
if (backend.type === "memory") {
|
|
62
|
-
distributedTransformCache = null;
|
|
63
|
-
logger.debug(`${LOG_PREFIX_MDX_LOADER} No distributed transform cache (memory only)`);
|
|
64
|
-
return null;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
distributedTransformCache = backend;
|
|
68
|
-
logger.debug(`${LOG_PREFIX_MDX_LOADER} Distributed transform cache initialized`, {
|
|
69
|
-
type: backend.type,
|
|
70
|
-
});
|
|
71
|
-
return backend;
|
|
72
|
-
} catch (error) {
|
|
73
|
-
logger.debug(`${LOG_PREFIX_MDX_LOADER} Failed to init distributed transform cache`, {
|
|
74
|
-
error,
|
|
75
|
-
});
|
|
76
|
-
distributedTransformCache = null;
|
|
77
|
-
return null;
|
|
78
|
-
}
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
|
|
82
44
|
/**
|
|
83
45
|
* Build cache key for transformed module.
|
|
84
46
|
* Includes content hash so cache invalidates when source changes.
|
|
@@ -554,7 +516,7 @@ async function doFetchAndCacheModule(
|
|
|
554
516
|
const transformCacheKey = getTransformCacheKey(projectId, normalizedPath, contentHash);
|
|
555
517
|
|
|
556
518
|
let moduleCode: string | null = null;
|
|
557
|
-
const distributedCache = await
|
|
519
|
+
const distributedCache = await getDistributedTransformBackend();
|
|
558
520
|
|
|
559
521
|
if (distributedCache) {
|
|
560
522
|
try {
|