veryfront 0.0.81 → 0.0.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -1
- package/esm/deno.js +1 -1
- package/esm/proxy/cache/index.d.ts +41 -0
- package/esm/proxy/cache/index.d.ts.map +1 -0
- package/esm/proxy/cache/index.js +75 -0
- package/esm/proxy/cache/memory-cache.d.ts +18 -0
- package/esm/proxy/cache/memory-cache.d.ts.map +1 -0
- package/esm/proxy/cache/memory-cache.js +100 -0
- package/esm/proxy/cache/redis-cache.d.ts +27 -0
- package/esm/proxy/cache/redis-cache.d.ts.map +1 -0
- package/esm/proxy/cache/redis-cache.js +183 -0
- package/esm/proxy/cache/resilient-cache.d.ts +44 -0
- package/esm/proxy/cache/resilient-cache.d.ts.map +1 -0
- package/esm/proxy/cache/resilient-cache.js +178 -0
- package/esm/proxy/cache/types.d.ts +65 -0
- package/esm/proxy/cache/types.d.ts.map +1 -0
- package/esm/proxy/cache/types.js +7 -0
- package/esm/proxy/handler.d.ts +81 -0
- package/esm/proxy/handler.d.ts.map +1 -0
- package/esm/proxy/handler.js +417 -0
- package/esm/proxy/logger.d.ts +29 -0
- package/esm/proxy/logger.d.ts.map +1 -0
- package/esm/proxy/logger.js +258 -0
- package/esm/proxy/oauth-client.d.ts +15 -0
- package/esm/proxy/oauth-client.d.ts.map +1 -0
- package/esm/proxy/oauth-client.js +52 -0
- package/esm/proxy/token-manager.d.ts +59 -0
- package/esm/proxy/token-manager.d.ts.map +1 -0
- package/esm/proxy/token-manager.js +125 -0
- package/esm/proxy/tracing.d.ts +39 -0
- package/esm/proxy/tracing.d.ts.map +1 -0
- package/esm/proxy/tracing.js +194 -0
- package/esm/src/cache/backend.d.ts +22 -0
- package/esm/src/cache/backend.d.ts.map +1 -1
- package/esm/src/cache/backend.js +59 -0
- package/esm/src/cache/cache-key-builder.d.ts +0 -4
- package/esm/src/cache/cache-key-builder.d.ts.map +1 -1
- package/esm/src/cache/cache-key-builder.js +0 -6
- package/esm/src/cache/hash.d.ts +107 -0
- package/esm/src/cache/hash.d.ts.map +1 -0
- package/esm/src/cache/hash.js +166 -0
- package/esm/src/cache/index.d.ts +3 -0
- package/esm/src/cache/index.d.ts.map +1 -1
- package/esm/src/cache/index.js +3 -0
- package/esm/src/cache/module-cache.d.ts +82 -0
- package/esm/src/cache/module-cache.d.ts.map +1 -0
- package/esm/src/cache/module-cache.js +214 -0
- package/esm/src/cache/multi-tier.d.ts +148 -0
- package/esm/src/cache/multi-tier.d.ts.map +1 -0
- package/esm/src/cache/multi-tier.js +326 -0
- package/esm/src/cli/app/actions.d.ts +26 -0
- package/esm/src/cli/app/actions.d.ts.map +1 -0
- package/esm/src/cli/app/actions.js +152 -0
- package/esm/src/cli/app/components/inline-input.d.ts +35 -0
- package/esm/src/cli/app/components/inline-input.d.ts.map +1 -0
- package/esm/src/cli/app/components/inline-input.js +220 -0
- package/esm/src/cli/app/components/list-select.d.ts +69 -0
- package/esm/src/cli/app/components/list-select.d.ts.map +1 -0
- package/esm/src/cli/app/components/list-select.js +137 -0
- package/esm/src/cli/app/index.d.ts +45 -0
- package/esm/src/cli/app/index.d.ts.map +1 -0
- package/esm/src/cli/app/index.js +1252 -0
- package/esm/src/cli/app/state.d.ts +122 -0
- package/esm/src/cli/app/state.d.ts.map +1 -0
- package/esm/src/cli/app/state.js +232 -0
- package/esm/src/cli/app/views/dashboard.d.ts +19 -0
- package/esm/src/cli/app/views/dashboard.d.ts.map +1 -0
- package/esm/src/cli/app/views/dashboard.js +178 -0
- package/esm/src/cli/index/command-router.d.ts.map +1 -1
- package/esm/src/cli/index/command-router.js +9 -39
- package/esm/src/cli/index/start-handler.d.ts +3 -0
- package/esm/src/cli/index/start-handler.d.ts.map +1 -0
- package/esm/src/cli/index/start-handler.js +145 -0
- package/esm/src/cli/mcp/index.d.ts +11 -0
- package/esm/src/cli/mcp/index.d.ts.map +1 -0
- package/esm/src/cli/mcp/index.js +10 -0
- package/esm/src/cli/templates/integration-loader.d.ts.map +1 -1
- package/esm/src/cli/templates/integration-loader.js +2 -4
- package/esm/src/middleware/builtin/security/redis-rate-limit.d.ts +2 -0
- package/esm/src/middleware/builtin/security/redis-rate-limit.d.ts.map +1 -1
- package/esm/src/middleware/builtin/security/redis-rate-limit.js +23 -9
- package/esm/src/modules/react-loader/ssr-module-loader/cache/redis.d.ts +10 -0
- package/esm/src/modules/react-loader/ssr-module-loader/cache/redis.d.ts.map +1 -1
- package/esm/src/modules/react-loader/ssr-module-loader/cache/redis.js +30 -42
- package/esm/src/modules/react-loader/ssr-module-loader/loader.d.ts.map +1 -1
- package/esm/src/modules/react-loader/ssr-module-loader/loader.js +148 -20
- package/esm/src/observability/tracing/span-names.d.ts +2 -0
- package/esm/src/observability/tracing/span-names.d.ts.map +1 -1
- package/esm/src/observability/tracing/span-names.js +2 -0
- package/esm/src/platform/adapters/fs/cache/file-cache.d.ts.map +1 -1
- package/esm/src/platform/adapters/fs/cache/file-cache.js +9 -3
- package/esm/src/rendering/orchestrator/module-loader/cache.d.ts +10 -2
- package/esm/src/rendering/orchestrator/module-loader/cache.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/module-loader/cache.js +11 -6
- package/esm/src/rendering/orchestrator/module-loader/index.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/module-loader/index.js +72 -77
- package/esm/src/server/context/cache-invalidation.d.ts.map +1 -1
- package/esm/src/server/context/cache-invalidation.js +4 -0
- package/esm/src/server/handlers/dev/dashboard/api.js +4 -0
- package/esm/src/server/handlers/dev/projects/ui-handler.d.ts.map +1 -1
- package/esm/src/server/handlers/dev/projects/ui-handler.js +6 -0
- package/esm/src/transforms/esm/http-cache.d.ts.map +1 -1
- package/esm/src/transforms/esm/http-cache.js +145 -93
- package/esm/src/transforms/esm/transform-cache.d.ts +25 -0
- package/esm/src/transforms/esm/transform-cache.d.ts.map +1 -1
- package/esm/src/transforms/esm/transform-cache.js +45 -0
- package/esm/src/transforms/mdx/esm-module-loader/module-fetcher/index.d.ts.map +1 -1
- package/esm/src/transforms/mdx/esm-module-loader/module-fetcher/index.js +2 -36
- package/esm/src/utils/constants/cache.d.ts +4 -0
- package/esm/src/utils/constants/cache.d.ts.map +1 -1
- package/esm/src/utils/constants/cache.js +14 -1
- package/esm/src/utils/index.d.ts +1 -1
- package/esm/src/utils/index.d.ts.map +1 -1
- package/esm/src/utils/index.js +1 -1
- package/package.json +2 -1
- package/src/deno.js +1 -1
- package/src/proxy/cache/index.ts +93 -0
- package/src/proxy/cache/memory-cache.ts +120 -0
- package/src/proxy/cache/redis-cache.ts +203 -0
- package/src/proxy/cache/resilient-cache.ts +205 -0
- package/src/proxy/cache/types.ts +72 -0
- package/src/proxy/handler.ts +593 -0
- package/src/proxy/logger.ts +329 -0
- package/src/proxy/oauth-client.ts +91 -0
- package/src/proxy/token-manager.ts +174 -0
- package/src/proxy/tracing.ts +237 -0
- package/src/src/cache/backend.ts +65 -0
- package/src/src/cache/cache-key-builder.ts +0 -9
- package/src/src/cache/hash.ts +205 -0
- package/src/src/cache/index.ts +3 -0
- package/src/src/cache/module-cache.ts +252 -0
- package/src/src/cache/multi-tier.ts +462 -0
- package/src/src/cli/app/actions.ts +190 -0
- package/src/src/cli/app/components/inline-input.ts +255 -0
- package/src/src/cli/app/components/list-select.ts +215 -0
- package/src/src/cli/app/index.ts +1471 -0
- package/src/src/cli/app/state.ts +385 -0
- package/src/src/cli/app/views/dashboard.ts +212 -0
- package/src/src/cli/index/command-router.ts +9 -40
- package/src/src/cli/index/start-handler.ts +195 -0
- package/src/src/cli/mcp/index.ts +11 -0
- package/src/src/cli/templates/integration-loader.ts +2 -8
- package/src/src/middleware/builtin/security/redis-rate-limit.ts +24 -11
- package/src/src/modules/react-loader/ssr-module-loader/cache/redis.ts +36 -50
- package/src/src/modules/react-loader/ssr-module-loader/loader.ts +168 -25
- package/src/src/observability/tracing/span-names.ts +2 -0
- package/src/src/platform/adapters/fs/cache/file-cache.ts +9 -3
- package/src/src/rendering/orchestrator/module-loader/cache.ts +14 -8
- package/src/src/rendering/orchestrator/module-loader/index.ts +94 -89
- package/src/src/server/context/cache-invalidation.ts +4 -0
- package/src/src/server/handlers/dev/dashboard/api.ts +2 -0
- package/src/src/server/handlers/dev/projects/ui-handler.ts +6 -0
- package/src/src/transforms/esm/http-cache.ts +160 -105
- package/src/src/transforms/esm/transform-cache.ts +53 -0
- package/src/src/transforms/mdx/esm-module-loader/module-fetcher/index.ts +2 -40
- package/src/src/utils/constants/cache.ts +21 -1
- package/src/src/utils/index.ts +0 -1
|
@@ -221,7 +221,9 @@ export class FileCache {
|
|
|
221
221
|
const serialized = JSON.stringify(entry);
|
|
222
222
|
// Update request-scoped cache so subsequent reads in same request see the new value
|
|
223
223
|
setInRequestCache(key, serialized);
|
|
224
|
-
backend.set(key, serialized, BACKEND_TTL_SECONDS).catch(() => {
|
|
224
|
+
backend.set(key, serialized, BACKEND_TTL_SECONDS).catch((error) => {
|
|
225
|
+
logger.debug("[FileCache] Backend set failed", { key, error });
|
|
226
|
+
});
|
|
225
227
|
return;
|
|
226
228
|
}
|
|
227
229
|
|
|
@@ -308,7 +310,9 @@ export class FileCache {
|
|
|
308
310
|
|
|
309
311
|
// Fire-and-forget backend deletion
|
|
310
312
|
// Note: prefix already includes "file:" from buildFileCacheKeyPrefix, don't add it again
|
|
311
|
-
cacheBackend?.delByPattern?.(`${prefix}*`).catch(() => {
|
|
313
|
+
cacheBackend?.delByPattern?.(`${prefix}*`).catch((error) => {
|
|
314
|
+
logger.debug("[FileCache] Backend invalidation failed", { prefix, error });
|
|
315
|
+
});
|
|
312
316
|
|
|
313
317
|
return count;
|
|
314
318
|
}
|
|
@@ -346,7 +350,9 @@ export class FileCache {
|
|
|
346
350
|
|
|
347
351
|
// Fire-and-forget backend deletion
|
|
348
352
|
// Note: prefix already includes "file:" from buildFileCacheKeyPrefix, don't add it again
|
|
349
|
-
cacheBackend?.delByPattern?.(`${prefix}*:${suffix}`).catch(() => {
|
|
353
|
+
cacheBackend?.delByPattern?.(`${prefix}*:${suffix}`).catch((error) => {
|
|
354
|
+
logger.debug("[FileCache] Backend invalidation failed", { prefix, suffix, error });
|
|
355
|
+
});
|
|
350
356
|
|
|
351
357
|
return count;
|
|
352
358
|
}
|
|
@@ -1,4 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module Loader Cache Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides hash generation and cache factory functions.
|
|
5
|
+
* Module caches are now pod-level singletons (see src/cache/module-cache.ts)
|
|
6
|
+
* to ensure caches persist across requests within the same pod.
|
|
7
|
+
*
|
|
8
|
+
* @module rendering/orchestrator/module-loader/cache
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
// Re-export pod-level cache factories
|
|
1
12
|
import * as dntShim from "../../../../_dnt.shims.js";
|
|
13
|
+
|
|
14
|
+
export { createEsmCache, createModuleCache } from "../../../cache/module-cache.js";
|
|
15
|
+
|
|
2
16
|
const HEX_CHARS = "0123456789abcdef";
|
|
3
17
|
|
|
4
18
|
export async function generateHash(str: string): Promise<string> {
|
|
@@ -13,11 +27,3 @@ export async function generateHash(str: string): Promise<string> {
|
|
|
13
27
|
}
|
|
14
28
|
return hex;
|
|
15
29
|
}
|
|
16
|
-
|
|
17
|
-
export function createModuleCache(): Map<string, string> {
|
|
18
|
-
return new Map();
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export function createEsmCache(): Map<string, string> {
|
|
22
|
-
return new Map();
|
|
23
|
-
}
|
|
@@ -7,80 +7,52 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { parallelMap, rendererLogger as logger } from "../../../utils/index.js";
|
|
10
|
-
import { Singleflight } from "../../../utils/singleflight.js";
|
|
11
10
|
import type { RuntimeAdapter } from "../../../platform/adapters/base.js";
|
|
12
|
-
import type { CacheBackend } from "../../../cache/backend.js";
|
|
13
11
|
import { getLocalAdapter } from "../../../platform/adapters/registry.js";
|
|
14
12
|
import { generateHash } from "./cache.js";
|
|
15
13
|
import { findLocalLibFile, findSourceFile } from "../file-resolver/index.js";
|
|
16
14
|
import { transformToESM } from "../../../transforms/esm-transform.js";
|
|
17
15
|
import { getProjectTmpDir } from "../../../modules/react-loader/index.js";
|
|
18
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
generateCacheKey as generateTransformCacheKey,
|
|
18
|
+
getOrComputeTransform,
|
|
19
|
+
initializeTransformCache,
|
|
20
|
+
setCachedTransformAsync,
|
|
21
|
+
} from "../../../transforms/esm/transform-cache.js";
|
|
22
|
+
import { hashString } from "../../../cache/hash.js";
|
|
23
|
+
import { TRANSFORM_DISTRIBUTED_TTL_SEC } from "../../../utils/constants/cache.js";
|
|
24
|
+
import { ensureHttpBundlesExist } from "../../../transforms/esm/http-cache.js";
|
|
25
|
+
import { getHttpBundleCacheDir } from "../../../utils/cache-dir.js";
|
|
19
26
|
|
|
20
27
|
// Re-export utilities
|
|
21
28
|
export { createEsmCache, createModuleCache, generateHash } from "./cache.js";
|
|
22
29
|
export { fetchEsmModule, rewriteEsmPaths } from "./esm-rewriter.js";
|
|
23
30
|
|
|
24
|
-
/**
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
let
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (distributedTransformCache !== undefined) {
|
|
39
|
-
return Promise.resolve(distributedTransformCache);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return distributedCacheInit.do("init", async () => {
|
|
43
|
-
try {
|
|
44
|
-
const { CacheBackends } = await import("../../../cache/backend.js");
|
|
45
|
-
const backend = await CacheBackends.transform();
|
|
46
|
-
|
|
47
|
-
// Only use distributed cache if API or Redis (not memory - that's per-process)
|
|
48
|
-
if (backend.type === "memory") {
|
|
49
|
-
distributedTransformCache = null;
|
|
50
|
-
logger.debug("[ModuleLoader] No distributed transform cache (memory only)");
|
|
51
|
-
return null;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
distributedTransformCache = backend;
|
|
55
|
-
logger.debug("[ModuleLoader] Distributed transform cache initialized", {
|
|
56
|
-
type: backend.type,
|
|
57
|
-
});
|
|
58
|
-
return backend;
|
|
59
|
-
} catch (error) {
|
|
60
|
-
logger.debug("[ModuleLoader] Failed to init distributed transform cache", { error });
|
|
61
|
-
distributedTransformCache = null;
|
|
62
|
-
return null;
|
|
31
|
+
/** Pattern to match HTTP bundle file:// paths in transformed code */
|
|
32
|
+
const HTTP_BUNDLE_PATTERN = /file:\/\/([^"'\s]+veryfront-http-bundle\/http-([a-f0-9]+)\.mjs)/gi;
|
|
33
|
+
|
|
34
|
+
/** Extract HTTP bundle paths from transformed code for proactive recovery */
|
|
35
|
+
function extractHttpBundlePaths(code: string): Array<{ path: string; hash: string }> {
|
|
36
|
+
const bundles: Array<{ path: string; hash: string }> = [];
|
|
37
|
+
const seen = new Set<string>();
|
|
38
|
+
let match;
|
|
39
|
+
while ((match = HTTP_BUNDLE_PATTERN.exec(code)) !== null) {
|
|
40
|
+
const path = match[1] as string;
|
|
41
|
+
const hash = match[2] as string;
|
|
42
|
+
if (!seen.has(hash)) {
|
|
43
|
+
seen.add(hash);
|
|
44
|
+
bundles.push({ path, hash });
|
|
63
45
|
}
|
|
64
|
-
}
|
|
46
|
+
}
|
|
47
|
+
HTTP_BUNDLE_PATTERN.lastIndex = 0;
|
|
48
|
+
return bundles;
|
|
65
49
|
}
|
|
66
50
|
|
|
67
|
-
/**
|
|
68
|
-
|
|
69
|
-
* Includes content hash so cache invalidates when source changes.
|
|
70
|
-
*/
|
|
71
|
-
function getTransformCacheKey(projectId: string, filePath: string, contentHash: string): string {
|
|
72
|
-
return `v${TRANSFORM_CACHE_VERSION}:${projectId}:${filePath}:${contentHash}`;
|
|
73
|
-
}
|
|
51
|
+
/** Cache for created directories to avoid repeated mkdir calls */
|
|
52
|
+
const createdDirs = new Set<string>();
|
|
74
53
|
|
|
75
|
-
/**
|
|
76
|
-
|
|
77
|
-
let hash = 0;
|
|
78
|
-
for (let i = 0; i < str.length; i++) {
|
|
79
|
-
hash = ((hash << 5) - hash) + str.charCodeAt(i);
|
|
80
|
-
hash &= hash;
|
|
81
|
-
}
|
|
82
|
-
return Math.abs(hash).toString(36);
|
|
83
|
-
}
|
|
54
|
+
/** TTL for cached transforms (uses centralized config) */
|
|
55
|
+
const TRANSFORM_CACHE_TTL_SECONDS = TRANSFORM_DISTRIBUTED_TTL_SEC;
|
|
84
56
|
|
|
85
57
|
export interface ModuleLoaderConfig {
|
|
86
58
|
projectDir: string;
|
|
@@ -223,39 +195,54 @@ export async function transformModuleWithDeps(
|
|
|
223
195
|
|
|
224
196
|
const contentHash = hashString(fileContent);
|
|
225
197
|
const effectiveProjectId = projectId ?? projectDir;
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
198
|
+
const scopedPath = `${effectiveProjectId}:${filePath}`;
|
|
199
|
+
const transformCacheKey = generateTransformCacheKey(scopedPath, contentHash, true); // ssr=true
|
|
200
|
+
|
|
201
|
+
// Initialize transform cache (lazy, only once per pod)
|
|
202
|
+
await initializeTransformCache();
|
|
203
|
+
|
|
204
|
+
// Use consolidated transform cache with getOrCompute pattern
|
|
205
|
+
let transformedCode = await getOrComputeTransform(
|
|
206
|
+
transformCacheKey,
|
|
207
|
+
() => {
|
|
208
|
+
logger.debug("[ModuleLoader] Transform cache miss, transforming", { filePath });
|
|
209
|
+
return transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
210
|
+
projectId: effectiveProjectId,
|
|
211
|
+
dev: mode === "development",
|
|
212
|
+
ssr: true,
|
|
213
|
+
});
|
|
214
|
+
},
|
|
215
|
+
TRANSFORM_CACHE_TTL_SECONDS,
|
|
216
|
+
);
|
|
230
217
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
218
|
+
// Proactively ensure HTTP bundles exist before writing the module.
|
|
219
|
+
// Cached transforms from a different pod may reference file:// paths
|
|
220
|
+
// to HTTP bundles that don't exist locally.
|
|
221
|
+
const bundlePaths = extractHttpBundlePaths(transformedCode);
|
|
222
|
+
if (bundlePaths.length > 0) {
|
|
223
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
224
|
+
const failed = await ensureHttpBundlesExist(bundlePaths, cacheDir);
|
|
225
|
+
if (failed.length > 0) {
|
|
226
|
+
logger.warn("[ModuleLoader] HTTP bundle recovery failed, re-transforming", {
|
|
227
|
+
filePath,
|
|
228
|
+
failed,
|
|
229
|
+
});
|
|
230
|
+
transformedCode = await transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
231
|
+
projectId: effectiveProjectId,
|
|
232
|
+
dev: mode === "development",
|
|
233
|
+
ssr: true,
|
|
234
|
+
});
|
|
235
|
+
setCachedTransformAsync(
|
|
236
|
+
transformCacheKey,
|
|
237
|
+
transformedCode,
|
|
238
|
+
contentHash,
|
|
239
|
+
TRANSFORM_CACHE_TTL_SECONDS,
|
|
240
|
+
).catch((error) => {
|
|
241
|
+
logger.debug("[ModuleLoader] Failed to update transform cache after re-transform", {
|
|
237
242
|
filePath,
|
|
238
|
-
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
} catch (error) {
|
|
242
|
-
logger.debug("[ModuleLoader] Distributed cache get failed", { filePath, error });
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if (!transformedCode) {
|
|
247
|
-
transformedCode = await transformToESM(fileContent, filePath, projectDir, adapter, {
|
|
248
|
-
projectId: effectiveProjectId,
|
|
249
|
-
dev: mode === "development",
|
|
250
|
-
ssr: true,
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
if (distributedCache) {
|
|
254
|
-
distributedCache
|
|
255
|
-
.set(transformCacheKey, transformedCode, TRANSFORM_CACHE_TTL_SECONDS)
|
|
256
|
-
.catch((error) => {
|
|
257
|
-
logger.debug("[ModuleLoader] Distributed cache set failed", { filePath, error });
|
|
243
|
+
error,
|
|
258
244
|
});
|
|
245
|
+
});
|
|
259
246
|
}
|
|
260
247
|
}
|
|
261
248
|
|
|
@@ -296,6 +283,24 @@ export async function loadModule(filePath: string, config: ModuleLoaderConfig):
|
|
|
296
283
|
try {
|
|
297
284
|
return await import(moduleUrl);
|
|
298
285
|
} catch (error) {
|
|
286
|
+
// If import fails due to missing HTTP bundle, try to recover and retry once
|
|
287
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
288
|
+
const bundleMatch = errorMsg.match(/veryfront-http-bundle\/http-([a-f0-9]+)\.mjs/);
|
|
289
|
+
if (bundleMatch) {
|
|
290
|
+
const hash = bundleMatch[1]!;
|
|
291
|
+
logger.warn("[ModuleLoader] Import failed due to missing HTTP bundle, attempting recovery", {
|
|
292
|
+
filePath,
|
|
293
|
+
hash,
|
|
294
|
+
});
|
|
295
|
+
const { recoverHttpBundleByHash } = await import("../../../transforms/esm/http-cache.js");
|
|
296
|
+
const cacheDir = getHttpBundleCacheDir();
|
|
297
|
+
const recovered = await recoverHttpBundleByHash(hash, cacheDir);
|
|
298
|
+
if (recovered) {
|
|
299
|
+
logger.info("[ModuleLoader] HTTP bundle recovered, retrying import", { hash });
|
|
300
|
+
return await import(`file://${tempFilePath}?t=${Date.now()}&retry=1`);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
299
304
|
logger.error("[ModuleLoader] Failed to import module:", {
|
|
300
305
|
filePath,
|
|
301
306
|
tempFilePath,
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
clearModulePathCache,
|
|
4
4
|
invalidateModulePaths,
|
|
5
5
|
} from "../../transforms/mdx/esm-module-loader/index.js";
|
|
6
|
+
import { clearModuleCacheForProject } from "../../cache/module-cache.js";
|
|
6
7
|
import {
|
|
7
8
|
clearSSRModuleCache,
|
|
8
9
|
clearSSRModuleCacheForProject,
|
|
@@ -63,6 +64,9 @@ export async function invalidateProjectCaches(
|
|
|
63
64
|
});
|
|
64
65
|
if (projectId) {
|
|
65
66
|
clearSSRModuleCacheForProject(projectId);
|
|
67
|
+
// Also clear the pod-level module cache (used by RenderPipeline)
|
|
68
|
+
// This was previously missed, causing stale renders despite SSR module cache clearing
|
|
69
|
+
clearModuleCacheForProject(projectId);
|
|
66
70
|
} else {
|
|
67
71
|
clearSSRModuleCache();
|
|
68
72
|
}
|
|
@@ -360,6 +360,7 @@ async function handleListFiles(req: dntShim.Request, ctx: HandlerContext): Promi
|
|
|
360
360
|
if (!projectDir) return errorResponse("No project directory configured", 500);
|
|
361
361
|
|
|
362
362
|
const relativePath = new URL(req.url).searchParams.get("path") || "";
|
|
363
|
+
if (relativePath.includes("..")) return errorResponse("Invalid path", 400);
|
|
363
364
|
const fullPath = relativePath ? `${projectDir}/${relativePath}` : projectDir;
|
|
364
365
|
|
|
365
366
|
try {
|
|
@@ -394,6 +395,7 @@ async function handleReadFileContent(req: dntShim.Request, ctx: HandlerContext):
|
|
|
394
395
|
|
|
395
396
|
const relativePath = new URL(req.url).searchParams.get("path") || "";
|
|
396
397
|
if (!relativePath) return errorResponse("path parameter is required", 400);
|
|
398
|
+
if (relativePath.includes("..")) return errorResponse("Invalid path", 400);
|
|
397
399
|
|
|
398
400
|
try {
|
|
399
401
|
const content = await adapter.fs.readFile(`${projectDir}/${relativePath}`);
|
|
@@ -83,6 +83,12 @@ export function handleProjectsUI(req: dntShim.Request): Promise<dntShim.Response
|
|
|
83
83
|
"server.dev.projectsUI.handle",
|
|
84
84
|
async () => {
|
|
85
85
|
const relativePath = pathname.replace("/_projects/ui/", "").replace(/\.js$/, "");
|
|
86
|
+
if (relativePath.includes("..")) {
|
|
87
|
+
return new dntShim.Response("Invalid path", {
|
|
88
|
+
status: 400,
|
|
89
|
+
headers: { "Content-Type": "text/plain" },
|
|
90
|
+
});
|
|
91
|
+
}
|
|
86
92
|
const uiDir = getUiDirectory();
|
|
87
93
|
|
|
88
94
|
const module = await readUiSource(uiDir, relativePath);
|
|
@@ -14,7 +14,6 @@ import { isAbsolute, join } from "../../platform/compat/path/index.js";
|
|
|
14
14
|
import { cwd } from "../../platform/compat/process.js";
|
|
15
15
|
import { rendererLogger as logger } from "../../utils/index.js";
|
|
16
16
|
import { simpleHash } from "../../utils/hash-utils.js";
|
|
17
|
-
import { Singleflight } from "../../utils/singleflight.js";
|
|
18
17
|
import { LRUCache } from "../../utils/lru-wrapper.js";
|
|
19
18
|
import { withSpan } from "../../observability/tracing/otlp-setup.js";
|
|
20
19
|
import { SpanNames } from "../../observability/tracing/span-names.js";
|
|
@@ -23,39 +22,20 @@ import type { ImportMapConfig } from "../../modules/import-map/types.js";
|
|
|
23
22
|
import { isDeno } from "../../platform/compat/runtime.js";
|
|
24
23
|
import { getDenoNpmReactMap, getReactImportMap, REACT_VERSION } from "./package-registry.js";
|
|
25
24
|
import { parseImports, replaceSpecifiers } from "./lexer.js";
|
|
26
|
-
import
|
|
25
|
+
import { CacheBackends, createDistributedCacheAccessor } from "../../cache/backend.js";
|
|
26
|
+
import {
|
|
27
|
+
HTTP_MODULE_CACHE_MAX_ENTRIES,
|
|
28
|
+
HTTP_MODULE_DISTRIBUTED_TTL_SEC,
|
|
29
|
+
} from "../../utils/constants/cache.js";
|
|
27
30
|
|
|
28
31
|
/** Lazy-loaded distributed cache backend for cross-pod sharing */
|
|
29
|
-
|
|
30
|
-
|
|
32
|
+
const getDistributedCache = createDistributedCacheAccessor(
|
|
33
|
+
() => CacheBackends.httpModule(),
|
|
34
|
+
"HTTP-CACHE",
|
|
35
|
+
);
|
|
31
36
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return distributedCacheInit.do("init", async () => {
|
|
36
|
-
try {
|
|
37
|
-
const { CacheBackends } = await import("../../cache/backend.js");
|
|
38
|
-
const backend = await CacheBackends.httpModule();
|
|
39
|
-
|
|
40
|
-
if (backend.type === "memory") {
|
|
41
|
-
distributedCache = null;
|
|
42
|
-
logger.debug("[HTTP-CACHE] No distributed cache available (memory only)");
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
distributedCache = backend;
|
|
47
|
-
logger.debug("[HTTP-CACHE] Distributed cache initialized", { type: backend.type });
|
|
48
|
-
return backend;
|
|
49
|
-
} catch (error) {
|
|
50
|
-
logger.debug("[HTTP-CACHE] Failed to initialize distributed cache", { error });
|
|
51
|
-
distributedCache = null;
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** TTL for cached modules in distributed cache (24 hours) */
|
|
58
|
-
const DISTRIBUTED_CACHE_TTL_SECONDS = 86400;
|
|
37
|
+
/** TTL for cached modules in distributed cache (uses centralized config) */
|
|
38
|
+
const DISTRIBUTED_CACHE_TTL_SECONDS = HTTP_MODULE_DISTRIBUTED_TTL_SEC;
|
|
59
39
|
|
|
60
40
|
type CacheOptions = {
|
|
61
41
|
cacheDir: string;
|
|
@@ -64,7 +44,7 @@ type CacheOptions = {
|
|
|
64
44
|
reactVersion?: string;
|
|
65
45
|
};
|
|
66
46
|
|
|
67
|
-
const cachedPaths = new LRUCache<string, string>({ maxEntries:
|
|
47
|
+
const cachedPaths = new LRUCache<string, string>({ maxEntries: HTTP_MODULE_CACHE_MAX_ENTRIES });
|
|
68
48
|
const processingStack = new Set<string>();
|
|
69
49
|
|
|
70
50
|
function ensureAbsoluteDir(path: string): string {
|
|
@@ -300,16 +280,21 @@ async function cacheHttpModule(url: string, options: CacheOptions): Promise<stri
|
|
|
300
280
|
await fs.writeTextFile(cachePath, code);
|
|
301
281
|
|
|
302
282
|
if (distributed) {
|
|
303
|
-
// Store code by URL, by hash (for direct recovery), and URL mapping (for debugging)
|
|
304
|
-
// Storing code by hash enables recovery without needing URL lookup
|
|
283
|
+
// Store code by URL, by hash (for direct recovery), and URL mapping (for debugging).
|
|
284
|
+
// Storing code by hash enables recovery without needing URL lookup.
|
|
285
|
+
// IMPORTANT: await the writes so other pods can recover this bundle immediately.
|
|
286
|
+
// Without await, a transform referencing this bundle could reach Redis before
|
|
287
|
+
// the bundle code does, causing ensureHttpBundlesExist on another pod to miss.
|
|
305
288
|
const hash = simpleHash(normalizedUrl);
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
289
|
+
try {
|
|
290
|
+
await Promise.all([
|
|
291
|
+
distributed.set(normalizedUrl, code, DISTRIBUTED_CACHE_TTL_SECONDS),
|
|
292
|
+
distributed.set(`code:${hash}`, code, DISTRIBUTED_CACHE_TTL_SECONDS),
|
|
293
|
+
distributed.set(`hash:${hash}`, normalizedUrl, DISTRIBUTED_CACHE_TTL_SECONDS),
|
|
294
|
+
]);
|
|
295
|
+
} catch (error) {
|
|
311
296
|
logger.debug("[HTTP-CACHE] Distributed cache set failed", { url: normalizedUrl, error });
|
|
312
|
-
}
|
|
297
|
+
}
|
|
313
298
|
}
|
|
314
299
|
|
|
315
300
|
cachedPaths.set(cacheKey, cachePath);
|
|
@@ -451,6 +436,28 @@ export async function recoverHttpBundleByHash(hash: string, cacheDir: string): P
|
|
|
451
436
|
await fs.mkdir(absoluteCacheDir, { recursive: true });
|
|
452
437
|
await fs.writeTextFile(cachePath, cachedCode);
|
|
453
438
|
logger.info("[HTTP-CACHE] Bundle recovery successful (direct)", { hash, path: cachePath });
|
|
439
|
+
|
|
440
|
+
// Proactively recover transitive deps so the import retry doesn't
|
|
441
|
+
// fail again with a different missing bundle.
|
|
442
|
+
const BUNDLE_RE = /file:\/\/([^"'\s]+veryfront-http-bundle\/http-([a-f0-9]+)\.mjs)/gi;
|
|
443
|
+
const transitiveDeps: Array<{ path: string; hash: string }> = [];
|
|
444
|
+
let m;
|
|
445
|
+
while ((m = BUNDLE_RE.exec(cachedCode)) !== null) {
|
|
446
|
+
const tHash = m[2]!;
|
|
447
|
+
if (tHash !== hash) {
|
|
448
|
+
transitiveDeps.push({
|
|
449
|
+
path: join(absoluteCacheDir, `http-${tHash}.mjs`),
|
|
450
|
+
hash: tHash,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (transitiveDeps.length > 0) {
|
|
455
|
+
logger.info("[HTTP-CACHE] Recovering transitive deps from last-resort recovery", {
|
|
456
|
+
count: transitiveDeps.length,
|
|
457
|
+
});
|
|
458
|
+
await ensureHttpBundlesExist(transitiveDeps, cacheDir);
|
|
459
|
+
}
|
|
460
|
+
|
|
454
461
|
return true;
|
|
455
462
|
}
|
|
456
463
|
|
|
@@ -494,81 +501,129 @@ export async function ensureHttpBundlesExist(
|
|
|
494
501
|
if (bundlePaths.length === 0) return [];
|
|
495
502
|
|
|
496
503
|
const fs = createFileSystem();
|
|
497
|
-
const
|
|
498
|
-
|
|
499
|
-
// Check which bundles exist locally
|
|
500
|
-
const existenceChecks = await Promise.all(
|
|
501
|
-
bundlePaths.map(async ({ path, hash }) => ({
|
|
502
|
-
path,
|
|
503
|
-
hash,
|
|
504
|
-
exists: await exists(path),
|
|
505
|
-
})),
|
|
506
|
-
);
|
|
504
|
+
const absoluteCacheDir = ensureAbsoluteDir(cacheDir);
|
|
507
505
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
506
|
+
// Use [a-f0-9]+ to match both hex and decimal hashes consistently
|
|
507
|
+
const BUNDLE_RE = /file:\/\/([^"'\s]+veryfront-http-bundle\/http-([a-f0-9]+)\.mjs)/gi;
|
|
508
|
+
|
|
509
|
+
const extractBundleRefs = (code: string): Array<{ hash: string }> => {
|
|
510
|
+
const refs: Array<{ hash: string }> = [];
|
|
511
|
+
const dedup = new Set<string>();
|
|
512
|
+
let match;
|
|
513
|
+
while ((match = BUNDLE_RE.exec(code)) !== null) {
|
|
514
|
+
const hash = match[2] as string;
|
|
515
|
+
if (!dedup.has(hash)) {
|
|
516
|
+
dedup.add(hash);
|
|
517
|
+
refs.push({ hash });
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
BUNDLE_RE.lastIndex = 0;
|
|
521
|
+
return refs;
|
|
522
|
+
};
|
|
513
523
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
});
|
|
524
|
+
const pending: Array<{ hash: string }> = bundlePaths.map((b) => ({ hash: b.hash }));
|
|
525
|
+
const seen = new Set<string>();
|
|
526
|
+
const failed = new Set<string>();
|
|
518
527
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
return missing.map((m) => m.hash);
|
|
523
|
-
}
|
|
528
|
+
while (pending.length > 0) {
|
|
529
|
+
const batch = pending.splice(0, pending.length).filter((b) => !seen.has(b.hash));
|
|
530
|
+
if (batch.length === 0) break;
|
|
524
531
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
532
|
+
for (const item of batch) {
|
|
533
|
+
seen.add(item.hash);
|
|
534
|
+
}
|
|
528
535
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
536
|
+
// Check which bundles exist locally using canonical paths derived from
|
|
537
|
+
// cacheDir + hash. Don't trust caller-provided paths — they may reference
|
|
538
|
+
// a different pod's absolute directory.
|
|
539
|
+
const existenceChecks = await Promise.all(
|
|
540
|
+
batch.map(async ({ hash }) => ({
|
|
541
|
+
hash,
|
|
542
|
+
canonicalPath: join(absoluteCacheDir, `http-${hash}.mjs`),
|
|
543
|
+
exists: await exists(join(absoluteCacheDir, `http-${hash}.mjs`)),
|
|
544
|
+
})),
|
|
545
|
+
);
|
|
546
|
+
|
|
547
|
+
const missing = existenceChecks.filter((b) => !b.exists);
|
|
548
|
+
if (missing.length === 0) continue;
|
|
549
|
+
|
|
550
|
+
logger.info("[HTTP-CACHE] Fetching missing bundles from distributed cache", {
|
|
551
|
+
missing: missing.length,
|
|
552
|
+
total: batch.length,
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
const distributed = await getDistributedCache();
|
|
556
|
+
if (!distributed) {
|
|
557
|
+
logger.error("[HTTP-CACHE] No distributed cache available for bundle recovery");
|
|
558
|
+
for (const m of missing) failed.add(m.hash);
|
|
559
|
+
continue;
|
|
538
560
|
}
|
|
539
|
-
} catch (error) {
|
|
540
|
-
logger.error("[HTTP-CACHE] Batch fetch from distributed cache failed", { error });
|
|
541
|
-
return missing.map((m) => m.hash);
|
|
542
|
-
}
|
|
543
561
|
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
missing.map(async ({ path, hash }) => {
|
|
548
|
-
const code = codes.get(`code:${hash}`);
|
|
549
|
-
if (!code) {
|
|
550
|
-
logger.warn("[HTTP-CACHE] Bundle not found in distributed cache", { hash });
|
|
551
|
-
failed.push(hash);
|
|
552
|
-
return;
|
|
553
|
-
}
|
|
562
|
+
// Batch fetch from distributed cache
|
|
563
|
+
const codeKeys = missing.map((m) => `code:${m.hash}`);
|
|
564
|
+
let codes: Map<string, string | null>;
|
|
554
565
|
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
await
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
566
|
+
try {
|
|
567
|
+
if (distributed.getBatch) {
|
|
568
|
+
codes = await distributed.getBatch(codeKeys);
|
|
569
|
+
} else {
|
|
570
|
+
const results = await Promise.all(
|
|
571
|
+
codeKeys.map(async (key) => [key, await distributed.get(key)] as const),
|
|
572
|
+
);
|
|
573
|
+
codes = new Map(results);
|
|
563
574
|
}
|
|
564
|
-
})
|
|
565
|
-
|
|
575
|
+
} catch (error) {
|
|
576
|
+
logger.error("[HTTP-CACHE] Batch fetch from distributed cache failed", { error });
|
|
577
|
+
for (const m of missing) failed.add(m.hash);
|
|
578
|
+
continue;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Write fetched bundles to disk using canonical paths and scan for transitive deps
|
|
582
|
+
await Promise.all(
|
|
583
|
+
missing.map(async ({ hash, canonicalPath }) => {
|
|
584
|
+
const code = codes.get(`code:${hash}`);
|
|
585
|
+
if (!code) {
|
|
586
|
+
// Try single-bundle recovery as last resort
|
|
587
|
+
const recovered = await recoverHttpBundleByHash(hash, absoluteCacheDir);
|
|
588
|
+
if (!recovered) {
|
|
589
|
+
failed.add(hash);
|
|
590
|
+
} else {
|
|
591
|
+
// Read the recovered bundle to scan for transitive deps
|
|
592
|
+
try {
|
|
593
|
+
const recoveredCode = await fs.readTextFile(canonicalPath);
|
|
594
|
+
for (const ref of extractBundleRefs(recoveredCode)) {
|
|
595
|
+
if (!seen.has(ref.hash)) pending.push(ref);
|
|
596
|
+
}
|
|
597
|
+
} catch { /* ignore read errors for dep scanning */ }
|
|
598
|
+
}
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
try {
|
|
603
|
+
await fs.mkdir(absoluteCacheDir, { recursive: true });
|
|
604
|
+
await fs.writeTextFile(canonicalPath, code);
|
|
605
|
+
logger.debug("[HTTP-CACHE] Wrote bundle to disk", { hash, path: canonicalPath });
|
|
606
|
+
|
|
607
|
+
// Scan recovered code for transitive HTTP bundle dependencies.
|
|
608
|
+
// HTTP bundles import other bundles (e.g., esm.sh packages depending
|
|
609
|
+
// on other packages). Without this, Pod B recovers only the top-level
|
|
610
|
+
// bundle and gets "Module not found" for transitive deps at import time.
|
|
611
|
+
for (const ref of extractBundleRefs(code)) {
|
|
612
|
+
if (!seen.has(ref.hash)) pending.push(ref);
|
|
613
|
+
}
|
|
614
|
+
} catch (error) {
|
|
615
|
+
logger.error("[HTTP-CACHE] Failed to write bundle to disk", { hash, error });
|
|
616
|
+
failed.add(hash);
|
|
617
|
+
}
|
|
618
|
+
}),
|
|
619
|
+
);
|
|
620
|
+
}
|
|
566
621
|
|
|
567
|
-
if (failed.
|
|
568
|
-
logger.warn("[HTTP-CACHE] Some bundles could not be recovered", {
|
|
569
|
-
|
|
570
|
-
|
|
622
|
+
if (failed.size > 0) {
|
|
623
|
+
logger.warn("[HTTP-CACHE] Some bundles could not be recovered", {
|
|
624
|
+
failed: Array.from(failed),
|
|
625
|
+
});
|
|
571
626
|
}
|
|
572
627
|
|
|
573
|
-
return failed;
|
|
628
|
+
return Array.from(failed);
|
|
574
629
|
}
|