veryfront 0.1.74 → 0.1.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/esm/cli/commands/files/command.d.ts +3 -3
- package/esm/cli/commands/knowledge/command.d.ts +2 -0
- package/esm/cli/commands/knowledge/command.d.ts.map +1 -1
- package/esm/cli/commands/knowledge/command.js +64 -1
- package/esm/deno.d.ts +7 -0
- package/esm/deno.js +13 -6
- package/esm/src/data/data-fetcher.d.ts +11 -1
- package/esm/src/data/data-fetcher.d.ts.map +1 -1
- package/esm/src/data/data-fetcher.js +5 -2
- package/esm/src/data/index.d.ts +1 -1
- package/esm/src/data/index.d.ts.map +1 -1
- package/esm/src/data/server-data-fetcher.d.ts +14 -1
- package/esm/src/data/server-data-fetcher.d.ts.map +1 -1
- package/esm/src/data/server-data-fetcher.js +65 -3
- package/esm/src/jobs/index.d.ts +34 -0
- package/esm/src/jobs/index.d.ts.map +1 -0
- package/esm/src/jobs/index.js +33 -0
- package/esm/src/jobs/jobs-client.d.ts +134 -0
- package/esm/src/jobs/jobs-client.d.ts.map +1 -0
- package/esm/src/jobs/jobs-client.js +218 -0
- package/esm/src/jobs/schemas.d.ts +1304 -0
- package/esm/src/jobs/schemas.d.ts.map +1 -0
- package/esm/src/jobs/schemas.js +159 -0
- package/esm/src/platform/adapters/veryfront-api-client/retry-handler.d.ts +4 -0
- package/esm/src/platform/adapters/veryfront-api-client/retry-handler.d.ts.map +1 -1
- package/esm/src/platform/adapters/veryfront-api-client/retry-handler.js +12 -6
- package/esm/src/proxy/handler.d.ts.map +1 -1
- package/esm/src/proxy/handler.js +21 -21
- package/esm/src/rendering/orchestrator/lifecycle.d.ts +4 -0
- package/esm/src/rendering/orchestrator/lifecycle.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/lifecycle.js +8 -0
- package/esm/src/rendering/orchestrator/pipeline.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/pipeline.js +6 -1
- package/esm/src/rendering/orchestrator/ssr-orchestrator.d.ts +26 -1
- package/esm/src/rendering/orchestrator/ssr-orchestrator.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/ssr-orchestrator.js +77 -1
- package/esm/src/routing/api/handler.d.ts.map +1 -1
- package/esm/src/routing/api/handler.js +6 -2
- package/esm/src/routing/api/route-executor.d.ts +8 -2
- package/esm/src/routing/api/route-executor.d.ts.map +1 -1
- package/esm/src/routing/api/route-executor.js +158 -3
- package/esm/src/security/deno-permissions.d.ts +7 -1
- package/esm/src/security/deno-permissions.d.ts.map +1 -1
- package/esm/src/security/deno-permissions.js +12 -1
- package/esm/src/security/sandbox/project-worker.d.ts +61 -0
- package/esm/src/security/sandbox/project-worker.d.ts.map +1 -0
- package/esm/src/security/sandbox/project-worker.js +318 -0
- package/esm/src/security/sandbox/worker-permissions.d.ts +30 -0
- package/esm/src/security/sandbox/worker-permissions.d.ts.map +1 -0
- package/esm/src/security/sandbox/worker-permissions.js +63 -0
- package/esm/src/security/sandbox/worker-pool.d.ts +87 -0
- package/esm/src/security/sandbox/worker-pool.d.ts.map +1 -0
- package/esm/src/security/sandbox/worker-pool.js +359 -0
- package/esm/src/security/sandbox/worker-types.d.ts +167 -0
- package/esm/src/security/sandbox/worker-types.d.ts.map +1 -0
- package/esm/src/security/sandbox/worker-types.js +19 -0
- package/esm/src/server/handlers/request/internal-tasks-list.handler.d.ts +11 -0
- package/esm/src/server/handlers/request/internal-tasks-list.handler.d.ts.map +1 -0
- package/esm/src/server/handlers/request/internal-tasks-list.handler.js +72 -0
- package/esm/src/server/project-env/storage.d.ts +6 -0
- package/esm/src/server/project-env/storage.d.ts.map +1 -1
- package/esm/src/server/project-env/storage.js +8 -0
- package/esm/src/server/runtime-handler/index.d.ts +1 -1
- package/esm/src/server/runtime-handler/index.d.ts.map +1 -1
- package/esm/src/server/runtime-handler/index.js +3 -0
- package/esm/src/server/runtime-handler/project-isolation.d.ts +5 -0
- package/esm/src/server/runtime-handler/project-isolation.d.ts.map +1 -1
- package/esm/src/server/runtime-handler/project-isolation.js +44 -0
- package/esm/src/server/shared/renderer/memory/pressure.d.ts +7 -0
- package/esm/src/server/shared/renderer/memory/pressure.d.ts.map +1 -1
- package/esm/src/server/shared/renderer/memory/pressure.js +7 -0
- package/esm/src/task/control-plane.d.ts +105 -0
- package/esm/src/task/control-plane.d.ts.map +1 -0
- package/esm/src/task/control-plane.js +52 -0
- package/esm/src/task/types.d.ts +6 -0
- package/esm/src/task/types.d.ts.map +1 -1
- package/esm/src/transforms/pipeline/stages/ssr-vf-modules/path-resolver.d.ts +4 -4
- package/esm/src/transforms/pipeline/stages/ssr-vf-modules/path-resolver.d.ts.map +1 -1
- package/esm/src/transforms/pipeline/stages/ssr-vf-modules/path-resolver.js +15 -15
- package/esm/src/utils/index.d.ts +10 -1
- package/esm/src/utils/index.d.ts.map +1 -1
- package/esm/src/utils/index.js +9 -1
- package/esm/src/utils/logger/index.d.ts +1 -1
- package/esm/src/utils/logger/index.d.ts.map +1 -1
- package/esm/src/utils/logger/index.js +1 -1
- package/esm/src/utils/logger/logger.d.ts +14 -0
- package/esm/src/utils/logger/logger.d.ts.map +1 -1
- package/esm/src/utils/logger/logger.js +17 -0
- package/esm/src/workflow/claude-code/tool.d.ts +5 -5
- package/package.json +8 -1
- package/src/cli/commands/knowledge/command.ts +76 -1
- package/src/deno.js +13 -6
- package/src/src/data/data-fetcher.ts +18 -2
- package/src/src/data/index.ts +1 -1
- package/src/src/data/server-data-fetcher.ts +106 -3
- package/src/src/jobs/index.ts +85 -0
- package/src/src/jobs/jobs-client.ts +503 -0
- package/src/src/jobs/schemas.ts +202 -0
- package/src/src/platform/adapters/veryfront-api-client/retry-handler.ts +15 -6
- package/src/src/proxy/handler.ts +27 -19
- package/src/src/rendering/orchestrator/lifecycle.ts +11 -0
- package/src/src/rendering/orchestrator/pipeline.ts +7 -2
- package/src/src/rendering/orchestrator/ssr-orchestrator.ts +119 -0
- package/src/src/routing/api/handler.ts +16 -3
- package/src/src/routing/api/route-executor.ts +258 -1
- package/src/src/security/deno-permissions.ts +13 -1
- package/src/src/security/sandbox/project-worker.ts +416 -0
- package/src/src/security/sandbox/worker-permissions.ts +77 -0
- package/src/src/security/sandbox/worker-pool.ts +459 -0
- package/src/src/security/sandbox/worker-types.ts +212 -0
- package/src/src/server/handlers/request/internal-tasks-list.handler.ts +103 -0
- package/src/src/server/project-env/storage.ts +9 -0
- package/src/src/server/runtime-handler/index.ts +3 -0
- package/src/src/server/runtime-handler/project-isolation.ts +53 -0
- package/src/src/server/shared/renderer/memory/pressure.ts +8 -0
- package/src/src/task/control-plane.ts +76 -0
- package/src/src/task/types.ts +6 -0
- package/src/src/transforms/pipeline/stages/ssr-vf-modules/path-resolver.ts +18 -12
- package/src/src/utils/index.ts +11 -0
- package/src/src/utils/logger/index.ts +1 -0
- package/src/src/utils/logger/logger.ts +34 -0
package/README.md
CHANGED
|
@@ -18,6 +18,8 @@ Purpose-built for TypeScript and React, Veryfront gives you everything you need
|
|
|
18
18
|
|
|
19
19
|
- [**Workflows**](https://veryfront.com/docs/code/guides/workflows) — Orchestrate multi-step AI pipelines with branching, parallelism, and human-in-the-loop approval gates.
|
|
20
20
|
|
|
21
|
+
- [**Jobs & Cron Jobs**](https://veryfront.com/docs/code/guides/jobs) — Run durable project-scoped background work now or on a schedule through the Veryfront platform.
|
|
22
|
+
|
|
21
23
|
- [**Multi-Agent**](https://veryfront.com/docs/code/guides/multi-agent) — Compose agents that delegate to each other as tools for complex, coordinated tasks.
|
|
22
24
|
|
|
23
25
|
- [**Memory & Streaming**](https://veryfront.com/docs/code/guides/memory-and-streaming) — Give agents conversation history and streaming responses. Built-in chat UI components for React.
|
|
@@ -31,8 +31,8 @@ declare const FilesGetArgsSchema: z.ZodObject<{
|
|
|
31
31
|
quiet: z.ZodDefault<z.ZodBoolean>;
|
|
32
32
|
}, "strip", z.ZodTypeAny, {
|
|
33
33
|
json: boolean;
|
|
34
|
-
quiet: boolean;
|
|
35
34
|
remotePath: string;
|
|
35
|
+
quiet: boolean;
|
|
36
36
|
projectSlug?: string | undefined;
|
|
37
37
|
projectDir?: string | undefined;
|
|
38
38
|
output?: string | undefined;
|
|
@@ -54,8 +54,8 @@ declare const FilesPutArgsSchema: z.ZodObject<{
|
|
|
54
54
|
}, "strip", z.ZodTypeAny, {
|
|
55
55
|
json: boolean;
|
|
56
56
|
from: string;
|
|
57
|
-
quiet: boolean;
|
|
58
57
|
remotePath: string;
|
|
58
|
+
quiet: boolean;
|
|
59
59
|
projectSlug?: string | undefined;
|
|
60
60
|
projectDir?: string | undefined;
|
|
61
61
|
}, {
|
|
@@ -74,8 +74,8 @@ declare const FilesDeleteArgsSchema: z.ZodObject<{
|
|
|
74
74
|
quiet: z.ZodDefault<z.ZodBoolean>;
|
|
75
75
|
}, "strip", z.ZodTypeAny, {
|
|
76
76
|
json: boolean;
|
|
77
|
-
quiet: boolean;
|
|
78
77
|
remotePath: string;
|
|
78
|
+
quiet: boolean;
|
|
79
79
|
projectSlug?: string | undefined;
|
|
80
80
|
projectDir?: string | undefined;
|
|
81
81
|
}, {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { type ApiClient } from "../../shared/config.js";
|
|
3
3
|
import type { ParsedArgs } from "../../shared/types.js";
|
|
4
|
+
import { type Logger } from "../../../src/utils/index.js";
|
|
4
5
|
export interface KnowledgeParserResult {
|
|
5
6
|
success: true;
|
|
6
7
|
source_path: string;
|
|
@@ -143,6 +144,7 @@ export declare function ingestResolvedSources(sources: KnowledgeSource[], option
|
|
|
143
144
|
uploadKnowledgeFile: (remotePath: string, localPath: string) => Promise<{
|
|
144
145
|
path: string;
|
|
145
146
|
}>;
|
|
147
|
+
eventLogger?: Logger | null;
|
|
146
148
|
}): Promise<KnowledgeIngestFileResult[]>;
|
|
147
149
|
export declare function knowledgeCommand(args: ParsedArgs): Promise<void>;
|
|
148
150
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/knowledge/command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAAE,KAAK,SAAS,EAA0C,MAAM,wBAAwB,CAAC;AAChG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/knowledge/command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAAE,KAAK,SAAS,EAA0C,MAAM,wBAAwB,CAAC;AAChG,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAuB,KAAK,MAAM,EAAgB,MAAM,6BAA6B,CAAC;AAkB7F,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,IAAI,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,KAAK,eAAe,GAChB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7E,KAAK,cAAc,GAAG;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAIhF,QAAA,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmD7B,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAoD/E,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,UAAU,GACf,CAAC,CAAC,mBAAmB,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAexD;AAED,wBAAgB,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAMrE;AAED,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAGpE;AAED,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtE;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAGxD;AAoBD,wBAAgB,iCAAiC,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAE3E;AAgED,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,GACpB,MAAM,CAQR;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,IAAI,CAAC,qBAAqB,EAAE,MAAM,GAAG,OAAO,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,CAAC,CAAC;CAChG,GAAG,yBAAyB,CAY5B;AAED,wBAAsB,kBAAkB,CAAC,KAAK,EAAE;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA8CjC;AAED,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,WAAW,CAAC,EAC/E,IAAI,EAAE;IACJ,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;CACvE,GACA,OAAO,CAAC,eAAe,EAAE,CAAC,CA0G5B;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,eAAe,EAAE,EAC1B,OAAO,EAAE,sBAAsB,EAC/B,IAAI,EAAE;IACJ,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,kBAAkB,CAAC;IACrC,mBAAmB,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1F,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,GACA,OAAO,CAAC,yBAAyB,EAAE,CAAC,CA6DtC;AAED,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CA+FtE"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
import { createFileSystem } from "../../../src/platform/index.js";
|
|
3
|
+
import { createFileSystem, getEnv } from "../../../src/platform/index.js";
|
|
4
4
|
import { basename, extname, join, normalize, relative } from "../../../src/platform/compat/path/index.js";
|
|
5
5
|
import { withSpan } from "../../../src/observability/tracing/otlp-setup.js";
|
|
6
6
|
import { cliLogger } from "../../utils/index.js";
|
|
@@ -8,6 +8,7 @@ import { createApiClient, resolveConfigWithAuth } from "../../shared/config.js";
|
|
|
8
8
|
import { downloadUploadToFile, listAllUploads } from "../uploads/command.js";
|
|
9
9
|
import { putRemoteFileFromLocal } from "../files/command.js";
|
|
10
10
|
import { knowledgeIngestPythonSource } from "./parser-source.js";
|
|
11
|
+
import { createJobUserLogger, serverLogger } from "../../../src/utils/index.js";
|
|
11
12
|
const SUPPORTED_EXTENSIONS = new Set([
|
|
12
13
|
".pdf",
|
|
13
14
|
".csv",
|
|
@@ -23,6 +24,7 @@ const SUPPORTED_EXTENSIONS = new Set([
|
|
|
23
24
|
".md",
|
|
24
25
|
".mdx",
|
|
25
26
|
]);
|
|
27
|
+
const knowledgeJobLogger = serverLogger.component("knowledge-ingest");
|
|
26
28
|
const KnowledgeIngestArgsSchema = z.object({
|
|
27
29
|
projectSlug: z.string().optional(),
|
|
28
30
|
projectDir: z.string().optional(),
|
|
@@ -84,6 +86,23 @@ function getBooleanArg(args, ...keys) {
|
|
|
84
86
|
function printJson(value) {
|
|
85
87
|
console.log(JSON.stringify(value, null, 2));
|
|
86
88
|
}
|
|
89
|
+
function createKnowledgeIngestEventLogger() {
|
|
90
|
+
const projectId = getEnv("TENANT_PROJECT_ID");
|
|
91
|
+
const jobId = getEnv("JOB_ID");
|
|
92
|
+
if (!projectId || !jobId) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
return createJobUserLogger(knowledgeJobLogger, {
|
|
96
|
+
projectId,
|
|
97
|
+
jobId,
|
|
98
|
+
batchId: getEnv("JOB_BATCH_ID") ?? undefined,
|
|
99
|
+
jobTarget: getEnv("JOB_TARGET") ?? undefined,
|
|
100
|
+
task: "knowledge-ingest",
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
function buildKnowledgeSourceName(source) {
|
|
104
|
+
return basename(source.kind === "upload" ? source.uploadPath : source.localPath);
|
|
105
|
+
}
|
|
87
106
|
function showKnowledgeUsage() {
|
|
88
107
|
console.log(`
|
|
89
108
|
Veryfront Knowledge
|
|
@@ -367,6 +386,12 @@ export async function ingestResolvedSources(sources, options, deps) {
|
|
|
367
386
|
const slugs = options.slug ? [options.slug] : ensureUniqueSlugs(sources);
|
|
368
387
|
const results = [];
|
|
369
388
|
for (const [index, source] of sources.entries()) {
|
|
389
|
+
deps.eventLogger?.info("Processing knowledge source", {
|
|
390
|
+
phase: "file_processing",
|
|
391
|
+
progress_current: index + 1,
|
|
392
|
+
progress_total: sources.length,
|
|
393
|
+
source_name: buildKnowledgeSourceName(source),
|
|
394
|
+
});
|
|
370
395
|
const parser = await deps.runParser({
|
|
371
396
|
filePath: source.localPath,
|
|
372
397
|
outputDir: deps.outputDir,
|
|
@@ -376,6 +401,23 @@ export async function ingestResolvedSources(sources, options, deps) {
|
|
|
376
401
|
});
|
|
377
402
|
const remotePath = deriveKnowledgeRemotePath(parser.sandbox_output_path, deps.outputDir, options.knowledgePath);
|
|
378
403
|
const uploaded = await deps.uploadKnowledgeFile(remotePath, parser.sandbox_output_path);
|
|
404
|
+
deps.eventLogger?.info("Knowledge source ingested", {
|
|
405
|
+
phase: "file_completed",
|
|
406
|
+
progress_current: index + 1,
|
|
407
|
+
progress_total: sources.length,
|
|
408
|
+
source_name: buildKnowledgeSourceName(source),
|
|
409
|
+
remote_path: uploaded.path,
|
|
410
|
+
warning_count: parser.warnings.length,
|
|
411
|
+
});
|
|
412
|
+
if (parser.warnings.length > 0) {
|
|
413
|
+
deps.eventLogger?.warn("Knowledge source emitted warnings", {
|
|
414
|
+
phase: "file_warning",
|
|
415
|
+
progress_current: index + 1,
|
|
416
|
+
progress_total: sources.length,
|
|
417
|
+
source_name: buildKnowledgeSourceName(source),
|
|
418
|
+
warning_count: parser.warnings.length,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
379
421
|
results.push(createKnowledgeIngestResult({
|
|
380
422
|
source: buildSourceReference(source),
|
|
381
423
|
localSourcePath: source.localPath,
|
|
@@ -407,19 +449,34 @@ export async function knowledgeCommand(args) {
|
|
|
407
449
|
const outputDir = options.outputDir ?? await defaultOutputRoot();
|
|
408
450
|
const shouldCleanupOutputDir = options.outputDir === undefined;
|
|
409
451
|
const downloadOutputDir = resolveKnowledgeDownloadOutputDir(outputDir);
|
|
452
|
+
const eventLogger = createKnowledgeIngestEventLogger();
|
|
410
453
|
try {
|
|
454
|
+
eventLogger?.info("Starting knowledge ingest", {
|
|
455
|
+
phase: "started",
|
|
456
|
+
mode: options.path ? "path_prefix" : "explicit_sources",
|
|
457
|
+
});
|
|
411
458
|
const sources = await collectKnowledgeSources(options, {
|
|
412
459
|
client,
|
|
413
460
|
projectSlug: config.projectSlug,
|
|
414
461
|
downloadUploads: (uploadPaths) => Promise.all(uploadPaths.map((uploadPath) => downloadUploadToFile(client, config.projectSlug, uploadPath, downloadOutputDir))),
|
|
415
462
|
});
|
|
463
|
+
eventLogger?.info("Resolved knowledge sources", {
|
|
464
|
+
phase: "sources_resolved",
|
|
465
|
+
progress_total: sources.length,
|
|
466
|
+
});
|
|
416
467
|
const results = await ingestResolvedSources(sources, options, {
|
|
417
468
|
client,
|
|
418
469
|
projectSlug: config.projectSlug,
|
|
419
470
|
outputDir,
|
|
420
471
|
runParser: runKnowledgeParser,
|
|
472
|
+
eventLogger,
|
|
421
473
|
uploadKnowledgeFile: (remotePath, localPath) => putRemoteFileFromLocal(client, config.projectSlug, remotePath, localPath),
|
|
422
474
|
});
|
|
475
|
+
eventLogger?.info("Completed knowledge ingest", {
|
|
476
|
+
phase: "completed",
|
|
477
|
+
progress_current: results.length,
|
|
478
|
+
progress_total: results.length,
|
|
479
|
+
});
|
|
423
480
|
if (options.json) {
|
|
424
481
|
printJson(results);
|
|
425
482
|
return;
|
|
@@ -431,6 +488,12 @@ export async function knowledgeCommand(args) {
|
|
|
431
488
|
}
|
|
432
489
|
}
|
|
433
490
|
}
|
|
491
|
+
catch (error) {
|
|
492
|
+
eventLogger?.error("Knowledge ingest failed", {
|
|
493
|
+
phase: "failed",
|
|
494
|
+
});
|
|
495
|
+
throw error;
|
|
496
|
+
}
|
|
434
497
|
finally {
|
|
435
498
|
if (shouldCleanupOutputDir) {
|
|
436
499
|
await Promise.all([
|
package/esm/deno.d.ts
CHANGED
|
@@ -22,8 +22,10 @@ declare namespace _default {
|
|
|
22
22
|
"./workflow/discovery": string;
|
|
23
23
|
"./prompt": string;
|
|
24
24
|
"./resource": string;
|
|
25
|
+
"./jobs": string;
|
|
25
26
|
"./mcp": string;
|
|
26
27
|
"./middleware": string;
|
|
28
|
+
"./utils": string;
|
|
27
29
|
"./oauth": string;
|
|
28
30
|
"./provider": string;
|
|
29
31
|
"./fs": string;
|
|
@@ -50,6 +52,7 @@ declare namespace _default {
|
|
|
50
52
|
"veryfront/components/ai": string;
|
|
51
53
|
"veryfront/sandbox": string;
|
|
52
54
|
"veryfront/embedding": string;
|
|
55
|
+
"veryfront/jobs": string;
|
|
53
56
|
"veryfront/agent/react": string;
|
|
54
57
|
"veryfront/agent/testing": string;
|
|
55
58
|
"veryfront/agent/middleware": string;
|
|
@@ -61,6 +64,7 @@ declare namespace _default {
|
|
|
61
64
|
"veryfront/workflow/claude-code": string;
|
|
62
65
|
"veryfront/workflow/claude-code/react": string;
|
|
63
66
|
"veryfront/workflow/discovery": string;
|
|
67
|
+
"veryfront/utils": string;
|
|
64
68
|
"veryfront/utils/box": string;
|
|
65
69
|
"veryfront/utils/case-utils": string;
|
|
66
70
|
"veryfront/utils/constants/server": string;
|
|
@@ -122,6 +126,7 @@ declare namespace _default {
|
|
|
122
126
|
"#veryfront/react": string;
|
|
123
127
|
"#veryfront/rendering": string;
|
|
124
128
|
"#veryfront/resource": string;
|
|
129
|
+
"#veryfront/jobs": string;
|
|
125
130
|
"#veryfront/routing": string;
|
|
126
131
|
"#veryfront/security": string;
|
|
127
132
|
"#veryfront/skill": string;
|
|
@@ -241,6 +246,7 @@ declare namespace _default {
|
|
|
241
246
|
"tailwindcss/colors": string;
|
|
242
247
|
redis: string;
|
|
243
248
|
pg: string;
|
|
249
|
+
jose: string;
|
|
244
250
|
"@opentelemetry/api": string;
|
|
245
251
|
"@opentelemetry/core": string;
|
|
246
252
|
"@opentelemetry/context-async-hooks": string;
|
|
@@ -289,6 +295,7 @@ declare namespace _default {
|
|
|
289
295
|
"test:coverage:integration": string;
|
|
290
296
|
"coverage:report": string;
|
|
291
297
|
"coverage:html": string;
|
|
298
|
+
bench: string;
|
|
292
299
|
clean: string;
|
|
293
300
|
lint: string;
|
|
294
301
|
fmt: string;
|
package/esm/deno.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export default {
|
|
2
2
|
"name": "veryfront",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.76",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"nodeModulesDir": "auto",
|
|
6
6
|
"exclude": [
|
|
@@ -32,8 +32,10 @@ export default {
|
|
|
32
32
|
"./workflow/discovery": "./src/workflow/discovery/index.ts",
|
|
33
33
|
"./prompt": "./src/prompt/index.ts",
|
|
34
34
|
"./resource": "./src/resource/index.ts",
|
|
35
|
+
"./jobs": "./src/jobs/index.ts",
|
|
35
36
|
"./mcp": "./src/mcp/index.ts",
|
|
36
37
|
"./middleware": "./src/middleware/index.ts",
|
|
38
|
+
"./utils": "./src/utils/index.ts",
|
|
37
39
|
"./oauth": "./src/oauth/index.ts",
|
|
38
40
|
"./provider": "./src/provider/index.ts",
|
|
39
41
|
"./fs": "./src/fs/index.ts",
|
|
@@ -60,6 +62,7 @@ export default {
|
|
|
60
62
|
"veryfront/components/ai": "./src/react/components/ai/index.ts",
|
|
61
63
|
"veryfront/sandbox": "./src/sandbox/index.ts",
|
|
62
64
|
"veryfront/embedding": "./src/embedding/index.ts",
|
|
65
|
+
"veryfront/jobs": "./src/jobs/index.ts",
|
|
63
66
|
"veryfront/agent/react": "./src/agent/react/index.ts",
|
|
64
67
|
"veryfront/agent/testing": "./src/agent/testing/index.ts",
|
|
65
68
|
"veryfront/agent/middleware": "./src/agent/middleware/index.ts",
|
|
@@ -71,6 +74,7 @@ export default {
|
|
|
71
74
|
"veryfront/workflow/claude-code": "./src/workflow/claude-code/index.ts",
|
|
72
75
|
"veryfront/workflow/claude-code/react": "./src/workflow/claude-code/react/index.ts",
|
|
73
76
|
"veryfront/workflow/discovery": "./src/workflow/discovery/index.ts",
|
|
77
|
+
"veryfront/utils": "./src/utils/index.ts",
|
|
74
78
|
"veryfront/utils/box": "./src/utils/box.ts",
|
|
75
79
|
"veryfront/utils/case-utils": "./src/utils/case-utils.ts",
|
|
76
80
|
"veryfront/utils/constants/server": "./src/utils/constants/server.ts",
|
|
@@ -132,6 +136,7 @@ export default {
|
|
|
132
136
|
"#veryfront/react": "./src/react/index.ts",
|
|
133
137
|
"#veryfront/rendering": "./src/rendering/index.ts",
|
|
134
138
|
"#veryfront/resource": "./src/resource/index.ts",
|
|
139
|
+
"#veryfront/jobs": "./src/jobs/index.ts",
|
|
135
140
|
"#veryfront/routing": "./src/routing/index.ts",
|
|
136
141
|
"#veryfront/security": "./src/security/index.ts",
|
|
137
142
|
"#veryfront/skill": "./src/skill/index.ts",
|
|
@@ -251,6 +256,7 @@ export default {
|
|
|
251
256
|
"tailwindcss/colors": "https://esm.sh/tailwindcss@4.1.8/colors",
|
|
252
257
|
"redis": "npm:redis@4.6.13",
|
|
253
258
|
"pg": "npm:pg",
|
|
259
|
+
"jose": "npm:jose@5.9.6",
|
|
254
260
|
"@opentelemetry/api": "npm:@opentelemetry/api@1",
|
|
255
261
|
"@opentelemetry/core": "npm:@opentelemetry/core@1",
|
|
256
262
|
"@opentelemetry/context-async-hooks": "npm:@opentelemetry/context-async-hooks@1",
|
|
@@ -288,11 +294,11 @@ export default {
|
|
|
288
294
|
"tasks": {
|
|
289
295
|
"setup": "deno run --allow-all scripts/setup.ts",
|
|
290
296
|
"generate": "deno run -A scripts/build/generate-templates-manifest.ts && deno run -A scripts/build/generate-dev-ui-manifest.ts && deno run -A scripts/build/prebundle-client-scripts.ts && deno run -A scripts/build/prebundle-bridge.ts",
|
|
291
|
-
"start": "deno task generate && deno run --allow-
|
|
292
|
-
"start:headless": "deno task generate && deno run --allow-
|
|
293
|
-
"proxy": "deno task generate && deno run --allow-
|
|
297
|
+
"start": "deno task generate && deno run --allow-read --allow-write --allow-net --allow-env --allow-run --allow-sys --unstable-worker-options --unstable-net cli/main.ts",
|
|
298
|
+
"start:headless": "deno task generate && deno run --allow-read --allow-write --allow-net --allow-env --allow-run --allow-sys --unstable-worker-options --unstable-net cli/main.ts --headless",
|
|
299
|
+
"proxy": "deno task generate && deno run --allow-read --allow-write --allow-net --allow-env --allow-run --allow-sys --unstable-worker-options --unstable-net cli/main.ts serve --mode=proxy",
|
|
294
300
|
"dev": "deno task generate && deno run --allow-all cli/main.ts dev",
|
|
295
|
-
"production": "deno task generate && deno run --allow-
|
|
301
|
+
"production": "deno task generate && deno run --allow-read --allow-write --allow-net --allow-env --allow-run --allow-sys --unstable-worker-options --unstable-net cli/main.ts serve --mode=production",
|
|
296
302
|
"build:prepare": "deno run -A scripts/build/generate-integrations-module.ts && deno task generate && deno run -A scripts/build/prepare-framework-sources.ts",
|
|
297
303
|
"build": "deno task build:prepare && deno run -A scripts/build/compile-binary.ts --output ./bin/veryfront",
|
|
298
304
|
"build:npm": "deno run -A scripts/build/generate-integrations-module.ts && deno task generate && deno run -A scripts/build/build-npm-dnt.ts",
|
|
@@ -307,11 +313,12 @@ export default {
|
|
|
307
313
|
"test:coverage:integration": "rm -rf coverage && VF_DISABLE_LRU_INTERVAL=1 SSR_TRANSFORM_PER_PROJECT_LIMIT=0 REVALIDATION_PER_PROJECT_LIMIT=0 NODE_ENV=production LOG_FORMAT=text deno test --no-check --parallel --fail-fast --allow-all --coverage=coverage '--ignore=tests/e2e,tests/integration/compiled-binary-e2e.test.ts' tests --unstable-worker-options --unstable-net || exit 1",
|
|
308
314
|
"coverage:report": "deno coverage coverage --include=src/ --exclude=tests --exclude=src/**/*_test.ts --exclude=src/**/*_test.tsx --exclude=src/**/*.test.ts --exclude=src/**/*.test.tsx --lcov > coverage/lcov.info && deno run --allow-read scripts/lint/check-coverage.ts 80",
|
|
309
315
|
"coverage:html": "deno coverage coverage --include=src/ --exclude=tests --exclude=src/**/*_test.ts --exclude=src/**/*_test.tsx --exclude=src/**/*.test.ts --exclude=src/**/*.test.tsx --html",
|
|
316
|
+
"bench": "VF_DISABLE_LRU_INTERVAL=1 NODE_ENV=production LOG_FORMAT=text deno bench --no-check --allow-all --unstable-worker-options --unstable-net $(find src -name '*.bench.ts')",
|
|
310
317
|
"clean": "rm -rf .cache/",
|
|
311
318
|
"lint": "DENO_NO_PACKAGE_JSON=1 deno lint src/ cli/",
|
|
312
319
|
"fmt": "deno fmt src/ cli/",
|
|
313
320
|
"fmt:check": "deno fmt --check src/ cli/",
|
|
314
|
-
"typecheck": "deno task generate && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/modules/index.ts src/proxy/main.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/tool/index.ts src/workflow/index.ts src/prompt/index.ts src/resource/index.ts src/mcp/index.ts src/provider/index.ts",
|
|
321
|
+
"typecheck": "deno task generate && deno check src/index.ts cli/main.ts src/server/index.ts src/routing/api/index.ts src/rendering/index.ts src/platform/index.ts src/platform/adapters/index.ts src/build/index.ts src/build/production-build/index.ts src/transforms/index.ts src/config/index.ts src/utils/index.ts src/data/index.ts src/security/index.ts src/middleware/index.ts src/server/handlers/dev/index.ts src/server/handlers/request/api/index.ts src/rendering/cache/index.ts src/rendering/cache/stores/index.ts src/rendering/rsc/actions/index.ts src/html/index.ts src/modules/index.ts src/proxy/main.ts src/chat/index.ts src/markdown/index.ts src/mdx/index.ts src/fs/index.ts src/oauth/index.ts src/agent/index.ts src/tool/index.ts src/workflow/index.ts src/prompt/index.ts src/resource/index.ts src/jobs/index.ts src/mcp/index.ts src/provider/index.ts",
|
|
315
322
|
"verify": "deno task generate && deno fmt --check src/ cli/ && DENO_NO_PACKAGE_JSON=1 deno lint src/ cli/ && deno task lint:style && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task docs:validate && deno task typecheck && deno task test && deno task test:e2e:binary",
|
|
316
323
|
"verify:quick": "deno task generate && deno fmt --check src/ cli/ && DENO_NO_PACKAGE_JSON=1 deno lint src/ cli/ && deno task lint:style && deno task lint:cli-boundary && deno task lint:wildcard-exports && deno task lint:barrel-jsdoc && deno task docs:validate && deno task typecheck",
|
|
317
324
|
"docs": "rm -rf docs/reference && deno run --allow-read --allow-write --allow-run scripts/docs/generate-api-reference.ts",
|
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import type { DataContext, DataResult, PageWithData, StaticPathsResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Options for isolated data fetching. Passed through to ServerDataFetcher
|
|
4
|
+
* when worker isolation is enabled.
|
|
5
|
+
*/
|
|
6
|
+
export interface FetchDataOptions {
|
|
7
|
+
/** Absolute path to the module containing getServerData */
|
|
8
|
+
modulePath?: string;
|
|
9
|
+
/** Project directory for worker scoping */
|
|
10
|
+
projectDir?: string;
|
|
11
|
+
}
|
|
2
12
|
export declare class DataFetcher {
|
|
3
13
|
private cacheManager;
|
|
4
14
|
private serverFetcher;
|
|
5
15
|
private staticFetcher;
|
|
6
16
|
private pathsFetcher;
|
|
7
17
|
constructor(_adapter?: unknown);
|
|
8
|
-
fetchData(pageModule: PageWithData, context: DataContext, mode?: "development" | "production"): Promise<DataResult>;
|
|
18
|
+
fetchData(pageModule: PageWithData, context: DataContext, mode?: "development" | "production", options?: FetchDataOptions): Promise<DataResult>;
|
|
9
19
|
getStaticPaths(pageModule: PageWithData): Promise<StaticPathsResult | null>;
|
|
10
20
|
clearCache(pattern?: string): void;
|
|
11
21
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data-fetcher.d.ts","sourceRoot":"","sources":["../../../src/src/data/data-fetcher.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE3F,qBAAa,WAAW;IACtB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,YAAY,CAAqB;gBAE7B,QAAQ,CAAC,EAAE,OAAO;IAO9B,SAAS,CACP,UAAU,EAAE,YAAY,EACxB,OAAO,EAAE,WAAW,EACpB,IAAI,GAAE,aAAa,GAAG,YAA4B,
|
|
1
|
+
{"version":3,"file":"data-fetcher.d.ts","sourceRoot":"","sources":["../../../src/src/data/data-fetcher.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE3F;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,YAAY,CAAqB;gBAE7B,QAAQ,CAAC,EAAE,OAAO;IAO9B,SAAS,CACP,UAAU,EAAE,YAAY,EACxB,OAAO,EAAE,WAAW,EACpB,IAAI,GAAE,aAAa,GAAG,YAA4B,EAClD,OAAO,CAAC,EAAE,gBAAgB,GACzB,OAAO,CAAC,UAAU,CAAC;IA8BtB,cAAc,CAAC,UAAU,EAAE,YAAY,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAI3E,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;CAQnC"}
|
|
@@ -15,7 +15,7 @@ export class DataFetcher {
|
|
|
15
15
|
this.staticFetcher = new StaticDataFetcher(this.cacheManager);
|
|
16
16
|
this.pathsFetcher = new StaticPathsFetcher();
|
|
17
17
|
}
|
|
18
|
-
fetchData(pageModule, context, mode = "development") {
|
|
18
|
+
fetchData(pageModule, context, mode = "development", options) {
|
|
19
19
|
const preferServerData = mode === "development" || !pageModule.getStaticData;
|
|
20
20
|
const useServer = preferServerData && !!pageModule.getServerData;
|
|
21
21
|
const useStatic = !useServer && !!pageModule.getStaticData;
|
|
@@ -24,9 +24,12 @@ export class DataFetcher {
|
|
|
24
24
|
: useStatic
|
|
25
25
|
? "static"
|
|
26
26
|
: "none";
|
|
27
|
+
const isolationOptions = options
|
|
28
|
+
? { modulePath: options.modulePath, projectDir: options.projectDir }
|
|
29
|
+
: undefined;
|
|
27
30
|
return withSpan(SpanNames.DATA_FETCH, () => {
|
|
28
31
|
if (useServer)
|
|
29
|
-
return this.serverFetcher.fetch(pageModule, context);
|
|
32
|
+
return this.serverFetcher.fetch(pageModule, context, isolationOptions);
|
|
30
33
|
if (useStatic)
|
|
31
34
|
return this.staticFetcher.fetch(pageModule, context);
|
|
32
35
|
return Promise.resolve({ props: {} });
|
package/esm/src/data/index.d.ts
CHANGED
|
@@ -5,6 +5,6 @@
|
|
|
5
5
|
* @module data
|
|
6
6
|
*/
|
|
7
7
|
export type { CacheEntry, DataContext, DataResult, InferGetServerDataProps, PageWithData, StaticPathsResult, } from "./types.js";
|
|
8
|
-
export { DataFetcher } from "./data-fetcher.js";
|
|
8
|
+
export { DataFetcher, type FetchDataOptions } from "./data-fetcher.js";
|
|
9
9
|
export { notFound, redirect } from "./helpers.js";
|
|
10
10
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/data/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,YAAY,EACV,UAAU,EACV,WAAW,EACX,UAAU,EACV,uBAAuB,EACvB,YAAY,EACZ,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/data/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,YAAY,EACV,UAAU,EACV,WAAW,EACX,UAAU,EACV,uBAAuB,EACvB,YAAY,EACZ,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import type { DataContext, DataResult, PageWithData } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Options for isolated data fetching through Worker pool.
|
|
4
|
+
*/
|
|
5
|
+
export interface ServerDataFetchOptions {
|
|
6
|
+
/** Absolute path to the module containing getServerData */
|
|
7
|
+
modulePath?: string;
|
|
8
|
+
/** Project directory for worker scoping */
|
|
9
|
+
projectDir?: string;
|
|
10
|
+
}
|
|
2
11
|
export declare class ServerDataFetcher {
|
|
3
|
-
fetch(pageModule: PageWithData, context: DataContext): Promise<DataResult>;
|
|
12
|
+
fetch(pageModule: PageWithData, context: DataContext, options?: ServerDataFetchOptions): Promise<DataResult>;
|
|
13
|
+
/**
|
|
14
|
+
* Execute getServerData in a per-project Worker.
|
|
15
|
+
*/
|
|
16
|
+
private fetchIsolated;
|
|
4
17
|
/**
|
|
5
18
|
* Log errors unconditionally. Production errors should always be logged.
|
|
6
19
|
* @see plans/architecture-audit/010-error-handling.md
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-data-fetcher.d.ts","sourceRoot":"","sources":["../../../src/src/data/server-data-fetcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"server-data-fetcher.d.ts","sourceRoot":"","sources":["../../../src/src/data/server-data-fetcher.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAYxE;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,qBAAa,iBAAiB;IAC5B,KAAK,CACH,UAAU,EAAE,YAAY,EACxB,OAAO,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,UAAU,CAAC;IA8EtB;;OAEG;YACW,aAAa;IAoE3B;;;OAGG;IACH,OAAO,CAAC,QAAQ;CAGjB"}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import * as dntShim from "../../_dnt.shims.js";
|
|
1
2
|
import { serverLogger } from "../utils/index.js";
|
|
2
3
|
import { DATA_FETCH_TIMEOUT_MS } from "../config/defaults.js";
|
|
3
4
|
import { TimeoutError, withTimeoutThrow } from "../rendering/utils/stream-utils.js";
|
|
4
5
|
import { withSpan } from "../observability/tracing/otlp-setup.js";
|
|
5
6
|
import { CircuitBreakerOpen, getCircuitBreaker } from "../utils/circuit-breaker.js";
|
|
7
|
+
import { getWorkerPool, isDataIsolationEnabled } from "../security/sandbox/worker-pool.js";
|
|
8
|
+
import { MAX_WORKER_BODY_BYTES, } from "../security/sandbox/worker-types.js";
|
|
6
9
|
export class ServerDataFetcher {
|
|
7
|
-
fetch(pageModule, context) {
|
|
10
|
+
fetch(pageModule, context, options) {
|
|
8
11
|
if (typeof pageModule.getServerData !== "function") {
|
|
9
12
|
return Promise.resolve({ props: {} });
|
|
10
13
|
}
|
|
@@ -15,10 +18,16 @@ export class ServerDataFetcher {
|
|
|
15
18
|
resetTimeoutMs: 30_000,
|
|
16
19
|
successThreshold: 2,
|
|
17
20
|
});
|
|
21
|
+
// Choose isolated or direct execution
|
|
22
|
+
const useIsolation = isDataIsolationEnabled() &&
|
|
23
|
+
!!options?.modulePath &&
|
|
24
|
+
!!options?.projectDir;
|
|
18
25
|
return withSpan("data.fetch_server", async () => {
|
|
19
26
|
const start = performance.now();
|
|
20
27
|
try {
|
|
21
|
-
const result = await circuitBreaker.execute(() => withTimeoutThrow(
|
|
28
|
+
const result = await circuitBreaker.execute(() => withTimeoutThrow(useIsolation
|
|
29
|
+
? this.fetchIsolated(options.modulePath, options.projectDir, context)
|
|
30
|
+
: Promise.resolve(pageModule.getServerData(context)), DATA_FETCH_TIMEOUT_MS, `getServerData for ${pathname}`));
|
|
22
31
|
if (result.redirect)
|
|
23
32
|
return { redirect: result.redirect };
|
|
24
33
|
if (result.notFound)
|
|
@@ -43,7 +52,11 @@ export class ServerDataFetcher {
|
|
|
43
52
|
});
|
|
44
53
|
throw error;
|
|
45
54
|
}
|
|
46
|
-
this.logError("DATA_FETCH_ERROR getServerData failed", error, {
|
|
55
|
+
this.logError("DATA_FETCH_ERROR getServerData failed", error, {
|
|
56
|
+
pathname,
|
|
57
|
+
durationMs,
|
|
58
|
+
isolated: useIsolation,
|
|
59
|
+
});
|
|
47
60
|
throw error;
|
|
48
61
|
}
|
|
49
62
|
}, {
|
|
@@ -51,8 +64,57 @@ export class ServerDataFetcher {
|
|
|
51
64
|
"data.pathname": pathname,
|
|
52
65
|
"data.timeout_ms": DATA_FETCH_TIMEOUT_MS,
|
|
53
66
|
"data.project_id": projectId,
|
|
67
|
+
"data.isolated": useIsolation,
|
|
54
68
|
});
|
|
55
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Execute getServerData in a per-project Worker.
|
|
72
|
+
*/
|
|
73
|
+
async fetchIsolated(modulePath, projectDir, context) {
|
|
74
|
+
const pool = getWorkerPool();
|
|
75
|
+
let body = null;
|
|
76
|
+
if (context.request?.body) {
|
|
77
|
+
// Fast path: reject before buffering if Content-Length is known
|
|
78
|
+
const contentLength = context.request.headers?.get("content-length");
|
|
79
|
+
if (contentLength) {
|
|
80
|
+
const bytes = parseInt(contentLength, 10);
|
|
81
|
+
if (bytes > MAX_WORKER_BODY_BYTES) {
|
|
82
|
+
throw new Error(`Request body too large for isolated data fetch (${(bytes / 1024 / 1024).toFixed(1)} MB, limit ${MAX_WORKER_BODY_BYTES / 1024 / 1024} MB)`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
body = new Uint8Array(await context.request.arrayBuffer());
|
|
86
|
+
// Fallback: check actual size for chunked/streaming bodies
|
|
87
|
+
if (body.byteLength > MAX_WORKER_BODY_BYTES) {
|
|
88
|
+
throw new Error(`Request body too large for isolated data fetch (${(body.byteLength / 1024 / 1024).toFixed(1)} MB, limit ${MAX_WORKER_BODY_BYTES / 1024 / 1024} MB)`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const workerResponse = await pool.execute(projectDir, [projectDir], {
|
|
92
|
+
type: "fetch-data",
|
|
93
|
+
id: dntShim.crypto.randomUUID(),
|
|
94
|
+
modulePath,
|
|
95
|
+
context: {
|
|
96
|
+
params: context.params,
|
|
97
|
+
query: context.query?.toString() ?? "",
|
|
98
|
+
request: {
|
|
99
|
+
url: context.request?.url ?? context.url?.toString() ?? "http://localhost",
|
|
100
|
+
method: context.request?.method ?? "GET",
|
|
101
|
+
headers: context.request ? [...context.request.headers.entries()] : [],
|
|
102
|
+
body,
|
|
103
|
+
},
|
|
104
|
+
url: context.url?.toString() ?? "http://localhost",
|
|
105
|
+
},
|
|
106
|
+
});
|
|
107
|
+
if (workerResponse.type === "error") {
|
|
108
|
+
const err = new Error(workerResponse.error.message);
|
|
109
|
+
err.name = workerResponse.error.name;
|
|
110
|
+
throw err;
|
|
111
|
+
}
|
|
112
|
+
if (workerResponse.type === "data-result") {
|
|
113
|
+
return workerResponse.result;
|
|
114
|
+
}
|
|
115
|
+
// Unexpected response type — shouldn't happen but be defensive
|
|
116
|
+
throw new Error(`Unexpected worker response type: ${workerResponse.type}`);
|
|
117
|
+
}
|
|
56
118
|
/**
|
|
57
119
|
* Log errors unconditionally. Production errors should always be logged.
|
|
58
120
|
* @see plans/architecture-audit/010-error-handling.md
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs module for durable project-scoped background execution.
|
|
3
|
+
*
|
|
4
|
+
* Provides a public SDK surface for one-off jobs, cron jobs, batch summaries,
|
|
5
|
+
* job target discovery, and the canonical split between user-visible `events`
|
|
6
|
+
* and raw debugging `logs`.
|
|
7
|
+
*
|
|
8
|
+
* @module
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { VeryfrontJobsClient } from "veryfront/jobs";
|
|
13
|
+
*
|
|
14
|
+
* const jobs = new VeryfrontJobsClient({
|
|
15
|
+
* authToken: process.env.VERYFRONT_API_TOKEN,
|
|
16
|
+
* projectReference: "my-project",
|
|
17
|
+
* });
|
|
18
|
+
*
|
|
19
|
+
* const job = await jobs.create({
|
|
20
|
+
* name: "Ingest 1 file",
|
|
21
|
+
* target: "task:knowledge-ingest",
|
|
22
|
+
* config: {
|
|
23
|
+
* file_count: 1,
|
|
24
|
+
* upload_ids: ["00000000-0000-0000-0000-000000000000"],
|
|
25
|
+
* },
|
|
26
|
+
* });
|
|
27
|
+
*
|
|
28
|
+
* const events = await jobs.events(job.id);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
import "../../_dnt.polyfills.js";
|
|
32
|
+
export { type CreateCronJobInput, type CreateJobInput, createJobsClient, type ListBatchJobsOptions, type ListCronJobsOptions, type ListJobEventsOptions, type ListJobsOptions, type ProjectScopedOptions, type UpdateCronJobInput, VeryfrontJobsClient, type VeryfrontJobsClientConfig, } from "./jobs-client.js";
|
|
33
|
+
export { type CronJob, CronJobSchema, type CronJobStatus, CronJobStatusSchema, type Job, type JobBatch, type JobBatchResult, JobBatchResultSchema, JobBatchSchema, type JobBatchStatusCounts, JobBatchStatusCountsSchema, type JobEvent, JobEventSchema, type JobEventsResponse, JobEventsResponseSchema, type JobLogsResponse, JobLogsResponseSchema, type JobResult, JobResultSchema, JobSchema, type JobStatus, JobStatusSchema, type JobTargetDefinition, JobTargetDefinitionSchema, type JobTargetDefinitionsResponse, JobTargetDefinitionsResponseSchema, type KnowledgeIngestBatchSource, KnowledgeIngestBatchSourceSchema, type KnowledgeIngestFileResult, KnowledgeIngestFileResultSchema, PageInfoSchema, type PaginatedCronJobsResponse, PaginatedCronJobsResponseSchema, type PaginatedJobsResponse, PaginatedJobsResponseSchema, type ReservedJobTargetFamily, ReservedJobTargetFamilySchema, } from "./schemas.js";
|
|
34
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,yBAAyB,CAAC;AAGjC,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EACnB,gBAAgB,EAChB,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,mBAAmB,EACnB,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,KAAK,OAAO,EACZ,aAAa,EACb,KAAK,aAAa,EAClB,mBAAmB,EACnB,KAAK,GAAG,EACR,KAAK,QAAQ,EACb,KAAK,cAAc,EACnB,oBAAoB,EACpB,cAAc,EACd,KAAK,oBAAoB,EACzB,0BAA0B,EAC1B,KAAK,QAAQ,EACb,cAAc,EACd,KAAK,iBAAiB,EACtB,uBAAuB,EACvB,KAAK,eAAe,EACpB,qBAAqB,EACrB,KAAK,SAAS,EACd,eAAe,EACf,SAAS,EACT,KAAK,SAAS,EACd,eAAe,EACf,KAAK,mBAAmB,EACxB,yBAAyB,EACzB,KAAK,4BAA4B,EACjC,kCAAkC,EAClC,KAAK,0BAA0B,EAC/B,gCAAgC,EAChC,KAAK,yBAAyB,EAC9B,+BAA+B,EAC/B,cAAc,EACd,KAAK,yBAAyB,EAC9B,+BAA+B,EAC/B,KAAK,qBAAqB,EAC1B,2BAA2B,EAC3B,KAAK,uBAAuB,EAC5B,6BAA6B,GAC9B,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jobs module for durable project-scoped background execution.
|
|
3
|
+
*
|
|
4
|
+
* Provides a public SDK surface for one-off jobs, cron jobs, batch summaries,
|
|
5
|
+
* job target discovery, and the canonical split between user-visible `events`
|
|
6
|
+
* and raw debugging `logs`.
|
|
7
|
+
*
|
|
8
|
+
* @module
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { VeryfrontJobsClient } from "veryfront/jobs";
|
|
13
|
+
*
|
|
14
|
+
* const jobs = new VeryfrontJobsClient({
|
|
15
|
+
* authToken: process.env.VERYFRONT_API_TOKEN,
|
|
16
|
+
* projectReference: "my-project",
|
|
17
|
+
* });
|
|
18
|
+
*
|
|
19
|
+
* const job = await jobs.create({
|
|
20
|
+
* name: "Ingest 1 file",
|
|
21
|
+
* target: "task:knowledge-ingest",
|
|
22
|
+
* config: {
|
|
23
|
+
* file_count: 1,
|
|
24
|
+
* upload_ids: ["00000000-0000-0000-0000-000000000000"],
|
|
25
|
+
* },
|
|
26
|
+
* });
|
|
27
|
+
*
|
|
28
|
+
* const events = await jobs.events(job.id);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
import "../../_dnt.polyfills.js";
|
|
32
|
+
export { createJobsClient, VeryfrontJobsClient, } from "./jobs-client.js";
|
|
33
|
+
export { CronJobSchema, CronJobStatusSchema, JobBatchResultSchema, JobBatchSchema, JobBatchStatusCountsSchema, JobEventSchema, JobEventsResponseSchema, JobLogsResponseSchema, JobResultSchema, JobSchema, JobStatusSchema, JobTargetDefinitionSchema, JobTargetDefinitionsResponseSchema, KnowledgeIngestBatchSourceSchema, KnowledgeIngestFileResultSchema, PageInfoSchema, PaginatedCronJobsResponseSchema, PaginatedJobsResponseSchema, ReservedJobTargetFamilySchema, } from "./schemas.js";
|