videobook 0.2.5 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/README.md +6 -2
  3. package/dist/archive-b2.js +1 -1
  4. package/dist/asset-media-projection.js +161 -0
  5. package/dist/cli/args.js +42 -3
  6. package/dist/cli/config-command.js +17 -1
  7. package/dist/cli/doctor-command.js +15 -0
  8. package/dist/cli/har-command.js +86 -0
  9. package/dist/cli/main.js +13 -1
  10. package/dist/command-reference.js +3 -1
  11. package/dist/content-store.js +2 -1
  12. package/dist/diagnostics.js +2 -2
  13. package/dist/env-secrets.js +1 -0
  14. package/dist/generation-har.js +106 -0
  15. package/dist/generation-reference-mentions.js +19 -0
  16. package/dist/goal-plan-v3.js +11 -1
  17. package/dist/goal-plan.js +1 -1
  18. package/dist/goal-store.js +2 -2
  19. package/dist/har-capture.js +175 -0
  20. package/dist/har-export.js +238 -0
  21. package/dist/har-index.js +98 -0
  22. package/dist/har-retention.js +73 -0
  23. package/dist/har-settings.js +27 -0
  24. package/dist/jobs/_alibaba-upload.js +2 -4
  25. package/dist/jobs/_fal-upload.js +2 -4
  26. package/dist/jobs/_image-edit-adapter.js +3 -1
  27. package/dist/jobs/_kie-upload.js +2 -4
  28. package/dist/jobs/assess-media-probes.js +94 -0
  29. package/dist/jobs/generate-media.js +10 -1
  30. package/dist/jobs/image-subjects.js +3 -1
  31. package/dist/jobs/kie-paygate.js +10 -3
  32. package/dist/jobs/register-all.js +2 -0
  33. package/dist/jobs/run-notebook.js +20 -2
  34. package/dist/jobs/xai-paygate.js +5 -2
  35. package/dist/library-asset-projection.js +141 -0
  36. package/dist/lightning-defaults.js +4 -4
  37. package/dist/media/adapters/kie.js +2 -1
  38. package/dist/media/catalog.js +8 -2
  39. package/dist/media/cost.js +1 -1
  40. package/dist/media/descriptor.js +7 -4
  41. package/dist/media/frontier-gaps.js +0 -1
  42. package/dist/media/har-endpoint.js +25 -0
  43. package/dist/media/json-schema.js +3 -0
  44. package/dist/media/preferred-defaults.js +161 -0
  45. package/dist/media/prompt-dialects.js +9 -1
  46. package/dist/media-library.js +158 -153
  47. package/dist/notebook-sticky-settings.js +60 -0
  48. package/dist/package-version.js +18 -0
  49. package/dist/posts/dropbox-auth.js +1 -1
  50. package/dist/posts/post-metrics.js +2 -1
  51. package/dist/provider-usage.js +1 -18
  52. package/dist/queue.js +12 -1
  53. package/dist/serve.js +2 -0
  54. package/dist/tools/api-status.js +1 -1
  55. package/dist/tools/generate-media.js +3 -0
  56. package/dist/tools/generation-payload.js +17 -2
  57. package/dist/tools/get-generation-har.js +43 -0
  58. package/dist/tools/index.js +4 -0
  59. package/dist/tools/list-cell-generations.js +19 -0
  60. package/dist/tools/reconcile-project-work.js +52 -193
  61. package/dist/tools/set-har-capture.js +30 -0
  62. package/dist/tools/videobook-domain.js +16 -1
  63. package/dist/upload.js +2 -1
  64. package/dist/usage-providers.js +48 -10
  65. package/dist/workspace-action-groups.js +2 -2
  66. package/dist/x/account.js +1 -1
  67. package/dist/x/authenticated-handle.js +1 -1
  68. package/dist/x/oauth-flow.js +1 -1
  69. package/dist/x/post-to-x.js +2 -1
  70. package/dist/x/x-tokens.js +2 -1
  71. package/package.json +15 -14
  72. package/ui/assets/{alive-captions-duTKsyll.js → alive-captions-CnLK22ED.js} +1 -1
  73. package/ui/assets/confirm-bodies-BaalBxp1.js +1 -0
  74. package/ui/assets/{fullscreen-COrmiHPn.js → fullscreen-DTJQJ6dX.js} +1 -1
  75. package/ui/assets/{goal-ui-D5Xsm2EY.js → goal-ui-CN-kZTEO.js} +2 -2
  76. package/ui/assets/index-CTPstLYq.css +1 -0
  77. package/ui/assets/index-Dp81yChE.js +37 -0
  78. package/ui/assets/inspector-jeyma7HL.js +1 -0
  79. package/ui/assets/{jobs-ui-DdP2Re41.js → jobs-ui-D6S3LJ4j.js} +1 -1
  80. package/ui/assets/{music-video-B709JZqa.js → music-video-CDqr0X6G.js} +1 -1
  81. package/ui/assets/ui-BBM3YCY6.css +1 -0
  82. package/ui/assets/ui-Cqs1-YNg.js +2 -0
  83. package/ui/assets/vendor-BWf-WCsY.js +19 -0
  84. package/ui/assets/{vendor-flow-CKiuzaeM.js → vendor-flow-DPvi8ynA.js} +1 -1
  85. package/ui/assets/vendor-icons-D3KrGMJY.js +1 -0
  86. package/ui/assets/vendor-mcp-BSR9acpG.js +60 -0
  87. package/ui/assets/videobook-b9VODNOA.js +21 -0
  88. package/ui/index.html +14 -14
  89. package/ui/assets/confirm-bodies--2sc9DbH.js +0 -1
  90. package/ui/assets/index-DEKl00St.js +0 -37
  91. package/ui/assets/index-DWfR79lA.css +0 -1
  92. package/ui/assets/inspector-EBp3IhvH.js +0 -1
  93. package/ui/assets/ui-Oirr9VOf.css +0 -1
  94. package/ui/assets/ui-jRPWsNUw.js +0 -2
  95. package/ui/assets/vendor-BFupLz37.js +0 -19
  96. package/ui/assets/vendor-icons-CGqhCIXV.js +0 -1
  97. package/ui/assets/vendor-mcp-DQEeVZH0.js +0 -67
  98. package/ui/assets/videobook-0DOLlhG2.js +0 -20
@@ -0,0 +1,106 @@
1
+ import { scrubHar } from "@apicity/har";
2
+ import { captureFileExists, captureFilePath, readCaptureFile, readHarIndex } from "./har-index.js";
3
+ import { harDirectory } from "./har-settings.js";
4
+ import { readProviderTaskRecord } from "./provider-task-record.js";
5
+ function bookIdOf(engine) {
6
+ try {
7
+ return engine.book.get().bookId;
8
+ }
9
+ catch {
10
+ return null;
11
+ }
12
+ }
13
+ export async function captureKeys(engine, ids) {
14
+ const known = new Set([...ids].filter((id) => typeof id === "string" && id.length > 0));
15
+ const taskIds = new Set();
16
+ for (const id of known) {
17
+ const task = await readProviderTaskRecord(engine, id).catch(() => null);
18
+ if (task && task.taskId !== "queued")
19
+ taskIds.add(task.taskId);
20
+ }
21
+ return { bookId: bookIdOf(engine), ids: known, taskIds };
22
+ }
23
+ export async function generationCaptureKeys(engine, generationId) {
24
+ let generation = null;
25
+ try {
26
+ generation = engine.generations.read(generationId);
27
+ }
28
+ catch {
29
+ generation = null;
30
+ }
31
+ const row = generation?.ok ? generation.value : null;
32
+ return captureKeys(engine, [generationId, row?.providerArtifactId, row?.outputArtifactId]);
33
+ }
34
+ function matchesCapture(row, keys) {
35
+ const context = row.context ?? {};
36
+ if (keys.bookId && typeof context.bookId === "string" && context.bookId !== keys.bookId)
37
+ return false;
38
+ return [context.artifactId, context.attemptId].some((id) => typeof id === "string" && keys.ids.has(id))
39
+ || (typeof context.providerTaskId === "string" && keys.taskIds.has(context.providerTaskId));
40
+ }
41
+ function chronological(a, b) {
42
+ return a.ts.localeCompare(b.ts) || a.callId.localeCompare(b.callId);
43
+ }
44
+ async function attemptCaptureRows(keys, rows, dir = harDirectory()) {
45
+ const matched = (rows ?? await readHarIndex(dir)).filter((row) => matchesCapture(row, keys)).sort(chronological);
46
+ const present = await Promise.all(matched.map((row) => captureFileExists(dir, row.file)));
47
+ return matched.filter((_row, index) => present[index]);
48
+ }
49
+ export async function generationHarStatus(keys, mode, rows, dir = harDirectory()) {
50
+ const captures = await attemptCaptureRows(keys, rows, dir);
51
+ const primary = [...captures].reverse().find((row) => row.outcome !== "ok") ?? captures.at(-1);
52
+ return {
53
+ mode,
54
+ hasCapture: captures.length > 0,
55
+ captures: captures.length,
56
+ path: primary ? captureFilePath(dir, primary.file) : null,
57
+ outcome: primary?.outcome ?? null,
58
+ };
59
+ }
60
+ function combineCaptures(captures, generationId) {
61
+ const [first] = captures;
62
+ if (!first || captures.length === 1)
63
+ return first?.har ?? null;
64
+ const pages = captures.map(({ row, har }) => ({
65
+ startedDateTime: har.log._apicity?.startedAt ?? row.ts,
66
+ id: har.log._apicity?.callId ?? row.callId,
67
+ title: `${row.provider} ${row.endpoint} (${row.outcome})`,
68
+ pageTimings: {},
69
+ ...(har.log._apicity ? { _apicity: har.log._apicity } : {}),
70
+ }));
71
+ return {
72
+ log: {
73
+ version: "1.2",
74
+ creator: first.har.log.creator,
75
+ comment: `videobook generation ${generationId}: ${captures.length} captured apicity calls, one page each`,
76
+ pages,
77
+ entries: captures.flatMap(({ row, har }) => har.log.entries.map((entry) => ({
78
+ ...entry,
79
+ pageref: har.log._apicity?.callId ?? row.callId,
80
+ }))),
81
+ },
82
+ };
83
+ }
84
+ export async function generationHar(keys, generationId, dir = harDirectory()) {
85
+ const loaded = [];
86
+ for (const row of await attemptCaptureRows(keys, undefined, dir)) {
87
+ const capture = await readCaptureFile(dir, row.file);
88
+ if (capture)
89
+ loaded.push({ row, path: capture.path, har: scrubHar(capture.har) });
90
+ }
91
+ const har = combineCaptures(loaded, generationId);
92
+ if (!har)
93
+ return null;
94
+ return {
95
+ captures: loaded.map(({ row, path }) => ({
96
+ path,
97
+ ts: row.ts,
98
+ callId: row.callId,
99
+ provider: row.provider,
100
+ endpoint: row.endpoint,
101
+ outcome: row.outcome,
102
+ status: row.status ?? null,
103
+ })),
104
+ har,
105
+ };
106
+ }
@@ -65,6 +65,25 @@ export function rewriteSeedanceAssetMentions(prompt, refs) {
65
65
  }
66
66
  return replaceNotebookMentions(prompt, replacements);
67
67
  }
68
+ function numberedMentions(refs, marker) {
69
+ return refMentionEntries(refs).map(({ ref, index }) => ({
70
+ reference: ref,
71
+ replacement: `@${marker}_${index + 1}`,
72
+ }));
73
+ }
74
+ export function rewriteKling4ReferenceMentions(prompt, imageRefs, videoRefs) {
75
+ return replaceNotebookMentions(prompt, [
76
+ ...numberedMentions(imageRefs, "image"),
77
+ ...numberedMentions(videoRefs, "video"),
78
+ ]);
79
+ }
80
+ export function rewriteKling4FrameMentions(prompt, frames) {
81
+ return replaceNotebookMentions(prompt, [
82
+ ...frames.first.slice(0, 1).map((reference) => ({ reference, replacement: "@first_frame" })),
83
+ ...numberedMentions(frames.keyframes, "keyframe"),
84
+ ...frames.last.slice(0, 1).map((reference) => ({ reference, replacement: "@last_frame" })),
85
+ ]);
86
+ }
68
87
  export function rewriteViduReferenceMentions(prompt, imageRefs, audioRefs) {
69
88
  return replaceNotebookMentions(prompt, [
70
89
  ...refMentionEntries(imageRefs).map(({ ref, index }) => ({
@@ -4,6 +4,8 @@ import { goalPromptWords } from "./goal-prompt-budget.js";
4
4
  import { LIGHTNING_COLD_OPEN_SECONDS, LIGHTNING_SCENE_ASPECT_RATIO, LIGHTNING_SCENE_MODEL, LIGHTNING_SCENE_OUTPUT_FORMAT, LIGHTNING_SCENE_PROMPT_WORDS, LIGHTNING_SCENE_RESOLUTION, LIGHTNING_VIDEO_DURATION_SECONDS, LIGHTNING_VIDEO_GENERATE_AUDIO, LIGHTNING_VIDEO_OUTPUT_FORMAT, LIGHTNING_VIDEO_PROMPT_WORDS, LIGHTNING_VIDEO_RESOLUTION, LIGHTNING_VIDEO_MODEL, LIGHTNING_WATERMARK_PROFILE, preservedPrompt, } from "./lightning-defaults.js";
5
5
  const IMAGE_LIMIT = 30;
6
6
  const VIDEO_LIMIT = 10;
7
+ const LEGACY_SCENE_RESOLUTION = "2K";
8
+ const SCENE_RESOLUTIONS = [LEGACY_SCENE_RESOLUTION, "4K"];
7
9
  export const goalMediaRefSchema = z.object({
8
10
  kind: z.enum(["image", "video"]),
9
11
  artifactId: z.string().min(1),
@@ -22,7 +24,7 @@ export const goalPlanV3Schema = z.object({
22
24
  tool: z.literal("generate_image"),
23
25
  model: z.literal(LIGHTNING_SCENE_MODEL),
24
26
  aspectRatio: z.enum(["16:9", "9:16", "1:1"]),
25
- resolution: z.literal(LIGHTNING_SCENE_RESOLUTION),
27
+ resolution: z.enum(SCENE_RESOLUTIONS),
26
28
  outputFormat: z.literal(LIGHTNING_SCENE_OUTPUT_FORMAT),
27
29
  imageRefs: z.array(goalMediaRefSchema).max(IMAGE_LIMIT),
28
30
  }).strict(),
@@ -59,6 +61,14 @@ export const goalPlanV3Schema = z.object({
59
61
  }).strict(),
60
62
  }).strict(),
61
63
  }).strict();
64
+ function withLegacySceneResolution(plan) {
65
+ if (typeof plan !== "object" || plan === null || !("version" in plan) || plan.version !== 3)
66
+ return plan;
67
+ if (!("scene" in plan) || typeof plan.scene !== "object" || plan.scene === null || "resolution" in plan.scene)
68
+ return plan;
69
+ return { ...plan, scene: { ...plan.scene, resolution: LEGACY_SCENE_RESOLUTION } };
70
+ }
71
+ export const savedGoalPlanV3Schema = z.preprocess(withLegacySceneResolution, goalPlanV3Schema);
62
72
  export function isGoalPlanV3(plan) {
63
73
  return plan?.version === 3;
64
74
  }
package/dist/goal-plan.js CHANGED
@@ -2,7 +2,7 @@ import { z } from "zod";
2
2
  import { LIGHTNING_VIDEO_MODEL } from "./lightning-defaults.js";
3
3
  import { GOAL_PROMPT_BUDGET, goalPromptWords } from "./goal-prompt-budget.js";
4
4
  import { GOAL_STYLE_IDS, goalStyleFragment, resolveGoalStyleOptions } from "./goal-styles.js";
5
- export { compileGoalV3Approval, compileGoalV3Steps, goalMediaRefSchema, goalPlanV3Schema, isGoalPlanV3, refreshGoalPlanV3 } from "./goal-plan-v3.js";
5
+ export { compileGoalV3Approval, compileGoalV3Steps, goalMediaRefSchema, goalPlanV3Schema, isGoalPlanV3, refreshGoalPlanV3, savedGoalPlanV3Schema } from "./goal-plan-v3.js";
6
6
  export { GOAL_PROMPT_BUDGET, goalPromptWords } from "./goal-prompt-budget.js";
7
7
  export const GOAL_VIDEO_TOOL = "generate_video";
8
8
  export const GOAL_VIDEO_MODEL = LIGHTNING_VIDEO_MODEL;
@@ -1,6 +1,6 @@
1
1
  import { z } from "zod";
2
2
  import { firstEmptyNotebookGridSlot, notebookOccupiedSlots } from "videobook-engine/notebook-grid";
3
- import { goalMediaRefSchema, goalPlanSchema, goalPlanV3Schema, goalStyleSchema, legacyGoalPlanSchema } from "./goal-plan.js";
3
+ import { goalMediaRefSchema, goalPlanSchema, goalStyleSchema, legacyGoalPlanSchema, savedGoalPlanV3Schema } from "./goal-plan.js";
4
4
  import { mutateNotebook } from "./notebook-mutations.js";
5
5
  import { arrangeGoalNotebook } from "./notebook-goal-layout.js";
6
6
  const KEY = "application.goals.v2";
@@ -68,7 +68,7 @@ function mediaFrom(value) {
68
68
  }
69
69
  export function goalContent(cell) {
70
70
  const inputs = cell.inputs ?? {};
71
- const plan = z.union([goalPlanV3Schema, goalPlanSchema, legacyGoalPlanSchema]).safeParse(inputs.plan);
71
+ const plan = z.union([savedGoalPlanV3Schema, goalPlanSchema, legacyGoalPlanSchema]).safeParse(inputs.plan);
72
72
  const requestedStyle = goalStyleSchema.safeParse(inputs.requestedStyle);
73
73
  const media = mediaFrom(inputs.media);
74
74
  return {
@@ -0,0 +1,175 @@
1
+ import { createHash } from "node:crypto";
2
+ import { AsyncLocalStorage } from "node:async_hooks";
3
+ import { createHarCapture } from "@apicity/har";
4
+ import { currentBookContext } from "./book-context.js";
5
+ import { harDirectory, readHarSettings } from "./har-settings.js";
6
+ import { logger } from "./logger.js";
7
+ const localFailures = new AsyncLocalStorage();
8
+ const pending = new Set();
9
+ let maintenance = Promise.resolve();
10
+ export async function maintainHarCaptures(operation) {
11
+ if (pending.size)
12
+ return false;
13
+ const next = maintenance.then(operation);
14
+ maintenance = next.catch(() => undefined);
15
+ await next;
16
+ return true;
17
+ }
18
+ export async function flushHarCaptures() {
19
+ while (pending.size)
20
+ await Promise.all([...pending]);
21
+ }
22
+ function captureInput(value, seen = new WeakSet()) {
23
+ if (value instanceof Blob)
24
+ return { _file: { size: value.size, contentType: value.type } };
25
+ if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
26
+ const bytes = value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
27
+ return { _apicity_placeholder: { bytes: bytes.byteLength, sha256: createHash("sha256").update(bytes).digest("hex") } };
28
+ }
29
+ if (!value || typeof value !== "object")
30
+ return value;
31
+ if (seen.has(value))
32
+ return "[circular]";
33
+ seen.add(value);
34
+ if (value instanceof FormData)
35
+ return Object.fromEntries(Array.from(value.entries(), ([key, entry]) => [key, captureInput(entry, seen)]));
36
+ if (Array.isArray(value))
37
+ return value.map((entry) => captureInput(entry, seen));
38
+ if (Object.getPrototypeOf(value) !== Object.prototype)
39
+ return value;
40
+ return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, key === "otp" ? "***" : captureInput(entry, seen)]));
41
+ }
42
+ export async function withHarCapture(meta, operation, base = globalThis.fetch, context = {}) {
43
+ for (;;) {
44
+ const barrier = maintenance;
45
+ await barrier;
46
+ if (barrier === maintenance)
47
+ break;
48
+ }
49
+ let finish;
50
+ const finished = new Promise((resolve) => { finish = resolve; });
51
+ pending.add(finished);
52
+ try {
53
+ return await runCapture(meta, operation, base, context);
54
+ }
55
+ catch (error) {
56
+ localFailures.getStore()?.add(error);
57
+ throw error;
58
+ }
59
+ finally {
60
+ finish();
61
+ pending.delete(finished);
62
+ }
63
+ }
64
+ export async function withHarLocalFailure(meta, operation) {
65
+ const failures = localFailures.getStore() ?? new Set();
66
+ return localFailures.run(failures, async () => {
67
+ try {
68
+ return await operation();
69
+ }
70
+ catch (error) {
71
+ const seen = new Set();
72
+ let cause = error;
73
+ while (!seen.has(cause)) {
74
+ if (failures.has(cause))
75
+ throw error;
76
+ seen.add(cause);
77
+ if (!(cause instanceof Error) || cause.cause === undefined)
78
+ break;
79
+ cause = cause.cause;
80
+ }
81
+ return withHarCapture(meta, () => { throw error; });
82
+ }
83
+ });
84
+ }
85
+ async function runCapture(meta, operation, base, context) {
86
+ const settings = await readHarSettings();
87
+ if (settings.mode === "off")
88
+ return operation(base);
89
+ let input;
90
+ try {
91
+ input = captureInput(meta.input);
92
+ }
93
+ catch (error) {
94
+ logger.warn("har-capture-failed", { error });
95
+ return operation(base);
96
+ }
97
+ const book = currentBookContext();
98
+ const capture = createHarCapture({
99
+ dir: harDirectory(),
100
+ mode: settings.mode,
101
+ context: {
102
+ bookId: book?.bookId,
103
+ operation: book?.operation,
104
+ artifactId: book?.artifactId,
105
+ attemptId: book?.attemptId ?? book?.artifactId,
106
+ providerTaskId: book?.providerTaskId,
107
+ runId: book?.runId,
108
+ model: book?.model,
109
+ ...context,
110
+ },
111
+ onError: (error) => logger.warn("har-capture-failed", { error }),
112
+ });
113
+ const host = { fetch: base };
114
+ capture.install(host);
115
+ let finish;
116
+ const finished = new Promise((resolve) => { finish = resolve; });
117
+ const done = finished.then(() => capture.close());
118
+ pending.add(done);
119
+ void done.finally(() => pending.delete(done));
120
+ try {
121
+ return await capture.call({ ...meta, input }, () => operation(host.fetch));
122
+ }
123
+ finally {
124
+ finish();
125
+ }
126
+ }
127
+ const HTTP_METHODS = new Set(["get", "post", "put", "patch", "delete", "head", "options"]);
128
+ const SDK_ONLY_NAMESPACES = new Map([["kie", new Set(["veo", "suno"])]]);
129
+ function endpointDotPath(provider, path) {
130
+ const [head = "", next = ""] = path;
131
+ if (path.length > 1 && HTTP_METHODS.has(head))
132
+ return path.slice(1).join(".");
133
+ if (path.length < 3 || !HTTP_METHODS.has(next))
134
+ return path.join(".");
135
+ const rest = path.slice(2);
136
+ return (SDK_ONLY_NAMESPACES.get(provider)?.has(head) ? rest : [head, ...rest]).join(".");
137
+ }
138
+ const providerCalls = new WeakMap();
139
+ export function withProviderCall(provider, meta, operation) {
140
+ const state = providerCalls.get(provider);
141
+ if (!state)
142
+ return operation(provider);
143
+ return withHarCapture(meta, (fetch) => state.transports.run(fetch, () => operation(state.original)), state.base);
144
+ }
145
+ export function capturedProvider(provider, factory, base = globalThis.fetch) {
146
+ const transports = new AsyncLocalStorage();
147
+ const original = factory((input, init) => {
148
+ const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
149
+ signal?.throwIfAborted();
150
+ return (transports.getStore() ?? base)(input, init);
151
+ });
152
+ function wrap(target, path) {
153
+ return new Proxy(target, {
154
+ apply(fn, receiver, args) {
155
+ return withHarCapture({ provider, endpoint: endpointDotPath(provider, path), input: args.length === 1 ? args[0] : args }, (fetch) => transports.run(fetch, () => Reflect.apply(fn, receiver, args)), base);
156
+ },
157
+ get(object, property, receiver) {
158
+ const value = Reflect.get(object, property, receiver);
159
+ if (typeof target === "function" && ["bind", "call", "apply"].includes(String(property)) && typeof value === "function")
160
+ return value.bind(receiver);
161
+ if (typeof property !== "string" || ["presign", "schema", "example", "examples"].includes(property))
162
+ return value;
163
+ const next = [...path, property];
164
+ if (typeof value === "function")
165
+ return wrap(value, next);
166
+ if (value && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype)
167
+ return wrap(value, next);
168
+ return value;
169
+ },
170
+ });
171
+ }
172
+ const providerClient = wrap(original, []);
173
+ providerCalls.set(providerClient, { original, base, transports });
174
+ return providerClient;
175
+ }
@@ -0,0 +1,238 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { createWriteStream } from "node:fs";
3
+ import { mkdir, readdir, readFile, rename, rm, rmdir, stat, writeFile } from "node:fs/promises";
4
+ import { createRequire } from "node:module";
5
+ import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
6
+ import { Readable } from "node:stream";
7
+ import { pipeline } from "node:stream/promises";
8
+ import { createGzip } from "node:zlib";
9
+ import { scrubHar } from "@apicity/har";
10
+ import { captureFilePath, readCaptureFile, readHarIndex } from "./har-index.js";
11
+ import { installedPackageVersion } from "./package-version.js";
12
+ const BLOCK = 512;
13
+ const OWN_VERSION = createRequire(import.meta.url)("../package.json").version;
14
+ function isHarArchivePath(path) {
15
+ return /\.(?:tar\.gz|tgz)$/iu.test(path);
16
+ }
17
+ export function selectHarRows(rows, selection) {
18
+ const seen = new Set();
19
+ return rows.filter((row) => {
20
+ if (seen.has(row.file))
21
+ return false;
22
+ seen.add(row.file);
23
+ return (selection.since == null || Date.parse(row.ts) >= selection.since)
24
+ && (!selection.failures || row.outcome !== "ok")
25
+ && (!selection.providers?.length || selection.providers.includes(row.provider));
26
+ });
27
+ }
28
+ function emptyOutcomes() {
29
+ return { ok: 0, provider_error: 0, local_error: 0, network_error: 0 };
30
+ }
31
+ function sha256(bytes) {
32
+ return createHash("sha256").update(bytes).digest("hex");
33
+ }
34
+ function verifiedMeta(row, har) {
35
+ const meta = har.log._apicity;
36
+ return {
37
+ ts: meta?.startedAt ?? row.ts,
38
+ callId: meta?.callId ?? row.callId,
39
+ provider: meta?.provider ?? row.provider,
40
+ endpoint: meta?.endpoint ?? row.endpoint,
41
+ outcome: meta?.outcome ?? row.outcome,
42
+ status: meta?.status ?? row.status ?? null,
43
+ context: meta?.context ?? row.context ?? null,
44
+ };
45
+ }
46
+ function manifestFor(captures, options) {
47
+ const outcomes = emptyOutcomes();
48
+ const providers = new Map();
49
+ const endpoints = new Map();
50
+ for (const { meta } of captures) {
51
+ outcomes[meta.outcome] += 1;
52
+ const provider = providers.get(meta.provider) ?? { provider: meta.provider, calls: 0, outcomes: emptyOutcomes() };
53
+ provider.calls += 1;
54
+ provider.outcomes[meta.outcome] += 1;
55
+ providers.set(meta.provider, provider);
56
+ const key = JSON.stringify([meta.provider, meta.endpoint]);
57
+ const endpoint = endpoints.get(key) ?? { provider: meta.provider, endpoint: meta.endpoint, calls: 0, outcomes: emptyOutcomes() };
58
+ endpoint.calls += 1;
59
+ endpoint.outcomes[meta.outcome] += 1;
60
+ endpoints.set(key, endpoint);
61
+ }
62
+ const times = captures.map(({ meta }) => meta.ts).sort();
63
+ return {
64
+ format: "videobook-har-export",
65
+ version: 1,
66
+ createdAt: (options.now ?? new Date()).toISOString(),
67
+ videobook: OWN_VERSION,
68
+ apicity: {
69
+ har: installedPackageVersion("@apicity/har"),
70
+ captured: [...new Set(captures.map((capture) => capture.apicityVersion))].sort(),
71
+ },
72
+ filters: {
73
+ since: options.since == null ? null : new Date(options.since).toISOString(),
74
+ failures: Boolean(options.failures),
75
+ providers: options.providers?.length ? [...options.providers] : null,
76
+ },
77
+ files: captures.length,
78
+ dateRange: { first: times[0] ?? null, last: times.at(-1) ?? null },
79
+ outcomes,
80
+ providers: [...providers.values()].sort((a, b) => a.provider.localeCompare(b.provider)),
81
+ endpoints: [...endpoints.values()].sort((a, b) => a.provider.localeCompare(b.provider) || a.endpoint.localeCompare(b.endpoint)),
82
+ };
83
+ }
84
+ function indexLines(captures) {
85
+ return captures.map(({ file, meta }) => `${JSON.stringify({
86
+ ts: meta.ts,
87
+ callId: meta.callId,
88
+ provider: meta.provider,
89
+ endpoint: meta.endpoint,
90
+ outcome: meta.outcome,
91
+ ...(meta.status === null ? {} : { status: meta.status }),
92
+ file,
93
+ ...(meta.context ? { context: meta.context } : {}),
94
+ })}\n`).join("");
95
+ }
96
+ async function* exportEntries(captures, manifest) {
97
+ yield { path: "manifest.json", bytes: Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`) };
98
+ yield { path: "index.jsonl", bytes: Buffer.from(indexLines(captures)) };
99
+ for (const capture of captures) {
100
+ const bytes = await readFile(capture.path);
101
+ if (sha256(bytes) !== capture.sha256)
102
+ throw new Error(`${capture.path} changed during the export; run it again`);
103
+ yield { path: capture.file, bytes };
104
+ }
105
+ }
106
+ function octal(value, width) {
107
+ return `${value.toString(8).padStart(width - 1, "0")}\0`;
108
+ }
109
+ function tarHeader(name, size, type, mtime) {
110
+ const header = Buffer.alloc(BLOCK);
111
+ header.write(name, 0, 100, "utf8");
112
+ header.write(octal(0o600, 8), 100, "ascii");
113
+ header.write(octal(0, 8), 108, "ascii");
114
+ header.write(octal(0, 8), 116, "ascii");
115
+ header.write(octal(size, 12), 124, "ascii");
116
+ header.write(octal(Math.floor(mtime / 1000), 12), 136, "ascii");
117
+ header.write(" ".repeat(8), 148, "ascii");
118
+ header.write(type, 156, "ascii");
119
+ header.write("ustar\0", 257, "ascii");
120
+ header.write("00", 263, "ascii");
121
+ const checksum = header.reduce((sum, byte) => sum + byte, 0);
122
+ header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
123
+ return header;
124
+ }
125
+ function paxPath(path) {
126
+ const body = ` path=${path}\n`;
127
+ const bytes = Buffer.byteLength(body);
128
+ let length = bytes + String(bytes).length;
129
+ while (length !== bytes + String(length).length)
130
+ length = bytes + String(length).length;
131
+ return Buffer.from(`${length}${body}`);
132
+ }
133
+ function padding(size) {
134
+ return Buffer.alloc((BLOCK - (size % BLOCK)) % BLOCK);
135
+ }
136
+ async function* tarBlocks(entries, root, mtime) {
137
+ for await (const entry of entries) {
138
+ const name = `${root}/${entry.path}`;
139
+ if (Buffer.byteLength(name) > 100) {
140
+ const pax = paxPath(name);
141
+ yield tarHeader("PaxHeader", pax.length, "x", mtime);
142
+ yield pax;
143
+ yield padding(pax.length);
144
+ }
145
+ yield tarHeader(name, entry.bytes.length, "0", mtime);
146
+ yield entry.bytes;
147
+ yield padding(entry.bytes.length);
148
+ }
149
+ yield Buffer.alloc(BLOCK * 2);
150
+ }
151
+ async function writeArchive(out, entries, mtime) {
152
+ const root = basename(out).replace(/\.(?:tar\.gz|tgz)$/iu, "") || "videobook-har";
153
+ const partial = join(dirname(out), `.${basename(out)}.${randomUUID()}.partial`);
154
+ try {
155
+ await pipeline(Readable.from(tarBlocks(entries, root, mtime)), createGzip(), createWriteStream(partial, { flags: "wx", mode: 0o600 }));
156
+ await rename(partial, out);
157
+ }
158
+ catch (error) {
159
+ await rm(partial, { force: true });
160
+ throw error;
161
+ }
162
+ }
163
+ async function writeDirectory(out, entries) {
164
+ const partial = join(dirname(out), `.${basename(out)}.${randomUUID()}.partial`);
165
+ try {
166
+ await mkdir(partial, { mode: 0o700 });
167
+ for await (const entry of entries) {
168
+ const target = join(partial, ...entry.path.split("/"));
169
+ await mkdir(dirname(target), { recursive: true, mode: 0o700 });
170
+ await writeFile(target, entry.bytes, { flag: "wx", mode: 0o600 });
171
+ }
172
+ await rmdir(out).catch((error) => {
173
+ if (error.code !== "ENOENT")
174
+ throw error;
175
+ });
176
+ await rename(partial, out);
177
+ }
178
+ catch (error) {
179
+ await rm(partial, { recursive: true, force: true });
180
+ throw error;
181
+ }
182
+ }
183
+ async function outputTaken(out, archive) {
184
+ try {
185
+ const info = await stat(out);
186
+ return archive || !info.isDirectory() || (await readdir(out)).length > 0;
187
+ }
188
+ catch (error) {
189
+ if (error.code === "ENOENT")
190
+ return false;
191
+ throw error;
192
+ }
193
+ }
194
+ function inside(parent, child) {
195
+ const location = relative(resolve(parent), resolve(child));
196
+ return location === "" || (location !== ".." && !location.startsWith(`..${sep}`) && !isAbsolute(location));
197
+ }
198
+ export async function exportHarCaptures(options) {
199
+ const archive = isHarArchivePath(options.out);
200
+ if (inside(options.dir, options.out))
201
+ return { status: "out-inside-captures", out: options.out };
202
+ if (await outputTaken(options.out, archive))
203
+ return { status: "out-exists", out: options.out };
204
+ const verified = [];
205
+ const skipped = [];
206
+ const needsScrub = [];
207
+ for (const row of selectHarRows(await readHarIndex(options.dir), options)) {
208
+ const capture = await readCaptureFile(options.dir, row.file);
209
+ if (!capture) {
210
+ skipped.push(captureFilePath(options.dir, row.file) ?? row.file);
211
+ continue;
212
+ }
213
+ const before = JSON.stringify(capture.har);
214
+ if (JSON.stringify(scrubHar(structuredClone(capture.har))) !== before) {
215
+ needsScrub.push(capture.path);
216
+ continue;
217
+ }
218
+ verified.push({
219
+ path: capture.path,
220
+ file: row.file.split(/[\\/]/u).join("/"),
221
+ sha256: sha256(capture.bytes),
222
+ meta: verifiedMeta(row, capture.har),
223
+ apicityVersion: capture.har.log.creator?.version ?? "unknown",
224
+ });
225
+ }
226
+ if (needsScrub.length > 0)
227
+ return { status: "needs-scrub", files: needsScrub };
228
+ if (verified.length === 0)
229
+ return { status: "empty", skipped };
230
+ const manifest = manifestFor(verified, options);
231
+ const entries = exportEntries(verified, manifest);
232
+ await mkdir(dirname(options.out), { recursive: true });
233
+ if (archive)
234
+ await writeArchive(options.out, entries, (options.now ?? new Date()).getTime());
235
+ else
236
+ await writeDirectory(options.out, entries);
237
+ return { status: "exported", out: options.out, archive, manifest, skipped };
238
+ }