videobook 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +6 -2
- package/dist/archive-b2.js +1 -1
- package/dist/cli/args.js +42 -3
- package/dist/cli/config-command.js +17 -1
- package/dist/cli/doctor-command.js +15 -0
- package/dist/cli/har-command.js +86 -0
- package/dist/cli/main.js +13 -1
- package/dist/content-store.js +2 -1
- package/dist/diagnostics.js +1 -1
- package/dist/env-secrets.js +1 -0
- package/dist/generation-har.js +106 -0
- package/dist/har-capture.js +142 -0
- package/dist/har-export.js +238 -0
- package/dist/har-index.js +98 -0
- package/dist/har-retention.js +73 -0
- package/dist/har-settings.js +27 -0
- package/dist/jobs/_alibaba-upload.js +2 -4
- package/dist/jobs/_fal-upload.js +2 -4
- package/dist/jobs/_image-edit-adapter.js +1 -1
- package/dist/jobs/_kie-upload.js +2 -4
- package/dist/jobs/complete-provider-task.js +5 -2
- package/dist/jobs/generate-media.js +9 -1
- package/dist/jobs/image-subjects.js +1 -1
- package/dist/jobs/kie-paygate.js +10 -3
- package/dist/jobs/run-notebook.js +4 -1
- package/dist/jobs/xai-paygate.js +5 -2
- package/dist/media/adapters/kie.js +2 -1
- package/dist/media/cost.js +1 -1
- package/dist/package-version.js +18 -0
- package/dist/posts/dropbox-auth.js +1 -1
- package/dist/posts/post-metrics.js +2 -1
- package/dist/provider-usage.js +1 -18
- package/dist/queue.js +12 -1
- package/dist/serve.js +2 -0
- package/dist/tools/api-status.js +1 -1
- package/dist/tools/generate-media.js +3 -0
- package/dist/tools/generation-payload.js +17 -2
- package/dist/tools/get-generation-har.js +43 -0
- package/dist/tools/index.js +4 -0
- package/dist/tools/list-cell-generations.js +19 -0
- package/dist/tools/reconcile-project-work.js +18 -12
- package/dist/tools/set-har-capture.js +30 -0
- package/dist/upload.js +2 -1
- package/dist/usage-providers.js +48 -10
- package/dist/workspace-action-groups.js +2 -2
- package/dist/x/account.js +1 -1
- package/dist/x/authenticated-handle.js +1 -1
- package/dist/x/oauth-flow.js +1 -1
- package/dist/x/post-to-x.js +2 -1
- package/dist/x/x-tokens.js +2 -1
- package/package.json +15 -14
- package/ui/assets/{alive-captions-duTKsyll.js → alive-captions-Cj8C1tCN.js} +1 -1
- package/ui/assets/{confirm-bodies--2sc9DbH.js → confirm-bodies-BxWXwDf0.js} +1 -1
- package/ui/assets/{fullscreen-COrmiHPn.js → fullscreen-DyEncSdm.js} +1 -1
- package/ui/assets/{goal-ui-D5Xsm2EY.js → goal-ui-BdFeoSWh.js} +2 -2
- package/ui/assets/{index-DEKl00St.js → index-QZ8lbemN.js} +9 -9
- package/ui/assets/inspector-BC6xWkJU.js +1 -0
- package/ui/assets/{jobs-ui-DdP2Re41.js → jobs-ui-BudI4w0r.js} +1 -1
- package/ui/assets/{music-video-B709JZqa.js → music-video-CT4C55ok.js} +1 -1
- package/ui/assets/ui-mq0uIS5P.css +1 -0
- package/ui/assets/ui-z5r-p0K9.js +2 -0
- package/ui/assets/{vendor-BFupLz37.js → vendor-D8XcMav4.js} +2 -2
- package/ui/assets/{vendor-flow-CKiuzaeM.js → vendor-flow-nqTFQmTh.js} +1 -1
- package/ui/assets/vendor-mcp-D_C3EZPv.js +60 -0
- package/ui/assets/videobook-CsW453sb.js +21 -0
- package/ui/index.html +12 -12
- package/ui/assets/inspector-EBp3IhvH.js +0 -1
- package/ui/assets/ui-Oirr9VOf.css +0 -1
- package/ui/assets/ui-jRPWsNUw.js +0 -2
- package/ui/assets/vendor-mcp-DQEeVZH0.js +0 -67
- package/ui/assets/videobook-0DOLlhG2.js +0 -20
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,29 @@ and the package uses [Semantic Versioning](https://semver.org/).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.3.0] - 2026-10-24
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Opt-in local provider HAR diagnostics with Settings, MCP and CLI controls, concurrent book attribution and retention limits.
|
|
14
|
+
Captured SDK calls are named by their apicity catalog dot path, such as
|
|
15
|
+
`api.v1.jobs.createTask`, so `apicity har report` attributes them to their
|
|
16
|
+
endpoints.
|
|
17
|
+
- The Request payload panel of a queued, generating or failed cell downloads
|
|
18
|
+
the attempt's scrubbed captures as one HAR file, or shows "Capture off" with
|
|
19
|
+
a link to Settings. `inspect_generation_request` reports whether an asset has
|
|
20
|
+
a capture, with its path and outcome; the new `get_generation_har` tool
|
|
21
|
+
returns the HAR for an asset or attempt id.
|
|
22
|
+
- `videobook har list` and `videobook har export --out <dir|file.tar.gz>`,
|
|
23
|
+
both with `--since`, `--failures` and `--provider`. Export re-runs the
|
|
24
|
+
`@apicity/har` scrubber on every capture and refuses, naming the file, when
|
|
25
|
+
scrubbing would change it. Otherwise it writes the captures with an index
|
|
26
|
+
and a manifest and prints the `apicity har report` hand-off.
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
|
|
30
|
+
- Upgrade all Apicity dependencies to 0.18.0.
|
|
31
|
+
|
|
9
32
|
## [0.2.5] - 2026-10-24
|
|
10
33
|
|
|
11
34
|
### Added
|
package/README.md
CHANGED
|
@@ -65,6 +65,9 @@ videobook gc-books # reclaim catalog storage in every book
|
|
|
65
65
|
videobook doctor # check Node, binaries, native modules and keys
|
|
66
66
|
videobook config # show where each provider key comes from
|
|
67
67
|
videobook config set FAL_API_KEY op://Private/FAL_API_KEY/password
|
|
68
|
+
videobook config set har_capture failures # opt in to provider HAR capture
|
|
69
|
+
videobook har list --failures # captured provider calls, newest first
|
|
70
|
+
videobook har export --out captures.tar.gz # scrub-checked captures for apicity
|
|
68
71
|
videobook setup 1password --op-vault Private
|
|
69
72
|
videobook --version
|
|
70
73
|
```
|
|
@@ -104,10 +107,11 @@ run while a server owns the library.
|
|
|
104
107
|
Installed data lives in `~/.videobook`:
|
|
105
108
|
|
|
106
109
|
- `books/`, the library of books
|
|
107
|
-
- `app-state.json`, connected X and Dropbox accounts
|
|
110
|
+
- `app-state.json`, connected X and Dropbox accounts and the HAR capture setting
|
|
108
111
|
- `.env`, provider keys, also editable from Settings in the web UI
|
|
112
|
+
- `har/`, opt-in provider HAR captures (off by default) with an `index.jsonl`
|
|
109
113
|
|
|
110
|
-
`VIDEOBOOK_HOME` or `--home` moves all
|
|
114
|
+
`VIDEOBOOK_HOME` or `--home` moves all four. `VIDEOBOOK_LIBRARY_DIR`,
|
|
111
115
|
`VIDEOBOOK_APP_STATE_PATH` and `VIDEOBOOK_ENV_PATH` move one each; `--home`
|
|
112
116
|
overrides any of them inherited from your shell. Set `VIDEOBOOK_HOME` and
|
|
113
117
|
`VIDEOBOOK_ENV_PATH` in the environment, not in `.env`, because they decide
|
package/dist/archive-b2.js
CHANGED
package/dist/cli/args.js
CHANGED
|
@@ -7,9 +7,10 @@ export class CliUsageError extends Error {
|
|
|
7
7
|
}
|
|
8
8
|
}
|
|
9
9
|
const LOG_LEVELS = new Set(["silent", "error", "warn", "info", "debug"]);
|
|
10
|
-
const COMMANDS = new Set(["serve", "mcp", "run", "tools", "gc-books", "doctor", "setup", "config", "version", "help"]);
|
|
10
|
+
const COMMANDS = new Set(["serve", "mcp", "run", "tools", "gc-books", "doctor", "setup", "config", "har", "version", "help"]);
|
|
11
11
|
const CONFIG_ACTIONS = new Set(["list", "get", "set", "unset"]);
|
|
12
|
-
const
|
|
12
|
+
const HAR_ACTIONS = new Set(["list", "export"]);
|
|
13
|
+
const POSITIONAL_COMMANDS = new Set(["run", "setup", "config", "har"]);
|
|
13
14
|
const DEFAULT_RUN_TIMEOUT_SECONDS = 600;
|
|
14
15
|
const GLOBAL_OPTIONS = {
|
|
15
16
|
port: { type: "string" },
|
|
@@ -28,6 +29,7 @@ const COMMAND_OPTIONS = {
|
|
|
28
29
|
doctor: { json: { type: "boolean" } },
|
|
29
30
|
setup: { "op-vault": { type: "string" }, "op-token": { type: "string" }, "no-verify": { type: "boolean" }, remove: { type: "boolean" } },
|
|
30
31
|
config: { json: { type: "boolean" } },
|
|
32
|
+
har: { out: { type: "string" }, since: { type: "string" }, failures: { type: "boolean" }, provider: { type: "string" }, json: { type: "boolean" } },
|
|
31
33
|
};
|
|
32
34
|
function splitCommand(argv) {
|
|
33
35
|
const [first, ...rest] = argv;
|
|
@@ -147,6 +149,35 @@ function buildConfig(values, positionals, flags) {
|
|
|
147
149
|
flags: { ...flags, json: values.json === true },
|
|
148
150
|
};
|
|
149
151
|
}
|
|
152
|
+
function buildHar(values, positionals, flags) {
|
|
153
|
+
const [action, ...extra] = positionals;
|
|
154
|
+
if (!action || !HAR_ACTIONS.has(action) || extra.length > 0) {
|
|
155
|
+
throw new CliUsageError("Usage: videobook har list [--since <date>] [--failures] [--provider <p>] [--json] | har export --out <dir|file.tar.gz> [--since <date>] [--failures] [--provider <p>]");
|
|
156
|
+
}
|
|
157
|
+
const out = stringValue(values.out);
|
|
158
|
+
if (action === "export" && out === undefined)
|
|
159
|
+
throw new CliUsageError("har export needs --out <dir|file.tar.gz>");
|
|
160
|
+
if (action === "list" && out !== undefined)
|
|
161
|
+
throw new CliUsageError("--out applies to har export only");
|
|
162
|
+
const since = stringValue(values.since);
|
|
163
|
+
if (since !== undefined && !Number.isFinite(Date.parse(since)))
|
|
164
|
+
throw new CliUsageError(`--since needs an ISO date or time, got "${since}".`);
|
|
165
|
+
const providers = typeof values.provider === "string"
|
|
166
|
+
? values.provider.split(",").map((provider) => provider.trim()).filter(Boolean)
|
|
167
|
+
: [];
|
|
168
|
+
return {
|
|
169
|
+
command: "har",
|
|
170
|
+
action: action,
|
|
171
|
+
flags: {
|
|
172
|
+
...flags,
|
|
173
|
+
...(out === undefined ? {} : { out }),
|
|
174
|
+
...(since === undefined ? {} : { since }),
|
|
175
|
+
failures: values.failures === true,
|
|
176
|
+
providers,
|
|
177
|
+
json: values.json === true,
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
|
150
181
|
function buildCommand(command, values, positionals) {
|
|
151
182
|
const flags = globalFlags(values);
|
|
152
183
|
switch (command) {
|
|
@@ -164,6 +195,8 @@ function buildCommand(command, values, positionals) {
|
|
|
164
195
|
return buildSetup(values, positionals, flags);
|
|
165
196
|
case "config":
|
|
166
197
|
return buildConfig(values, positionals, flags);
|
|
198
|
+
case "har":
|
|
199
|
+
return buildHar(values, positionals, flags);
|
|
167
200
|
default:
|
|
168
201
|
return { command, flags };
|
|
169
202
|
}
|
|
@@ -215,6 +248,8 @@ export function helpText() {
|
|
|
215
248
|
" gc-books Reclaim catalog storage in every book",
|
|
216
249
|
" doctor Check Node, media binaries, native modules and keys",
|
|
217
250
|
" config [action] list | get <KEY> | set <KEY> <VALUE> | unset <KEY> in the data-home .env",
|
|
251
|
+
" har list List provider HAR captures, newest first",
|
|
252
|
+
" har export Copy scrub-checked captures and a manifest to share with apicity (--out)",
|
|
218
253
|
" setup 1password Read keys from a 1Password vault (--op-vault, --op-token)",
|
|
219
254
|
" version Print versions",
|
|
220
255
|
" help Print this help",
|
|
@@ -228,7 +263,11 @@ export function helpText() {
|
|
|
228
263
|
" --open serve: open the web UI in a browser",
|
|
229
264
|
" --no-wait run: do not wait for queued jobs",
|
|
230
265
|
" --timeout <s> run: seconds to wait for jobs (default 600)",
|
|
231
|
-
" --json doctor, config: print machine-readable results",
|
|
266
|
+
" --json doctor, config, har: print machine-readable results",
|
|
267
|
+
" --out <path> har export: a new directory, or a .tar.gz/.tgz file",
|
|
268
|
+
" --since <date> har: only calls at or after this ISO date or time",
|
|
269
|
+
" --failures har: only failed calls",
|
|
270
|
+
" --provider <p> har: only these providers, comma separated",
|
|
232
271
|
" --op-vault <name> setup 1password: vault whose items are named after the keys",
|
|
233
272
|
" --op-token <tok> setup 1password: service-account token, best as env:VAR",
|
|
234
273
|
" --no-verify setup 1password: skip the op item list check",
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readHarSettings, writeHarSettings, harDirectory } from "../har-settings.js";
|
|
1
2
|
import { hasControlCharacters, readEnvValues, updateEnvFile } from "../env-file.js";
|
|
2
3
|
import { MANAGED_ENV_KEYS, getDefaultSecretStore, isSecretEnvKey, maskEnvValue, } from "../env-secrets.js";
|
|
3
4
|
import { OP_TOKEN_SETTING, OP_VAULT_SETTING, isOpReference, vaultReference } from "../one-password.js";
|
|
@@ -83,10 +84,25 @@ async function unsetKey(key, envPath) {
|
|
|
83
84
|
return 0;
|
|
84
85
|
}
|
|
85
86
|
export async function runConfigCommand(command) {
|
|
87
|
+
if (command.key === "har_capture") {
|
|
88
|
+
try {
|
|
89
|
+
const current = await readHarSettings();
|
|
90
|
+
const settings = command.action === "set" || command.action === "unset"
|
|
91
|
+
? await writeHarSettings({ ...await readHarSettings(undefined, {}), mode: command.action === "unset" ? "off" : command.value })
|
|
92
|
+
: current;
|
|
93
|
+
process.stdout.write(`${JSON.stringify({ harCapture: settings, dir: harDirectory(), overridden: Boolean(process.env.VIDEOBOOK_HAR_CAPTURE?.trim()) })}\n`);
|
|
94
|
+
return 0;
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
98
|
+
return 1;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
86
101
|
const envPath = getDefaultSecretStore().envPath();
|
|
87
102
|
if (command.action === "list") {
|
|
88
103
|
const file = readEnvValues(envPath);
|
|
89
|
-
|
|
104
|
+
const har = await readHarSettings();
|
|
105
|
+
printRows([...CONFIG_KEYS.map((key) => configRow(key, file)), { key: "har_capture", source: process.env.VIDEOBOOK_HAR_CAPTURE?.trim() ? "environment" : "app state", value: har.mode }], file, command.flags.json, envPath);
|
|
90
106
|
return 0;
|
|
91
107
|
}
|
|
92
108
|
const key = command.key ?? "";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { harDirectory, readHarSettings } from "../har-settings.js";
|
|
1
2
|
import { execFile } from "node:child_process";
|
|
2
3
|
import { access, constants, readdir } from "node:fs/promises";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
@@ -72,6 +73,19 @@ async function dataHomeCheck() {
|
|
|
72
73
|
});
|
|
73
74
|
}
|
|
74
75
|
}
|
|
76
|
+
async function harCaptureCheck() {
|
|
77
|
+
const settings = await readHarSettings();
|
|
78
|
+
const dir = harDirectory();
|
|
79
|
+
let writable = true;
|
|
80
|
+
try {
|
|
81
|
+
await access(await nearestExisting(dir), constants.W_OK);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
writable = false;
|
|
85
|
+
}
|
|
86
|
+
return check({ id: "har-capture", label: "HAR capture", status: writable || settings.mode === "off" ? "ok" : "warn",
|
|
87
|
+
detail: `${settings.mode}; ${dir}; writable: ${writable ? "yes" : "no"}`, required: false });
|
|
88
|
+
}
|
|
75
89
|
async function libraryCheck() {
|
|
76
90
|
const libraryDir = bookManager.rootDir;
|
|
77
91
|
try {
|
|
@@ -255,6 +269,7 @@ async function collectDoctorChecks() {
|
|
|
255
269
|
return [
|
|
256
270
|
nodeCheck(),
|
|
257
271
|
await dataHomeCheck(),
|
|
272
|
+
await harCaptureCheck(),
|
|
258
273
|
await libraryCheck(),
|
|
259
274
|
await binaryCheck("ffmpeg", "ffmpeg", FFMPEG, bundledPath("ffmpeg-static"), "VIDEOBOOK_FFMPEG_PATH"),
|
|
260
275
|
await binaryCheck("ffprobe", "ffprobe", FFPROBE, bundledFfprobe(), "VIDEOBOOK_FFPROBE_PATH"),
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { dirname, resolve } from "node:path";
|
|
2
|
+
import { exportHarCaptures, selectHarRows } from "../har-export.js";
|
|
3
|
+
import { readHarIndex } from "../har-index.js";
|
|
4
|
+
import { harDirectory } from "../har-settings.js";
|
|
5
|
+
const LISTED_SKIPS = 10;
|
|
6
|
+
function selection(flags) {
|
|
7
|
+
return {
|
|
8
|
+
since: flags.since === undefined ? null : Date.parse(flags.since),
|
|
9
|
+
failures: flags.failures,
|
|
10
|
+
providers: flags.providers,
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
async function listCaptures(flags) {
|
|
14
|
+
const dir = harDirectory();
|
|
15
|
+
const rows = selectHarRows(await readHarIndex(dir), selection(flags)).sort((a, b) => b.ts.localeCompare(a.ts));
|
|
16
|
+
if (flags.json) {
|
|
17
|
+
process.stdout.write(`${JSON.stringify({ dir, captures: rows }, null, 2)}\n`);
|
|
18
|
+
return 0;
|
|
19
|
+
}
|
|
20
|
+
const lines = rows.map((row) => [row.ts, row.provider, row.endpoint, row.outcome, row.status ?? "-", row.file].join("\t"));
|
|
21
|
+
process.stdout.write(`${["ts\tprovider\tendpoint\toutcome\tstatus\tfile", ...lines].join("\n")}\n`);
|
|
22
|
+
if (rows.length === 0)
|
|
23
|
+
process.stderr.write(`No matching captures in ${dir}\n`);
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
function outcomeSummary(outcomes) {
|
|
27
|
+
return Object.entries(outcomes).filter(([, count]) => count > 0).map(([outcome, count]) => `${outcome} ${count}`).join(", ");
|
|
28
|
+
}
|
|
29
|
+
function shellPath(path) {
|
|
30
|
+
return /^[\w./:@%+=,-]+$/u.test(path) ? path : `'${path.replaceAll("'", "'\\''")}'`;
|
|
31
|
+
}
|
|
32
|
+
function handoff(out, archive, manifest) {
|
|
33
|
+
const target = shellPath(archive ? out.replace(/\.(?:tar\.gz|tgz)$/iu, "") : out);
|
|
34
|
+
return [
|
|
35
|
+
`Exported ${manifest.files} capture${manifest.files === 1 ? "" : "s"} (${manifest.dateRange.first} to ${manifest.dateRange.last}) to ${out}`,
|
|
36
|
+
...manifest.providers.map((provider) => ` ${provider.provider}: ${provider.calls} (${outcomeSummary(provider.outcomes)})`),
|
|
37
|
+
`Every file passed the @apicity/har ${manifest.apicity.har} scrub check. The captures still contain prompts and media URLs; review them before sharing.`,
|
|
38
|
+
"",
|
|
39
|
+
"Hand-off to an apicity maintainer:",
|
|
40
|
+
...(archive ? [` tar -xzf ${shellPath(out)} -C ${shellPath(dirname(out))}`] : []),
|
|
41
|
+
` apicity har report ${target} > report.md`,
|
|
42
|
+
" apicity har scrub --check <capture.har>",
|
|
43
|
+
" pnpm run har:promote -- <capture.har> --name <provider>/<test-name> --dry-run (in the apicity repository, for a successful call)",
|
|
44
|
+
"",
|
|
45
|
+
].join("\n");
|
|
46
|
+
}
|
|
47
|
+
function skippedWarning(skipped) {
|
|
48
|
+
if (skipped.length === 0)
|
|
49
|
+
return "";
|
|
50
|
+
const listed = skipped.slice(0, LISTED_SKIPS).map((file) => ` ${file}\n`).join("");
|
|
51
|
+
const more = skipped.length > LISTED_SKIPS ? ` and ${skipped.length - LISTED_SKIPS} more\n` : "";
|
|
52
|
+
return `Skipped ${skipped.length} indexed capture${skipped.length === 1 ? "" : "s"} that are missing or unreadable:\n${listed}${more}`;
|
|
53
|
+
}
|
|
54
|
+
async function exportCaptures(flags) {
|
|
55
|
+
const dir = harDirectory();
|
|
56
|
+
const out = resolve(process.cwd(), flags.out ?? "");
|
|
57
|
+
const result = await exportHarCaptures({ dir, out, ...selection(flags) });
|
|
58
|
+
switch (result.status) {
|
|
59
|
+
case "out-inside-captures":
|
|
60
|
+
process.stderr.write(`--out must be outside the capture directory ${dir}\n`);
|
|
61
|
+
return 2;
|
|
62
|
+
case "out-exists":
|
|
63
|
+
process.stderr.write(`${out} already exists; choose a new file or an empty directory\n`);
|
|
64
|
+
return 1;
|
|
65
|
+
case "needs-scrub":
|
|
66
|
+
process.stderr.write([
|
|
67
|
+
"Refusing to export: @apicity/har scrubHar would change these captures:",
|
|
68
|
+
...result.files.map((file) => ` ${file}`),
|
|
69
|
+
"A scrub rule is missing upstream. Report it to apicity instead of editing the captures. Nothing was written.",
|
|
70
|
+
"",
|
|
71
|
+
].join("\n"));
|
|
72
|
+
return 1;
|
|
73
|
+
case "empty":
|
|
74
|
+
process.stderr.write(`${skippedWarning(result.skipped)}No captures in ${dir} match the filters. Nothing was written.\n`);
|
|
75
|
+
return 1;
|
|
76
|
+
case "exported":
|
|
77
|
+
process.stderr.write(skippedWarning(result.skipped));
|
|
78
|
+
process.stdout.write(flags.json
|
|
79
|
+
? `${JSON.stringify({ out: result.out, archive: result.archive, manifest: result.manifest, skipped: result.skipped }, null, 2)}\n`
|
|
80
|
+
: handoff(result.out, result.archive, result.manifest));
|
|
81
|
+
return 0;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
export async function runHarCommand(command) {
|
|
85
|
+
return command.action === "export" ? exportCaptures(command.flags) : listCaptures(command.flags);
|
|
86
|
+
}
|
package/dist/cli/main.js
CHANGED
|
@@ -54,6 +54,10 @@ async function runCommand(parsed) {
|
|
|
54
54
|
const { runConfigCommand } = await import("./config-command.js");
|
|
55
55
|
return runConfigCommand(parsed);
|
|
56
56
|
}
|
|
57
|
+
case "har": {
|
|
58
|
+
const { runHarCommand } = await import("./har-command.js");
|
|
59
|
+
return runHarCommand(parsed);
|
|
60
|
+
}
|
|
57
61
|
}
|
|
58
62
|
}
|
|
59
63
|
async function loadDotenv() {
|
|
@@ -77,7 +81,15 @@ async function main(argv) {
|
|
|
77
81
|
redirectConsoleToStderr();
|
|
78
82
|
if (parsed.command !== "setup" && parsed.command !== "config")
|
|
79
83
|
await loadDotenv();
|
|
80
|
-
|
|
84
|
+
try {
|
|
85
|
+
return await runCommand(parsed);
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
if (parsed.command === "run" || parsed.command === "mcp") {
|
|
89
|
+
const { flushHarCaptures } = await import("../har-capture.js");
|
|
90
|
+
await flushHarCaptures();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
81
93
|
}
|
|
82
94
|
function flushed(stream) {
|
|
83
95
|
return new Promise((resolveFlush) => stream.write("", () => resolveFlush()));
|
package/dist/content-store.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
-
import {
|
|
2
|
+
import { createB2 } from "./usage-providers.js";
|
|
3
|
+
import { B2Error } from "@apicity/b2";
|
|
3
4
|
function createB2ContentStore(config) {
|
|
4
5
|
const provider = createB2({
|
|
5
6
|
accessKeyId: config.accessKeyId,
|
package/dist/diagnostics.js
CHANGED
|
@@ -4,7 +4,7 @@ const EVENTS = [
|
|
|
4
4
|
"book-open", "book-open-slow", "book-catalog-gc", "provider-poll-failed",
|
|
5
5
|
"ace-server-exited", "scheduled-post-failed", "scheduled-post-rearm-failed",
|
|
6
6
|
"generation-record-failed", "generation-update-failed", "generation-list-failed",
|
|
7
|
-
"provider-task-record-failed",
|
|
7
|
+
"provider-task-record-failed", "har-capture-failed",
|
|
8
8
|
"prompt-record-failed",
|
|
9
9
|
"waveform-empty", "waveform-failed", "heartbeat-failed", "subtitle-cache-failed",
|
|
10
10
|
"subtitle-translation-failed", "overlay-skipped", "kie-upload-fallback",
|
package/dist/env-secrets.js
CHANGED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { scrubHar } from "@apicity/har";
|
|
2
|
+
import { captureFileExists, captureFilePath, readCaptureFile, readHarIndex } from "./har-index.js";
|
|
3
|
+
import { harDirectory } from "./har-settings.js";
|
|
4
|
+
import { readProviderTaskRecord } from "./provider-task-record.js";
|
|
5
|
+
function bookIdOf(engine) {
|
|
6
|
+
try {
|
|
7
|
+
return engine.book.get().bookId;
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export async function captureKeys(engine, ids) {
|
|
14
|
+
const known = new Set([...ids].filter((id) => typeof id === "string" && id.length > 0));
|
|
15
|
+
const taskIds = new Set();
|
|
16
|
+
for (const id of known) {
|
|
17
|
+
const task = await readProviderTaskRecord(engine, id).catch(() => null);
|
|
18
|
+
if (task && task.taskId !== "queued")
|
|
19
|
+
taskIds.add(task.taskId);
|
|
20
|
+
}
|
|
21
|
+
return { bookId: bookIdOf(engine), ids: known, taskIds };
|
|
22
|
+
}
|
|
23
|
+
export async function generationCaptureKeys(engine, generationId) {
|
|
24
|
+
let generation = null;
|
|
25
|
+
try {
|
|
26
|
+
generation = engine.generations.read(generationId);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
generation = null;
|
|
30
|
+
}
|
|
31
|
+
const row = generation?.ok ? generation.value : null;
|
|
32
|
+
return captureKeys(engine, [generationId, row?.providerArtifactId, row?.outputArtifactId]);
|
|
33
|
+
}
|
|
34
|
+
function matchesCapture(row, keys) {
|
|
35
|
+
const context = row.context ?? {};
|
|
36
|
+
if (keys.bookId && typeof context.bookId === "string" && context.bookId !== keys.bookId)
|
|
37
|
+
return false;
|
|
38
|
+
return [context.artifactId, context.attemptId].some((id) => typeof id === "string" && keys.ids.has(id))
|
|
39
|
+
|| (typeof context.providerTaskId === "string" && keys.taskIds.has(context.providerTaskId));
|
|
40
|
+
}
|
|
41
|
+
function chronological(a, b) {
|
|
42
|
+
return a.ts.localeCompare(b.ts) || a.callId.localeCompare(b.callId);
|
|
43
|
+
}
|
|
44
|
+
async function attemptCaptureRows(keys, rows, dir = harDirectory()) {
|
|
45
|
+
const matched = (rows ?? await readHarIndex(dir)).filter((row) => matchesCapture(row, keys)).sort(chronological);
|
|
46
|
+
const present = await Promise.all(matched.map((row) => captureFileExists(dir, row.file)));
|
|
47
|
+
return matched.filter((_row, index) => present[index]);
|
|
48
|
+
}
|
|
49
|
+
export async function generationHarStatus(keys, mode, rows, dir = harDirectory()) {
|
|
50
|
+
const captures = await attemptCaptureRows(keys, rows, dir);
|
|
51
|
+
const primary = [...captures].reverse().find((row) => row.outcome !== "ok") ?? captures.at(-1);
|
|
52
|
+
return {
|
|
53
|
+
mode,
|
|
54
|
+
hasCapture: captures.length > 0,
|
|
55
|
+
captures: captures.length,
|
|
56
|
+
path: primary ? captureFilePath(dir, primary.file) : null,
|
|
57
|
+
outcome: primary?.outcome ?? null,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function combineCaptures(captures, generationId) {
|
|
61
|
+
const [first] = captures;
|
|
62
|
+
if (!first || captures.length === 1)
|
|
63
|
+
return first?.har ?? null;
|
|
64
|
+
const pages = captures.map(({ row, har }) => ({
|
|
65
|
+
startedDateTime: har.log._apicity?.startedAt ?? row.ts,
|
|
66
|
+
id: har.log._apicity?.callId ?? row.callId,
|
|
67
|
+
title: `${row.provider} ${row.endpoint} (${row.outcome})`,
|
|
68
|
+
pageTimings: {},
|
|
69
|
+
...(har.log._apicity ? { _apicity: har.log._apicity } : {}),
|
|
70
|
+
}));
|
|
71
|
+
return {
|
|
72
|
+
log: {
|
|
73
|
+
version: "1.2",
|
|
74
|
+
creator: first.har.log.creator,
|
|
75
|
+
comment: `videobook generation ${generationId}: ${captures.length} captured apicity calls, one page each`,
|
|
76
|
+
pages,
|
|
77
|
+
entries: captures.flatMap(({ row, har }) => har.log.entries.map((entry) => ({
|
|
78
|
+
...entry,
|
|
79
|
+
pageref: har.log._apicity?.callId ?? row.callId,
|
|
80
|
+
}))),
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export async function generationHar(keys, generationId, dir = harDirectory()) {
|
|
85
|
+
const loaded = [];
|
|
86
|
+
for (const row of await attemptCaptureRows(keys, undefined, dir)) {
|
|
87
|
+
const capture = await readCaptureFile(dir, row.file);
|
|
88
|
+
if (capture)
|
|
89
|
+
loaded.push({ row, path: capture.path, har: scrubHar(capture.har) });
|
|
90
|
+
}
|
|
91
|
+
const har = combineCaptures(loaded, generationId);
|
|
92
|
+
if (!har)
|
|
93
|
+
return null;
|
|
94
|
+
return {
|
|
95
|
+
captures: loaded.map(({ row, path }) => ({
|
|
96
|
+
path,
|
|
97
|
+
ts: row.ts,
|
|
98
|
+
callId: row.callId,
|
|
99
|
+
provider: row.provider,
|
|
100
|
+
endpoint: row.endpoint,
|
|
101
|
+
outcome: row.outcome,
|
|
102
|
+
status: row.status ?? null,
|
|
103
|
+
})),
|
|
104
|
+
har,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
3
|
+
import { createHarCapture } from "@apicity/har";
|
|
4
|
+
import { currentBookContext } from "./book-context.js";
|
|
5
|
+
import { harDirectory, readHarSettings } from "./har-settings.js";
|
|
6
|
+
import { logger } from "./logger.js";
|
|
7
|
+
const pending = new Set();
|
|
8
|
+
let maintenance = Promise.resolve();
|
|
9
|
+
export async function maintainHarCaptures(operation) {
|
|
10
|
+
if (pending.size)
|
|
11
|
+
return false;
|
|
12
|
+
const next = maintenance.then(operation);
|
|
13
|
+
maintenance = next.catch(() => undefined);
|
|
14
|
+
await next;
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
export async function flushHarCaptures() {
|
|
18
|
+
while (pending.size)
|
|
19
|
+
await Promise.all([...pending]);
|
|
20
|
+
}
|
|
21
|
+
function captureInput(value, seen = new WeakSet()) {
|
|
22
|
+
if (value instanceof Blob)
|
|
23
|
+
return { _file: { size: value.size, contentType: value.type } };
|
|
24
|
+
if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
|
|
25
|
+
const bytes = value instanceof ArrayBuffer ? new Uint8Array(value) : new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
|
|
26
|
+
return { _apicity_placeholder: { bytes: bytes.byteLength, sha256: createHash("sha256").update(bytes).digest("hex") } };
|
|
27
|
+
}
|
|
28
|
+
if (!value || typeof value !== "object")
|
|
29
|
+
return value;
|
|
30
|
+
if (seen.has(value))
|
|
31
|
+
return "[circular]";
|
|
32
|
+
seen.add(value);
|
|
33
|
+
if (value instanceof FormData)
|
|
34
|
+
return Object.fromEntries(Array.from(value.entries(), ([key, entry]) => [key, captureInput(entry, seen)]));
|
|
35
|
+
if (Array.isArray(value))
|
|
36
|
+
return value.map((entry) => captureInput(entry, seen));
|
|
37
|
+
if (Object.getPrototypeOf(value) !== Object.prototype)
|
|
38
|
+
return value;
|
|
39
|
+
return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, key === "otp" ? "***" : captureInput(entry, seen)]));
|
|
40
|
+
}
|
|
41
|
+
export async function withHarCapture(meta, operation, base = globalThis.fetch, context = {}) {
|
|
42
|
+
for (;;) {
|
|
43
|
+
const barrier = maintenance;
|
|
44
|
+
await barrier;
|
|
45
|
+
if (barrier === maintenance)
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
let finish;
|
|
49
|
+
const finished = new Promise((resolve) => { finish = resolve; });
|
|
50
|
+
pending.add(finished);
|
|
51
|
+
try {
|
|
52
|
+
return await runCapture(meta, operation, base, context);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
finish();
|
|
56
|
+
pending.delete(finished);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async function runCapture(meta, operation, base, context) {
|
|
60
|
+
const settings = await readHarSettings();
|
|
61
|
+
if (settings.mode === "off")
|
|
62
|
+
return operation(base);
|
|
63
|
+
let input;
|
|
64
|
+
try {
|
|
65
|
+
input = captureInput(meta.input);
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
logger.warn("har-capture-failed", { error });
|
|
69
|
+
return operation(base);
|
|
70
|
+
}
|
|
71
|
+
const book = currentBookContext();
|
|
72
|
+
const capture = createHarCapture({
|
|
73
|
+
dir: harDirectory(),
|
|
74
|
+
mode: settings.mode,
|
|
75
|
+
context: {
|
|
76
|
+
bookId: book?.bookId,
|
|
77
|
+
operation: book?.operation,
|
|
78
|
+
artifactId: book?.artifactId,
|
|
79
|
+
attemptId: book?.attemptId ?? book?.artifactId,
|
|
80
|
+
providerTaskId: book?.providerTaskId,
|
|
81
|
+
runId: book?.runId,
|
|
82
|
+
model: book?.model,
|
|
83
|
+
...context,
|
|
84
|
+
},
|
|
85
|
+
onError: (error) => logger.warn("har-capture-failed", { error }),
|
|
86
|
+
});
|
|
87
|
+
const host = { fetch: base };
|
|
88
|
+
capture.install(host);
|
|
89
|
+
let finish;
|
|
90
|
+
const finished = new Promise((resolve) => { finish = resolve; });
|
|
91
|
+
const done = finished.then(() => capture.close());
|
|
92
|
+
pending.add(done);
|
|
93
|
+
void done.finally(() => pending.delete(done));
|
|
94
|
+
try {
|
|
95
|
+
return await capture.call({ ...meta, input }, () => operation(host.fetch));
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
finish();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const HTTP_METHODS = new Set(["get", "post", "put", "patch", "delete", "head", "options"]);
|
|
102
|
+
function endpointDotPath(path) {
|
|
103
|
+
return (path.length > 1 && HTTP_METHODS.has(path[0] ?? "") ? path.slice(1) : path).join(".");
|
|
104
|
+
}
|
|
105
|
+
const providerCalls = new WeakMap();
|
|
106
|
+
export function withProviderCall(provider, meta, operation) {
|
|
107
|
+
const state = providerCalls.get(provider);
|
|
108
|
+
if (!state)
|
|
109
|
+
return operation(provider);
|
|
110
|
+
return withHarCapture(meta, (fetch) => state.transports.run(fetch, () => operation(state.original)), state.base);
|
|
111
|
+
}
|
|
112
|
+
export function capturedProvider(provider, factory, base = globalThis.fetch) {
|
|
113
|
+
const transports = new AsyncLocalStorage();
|
|
114
|
+
const original = factory((input, init) => {
|
|
115
|
+
const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
|
|
116
|
+
signal?.throwIfAborted();
|
|
117
|
+
return (transports.getStore() ?? base)(input, init);
|
|
118
|
+
});
|
|
119
|
+
function wrap(target, path) {
|
|
120
|
+
return new Proxy(target, {
|
|
121
|
+
apply(fn, receiver, args) {
|
|
122
|
+
return withHarCapture({ provider, endpoint: endpointDotPath(path), input: args.length === 1 ? args[0] : args }, (fetch) => transports.run(fetch, () => Reflect.apply(fn, receiver, args)), base);
|
|
123
|
+
},
|
|
124
|
+
get(object, property, receiver) {
|
|
125
|
+
const value = Reflect.get(object, property, receiver);
|
|
126
|
+
if (typeof target === "function" && ["bind", "call", "apply"].includes(String(property)) && typeof value === "function")
|
|
127
|
+
return value.bind(receiver);
|
|
128
|
+
if (typeof property !== "string" || ["presign", "schema", "example", "examples"].includes(property))
|
|
129
|
+
return value;
|
|
130
|
+
const next = [...path, property];
|
|
131
|
+
if (typeof value === "function")
|
|
132
|
+
return wrap(value, next);
|
|
133
|
+
if (value && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype)
|
|
134
|
+
return wrap(value, next);
|
|
135
|
+
return value;
|
|
136
|
+
},
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
const providerClient = wrap(original, []);
|
|
140
|
+
providerCalls.set(providerClient, { original, base, transports });
|
|
141
|
+
return providerClient;
|
|
142
|
+
}
|